From openmp-commits at lists.llvm.org  Sat Aug  1 22:00:51 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Sat, 01 Aug 2026 22:00:51 -0700 (PDT)
Subject: [Openmp-commits] [flang] [llvm] [openmp] [OpenMP][Flang] Add
 integer-kind wrappers to omp_lib (PR #213505)
Message-ID: <llvm.org/llvm/llvm-project/pull/213505@github.com>


https://github.com/shivaramaarao created https://github.com/llvm/llvm-project/pull/213505

Many OpenMP routines in omp_lib currently accept only integer(kind=omp_integer_kind) arguments, where omp_integer_kind maps to c_int. As a result, Flang's type checking rejects calls that pass integer(1), integer(2), or integer(8) arguments. To improve compatibility, wrapper procedures have been added in the omp_lib module for applicable routines, enabling support for integer(1), integer(2), integer(4), and integer(8) argument types.

Fixes #123948.

Assisted-by: Cursor

>From b5dc85377e472e601fee054c620944e4453489e2 Mon Sep 17 00:00:00 2001
From: Shivarama Rao <shivarama.rao at amd.com>
Date: Sun, 2 Aug 2026 10:20:39 +0530
Subject: [PATCH] [OpenMP][Flang] Add integer-kind wrappers to omp_lib

Many OpenMP routines in omp_lib only accept integer(kind=omp_integer_kind),
which is c_int. Calls with integer(1), integer(2), or integer(8) arguments
therefore fail Flang's type checking. These wrapper functions are added
to omp_lib module file to resolve this issue.

fixes issue #123948

Co-authored-by: Cursor <cursoragent at cursor.com>
---
 flang-rt/lib/runtime/CMakeLists.txt           |  12 +
 .../Lower/OpenMP/omp-lib-integer-wrappers.f90 | 237 ++++++
 .../test/Lower/OpenMP/omp-lib-num-threads.f90 |   2 +-
 openmp/module/omp_lib.F90.var                 | 783 ++++++++++++++++--
 4 files changed, 986 insertions(+), 48 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/omp-lib-integer-wrappers.f90

diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index ae9a6e7fb7188..4cd3090113af3 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -322,6 +322,13 @@ else ()
   set(sources ${supported_sources} ${host_sources} ${f128_sources})
 endif ()
 
+# Include the Fortran OpenMP module implementation (omp_lib.o) in
+# libflang_rt.runtime when the OpenMP runtime is also being built.
+# Skip for GPU-default targets: omp_lib is a host-side Fortran module.
+if (TARGET libomp-mod AND NOT "${LLVM_DEFAULT_TARGET_TRIPLE}" MATCHES "^amdgcn|^nvptx")
+  list(APPEND sources "$<TARGET_OBJECTS:libomp-mod>")
+endif ()
+
 
 if (NOT WIN32)
   add_flangrt_library(flang_rt.runtime STATIC SHARED
@@ -333,6 +340,11 @@ if (NOT WIN32)
 
   enable_cuda_compilation(flang_rt.runtime "${sources}")
 
+  if (TARGET libomp-mod AND TARGET flang_rt.runtime.shared AND TARGET omp)
+    # Resolve bind(c) references from omp_lib when building the shared runtime.
+    target_link_libraries(flang_rt.runtime.shared PUBLIC omp)
+  endif ()
+
   # Select a default runtime, which is used for unit and regression tests.
   if (TARGET flang_rt.runtime.default)
     get_target_property(default_target flang_rt.runtime.default ALIASED_TARGET)
diff --git a/flang/test/Lower/OpenMP/omp-lib-integer-wrappers.f90 b/flang/test/Lower/OpenMP/omp-lib-integer-wrappers.f90
new file mode 100644
index 0000000000000..9881fe56261fa
--- /dev/null
+++ b/flang/test/Lower/OpenMP/omp-lib-integer-wrappers.f90
@@ -0,0 +1,237 @@
+! REQUIRES: openmp_runtime
+
+! RUN: %flang_fc1 -emit-hlfir %openmp_flags %s -o - 2>&1 | FileCheck %s
+! RUN: bbc %openmp_flags -emit-hlfir -o - %s 2>&1 | FileCheck %s
+! RUN: %flang_fc1 -emit-fir %openmp_flags %s -o - 2>&1 | FileCheck %s
+! RUN: bbc -emit-fir %openmp_flags -o - %s 2>&1 | FileCheck %s
+!
+! Test that omp_lib integer-kind wrapper procedures lower to module
+! procedures instead of failing intrinsic resolution.
+
+program main
+  use omp_lib
+  integer(1) :: i1 = 1_1
+  integer(2) :: i2 = 2_2
+  integer(4) :: i4 = 4_4
+  integer(8) :: i8 = 8_8
+  integer(omp_integer_kind) :: ires
+  integer(omp_sched_kind) :: sched_kind
+  integer(kmp_affinity_mask_kind) :: mask
+
+  call omp_set_num_threads(i1)
+  call omp_set_num_threads(i2)
+  call omp_set_num_threads(i4)
+  call omp_set_num_threads(i8)
+
+  call omp_set_max_active_levels(i1)
+  call omp_set_max_active_levels(i2)
+  call omp_set_max_active_levels(i4)
+  call omp_set_max_active_levels(i8)
+
+  call omp_set_default_device(i1)
+  call omp_set_default_device(i2)
+  call omp_set_default_device(i4)
+  call omp_set_default_device(i8)
+
+  call omp_set_num_teams(i1)
+  call omp_set_num_teams(i2)
+  call omp_set_num_teams(i4)
+  call omp_set_num_teams(i8)
+
+  call omp_set_teams_thread_limit(i1)
+  call omp_set_teams_thread_limit(i2)
+  call omp_set_teams_thread_limit(i4)
+  call omp_set_teams_thread_limit(i8)
+
+  call kmp_set_stacksize(i1)
+  call kmp_set_stacksize(i2)
+  call kmp_set_stacksize(i4)
+  call kmp_set_stacksize(i8)
+
+  call kmp_set_blocktime(i1)
+  call kmp_set_blocktime(i2)
+  call kmp_set_blocktime(i4)
+  call kmp_set_blocktime(i8)
+
+  call kmp_set_library(i1)
+  call kmp_set_library(i2)
+  call kmp_set_library(i4)
+  call kmp_set_library(i8)
+
+  call kmp_set_disp_num_buffers(i1)
+  call kmp_set_disp_num_buffers(i2)
+  call kmp_set_disp_num_buffers(i4)
+  call kmp_set_disp_num_buffers(i8)
+
+  ires = omp_get_ancestor_thread_num(i1)
+  ires = omp_get_ancestor_thread_num(i2)
+  ires = omp_get_ancestor_thread_num(i4)
+  ires = omp_get_ancestor_thread_num(i8)
+
+  ires = omp_get_team_size(i1)
+  ires = omp_get_team_size(i2)
+  ires = omp_get_team_size(i4)
+  ires = omp_get_team_size(i8)
+
+  ires = omp_get_place_num_procs(i1)
+  ires = omp_get_place_num_procs(i2)
+  ires = omp_get_place_num_procs(i4)
+  ires = omp_get_place_num_procs(i8)
+
+  call omp_set_schedule(omp_sched_static, i1)
+  call omp_set_schedule(omp_sched_static, i2)
+  call omp_set_schedule(omp_sched_static, i4)
+  call omp_set_schedule(omp_sched_static, i8)
+
+  call omp_get_schedule(sched_kind, i1)
+  call omp_get_schedule(sched_kind, i2)
+  call omp_get_schedule(sched_kind, i4)
+  call omp_get_schedule(sched_kind, i8)
+
+  ires = omp_pause_resource(omp_pause_soft, i1)
+  ires = omp_pause_resource(omp_pause_soft, i2)
+  ires = omp_pause_resource(omp_pause_soft, i4)
+  ires = omp_pause_resource(omp_pause_soft, i8)
+
+  call kmp_create_affinity_mask(mask)
+  ires = kmp_set_affinity_mask_proc(i1, mask)
+  ires = kmp_set_affinity_mask_proc(i2, mask)
+  ires = kmp_set_affinity_mask_proc(i4, mask)
+  ires = kmp_set_affinity_mask_proc(i8, mask)
+
+  ires = kmp_unset_affinity_mask_proc(i1, mask)
+  ires = kmp_unset_affinity_mask_proc(i2, mask)
+  ires = kmp_unset_affinity_mask_proc(i4, mask)
+  ires = kmp_unset_affinity_mask_proc(i8, mask)
+
+  ires = kmp_get_affinity_mask_proc(i1, mask)
+  ires = kmp_get_affinity_mask_proc(i2, mask)
+  ires = kmp_get_affinity_mask_proc(i4, mask)
+  ires = kmp_get_affinity_mask_proc(i8, mask)
+
+  block
+    integer(4) :: ids4(1)
+    integer(8) :: ids8(1)
+    integer(4) :: parts4(1)
+    integer(8) :: parts8(1)
+    call omp_get_place_proc_ids(i4, ids4)
+    call omp_get_place_proc_ids(i8, ids8)
+    call omp_get_partition_place_nums(parts4)
+    call omp_get_partition_place_nums(parts8)
+  end block
+end program
+
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_max_active_levels
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_default_device
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_num_teams
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_teams_thread_limit
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_stacksize
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_blocktime
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_library
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_disp_num_buffers
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_ancestor_thread_num
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_team_size
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_place_num_procs
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_schedule
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_schedule
+!CHECK-NOT: not yet implemented: intrinsic: omp_pause_resource
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_affinity_mask_proc
+!CHECK-NOT: not yet implemented: intrinsic: kmp_unset_affinity_mask_proc
+!CHECK-NOT: not yet implemented: intrinsic: kmp_get_affinity_mask_proc
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_place_proc_ids
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_partition_place_nums
+
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i1
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i2
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i4
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_max_active_levels_i1
+!CHECK: fir.call @_QMomp_libPomp_set_max_active_levels_i2
+!CHECK: fir.call @_QMomp_libPomp_set_max_active_levels_i4
+!CHECK: fir.call @_QMomp_libPomp_set_max_active_levels_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_default_device_i1
+!CHECK: fir.call @_QMomp_libPomp_set_default_device_i2
+!CHECK: fir.call @_QMomp_libPomp_set_default_device_i4
+!CHECK: fir.call @_QMomp_libPomp_set_default_device_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_num_teams_i1
+!CHECK: fir.call @_QMomp_libPomp_set_num_teams_i2
+!CHECK: fir.call @_QMomp_libPomp_set_num_teams_i4
+!CHECK: fir.call @_QMomp_libPomp_set_num_teams_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_teams_thread_limit_i1
+!CHECK: fir.call @_QMomp_libPomp_set_teams_thread_limit_i2
+!CHECK: fir.call @_QMomp_libPomp_set_teams_thread_limit_i4
+!CHECK: fir.call @_QMomp_libPomp_set_teams_thread_limit_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_stacksize_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_stacksize_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_stacksize_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_stacksize_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_blocktime_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_blocktime_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_blocktime_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_blocktime_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_library_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_library_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_library_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_library_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_disp_num_buffers_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_disp_num_buffers_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_disp_num_buffers_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_disp_num_buffers_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_ancestor_thread_num_i1
+!CHECK: fir.call @_QMomp_libPomp_get_ancestor_thread_num_i2
+!CHECK: fir.call @_QMomp_libPomp_get_ancestor_thread_num_i4
+!CHECK: fir.call @_QMomp_libPomp_get_ancestor_thread_num_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_team_size_i1
+!CHECK: fir.call @_QMomp_libPomp_get_team_size_i2
+!CHECK: fir.call @_QMomp_libPomp_get_team_size_i4
+!CHECK: fir.call @_QMomp_libPomp_get_team_size_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_place_num_procs_i1
+!CHECK: fir.call @_QMomp_libPomp_get_place_num_procs_i2
+!CHECK: fir.call @_QMomp_libPomp_get_place_num_procs_i4
+!CHECK: fir.call @_QMomp_libPomp_get_place_num_procs_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_schedule_i1
+!CHECK: fir.call @_QMomp_libPomp_set_schedule_i2
+!CHECK: fir.call @_QMomp_libPomp_set_schedule_i4
+!CHECK: fir.call @_QMomp_libPomp_set_schedule_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_schedule_i1
+!CHECK: fir.call @_QMomp_libPomp_get_schedule_i2
+!CHECK: fir.call @_QMomp_libPomp_get_schedule_i4
+!CHECK: fir.call @_QMomp_libPomp_get_schedule_i8
+
+!CHECK: fir.call @_QMomp_libPomp_pause_resource_i1
+!CHECK: fir.call @_QMomp_libPomp_pause_resource_i2
+!CHECK: fir.call @_QMomp_libPomp_pause_resource_i4
+!CHECK: fir.call @_QMomp_libPomp_pause_resource_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_affinity_mask_proc_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_affinity_mask_proc_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_affinity_mask_proc_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_affinity_mask_proc_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_unset_affinity_mask_proc_i1
+!CHECK: fir.call @_QMomp_libPkmp_unset_affinity_mask_proc_i2
+!CHECK: fir.call @_QMomp_libPkmp_unset_affinity_mask_proc_i4
+!CHECK: fir.call @_QMomp_libPkmp_unset_affinity_mask_proc_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_get_affinity_mask_proc_i1
+!CHECK: fir.call @_QMomp_libPkmp_get_affinity_mask_proc_i2
+!CHECK: fir.call @_QMomp_libPkmp_get_affinity_mask_proc_i4
+!CHECK: fir.call @_QMomp_libPkmp_get_affinity_mask_proc_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_place_proc_ids_i4
+!CHECK: fir.call @_QMomp_libPomp_get_place_proc_ids_i8
+!CHECK: fir.call @_QMomp_libPomp_get_partition_place_nums_i4
+!CHECK: fir.call @_QMomp_libPomp_get_partition_place_nums_i8
diff --git a/flang/test/Lower/OpenMP/omp-lib-num-threads.f90 b/flang/test/Lower/OpenMP/omp-lib-num-threads.f90
index 6e76dce473057..a2ec031a62d88 100644
--- a/flang/test/Lower/OpenMP/omp-lib-num-threads.f90
+++ b/flang/test/Lower/OpenMP/omp-lib-num-threads.f90
@@ -21,5 +21,5 @@ program main
 
 !CHECK-NOT: not yet implemented: intrinsic: omp_set_num_threads
 !CHECK-NOT: not yet implemented: intrinsic: omp_get_num_threads
-!CHECK: fir.call @omp_set_num_threads
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i4
 !CHECK: fir.call @omp_get_num_threads
diff --git a/openmp/module/omp_lib.F90.var b/openmp/module/omp_lib.F90.var
index a41b2faad9d3d..f3ef93a03ae01 100644
--- a/openmp/module/omp_lib.F90.var
+++ b/openmp/module/omp_lib.F90.var
@@ -63,6 +63,27 @@
         ! to be exported by this MODULE.
         private
 
+        private :: kmp_get_affinity_mask_proc_c
+        private :: kmp_set_affinity_mask_proc_c
+        private :: kmp_set_blocktime_c
+        private :: kmp_set_disp_num_buffers_c
+        private :: kmp_set_library_c
+        private :: kmp_set_stacksize_c
+        private :: kmp_unset_affinity_mask_proc_c
+        private :: omp_get_ancestor_thread_num_c
+        private :: omp_get_partition_place_nums_c
+        private :: omp_get_place_num_procs_c
+        private :: omp_get_place_proc_ids_c
+        private :: omp_get_schedule_c
+        private :: omp_get_team_size_c
+        private :: omp_pause_resource_c
+        private :: omp_set_default_device_c
+        private :: omp_set_max_active_levels_c
+        private :: omp_set_num_teams_c
+        private :: omp_set_num_threads_c
+        private :: omp_set_schedule_c
+        private :: omp_set_teams_thread_limit_c
+
         ! Re-export definitions in omp_lib_kinds
         public :: omp_integer_kind
         public :: omp_logical_kind
@@ -250,10 +271,10 @@
 !         *** omp_* entry points
 !         ***
 
-          subroutine omp_set_num_threads(num_threads) bind(c)
+          subroutine omp_set_num_threads_c(num_threads) bind(c, name='omp_set_num_threads')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: num_threads
-          end subroutine omp_set_num_threads
+          end subroutine omp_set_num_threads_c
 
           subroutine omp_set_dynamic(dynamic_threads) bind(c)
             use omp_lib_kinds
@@ -310,10 +331,10 @@
             integer (kind=omp_integer_kind) omp_get_thread_limit
           end function omp_get_thread_limit
 
-          subroutine omp_set_max_active_levels(max_levels) bind(c)
+          subroutine omp_set_max_active_levels_c(max_levels) bind(c, name='omp_set_max_active_levels')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: max_levels
-          end subroutine omp_set_max_active_levels
+          end subroutine omp_set_max_active_levels_c
 
           function omp_get_max_active_levels() bind(c)
             use omp_lib_kinds
@@ -330,29 +351,29 @@
             integer (kind=omp_integer_kind) omp_get_active_level
           end function omp_get_active_level
 
-          function omp_get_ancestor_thread_num(level) bind(c)
+          function omp_get_ancestor_thread_num_c(level) bind(c, name='omp_get_ancestor_thread_num')
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) omp_get_ancestor_thread_num
+            integer (kind=omp_integer_kind) omp_get_ancestor_thread_num_c
             integer (kind=omp_integer_kind), value :: level
-          end function omp_get_ancestor_thread_num
+          end function omp_get_ancestor_thread_num_c
 
-          function omp_get_team_size(level) bind(c)
+          function omp_get_team_size_c(level) bind(c, name='omp_get_team_size')
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) omp_get_team_size
+            integer (kind=omp_integer_kind) omp_get_team_size_c
             integer (kind=omp_integer_kind), value :: level
-          end function omp_get_team_size
+          end function omp_get_team_size_c
 
-          subroutine omp_set_schedule(kind, chunk_size) bind(c)
+          subroutine omp_set_schedule_c(kind, chunk_size) bind(c, name='omp_set_schedule')
             use omp_lib_kinds
             integer (kind=omp_sched_kind), value :: kind
             integer (kind=omp_integer_kind), value :: chunk_size
-          end subroutine omp_set_schedule
+          end subroutine omp_set_schedule_c
 
-          subroutine omp_get_schedule(kind, chunk_size) bind(c)
+          subroutine omp_get_schedule_c(kind, chunk_size) bind(c, name='omp_get_schedule')
             use omp_lib_kinds
             integer (kind=omp_sched_kind) kind
             integer (kind=omp_integer_kind) chunk_size
-          end subroutine omp_get_schedule
+          end subroutine omp_get_schedule_c
 
           function omp_get_proc_bind() bind(c)
             use omp_lib_kinds
@@ -364,17 +385,17 @@
             integer (kind=omp_integer_kind) omp_get_num_places
           end function omp_get_num_places
 
-          function omp_get_place_num_procs(place_num) bind(c)
+          function omp_get_place_num_procs_c(place_num) bind(c, name='omp_get_place_num_procs')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: place_num
-            integer (kind=omp_integer_kind) omp_get_place_num_procs
-          end function omp_get_place_num_procs
+            integer (kind=omp_integer_kind) omp_get_place_num_procs_c
+          end function omp_get_place_num_procs_c
 
-          subroutine omp_get_place_proc_ids(place_num, ids) bind(c)
+          subroutine omp_get_place_proc_ids_c(place_num, ids) bind(c, name='omp_get_place_proc_ids')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: place_num
             integer (kind=omp_integer_kind) ids(*)
-          end subroutine omp_get_place_proc_ids
+          end subroutine omp_get_place_proc_ids_c
 
           function omp_get_place_num() bind(c)
             use omp_lib_kinds
@@ -386,10 +407,10 @@
             integer (kind=omp_integer_kind) omp_get_partition_num_places
           end function omp_get_partition_num_places
 
-          subroutine omp_get_partition_place_nums(place_nums) bind(c)
+          subroutine omp_get_partition_place_nums_c(place_nums) bind(c, name='omp_get_partition_place_nums')
             use omp_lib_kinds
             integer (kind=omp_integer_kind) place_nums(*)
-          end subroutine omp_get_partition_place_nums
+          end subroutine omp_get_partition_place_nums_c
 
           function omp_get_wtime() bind(c)
             use omp_lib_kinds
@@ -406,10 +427,10 @@
             integer (kind=omp_integer_kind) omp_get_default_device
           end function omp_get_default_device
 
-          subroutine omp_set_default_device(device_num) bind(c)
+          subroutine omp_set_default_device_c(device_num) bind(c, name='omp_set_default_device')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: device_num
-          end subroutine omp_set_default_device
+          end subroutine omp_set_default_device_c
 
           function omp_get_num_devices() bind(c)
             use omp_lib_kinds
@@ -446,12 +467,12 @@
             integer (kind=omp_integer_kind) omp_get_device_num
           end function omp_get_device_num
 
-          function omp_pause_resource(kind, device_num) bind(c)
+          function omp_pause_resource_c(kind, device_num) bind(c, name='omp_pause_resource')
             use omp_lib_kinds
             integer (kind=omp_pause_resource_kind), value :: kind
             integer (kind=omp_integer_kind), value :: device_num
-            integer (kind=omp_integer_kind) omp_pause_resource
-          end function omp_pause_resource
+            integer (kind=omp_integer_kind) omp_pause_resource_c
+          end function omp_pause_resource_c
 
           function omp_pause_resource_all(kind) bind(c)
             use omp_lib_kinds
@@ -640,20 +661,20 @@
             integer (kind=kmp_size_t_kind) :: omp_capture_affinity
           end function omp_capture_affinity
 
-          subroutine omp_set_num_teams(num_teams) bind(c)
+          subroutine omp_set_num_teams_c(num_teams) bind(c, name='omp_set_num_teams')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: num_teams
-          end subroutine omp_set_num_teams
+          end subroutine omp_set_num_teams_c
 
           function omp_get_max_teams() bind(c)
             use omp_lib_kinds
             integer (kind=omp_integer_kind) omp_get_max_teams
           end function omp_get_max_teams
 
-          subroutine omp_set_teams_thread_limit(thread_limit) bind(c)
+          subroutine omp_set_teams_thread_limit_c(thread_limit) bind(c, name='omp_set_teams_thread_limit')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: thread_limit
-          end subroutine omp_set_teams_thread_limit
+          end subroutine omp_set_teams_thread_limit_c
 
           function omp_get_teams_thread_limit() bind(c)
             use omp_lib_kinds
@@ -937,20 +958,20 @@
 !         *** kmp_* entry points
 !         ***
 
-          subroutine kmp_set_stacksize(size) bind(c)
+          subroutine kmp_set_stacksize_c(size) bind(c, name='kmp_set_stacksize')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: size
-          end subroutine kmp_set_stacksize
+          end subroutine kmp_set_stacksize_c
 
           subroutine kmp_set_stacksize_s(size) bind(c)
             use omp_lib_kinds
             integer (kind=kmp_size_t_kind), value :: size
           end subroutine kmp_set_stacksize_s
 
-          subroutine kmp_set_blocktime(msec) bind(c)
+          subroutine kmp_set_blocktime_c(msec) bind(c, name='kmp_set_blocktime')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: msec
-          end subroutine kmp_set_blocktime
+          end subroutine kmp_set_blocktime_c
 
           subroutine kmp_set_library_serial() bind(c)
           end subroutine kmp_set_library_serial
@@ -961,10 +982,10 @@
           subroutine kmp_set_library_throughput() bind(c)
           end subroutine kmp_set_library_throughput
 
-          subroutine kmp_set_library(libnum) bind(c)
+          subroutine kmp_set_library_c(libnum) bind(c, name='kmp_set_library')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: libnum
-          end subroutine kmp_set_library
+          end subroutine kmp_set_library_c
 
           subroutine kmp_set_defaults(string)
             character (len=*) :: string
@@ -990,10 +1011,10 @@
             integer (kind=omp_integer_kind) kmp_get_library
           end function kmp_get_library
 
-          subroutine kmp_set_disp_num_buffers(num) bind(c)
+          subroutine kmp_set_disp_num_buffers_c(num) bind(c, name='kmp_set_disp_num_buffers')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: num
-          end subroutine kmp_set_disp_num_buffers
+          end subroutine kmp_set_disp_num_buffers_c
 
           function kmp_set_affinity(mask) bind(c)
             use omp_lib_kinds
@@ -1022,26 +1043,26 @@
             integer (kind=kmp_affinity_mask_kind) mask
           end subroutine kmp_destroy_affinity_mask
 
-          function kmp_set_affinity_mask_proc(proc, mask) bind(c)
+          function kmp_set_affinity_mask_proc_c(proc, mask) bind(c, name='kmp_set_affinity_mask_proc')
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) kmp_set_affinity_mask_proc
+            integer (kind=omp_integer_kind) kmp_set_affinity_mask_proc_c
             integer (kind=omp_integer_kind), value :: proc
             integer (kind=kmp_affinity_mask_kind) mask
-          end function kmp_set_affinity_mask_proc
+          end function kmp_set_affinity_mask_proc_c
 
-          function kmp_unset_affinity_mask_proc(proc, mask) bind(c)
+          function kmp_unset_affinity_mask_proc_c(proc, mask) bind(c, name='kmp_unset_affinity_mask_proc')
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) kmp_unset_affinity_mask_proc
+            integer (kind=omp_integer_kind) kmp_unset_affinity_mask_proc_c
             integer (kind=omp_integer_kind), value :: proc
             integer (kind=kmp_affinity_mask_kind) mask
-          end function kmp_unset_affinity_mask_proc
+          end function kmp_unset_affinity_mask_proc_c
 
-          function kmp_get_affinity_mask_proc(proc, mask) bind(c)
+          function kmp_get_affinity_mask_proc_c(proc, mask) bind(c, name='kmp_get_affinity_mask_proc')
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) kmp_get_affinity_mask_proc
+            integer (kind=omp_integer_kind) kmp_get_affinity_mask_proc_c
             integer (kind=omp_integer_kind), value :: proc
             integer (kind=kmp_affinity_mask_kind) mask
-          end function kmp_get_affinity_mask_proc
+          end function kmp_get_affinity_mask_proc_c
 
           function kmp_malloc(size) bind(c)
             use omp_lib_kinds
@@ -1150,6 +1171,69 @@
 
         end interface
 
+        ! Generic interfaces for multiple integer kinds
+        interface omp_set_num_threads
+          module procedure omp_set_num_threads_i1, omp_set_num_threads_i2, omp_set_num_threads_i4, omp_set_num_threads_i8
+        end interface omp_set_num_threads
+        interface omp_set_max_active_levels
+          module procedure omp_set_max_active_levels_i1, omp_set_max_active_levels_i2, omp_set_max_active_levels_i4, omp_set_max_active_levels_i8
+        end interface omp_set_max_active_levels
+        interface omp_set_default_device
+          module procedure omp_set_default_device_i1, omp_set_default_device_i2, omp_set_default_device_i4, omp_set_default_device_i8
+        end interface omp_set_default_device
+        interface omp_set_num_teams
+          module procedure omp_set_num_teams_i1, omp_set_num_teams_i2, omp_set_num_teams_i4, omp_set_num_teams_i8
+        end interface omp_set_num_teams
+        interface omp_set_teams_thread_limit
+          module procedure omp_set_teams_thread_limit_i1, omp_set_teams_thread_limit_i2, omp_set_teams_thread_limit_i4, omp_set_teams_thread_limit_i8
+        end interface omp_set_teams_thread_limit
+        interface kmp_set_stacksize
+          module procedure kmp_set_stacksize_i1, kmp_set_stacksize_i2, kmp_set_stacksize_i4, kmp_set_stacksize_i8
+        end interface kmp_set_stacksize
+        interface kmp_set_blocktime
+          module procedure kmp_set_blocktime_i1, kmp_set_blocktime_i2, kmp_set_blocktime_i4, kmp_set_blocktime_i8
+        end interface kmp_set_blocktime
+        interface kmp_set_library
+          module procedure kmp_set_library_i1, kmp_set_library_i2, kmp_set_library_i4, kmp_set_library_i8
+        end interface kmp_set_library
+        interface kmp_set_disp_num_buffers
+          module procedure kmp_set_disp_num_buffers_i1, kmp_set_disp_num_buffers_i2, kmp_set_disp_num_buffers_i4, kmp_set_disp_num_buffers_i8
+        end interface kmp_set_disp_num_buffers
+        interface omp_get_ancestor_thread_num
+          module procedure omp_get_ancestor_thread_num_i1, omp_get_ancestor_thread_num_i2, omp_get_ancestor_thread_num_i4, omp_get_ancestor_thread_num_i8
+        end interface omp_get_ancestor_thread_num
+        interface omp_get_team_size
+          module procedure omp_get_team_size_i1, omp_get_team_size_i2, omp_get_team_size_i4, omp_get_team_size_i8
+        end interface omp_get_team_size
+        interface omp_get_place_num_procs
+          module procedure omp_get_place_num_procs_i1, omp_get_place_num_procs_i2, omp_get_place_num_procs_i4, omp_get_place_num_procs_i8
+        end interface omp_get_place_num_procs
+        interface omp_set_schedule
+          module procedure omp_set_schedule_i1, omp_set_schedule_i2, omp_set_schedule_i4, omp_set_schedule_i8
+        end interface omp_set_schedule
+        interface omp_get_schedule
+          module procedure omp_get_schedule_i1, omp_get_schedule_i2, omp_get_schedule_i4, omp_get_schedule_i8
+        end interface omp_get_schedule
+        interface omp_pause_resource
+          module procedure omp_pause_resource_i1, omp_pause_resource_i2, omp_pause_resource_i4, omp_pause_resource_i8
+        end interface omp_pause_resource
+        interface omp_get_place_proc_ids
+          module procedure omp_get_place_proc_ids_i4, omp_get_place_proc_ids_i8
+        end interface omp_get_place_proc_ids
+        interface omp_get_partition_place_nums
+          module procedure omp_get_partition_place_nums_i4, omp_get_partition_place_nums_i8
+        end interface omp_get_partition_place_nums
+        interface kmp_set_affinity_mask_proc
+          module procedure kmp_set_affinity_mask_proc_i1, kmp_set_affinity_mask_proc_i2, kmp_set_affinity_mask_proc_i4, kmp_set_affinity_mask_proc_i8
+        end interface kmp_set_affinity_mask_proc
+        interface kmp_unset_affinity_mask_proc
+          module procedure kmp_unset_affinity_mask_proc_i1, kmp_unset_affinity_mask_proc_i2, kmp_unset_affinity_mask_proc_i4, kmp_unset_affinity_mask_proc_i8
+        end interface kmp_unset_affinity_mask_proc
+        interface kmp_get_affinity_mask_proc
+          module procedure kmp_get_affinity_mask_proc_i1, kmp_get_affinity_mask_proc_i2, kmp_get_affinity_mask_proc_i4, kmp_get_affinity_mask_proc_i8
+        end interface kmp_get_affinity_mask_proc
+
+
         ! make the above routine definitions public
         public :: omp_set_num_threads
         public :: omp_set_dynamic
@@ -1277,4 +1361,609 @@
         public :: omp_get_interop_type_desc
         public :: omp_get_interop_rc_desc
 
+
+        contains
+
+!===----------------------------------------------------------------------===!
+! Integer-kind wrapper procedures for omp_lib (generated).
+!===----------------------------------------------------------------------===!
+
+
+        subroutine omp_set_num_threads_i1(num_threads)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: num_threads
+          call omp_set_num_threads_c(int(num_threads, kind=c_int))
+        end subroutine omp_set_num_threads_i1
+
+        subroutine omp_set_num_threads_i2(num_threads)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: num_threads
+          call omp_set_num_threads_c(int(num_threads, kind=c_int))
+        end subroutine omp_set_num_threads_i2
+
+        subroutine omp_set_num_threads_i4(num_threads)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: num_threads
+          call omp_set_num_threads_c(int(num_threads, kind=c_int))
+        end subroutine omp_set_num_threads_i4
+
+        subroutine omp_set_num_threads_i8(num_threads)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: num_threads
+          call omp_set_num_threads_c(int(num_threads, kind=c_int))
+        end subroutine omp_set_num_threads_i8
+
+        subroutine omp_set_max_active_levels_i1(max_levels)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: max_levels
+          call omp_set_max_active_levels_c(int(max_levels, kind=c_int))
+        end subroutine omp_set_max_active_levels_i1
+
+        subroutine omp_set_max_active_levels_i2(max_levels)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: max_levels
+          call omp_set_max_active_levels_c(int(max_levels, kind=c_int))
+        end subroutine omp_set_max_active_levels_i2
+
+        subroutine omp_set_max_active_levels_i4(max_levels)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: max_levels
+          call omp_set_max_active_levels_c(int(max_levels, kind=c_int))
+        end subroutine omp_set_max_active_levels_i4
+
+        subroutine omp_set_max_active_levels_i8(max_levels)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: max_levels
+          call omp_set_max_active_levels_c(int(max_levels, kind=c_int))
+        end subroutine omp_set_max_active_levels_i8
+
+        subroutine omp_set_default_device_i1(device_num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: device_num
+          call omp_set_default_device_c(int(device_num, kind=c_int))
+        end subroutine omp_set_default_device_i1
+
+        subroutine omp_set_default_device_i2(device_num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: device_num
+          call omp_set_default_device_c(int(device_num, kind=c_int))
+        end subroutine omp_set_default_device_i2
+
+        subroutine omp_set_default_device_i4(device_num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: device_num
+          call omp_set_default_device_c(int(device_num, kind=c_int))
+        end subroutine omp_set_default_device_i4
+
+        subroutine omp_set_default_device_i8(device_num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: device_num
+          call omp_set_default_device_c(int(device_num, kind=c_int))
+        end subroutine omp_set_default_device_i8
+
+        subroutine omp_set_num_teams_i1(num_teams)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: num_teams
+          call omp_set_num_teams_c(int(num_teams, kind=c_int))
+        end subroutine omp_set_num_teams_i1
+
+        subroutine omp_set_num_teams_i2(num_teams)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: num_teams
+          call omp_set_num_teams_c(int(num_teams, kind=c_int))
+        end subroutine omp_set_num_teams_i2
+
+        subroutine omp_set_num_teams_i4(num_teams)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: num_teams
+          call omp_set_num_teams_c(int(num_teams, kind=c_int))
+        end subroutine omp_set_num_teams_i4
+
+        subroutine omp_set_num_teams_i8(num_teams)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: num_teams
+          call omp_set_num_teams_c(int(num_teams, kind=c_int))
+        end subroutine omp_set_num_teams_i8
+
+        subroutine omp_set_teams_thread_limit_i1(thread_limit)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: thread_limit
+          call omp_set_teams_thread_limit_c(int(thread_limit, kind=c_int))
+        end subroutine omp_set_teams_thread_limit_i1
+
+        subroutine omp_set_teams_thread_limit_i2(thread_limit)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: thread_limit
+          call omp_set_teams_thread_limit_c(int(thread_limit, kind=c_int))
+        end subroutine omp_set_teams_thread_limit_i2
+
+        subroutine omp_set_teams_thread_limit_i4(thread_limit)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: thread_limit
+          call omp_set_teams_thread_limit_c(int(thread_limit, kind=c_int))
+        end subroutine omp_set_teams_thread_limit_i4
+
+        subroutine omp_set_teams_thread_limit_i8(thread_limit)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: thread_limit
+          call omp_set_teams_thread_limit_c(int(thread_limit, kind=c_int))
+        end subroutine omp_set_teams_thread_limit_i8
+
+        subroutine kmp_set_stacksize_i1(size)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: size
+          call kmp_set_stacksize_c(int(size, kind=c_int))
+        end subroutine kmp_set_stacksize_i1
+
+        subroutine kmp_set_stacksize_i2(size)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: size
+          call kmp_set_stacksize_c(int(size, kind=c_int))
+        end subroutine kmp_set_stacksize_i2
+
+        subroutine kmp_set_stacksize_i4(size)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: size
+          call kmp_set_stacksize_c(int(size, kind=c_int))
+        end subroutine kmp_set_stacksize_i4
+
+        subroutine kmp_set_stacksize_i8(size)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: size
+          call kmp_set_stacksize_c(int(size, kind=c_int))
+        end subroutine kmp_set_stacksize_i8
+
+        subroutine kmp_set_blocktime_i1(msec)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: msec
+          call kmp_set_blocktime_c(int(msec, kind=c_int))
+        end subroutine kmp_set_blocktime_i1
+
+        subroutine kmp_set_blocktime_i2(msec)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: msec
+          call kmp_set_blocktime_c(int(msec, kind=c_int))
+        end subroutine kmp_set_blocktime_i2
+
+        subroutine kmp_set_blocktime_i4(msec)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: msec
+          call kmp_set_blocktime_c(int(msec, kind=c_int))
+        end subroutine kmp_set_blocktime_i4
+
+        subroutine kmp_set_blocktime_i8(msec)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: msec
+          call kmp_set_blocktime_c(int(msec, kind=c_int))
+        end subroutine kmp_set_blocktime_i8
+
+        subroutine kmp_set_library_i1(libnum)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: libnum
+          call kmp_set_library_c(int(libnum, kind=c_int))
+        end subroutine kmp_set_library_i1
+
+        subroutine kmp_set_library_i2(libnum)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: libnum
+          call kmp_set_library_c(int(libnum, kind=c_int))
+        end subroutine kmp_set_library_i2
+
+        subroutine kmp_set_library_i4(libnum)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: libnum
+          call kmp_set_library_c(int(libnum, kind=c_int))
+        end subroutine kmp_set_library_i4
+
+        subroutine kmp_set_library_i8(libnum)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: libnum
+          call kmp_set_library_c(int(libnum, kind=c_int))
+        end subroutine kmp_set_library_i8
+
+        subroutine kmp_set_disp_num_buffers_i1(num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: num
+          call kmp_set_disp_num_buffers_c(int(num, kind=c_int))
+        end subroutine kmp_set_disp_num_buffers_i1
+
+        subroutine kmp_set_disp_num_buffers_i2(num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: num
+          call kmp_set_disp_num_buffers_c(int(num, kind=c_int))
+        end subroutine kmp_set_disp_num_buffers_i2
+
+        subroutine kmp_set_disp_num_buffers_i4(num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: num
+          call kmp_set_disp_num_buffers_c(int(num, kind=c_int))
+        end subroutine kmp_set_disp_num_buffers_i4
+
+        subroutine kmp_set_disp_num_buffers_i8(num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: num
+          call kmp_set_disp_num_buffers_c(int(num, kind=c_int))
+        end subroutine kmp_set_disp_num_buffers_i8
+
+        function omp_get_ancestor_thread_num_i1(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_ancestor_thread_num_i1
+          omp_get_ancestor_thread_num_i1 = omp_get_ancestor_thread_num_c(int(level, kind=c_int))
+        end function omp_get_ancestor_thread_num_i1
+
+        function omp_get_ancestor_thread_num_i2(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_ancestor_thread_num_i2
+          omp_get_ancestor_thread_num_i2 = omp_get_ancestor_thread_num_c(int(level, kind=c_int))
+        end function omp_get_ancestor_thread_num_i2
+
+        function omp_get_ancestor_thread_num_i4(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_ancestor_thread_num_i4
+          omp_get_ancestor_thread_num_i4 = omp_get_ancestor_thread_num_c(int(level, kind=c_int))
+        end function omp_get_ancestor_thread_num_i4
+
+        function omp_get_ancestor_thread_num_i8(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_ancestor_thread_num_i8
+          omp_get_ancestor_thread_num_i8 = omp_get_ancestor_thread_num_c(int(level, kind=c_int))
+        end function omp_get_ancestor_thread_num_i8
+
+        function omp_get_team_size_i1(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_team_size_i1
+          omp_get_team_size_i1 = omp_get_team_size_c(int(level, kind=c_int))
+        end function omp_get_team_size_i1
+
+        function omp_get_team_size_i2(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_team_size_i2
+          omp_get_team_size_i2 = omp_get_team_size_c(int(level, kind=c_int))
+        end function omp_get_team_size_i2
+
+        function omp_get_team_size_i4(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_team_size_i4
+          omp_get_team_size_i4 = omp_get_team_size_c(int(level, kind=c_int))
+        end function omp_get_team_size_i4
+
+        function omp_get_team_size_i8(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_team_size_i8
+          omp_get_team_size_i8 = omp_get_team_size_c(int(level, kind=c_int))
+        end function omp_get_team_size_i8
+
+        function omp_get_place_num_procs_i1(place_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: place_num
+          integer(kind=omp_integer_kind) :: omp_get_place_num_procs_i1
+          omp_get_place_num_procs_i1 = omp_get_place_num_procs_c(int(place_num, kind=c_int))
+        end function omp_get_place_num_procs_i1
+
+        function omp_get_place_num_procs_i2(place_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: place_num
+          integer(kind=omp_integer_kind) :: omp_get_place_num_procs_i2
+          omp_get_place_num_procs_i2 = omp_get_place_num_procs_c(int(place_num, kind=c_int))
+        end function omp_get_place_num_procs_i2
+
+        function omp_get_place_num_procs_i4(place_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: place_num
+          integer(kind=omp_integer_kind) :: omp_get_place_num_procs_i4
+          omp_get_place_num_procs_i4 = omp_get_place_num_procs_c(int(place_num, kind=c_int))
+        end function omp_get_place_num_procs_i4
+
+        function omp_get_place_num_procs_i8(place_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: place_num
+          integer(kind=omp_integer_kind) :: omp_get_place_num_procs_i8
+          omp_get_place_num_procs_i8 = omp_get_place_num_procs_c(int(place_num, kind=c_int))
+        end function omp_get_place_num_procs_i8
+
+        subroutine omp_set_schedule_i1(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(kind=omp_sched_kind), value :: kind
+          integer(c_int8_t), value :: chunk_size
+          call omp_set_schedule_c(kind, int(chunk_size, kind=c_int))
+        end subroutine omp_set_schedule_i1
+
+        subroutine omp_set_schedule_i2(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(kind=omp_sched_kind), value :: kind
+          integer(c_int16_t), value :: chunk_size
+          call omp_set_schedule_c(kind, int(chunk_size, kind=c_int))
+        end subroutine omp_set_schedule_i2
+
+        subroutine omp_set_schedule_i4(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(kind=omp_sched_kind), value :: kind
+          integer(c_int), value :: chunk_size
+          call omp_set_schedule_c(kind, int(chunk_size, kind=c_int))
+        end subroutine omp_set_schedule_i4
+
+        subroutine omp_set_schedule_i8(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(kind=omp_sched_kind), value :: kind
+          integer(c_int64_t), value :: chunk_size
+          call omp_set_schedule_c(kind, int(chunk_size, kind=c_int))
+        end subroutine omp_set_schedule_i8
+
+        subroutine omp_get_schedule_i1(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(kind=omp_sched_kind), intent(out) :: kind
+          integer(c_int8_t), intent(out) :: chunk_size
+          integer(c_int) :: kind_c, chunk_c
+          call omp_get_schedule_c(kind_c, chunk_c)
+          kind = kind_c
+          chunk_size = chunk_c
+        end subroutine omp_get_schedule_i1
+
+        subroutine omp_get_schedule_i2(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(kind=omp_sched_kind), intent(out) :: kind
+          integer(c_int16_t), intent(out) :: chunk_size
+          integer(c_int) :: kind_c, chunk_c
+          call omp_get_schedule_c(kind_c, chunk_c)
+          kind = kind_c
+          chunk_size = chunk_c
+        end subroutine omp_get_schedule_i2
+
+        subroutine omp_get_schedule_i4(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(kind=omp_sched_kind), intent(out) :: kind
+          integer(c_int), intent(out) :: chunk_size
+          integer(c_int) :: kind_c, chunk_c
+          call omp_get_schedule_c(kind_c, chunk_c)
+          kind = kind_c
+          chunk_size = chunk_c
+        end subroutine omp_get_schedule_i4
+
+        subroutine omp_get_schedule_i8(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(kind=omp_sched_kind), intent(out) :: kind
+          integer(c_int64_t), intent(out) :: chunk_size
+          integer(c_int) :: kind_c, chunk_c
+          call omp_get_schedule_c(kind_c, chunk_c)
+          kind = kind_c
+          chunk_size = chunk_c
+        end subroutine omp_get_schedule_i8
+
+        function omp_pause_resource_i1(kind, device_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(kind=omp_pause_resource_kind), value :: kind
+          integer(c_int8_t), value :: device_num
+          integer(kind=omp_integer_kind) :: omp_pause_resource_i1
+          omp_pause_resource_i1 = &
+              omp_pause_resource_c(kind, int(device_num, kind=c_int))
+        end function omp_pause_resource_i1
+
+        function omp_pause_resource_i2(kind, device_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(kind=omp_pause_resource_kind), value :: kind
+          integer(c_int16_t), value :: device_num
+          integer(kind=omp_integer_kind) :: omp_pause_resource_i2
+          omp_pause_resource_i2 = &
+              omp_pause_resource_c(kind, int(device_num, kind=c_int))
+        end function omp_pause_resource_i2
+
+        function omp_pause_resource_i4(kind, device_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(kind=omp_pause_resource_kind), value :: kind
+          integer(c_int), value :: device_num
+          integer(kind=omp_integer_kind) :: omp_pause_resource_i4
+          omp_pause_resource_i4 = &
+              omp_pause_resource_c(kind, int(device_num, kind=c_int))
+        end function omp_pause_resource_i4
+
+        function omp_pause_resource_i8(kind, device_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(kind=omp_pause_resource_kind), value :: kind
+          integer(c_int64_t), value :: device_num
+          integer(kind=omp_integer_kind) :: omp_pause_resource_i8
+          omp_pause_resource_i8 = &
+              omp_pause_resource_c(kind, int(device_num, kind=c_int))
+        end function omp_pause_resource_i8
+
+        function kmp_set_affinity_mask_proc_i1(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_set_affinity_mask_proc_i1
+          kmp_set_affinity_mask_proc_i1 = kmp_set_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end function kmp_set_affinity_mask_proc_i1
+
+        function kmp_set_affinity_mask_proc_i2(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_set_affinity_mask_proc_i2
+          kmp_set_affinity_mask_proc_i2 = kmp_set_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end function kmp_set_affinity_mask_proc_i2
+
+        function kmp_set_affinity_mask_proc_i4(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_set_affinity_mask_proc_i4
+          kmp_set_affinity_mask_proc_i4 = kmp_set_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end function kmp_set_affinity_mask_proc_i4
+
+        function kmp_set_affinity_mask_proc_i8(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_set_affinity_mask_proc_i8
+          kmp_set_affinity_mask_proc_i8 = kmp_set_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end function kmp_set_affinity_mask_proc_i8
+
+        function kmp_unset_affinity_mask_proc_i1(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_unset_affinity_mask_proc_i1
+          kmp_unset_affinity_mask_proc_i1 = kmp_unset_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end function kmp_unset_affinity_mask_proc_i1
+
+        function kmp_unset_affinity_mask_proc_i2(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_unset_affinity_mask_proc_i2
+          kmp_unset_affinity_mask_proc_i2 = kmp_unset_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end function kmp_unset_affinity_mask_proc_i2
+
+        function kmp_unset_affinity_mask_proc_i4(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_unset_affinity_mask_proc_i4
+          kmp_unset_affinity_mask_proc_i4 = kmp_unset_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end function kmp_unset_affinity_mask_proc_i4
+
+        function kmp_unset_affinity_mask_proc_i8(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_unset_affinity_mask_proc_i8
+          kmp_unset_affinity_mask_proc_i8 = kmp_unset_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end function kmp_unset_affinity_mask_proc_i8
+
+        function kmp_get_affinity_mask_proc_i1(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_get_affinity_mask_proc_i1
+          kmp_get_affinity_mask_proc_i1 = kmp_get_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end function kmp_get_affinity_mask_proc_i1
+
+        function kmp_get_affinity_mask_proc_i2(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_get_affinity_mask_proc_i2
+          kmp_get_affinity_mask_proc_i2 = kmp_get_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end function kmp_get_affinity_mask_proc_i2
+
+        function kmp_get_affinity_mask_proc_i4(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_get_affinity_mask_proc_i4
+          kmp_get_affinity_mask_proc_i4 = kmp_get_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end function kmp_get_affinity_mask_proc_i4
+
+        function kmp_get_affinity_mask_proc_i8(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_get_affinity_mask_proc_i8
+          kmp_get_affinity_mask_proc_i8 = kmp_get_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end function kmp_get_affinity_mask_proc_i8
+
+        subroutine omp_get_place_proc_ids_i4(place_num, ids)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: place_num
+          integer(c_int), intent(out) :: ids(*)
+          integer(c_int) :: n, i
+          integer(c_int), allocatable :: ids_c(:)
+          n = omp_get_place_num_procs_c(int(place_num, kind=c_int))
+          allocate(ids_c(n))
+          call omp_get_place_proc_ids_c(int(place_num, kind=c_int), ids_c)
+          do i = 1, n
+            ids(i) = ids_c(i)
+          end do
+          deallocate(ids_c)
+        end subroutine omp_get_place_proc_ids_i4
+
+        subroutine omp_get_place_proc_ids_i8(place_num, ids)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: place_num
+          integer(c_int64_t), intent(out) :: ids(*)
+          integer(c_int) :: n, i
+          integer(c_int), allocatable :: ids_c(:)
+          n = omp_get_place_num_procs_c(int(place_num, kind=c_int))
+          allocate(ids_c(n))
+          call omp_get_place_proc_ids_c(int(place_num, kind=c_int), ids_c)
+          do i = 1, n
+            ids(i) = ids_c(i)
+          end do
+          deallocate(ids_c)
+        end subroutine omp_get_place_proc_ids_i8
+
+        subroutine omp_get_partition_place_nums_i4(place_nums)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), intent(out) :: place_nums(*)
+          integer(c_int) :: n, i
+          integer(c_int), allocatable :: place_nums_c(:)
+          n = omp_get_partition_num_places()
+          allocate(place_nums_c(n))
+          call omp_get_partition_place_nums_c(place_nums_c)
+          do i = 1, n
+            place_nums(i) = place_nums_c(i)
+          end do
+          deallocate(place_nums_c)
+        end subroutine omp_get_partition_place_nums_i4
+
+        subroutine omp_get_partition_place_nums_i8(place_nums)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), intent(out) :: place_nums(*)
+          integer(c_int) :: n, i
+          integer(c_int), allocatable :: place_nums_c(:)
+          n = omp_get_partition_num_places()
+          allocate(place_nums_c(n))
+          call omp_get_partition_place_nums_c(place_nums_c)
+          do i = 1, n
+            place_nums(i) = place_nums_c(i)
+          end do
+          deallocate(place_nums_c)
+        end subroutine omp_get_partition_place_nums_i8
       end module omp_lib


From openmp-commits at lists.llvm.org  Sat Aug  1 22:01:34 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Sat, 01 Aug 2026 22:01:34 -0700 (PDT)
Subject: [Openmp-commits] [flang] [llvm] [openmp] [OpenMP][Flang] Add
 integer-kind wrappers to omp_lib (PR #213505)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213505@github.com>
Message-ID: <6a6ecf2e.d76d81af.e9be0.8856@mx.google.com>


llvmorg-github-actions[bot] wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-openmp

Author: shivaramaarao

<details>
<summary>Changes</summary>

Many OpenMP routines in omp_lib currently accept only integer(kind=omp_integer_kind) arguments, where omp_integer_kind maps to c_int. As a result, Flang's type checking rejects calls that pass integer(1), integer(2), or integer(8) arguments. To improve compatibility, wrapper procedures have been added in the omp_lib module for applicable routines, enabling support for integer(1), integer(2), integer(4), and integer(8) argument types.

Fixes #<!-- -->123948.

Assisted-by: Cursor

---

Patch is 56.47 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/213505.diff


4 Files Affected:

- (modified) flang-rt/lib/runtime/CMakeLists.txt (+12) 
- (added) flang/test/Lower/OpenMP/omp-lib-integer-wrappers.f90 (+237) 
- (modified) flang/test/Lower/OpenMP/omp-lib-num-threads.f90 (+1-1) 
- (modified) openmp/module/omp_lib.F90.var (+736-47) 


``````````diff
diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index ae9a6e7fb7188..4cd3090113af3 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -322,6 +322,13 @@ else ()
   set(sources ${supported_sources} ${host_sources} ${f128_sources})
 endif ()
 
+# Include the Fortran OpenMP module implementation (omp_lib.o) in
+# libflang_rt.runtime when the OpenMP runtime is also being built.
+# Skip for GPU-default targets: omp_lib is a host-side Fortran module.
+if (TARGET libomp-mod AND NOT "${LLVM_DEFAULT_TARGET_TRIPLE}" MATCHES "^amdgcn|^nvptx")
+  list(APPEND sources "$<TARGET_OBJECTS:libomp-mod>")
+endif ()
+
 
 if (NOT WIN32)
   add_flangrt_library(flang_rt.runtime STATIC SHARED
@@ -333,6 +340,11 @@ if (NOT WIN32)
 
   enable_cuda_compilation(flang_rt.runtime "${sources}")
 
+  if (TARGET libomp-mod AND TARGET flang_rt.runtime.shared AND TARGET omp)
+    # Resolve bind(c) references from omp_lib when building the shared runtime.
+    target_link_libraries(flang_rt.runtime.shared PUBLIC omp)
+  endif ()
+
   # Select a default runtime, which is used for unit and regression tests.
   if (TARGET flang_rt.runtime.default)
     get_target_property(default_target flang_rt.runtime.default ALIASED_TARGET)
diff --git a/flang/test/Lower/OpenMP/omp-lib-integer-wrappers.f90 b/flang/test/Lower/OpenMP/omp-lib-integer-wrappers.f90
new file mode 100644
index 0000000000000..9881fe56261fa
--- /dev/null
+++ b/flang/test/Lower/OpenMP/omp-lib-integer-wrappers.f90
@@ -0,0 +1,237 @@
+! REQUIRES: openmp_runtime
+
+! RUN: %flang_fc1 -emit-hlfir %openmp_flags %s -o - 2>&1 | FileCheck %s
+! RUN: bbc %openmp_flags -emit-hlfir -o - %s 2>&1 | FileCheck %s
+! RUN: %flang_fc1 -emit-fir %openmp_flags %s -o - 2>&1 | FileCheck %s
+! RUN: bbc -emit-fir %openmp_flags -o - %s 2>&1 | FileCheck %s
+!
+! Test that omp_lib integer-kind wrapper procedures lower to module
+! procedures instead of failing intrinsic resolution.
+
+program main
+  use omp_lib
+  integer(1) :: i1 = 1_1
+  integer(2) :: i2 = 2_2
+  integer(4) :: i4 = 4_4
+  integer(8) :: i8 = 8_8
+  integer(omp_integer_kind) :: ires
+  integer(omp_sched_kind) :: sched_kind
+  integer(kmp_affinity_mask_kind) :: mask
+
+  call omp_set_num_threads(i1)
+  call omp_set_num_threads(i2)
+  call omp_set_num_threads(i4)
+  call omp_set_num_threads(i8)
+
+  call omp_set_max_active_levels(i1)
+  call omp_set_max_active_levels(i2)
+  call omp_set_max_active_levels(i4)
+  call omp_set_max_active_levels(i8)
+
+  call omp_set_default_device(i1)
+  call omp_set_default_device(i2)
+  call omp_set_default_device(i4)
+  call omp_set_default_device(i8)
+
+  call omp_set_num_teams(i1)
+  call omp_set_num_teams(i2)
+  call omp_set_num_teams(i4)
+  call omp_set_num_teams(i8)
+
+  call omp_set_teams_thread_limit(i1)
+  call omp_set_teams_thread_limit(i2)
+  call omp_set_teams_thread_limit(i4)
+  call omp_set_teams_thread_limit(i8)
+
+  call kmp_set_stacksize(i1)
+  call kmp_set_stacksize(i2)
+  call kmp_set_stacksize(i4)
+  call kmp_set_stacksize(i8)
+
+  call kmp_set_blocktime(i1)
+  call kmp_set_blocktime(i2)
+  call kmp_set_blocktime(i4)
+  call kmp_set_blocktime(i8)
+
+  call kmp_set_library(i1)
+  call kmp_set_library(i2)
+  call kmp_set_library(i4)
+  call kmp_set_library(i8)
+
+  call kmp_set_disp_num_buffers(i1)
+  call kmp_set_disp_num_buffers(i2)
+  call kmp_set_disp_num_buffers(i4)
+  call kmp_set_disp_num_buffers(i8)
+
+  ires = omp_get_ancestor_thread_num(i1)
+  ires = omp_get_ancestor_thread_num(i2)
+  ires = omp_get_ancestor_thread_num(i4)
+  ires = omp_get_ancestor_thread_num(i8)
+
+  ires = omp_get_team_size(i1)
+  ires = omp_get_team_size(i2)
+  ires = omp_get_team_size(i4)
+  ires = omp_get_team_size(i8)
+
+  ires = omp_get_place_num_procs(i1)
+  ires = omp_get_place_num_procs(i2)
+  ires = omp_get_place_num_procs(i4)
+  ires = omp_get_place_num_procs(i8)
+
+  call omp_set_schedule(omp_sched_static, i1)
+  call omp_set_schedule(omp_sched_static, i2)
+  call omp_set_schedule(omp_sched_static, i4)
+  call omp_set_schedule(omp_sched_static, i8)
+
+  call omp_get_schedule(sched_kind, i1)
+  call omp_get_schedule(sched_kind, i2)
+  call omp_get_schedule(sched_kind, i4)
+  call omp_get_schedule(sched_kind, i8)
+
+  ires = omp_pause_resource(omp_pause_soft, i1)
+  ires = omp_pause_resource(omp_pause_soft, i2)
+  ires = omp_pause_resource(omp_pause_soft, i4)
+  ires = omp_pause_resource(omp_pause_soft, i8)
+
+  call kmp_create_affinity_mask(mask)
+  ires = kmp_set_affinity_mask_proc(i1, mask)
+  ires = kmp_set_affinity_mask_proc(i2, mask)
+  ires = kmp_set_affinity_mask_proc(i4, mask)
+  ires = kmp_set_affinity_mask_proc(i8, mask)
+
+  ires = kmp_unset_affinity_mask_proc(i1, mask)
+  ires = kmp_unset_affinity_mask_proc(i2, mask)
+  ires = kmp_unset_affinity_mask_proc(i4, mask)
+  ires = kmp_unset_affinity_mask_proc(i8, mask)
+
+  ires = kmp_get_affinity_mask_proc(i1, mask)
+  ires = kmp_get_affinity_mask_proc(i2, mask)
+  ires = kmp_get_affinity_mask_proc(i4, mask)
+  ires = kmp_get_affinity_mask_proc(i8, mask)
+
+  block
+    integer(4) :: ids4(1)
+    integer(8) :: ids8(1)
+    integer(4) :: parts4(1)
+    integer(8) :: parts8(1)
+    call omp_get_place_proc_ids(i4, ids4)
+    call omp_get_place_proc_ids(i8, ids8)
+    call omp_get_partition_place_nums(parts4)
+    call omp_get_partition_place_nums(parts8)
+  end block
+end program
+
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_max_active_levels
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_default_device
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_num_teams
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_teams_thread_limit
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_stacksize
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_blocktime
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_library
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_disp_num_buffers
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_ancestor_thread_num
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_team_size
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_place_num_procs
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_schedule
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_schedule
+!CHECK-NOT: not yet implemented: intrinsic: omp_pause_resource
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_affinity_mask_proc
+!CHECK-NOT: not yet implemented: intrinsic: kmp_unset_affinity_mask_proc
+!CHECK-NOT: not yet implemented: intrinsic: kmp_get_affinity_mask_proc
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_place_proc_ids
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_partition_place_nums
+
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i1
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i2
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i4
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_max_active_levels_i1
+!CHECK: fir.call @_QMomp_libPomp_set_max_active_levels_i2
+!CHECK: fir.call @_QMomp_libPomp_set_max_active_levels_i4
+!CHECK: fir.call @_QMomp_libPomp_set_max_active_levels_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_default_device_i1
+!CHECK: fir.call @_QMomp_libPomp_set_default_device_i2
+!CHECK: fir.call @_QMomp_libPomp_set_default_device_i4
+!CHECK: fir.call @_QMomp_libPomp_set_default_device_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_num_teams_i1
+!CHECK: fir.call @_QMomp_libPomp_set_num_teams_i2
+!CHECK: fir.call @_QMomp_libPomp_set_num_teams_i4
+!CHECK: fir.call @_QMomp_libPomp_set_num_teams_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_teams_thread_limit_i1
+!CHECK: fir.call @_QMomp_libPomp_set_teams_thread_limit_i2
+!CHECK: fir.call @_QMomp_libPomp_set_teams_thread_limit_i4
+!CHECK: fir.call @_QMomp_libPomp_set_teams_thread_limit_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_stacksize_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_stacksize_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_stacksize_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_stacksize_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_blocktime_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_blocktime_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_blocktime_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_blocktime_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_library_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_library_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_library_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_library_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_disp_num_buffers_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_disp_num_buffers_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_disp_num_buffers_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_disp_num_buffers_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_ancestor_thread_num_i1
+!CHECK: fir.call @_QMomp_libPomp_get_ancestor_thread_num_i2
+!CHECK: fir.call @_QMomp_libPomp_get_ancestor_thread_num_i4
+!CHECK: fir.call @_QMomp_libPomp_get_ancestor_thread_num_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_team_size_i1
+!CHECK: fir.call @_QMomp_libPomp_get_team_size_i2
+!CHECK: fir.call @_QMomp_libPomp_get_team_size_i4
+!CHECK: fir.call @_QMomp_libPomp_get_team_size_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_place_num_procs_i1
+!CHECK: fir.call @_QMomp_libPomp_get_place_num_procs_i2
+!CHECK: fir.call @_QMomp_libPomp_get_place_num_procs_i4
+!CHECK: fir.call @_QMomp_libPomp_get_place_num_procs_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_schedule_i1
+!CHECK: fir.call @_QMomp_libPomp_set_schedule_i2
+!CHECK: fir.call @_QMomp_libPomp_set_schedule_i4
+!CHECK: fir.call @_QMomp_libPomp_set_schedule_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_schedule_i1
+!CHECK: fir.call @_QMomp_libPomp_get_schedule_i2
+!CHECK: fir.call @_QMomp_libPomp_get_schedule_i4
+!CHECK: fir.call @_QMomp_libPomp_get_schedule_i8
+
+!CHECK: fir.call @_QMomp_libPomp_pause_resource_i1
+!CHECK: fir.call @_QMomp_libPomp_pause_resource_i2
+!CHECK: fir.call @_QMomp_libPomp_pause_resource_i4
+!CHECK: fir.call @_QMomp_libPomp_pause_resource_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_affinity_mask_proc_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_affinity_mask_proc_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_affinity_mask_proc_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_affinity_mask_proc_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_unset_affinity_mask_proc_i1
+!CHECK: fir.call @_QMomp_libPkmp_unset_affinity_mask_proc_i2
+!CHECK: fir.call @_QMomp_libPkmp_unset_affinity_mask_proc_i4
+!CHECK: fir.call @_QMomp_libPkmp_unset_affinity_mask_proc_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_get_affinity_mask_proc_i1
+!CHECK: fir.call @_QMomp_libPkmp_get_affinity_mask_proc_i2
+!CHECK: fir.call @_QMomp_libPkmp_get_affinity_mask_proc_i4
+!CHECK: fir.call @_QMomp_libPkmp_get_affinity_mask_proc_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_place_proc_ids_i4
+!CHECK: fir.call @_QMomp_libPomp_get_place_proc_ids_i8
+!CHECK: fir.call @_QMomp_libPomp_get_partition_place_nums_i4
+!CHECK: fir.call @_QMomp_libPomp_get_partition_place_nums_i8
diff --git a/flang/test/Lower/OpenMP/omp-lib-num-threads.f90 b/flang/test/Lower/OpenMP/omp-lib-num-threads.f90
index 6e76dce473057..a2ec031a62d88 100644
--- a/flang/test/Lower/OpenMP/omp-lib-num-threads.f90
+++ b/flang/test/Lower/OpenMP/omp-lib-num-threads.f90
@@ -21,5 +21,5 @@ program main
 
 !CHECK-NOT: not yet implemented: intrinsic: omp_set_num_threads
 !CHECK-NOT: not yet implemented: intrinsic: omp_get_num_threads
-!CHECK: fir.call @omp_set_num_threads
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i4
 !CHECK: fir.call @omp_get_num_threads
diff --git a/openmp/module/omp_lib.F90.var b/openmp/module/omp_lib.F90.var
index a41b2faad9d3d..f3ef93a03ae01 100644
--- a/openmp/module/omp_lib.F90.var
+++ b/openmp/module/omp_lib.F90.var
@@ -63,6 +63,27 @@
         ! to be exported by this MODULE.
         private
 
+        private :: kmp_get_affinity_mask_proc_c
+        private :: kmp_set_affinity_mask_proc_c
+        private :: kmp_set_blocktime_c
+        private :: kmp_set_disp_num_buffers_c
+        private :: kmp_set_library_c
+        private :: kmp_set_stacksize_c
+        private :: kmp_unset_affinity_mask_proc_c
+        private :: omp_get_ancestor_thread_num_c
+        private :: omp_get_partition_place_nums_c
+        private :: omp_get_place_num_procs_c
+        private :: omp_get_place_proc_ids_c
+        private :: omp_get_schedule_c
+        private :: omp_get_team_size_c
+        private :: omp_pause_resource_c
+        private :: omp_set_default_device_c
+        private :: omp_set_max_active_levels_c
+        private :: omp_set_num_teams_c
+        private :: omp_set_num_threads_c
+        private :: omp_set_schedule_c
+        private :: omp_set_teams_thread_limit_c
+
         ! Re-export definitions in omp_lib_kinds
         public :: omp_integer_kind
         public :: omp_logical_kind
@@ -250,10 +271,10 @@
 !         *** omp_* entry points
 !         ***
 
-          subroutine omp_set_num_threads(num_threads) bind(c)
+          subroutine omp_set_num_threads_c(num_threads) bind(c, name='omp_set_num_threads')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: num_threads
-          end subroutine omp_set_num_threads
+          end subroutine omp_set_num_threads_c
 
           subroutine omp_set_dynamic(dynamic_threads) bind(c)
             use omp_lib_kinds
@@ -310,10 +331,10 @@
             integer (kind=omp_integer_kind) omp_get_thread_limit
           end function omp_get_thread_limit
 
-          subroutine omp_set_max_active_levels(max_levels) bind(c)
+          subroutine omp_set_max_active_levels_c(max_levels) bind(c, name='omp_set_max_active_levels')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: max_levels
-          end subroutine omp_set_max_active_levels
+          end subroutine omp_set_max_active_levels_c
 
           function omp_get_max_active_levels() bind(c)
             use omp_lib_kinds
@@ -330,29 +351,29 @@
             integer (kind=omp_integer_kind) omp_get_active_level
           end function omp_get_active_level
 
-          function omp_get_ancestor_thread_num(level) bind(c)
+          function omp_get_ancestor_thread_num_c(level) bind(c, name='omp_get_ancestor_thread_num')
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) omp_get_ancestor_thread_num
+            integer (kind=omp_integer_kind) omp_get_ancestor_thread_num_c
             integer (kind=omp_integer_kind), value :: level
-          end function omp_get_ancestor_thread_num
+          end function omp_get_ancestor_thread_num_c
 
-          function omp_get_team_size(level) bind(c)
+          function omp_get_team_size_c(level) bind(c, name='omp_get_team_size')
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) omp_get_team_size
+            integer (kind=omp_integer_kind) omp_get_team_size_c
             integer (kind=omp_integer_kind), value :: level
-          end function omp_get_team_size
+          end function omp_get_team_size_c
 
-          subroutine omp_set_schedule(kind, chunk_size) bind(c)
+          subroutine omp_set_schedule_c(kind, chunk_size) bind(c, name='omp_set_schedule')
             use omp_lib_kinds
             integer (kind=omp_sched_kind), value :: kind
             integer (kind=omp_integer_kind), value :: chunk_size
-          end subroutine omp_set_schedule
+          end subroutine omp_set_schedule_c
 
-          subroutine omp_get_schedule(kind, chunk_size) bind(c)
+          subroutine omp_get_schedule_c(kind, chunk_size) bind(c, name='omp_get_schedule')
             use omp_lib_kinds
             integer (kind=omp_sched_kind) kind
             integer (kind=omp_integer_kind) chunk_size
-          end subroutine omp_get_schedule
+          end subroutine omp_get_schedule_c
 
           function omp_get_proc_bind() bind(c)
             use omp_lib_kinds
@@ -364,17 +385,17 @@
             integer (kind=omp_integer_kind) omp_get_num_places
           end function omp_get_num_places
 
-          function omp_get_place_num_procs(place_num) bind(c)
+          function omp_get_place_num_procs_c(place_num) bind(c, name='omp_get_place_num_procs')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: place_num
-            integer (kind=omp_integer_kind) omp_get_place_num_procs
-          end function omp_get_place_num_procs
+            integer (kind=omp_integer_kind) omp_get_place_num_procs_c
+          end function omp_get_place_num_procs_c
 
-          subroutine omp_get_place_proc_ids(place_num, ids) bind(c)
+          subroutine omp_get_place_proc_ids_c(place_num, ids) bind(c, name='omp_get_place_proc_ids')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: place_num
             integer (kind=omp_integer_kind) ids(*)
-          end subroutine omp_get_place_proc_ids
+          end subroutine omp_get_place_proc_ids_c
 
           function omp_get_place_num() bind(c)
             use omp_lib_kinds
@@ -386,10 +407,10 @@
             integer (kind=omp_integer_kind) omp_get_partition_num_places
           end function omp_get_partition_num_places
 
-          subroutine omp_get_partition_place_nums(place_nums) bind(c)
+          subroutine omp_get_partition_place_nums_c(place_nums) bind(c, name='omp_get_partition_place_nums')
             use omp_lib_kinds
             integer (kind=omp_integer_kind) place_nums(*)
-          end subroutine omp_get_partition_place_nums
+          end subroutine omp_get_partition_place_nums_c
 
           function omp_get_wtime() bind(c)
             use omp_lib_kinds
@@ -406,10 +427,10 @@
             integer (kind=omp_integer_kind) omp_get_default_device
           end function omp_get_default_device
 
-          subroutine omp_set_default_device(device_num) bind(c)
+          subroutine omp_set_default_device_c(device_num) bind(c, name='omp_set_default_device')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: device_num
-          end subroutine omp_set_default_device
+          end subroutine omp_set_default_device_c
 
           function omp_get_num_devices() bind(c)
             use omp_lib_kinds
@@ -446,12 +467,12 @@
             integer (kind=omp_integer_kind) omp_get_device_num
           end function omp_get_device_num
 
-          function omp_pause_resource(kind, device_num) bind(c)
+          function omp_pause_resource_c(kind, device_num) bind(c, name='omp_pause_resource')
             use omp_lib_kinds
             integer (kind=omp_pause_resource_kind), value :: kind
             integer (kind=omp_integer_kind), value :: device_num
-            integer (kind=omp_integer_kind) omp_pause_resource
-          end function omp_pause_resource
+            integer (kind=omp_integer_kind) omp_pause_resource_c
+          end function omp_pause_resource_c
 
           function omp_pause_resource_all(kind) bind(c)
             use omp_lib_kinds
@@ -640,20 +661,20 @@
             integer (kind=kmp_size_t_kind) :: omp_capture_affinity
           end function omp_capture_affinity
 
-          subroutine omp_set_num_teams(num_teams) bind(c)
+          subroutine omp_set_num_teams_c(num_teams) bind(c, name='omp_set_num_teams')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: num_teams
-          end subroutine omp_set_num_teams
+          end subroutine omp_set_num_teams_c
 
           function omp_get_max_teams() bind(c)
             use omp_lib_kinds
             integer (kind=omp_integer_kind) omp_get_max_teams
           end function omp_get_max_teams
 
-          subroutine omp_set_teams_thread_limit(thread_limit) bind(c)
+          subroutine omp_set_teams_thread_limit_c(thread_limit) bind(c, name='omp_set_teams_thread_limit')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: thread_limit
-          end subroutine omp_set_teams_thread_limit
+          end subroutine omp_set_teams_thread_limit_c
 
           function omp_get_teams_thread_limit() bind(c)
             use omp_lib_kinds
@@ -937,20 +958,20 @@
 !         *** kmp_* entry points
 !         ***
 
-          subroutine kmp_set_stacksize(size) bind(c)
+          subroutine kmp_set_stacksize_c(size) bind(c, name='kmp_set_stacksize')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: size
-          end subroutine kmp_set_stacksize
+          end subroutine kmp_set_stacksize_c
 
           subroutine kmp_set_stac...
[truncated]

``````````

</details>


https://github.com/llvm/llvm-project/pull/213505

From openmp-commits at lists.llvm.org  Sun Aug  2 05:01:19 2026
From: openmp-commits at lists.llvm.org (Robert Imschweiler via Openmp-commits)
Date: Sun, 02 Aug 2026 05:01:19 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [offload][OpenMP] Add
 atomic cross-team reductions (PR #209298)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/209298@github.com>
Message-ID: <6a6f318f.4987c784.20091a.93ab@mx.google.com>


ro-i wrote:

> Are you planning a follow up to turn this on depending on the target? [...]  I would like us to turn this on if we have reason to believe it's the faster path

Yes.

> you have good data for some HPC devices

Let me see if I can collect some more data about when/where atomic instructions are favorable to the default path. At the moment, I only have data for gfx90a and gfx942, which is not bad, but maybe I can get my hands on some more.

https://github.com/llvm/llvm-project/pull/209298

From openmp-commits at lists.llvm.org  Sun Aug  2 05:01:38 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Sun, 02 Aug 2026 05:01:38 -0700 (PDT)
Subject: [Openmp-commits] [openmp] 29a956e - [offload][OpenMP] Add atomic
 cross-team reductions (#209298)
Message-ID: <6a6f31a2.824e11a6.18590a.021d@mx.google.com>


Author: Robert Imschweiler
Date: 2026-08-02T14:01:32+02:00
New Revision: 29a956e40a1f617887ecfeb02ae844102c4a5a48

URL: https://github.com/llvm/llvm-project/commit/29a956e40a1f617887ecfeb02ae844102c4a5a48
DIFF: https://github.com/llvm/llvm-project/commit/29a956e40a1f617887ecfeb02ae844102c4a5a48.diff

LOG: [offload][OpenMP] Add atomic cross-team reductions (#209298)

Regular cross-team reductions have two phases: the intra-team reduction
and the inter-team reduction. Atomic cross-team reductions replace the
second phase with a atomic instruction which is used by the main thread
of each team to directly fold the result of the intra-team reduction
into the final result. Since this requires a combination of "data type"
and "combine operation" for which an atomic instruction is available,
only some (but very common) reductions can be transformed to atomic
reductions. In cases where multiple reductions are performed on the same
construct, the atomic path is only taken if all reductions can be
transformed. Otherwise, we fall back to the regular cross-team reduction
using a buffer with per-team slots. This is not strictly necessary, but
hybrid reductions would induce more complexity with questionable
benefit.

Selecting an atomic path might not be the best option for every
situation, which is why it is not enabled by default. Instead, it can be
enabled via `-fopenmp-target-atomic-reduction`. Note that enabling the
atomic path will not *force* atomic reductions. They will only be
applied if possible, as described above.

The performance (measured with https://github.com/ro-i/xteam-test @
c71339705091500f731e2a39f247d2660bacbdce, array size 177,777,777) is up
to +15% faster (aka, more throughput) for supported reductions on a
gfx942, with no noticeable regressions.
Example:
- sum reduction, type double: +10.22% faster
- sum reduction, type uint: +15.57% faster
- sum reduction, type ulong: +13.31% faster

On a gfx90a, there is little to negative benefit:
- sum reduction, type double: -4.32% faster (aka, slower)
- sum reduction, type uint: +3.08% faster
- sum reduction, type ulong: +1.68% faster

Claude assisted with this patch.

Added: 
    clang/test/Driver/openmp-target-atomic-reduction-flag.c
    clang/test/OpenMP/target_teams_atomic_reduction_codegen.cpp
    offload/test/offloading/xteam_atomic_reduction_usm.cpp

Modified: 
    clang/include/clang/Basic/LangOptions.def
    clang/include/clang/Options/Options.td
    clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
    clang/lib/Driver/ToolChains/Clang.cpp
    llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
    llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
    offload/test/offloading/multiple_reductions.cpp
    openmp/device/include/Interface.h
    openmp/device/src/Kernel.cpp

Removed: 
    


################################################################################
diff  --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def
index 3d63b9677e4df..0519514327355 100644
--- a/clang/include/clang/Basic/LangOptions.def
+++ b/clang/include/clang/Basic/LangOptions.def
@@ -241,6 +241,7 @@ LANGOPT(OpenMPNoThreadState  , 1, 0, NotCompatible, "Assume that no thread in a
 LANGOPT(OpenMPNoNestedParallelism  , 1, 0, NotCompatible, "Assume that no thread in a parallel region will encounter a parallel region")
 LANGOPT(OpenMPOffloadMandatory  , 1, 0, NotCompatible, "Assert that offloading is mandatory and do not create a host fallback.")
 LANGOPT(OpenMPForceUSM     , 1, 0, NotCompatible, "Enable OpenMP unified shared memory mode via compiler.")
+LANGOPT(OpenMPTargetAtomicReduction , 1, 0, NotCompatible, "Use atomic operations for OpenMP GPU cross-team reductions where supported.")
 LANGOPT(NoGPULib  , 1, 0, NotCompatible, "Indicate a build without the standard GPU libraries.")
 
 LANGOPT(HLSL, 1, 0, NotCompatible, "HLSL")

diff  --git a/clang/include/clang/Options/Options.td b/clang/include/clang/Options/Options.td
index 2defcee88f741..2467ebd0abe19 100644
--- a/clang/include/clang/Options/Options.td
+++ b/clang/include/clang/Options/Options.td
@@ -4306,6 +4306,15 @@ def fopenmp_target_fast : Flag<["-"], "fopenmp-target-fast">,
 def fno_openmp_target_fast : Flag<["-"], "fno-openmp-target-fast">,
   Group<f_Group>, Flags<[NoArgumentUnused, HelpHidden]>,
   Visibility<[ClangOption, FlangOption]>;
+def fopenmp_target_atomic_reduction : Flag<["-"], "fopenmp-target-atomic-reduction">,
+  Group<f_Group>, Flags<[NoArgumentUnused]>,
+  Visibility<[ClangOption, CC1Option]>,
+  HelpText<"Use atomic operations for OpenMP GPU cross-team reductions where "
+           "supported">,
+  MarshallingInfoFlag<LangOpts<"OpenMPTargetAtomicReduction">>;
+def fno_openmp_target_atomic_reduction : Flag<["-"], "fno-openmp-target-atomic-reduction">,
+  Group<f_Group>, Flags<[NoArgumentUnused, HelpHidden]>,
+  Visibility<[ClangOption, CC1Option]>;
 defm openmp_optimistic_collapse : BoolFOption<"openmp-optimistic-collapse",
   LangOpts<"OpenMPOptimisticCollapse">, DefaultFalse,
   PosFlag<SetTrue, [], [ClangOption, CC1Option]>,

diff  --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
index 79b2b275ba100..8f9000660d86b 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
@@ -14,6 +14,7 @@
 #include "CGOpenMPRuntimeGPU.h"
 #include "CGDebugInfo.h"
 #include "CodeGenFunction.h"
+#include "TargetInfo.h"
 #include "clang/AST/Attr.h"
 #include "clang/AST/DeclOpenMP.h"
 #include "clang/AST/OpenMPClause.h"
@@ -22,6 +23,8 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/Frontend/OpenMP/OMPDeviceConstants.h"
 #include "llvm/Frontend/OpenMP/OMPGridValues.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instructions.h"
 #include "llvm/TargetParser/NVPTXTargetParser.h"
 
 using namespace clang;
@@ -1445,6 +1448,59 @@ static llvm::Value *castValueToType(CodeGenFunction &CGF, llvm::Value *Val,
                               TBAAAccessInfo());
 }
 
+/// Extracts the built-in reduction operator from a combiner of the form `x = x
+/// <op> rhs` (or the min/max conditional), or nullopt if the shape is not
+/// recognized (e.g. user-defined reductions).
+static std::optional<BinaryOperatorKind>
+getReductionBinOpKind(const Expr *ReductionOp) {
+  const auto *Assign = dyn_cast<BinaryOperator>(ReductionOp);
+  if (!Assign || Assign->getOpcode() != BO_Assign)
+    return std::nullopt;
+  const Expr *RHS = Assign->getRHS();
+  // min/max are lowered as `x <cmp> rhs ? x : rhs`; the comparison identifies
+  // it.
+  if (const auto *ACO =
+          dyn_cast<AbstractConditionalOperator>(RHS->IgnoreParenImpCasts()))
+    RHS = ACO->getCond();
+  if (const auto *BO = dyn_cast<BinaryOperator>(RHS->IgnoreParenImpCasts()))
+    return BO->getOpcode();
+  return std::nullopt;
+}
+
+/// Maps a built-in reduction operator to an atomicrmw opcode for the atomic
+/// cross-team reduction fast path, or nullopt if there is no direct atomicrmw
+/// (e.g. user-defined, complex, fp min/max) so the buffer path is used instead.
+static std::optional<llvm::AtomicRMWInst::BinOp>
+getReductionAtomicRMWOp(BinaryOperatorKind BOK, QualType Ty) {
+  bool IsInt = Ty->isIntegerType();
+  bool IsSigned = Ty->hasSignedIntegerRepresentation();
+  switch (BOK) {
+  case BO_Add:
+  case BO_Sub: // A `-` reduction sums the partials, so it accumulates with add.
+    if (IsInt)
+      return llvm::AtomicRMWInst::Add;
+    if (Ty->isFloatingType())
+      return llvm::AtomicRMWInst::FAdd;
+    return std::nullopt;
+  case BO_And:
+    return IsInt ? std::optional(llvm::AtomicRMWInst::And) : std::nullopt;
+  case BO_Or:
+    return IsInt ? std::optional(llvm::AtomicRMWInst::Or) : std::nullopt;
+  case BO_Xor:
+    return IsInt ? std::optional(llvm::AtomicRMWInst::Xor) : std::nullopt;
+  case BO_LT: // min
+    if (IsInt)
+      return IsSigned ? llvm::AtomicRMWInst::Min : llvm::AtomicRMWInst::UMin;
+    return std::nullopt;
+  case BO_GT: // max
+    if (IsInt)
+      return IsSigned ? llvm::AtomicRMWInst::Max : llvm::AtomicRMWInst::UMax;
+    return std::nullopt;
+  default:
+    return std::nullopt;
+  }
+}
+
 ///
 /// Design of OpenMP reductions on the GPU
 ///
@@ -1716,8 +1772,13 @@ void CGOpenMPRuntimeGPU::emitReduction(
   const RecordDecl *ReductionRec = ::buildRecordForGlobalizedVars(
       CGM.getContext(), PrivatesReductions, {}, VarFieldMap, 1);
 
-  if (TeamsReduction)
-    TeamsReductions.push_back(ReductionRec);
+  // The atomic cross-team reduction fast path is opt-in. Hand each eligible
+  // scalar reduction an atomic combiner; createReductionsGPU uses the atomic
+  // path only if every reduction in the set has one. Track whether that holds
+  // so we can skip the (then unused) per-team buffer registration.
+  bool UseAtomicReduction =
+      TeamsReduction && CGM.getLangOpts().OpenMPTargetAtomicReduction;
+  bool AllAtomicable = UseAtomicReduction;
 
   // Source location for the ident struct
   llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
@@ -1780,6 +1841,45 @@ void CGOpenMPRuntimeGPU::emitReduction(
       return InsertPointTy(CGF.Builder.GetInsertBlock(),
                            CGF.Builder.GetInsertPoint());
     };
+
+    // For the atomic fast path, hand this reduction an atomic combiner if it is
+    // a scalar with a direct atomicrmw; otherwise the set is not fully
+    // atomicable and falls back to the buffer path.
+    if (UseAtomicReduction) {
+      std::optional<llvm::AtomicRMWInst::BinOp> AtomicOp;
+      if (EvalKind == llvm::OpenMPIRBuilder::EvalKind::Scalar) {
+        if (std::optional<BinaryOperatorKind> BOK =
+                getReductionBinOpKind(ReductionOps[Idx]))
+          AtomicOp = getReductionAtomicRMWOp(*BOK, Private->getType());
+      }
+      if (!AtomicOp) {
+        AllAtomicable = false;
+      } else {
+        llvm::AtomicRMWInst::BinOp Op = *AtomicOp;
+        llvm::Align Alignment =
+            CGM.getModule().getDataLayout().getPrefTypeAlign(ElementType);
+        // Device (agent) scope suffices: all teams accumulate on-device and the
+        // host reads the result only after the kernel (via map-back), so the
+        // far costlier system scope is unnecessary. The
+        // no.fine.grained/no.remote memory metadata is omitted so the atomic
+        // stays correct under USM.
+        llvm::SyncScope::ID SSID = CGF.getTargetHooks().getLLVMSyncScopeID(
+            CGF.getLangOpts(), SyncScope::DeviceScope,
+            llvm::AtomicOrdering::Monotonic, CGF.getLLVMContext());
+        AtomicReductionGen = [Op, Alignment,
+                              SSID](InsertPointTy IP, llvm::Type *EltTy,
+                                    llvm::Value *LHS, llvm::Value *RHS)
+            -> llvm::OpenMPIRBuilder::InsertPointOrErrorTy {
+          llvm::IRBuilder<> Builder(IP.getBlock(), IP.getPoint());
+          llvm::Value *Val = Builder.CreateLoad(EltTy, RHS);
+          Builder.CreateAtomicRMW(Op, LHS, Val, Alignment,
+                                  llvm::AtomicOrdering::Monotonic, SSID);
+          return InsertPointTy(Builder.GetInsertBlock(),
+                               Builder.GetInsertPoint());
+        };
+      }
+    }
+
     ReductionInfos.emplace_back(llvm::OpenMPIRBuilder::ReductionInfo(
         ElementType, Variable, PrivateVariable, EvalKind,
         /*ReductionGen=*/nullptr, ReductionGen, AtomicReductionGen,
@@ -1787,6 +1887,11 @@ void CGOpenMPRuntimeGPU::emitReduction(
     Idx++;
   }
 
+  // The atomic path folds directly into the mapped variable and needs no
+  // per-team buffer; register the record for buffer allocation otherwise.
+  if (TeamsReduction && !AllAtomicable)
+    TeamsReductions.push_back(ReductionRec);
+
   bool IsSPMD = getExecutionMode() == CGOpenMPRuntimeGPU::EM_SPMD;
   llvm::OpenMPIRBuilder::InsertPointTy AfterIP =
       cantFail(OMPBuilder.createReductionsGPU(

diff  --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 7f3b503153ab1..9e90994c178dd 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -7128,6 +7128,12 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
                        /*Default=*/TargetFastUsed))
         CmdArgs.push_back("-fopenmp-assume-no-nested-parallelism");
 
+      // Handle -fopenmp-target-atomic-reduction.
+      if (Args.hasFlag(options::OPT_fopenmp_target_atomic_reduction,
+                       options::OPT_fno_openmp_target_atomic_reduction,
+                       /*Default=*/false))
+        CmdArgs.push_back("-fopenmp-target-atomic-reduction");
+
       if (Args.hasArg(options::OPT_fopenmp_offload_mandatory))
         CmdArgs.push_back("-fopenmp-offload-mandatory");
       if (Args.hasArg(options::OPT_fopenmp_force_usm))

diff  --git a/clang/test/Driver/openmp-target-atomic-reduction-flag.c b/clang/test/Driver/openmp-target-atomic-reduction-flag.c
new file mode 100644
index 0000000000000..59362ddd78e94
--- /dev/null
+++ b/clang/test/Driver/openmp-target-atomic-reduction-flag.c
@@ -0,0 +1,18 @@
+// REQUIRES: x86-registered-target, amdgpu-registered-target
+
+// Not passed by default.
+// RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=amdgcn-amd-amdhsa -Xopenmp-target=amdgcn-amd-amdhsa -march=gfx90a -nogpulib %s 2>&1 \
+// RUN:   | FileCheck -check-prefix=DEFAULT %s
+
+// Passed through to -cc1 when requested.
+// RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=amdgcn-amd-amdhsa -Xopenmp-target=amdgcn-amd-amdhsa -march=gfx90a -nogpulib -fopenmp-target-atomic-reduction %s 2>&1 \
+// RUN:   | FileCheck -check-prefix=ENABLE %s
+
+// Explicit disable wins over a preceding enable and is not passed through.
+// RUN:   %clang -### -fopenmp=libomp -fopenmp-targets=amdgcn-amd-amdhsa -Xopenmp-target=amdgcn-amd-amdhsa -march=gfx90a -nogpulib -fopenmp-target-atomic-reduction -fno-openmp-target-atomic-reduction %s 2>&1 \
+// RUN:   | FileCheck -check-prefix=DEFAULT %s
+
+// DEFAULT-NOT: {{"-f(no-)?openmp-target-atomic-reduction"}}
+
+// ENABLE: "-fopenmp-target-atomic-reduction"
+// ENABLE-NOT: "-fno-openmp-target-atomic-reduction"

diff  --git a/clang/test/OpenMP/target_teams_atomic_reduction_codegen.cpp b/clang/test/OpenMP/target_teams_atomic_reduction_codegen.cpp
new file mode 100644
index 0000000000000..dea0ecff52be4
--- /dev/null
+++ b/clang/test/OpenMP/target_teams_atomic_reduction_codegen.cpp
@@ -0,0 +1,90 @@
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple x86_64-unknown-linux-gnu \
+// RUN:   -fopenmp-targets=amdgcn-amd-amdhsa -emit-llvm-bc %s -o %t-host.bc
+// RUN: %clang_cc1 -verify -fopenmp -x c++ -triple amdgcn-amd-amdhsa \
+// RUN:   -fopenmp-targets=amdgcn-amd-amdhsa -fopenmp-is-target-device \
+// RUN:   -fopenmp-target-atomic-reduction -fopenmp-host-ir-file-path %t-host.bc \
+// RUN:   -emit-llvm %s -o %t.ll
+// RUN: FileCheck --check-prefix=ATOMIC %s < %t.ll
+// RUN: FileCheck --check-prefix=BUFFER %s < %t.ll
+// RUN: FileCheck %s < %t.ll
+
+// expected-no-diagnostics
+
+// ATOMIC-DAG: call i32 @__kmpc_is_team_main_thread(
+// ATOMIC-DAG: atomicrmw add ptr {{.*}} syncscope("agent") monotonic
+// ATOMIC-DAG: atomicrmw add ptr {{.*}} syncscope("agent") monotonic
+// ATOMIC-DAG: atomicrmw and ptr {{.*}} syncscope("agent") monotonic
+// ATOMIC-DAG: atomicrmw or ptr {{.*}} syncscope("agent") monotonic
+// ATOMIC-DAG: atomicrmw xor ptr {{.*}} syncscope("agent") monotonic
+// ATOMIC-DAG: atomicrmw min ptr {{.*}} syncscope("agent") monotonic
+// ATOMIC-DAG: atomicrmw max ptr {{.*}} syncscope("agent") monotonic
+// ATOMIC-DAG: atomicrmw umin ptr {{.*}} syncscope("agent") monotonic
+// ATOMIC-DAG: atomicrmw umax ptr {{.*}} syncscope("agent") monotonic
+// ATOMIC-DAG: atomicrmw fadd ptr {{.*}} syncscope("agent") monotonic
+void atomicable_matrix(int n) {
+  int si = 0, su = 0, an = ~0, orv = 0, xr = 0, smn = 0x7fffffff,
+      smx = -0x7fffffff - 1;
+  unsigned umn = ~0u, umx = 0u;
+  float f = 0.0f;
+#pragma omp target teams distribute parallel for reduction(+ : si)              \
+    reduction(- : su) reduction(& : an) reduction(| : orv) reduction(^ : xr)    \
+    reduction(min : smn) reduction(max : smx) reduction(min : umn)              \
+    reduction(max : umx) reduction(+ : f)
+  for (int i = 0; i < n; ++i) {
+    si += i;
+    su -= i;
+    an &= i;
+    orv |= i;
+    xr ^= i;
+    smn = i < smn ? i : smn;
+    smx = i > smx ? i : smx;
+    umn = (unsigned)i < umn ? (unsigned)i : umn;
+    umx = (unsigned)i > umx ? (unsigned)i : umx;
+    f += i;
+  }
+}
+
+// A fp multiply and a fp max reduction have no direct atomicrmw, so each falls
+// back to the buffer path even with -fopenmp-target-atomic-reduction. Together with
+// the mixed construct below there are exactly three buffered writebacks.
+//
+// BUFFER-COUNT-3: call i32 @__kmpc_gpu_xteam_reduce_nowait(
+// BUFFER-NOT:     call i32 @__kmpc_gpu_xteam_reduce_nowait(
+double nonatomicable_mul(int n) {
+  double p = 1.0;
+#pragma omp target teams distribute parallel for reduction(* : p)
+  for (int i = 0; i < n; ++i)
+    p *= 1.0;
+  return p;
+}
+
+double nonatomicable_fpmax(int n) {
+  double mx = 0.0;
+#pragma omp target teams distribute parallel for reduction(max : mx)
+  for (int i = 0; i < n; ++i)
+    mx = (double)i > mx ? (double)i : mx;
+  return mx;
+}
+
+// A single non-atomicable reduction (fp multiply) poisons the whole set: the
+// atomic path requires *all* reductions to have an atomicrmw, so even the int
+// sum here is combined through the buffer, not atomically. Hence no team-main
+// guard and no `atomicrmw add` for `s` from this construct (asserted by the
+// module-wide counts above).
+//
+// CHECK-COUNT-1: call i32 @__kmpc_is_team_main_thread(
+// CHECK-NOT:     call i32 @__kmpc_is_team_main_thread(
+// CHECK-NOT: atomicrmw fmul
+// CHECK-NOT: atomicrmw fmax
+
+double mixed(int n) {
+  int s = 0;
+  double p = 1.0;
+#pragma omp target teams distribute parallel for reduction(+ : s)               \
+    reduction(* : p)
+  for (int i = 0; i < n; ++i) {
+    s += i;
+    p *= 1.0;
+  }
+  return s + p;
+}

diff  --git a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
index e253c838e5e28..ea1cef97900f5 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
@@ -505,6 +505,7 @@ __OMP_RTL(__kmpc_end_sharing_variables, false, Void, )
 __OMP_RTL(__kmpc_get_shared_variables, false, Void, VoidPtrPtrPtr)
 __OMP_RTL(__kmpc_parallel_level, false, Int16, IdentPtr, Int32)
 __OMP_RTL(__kmpc_is_spmd_exec_mode, false, Int8, )
+__OMP_RTL(__kmpc_is_team_main_thread, false, Int32, )
 __OMP_RTL(__kmpc_barrier_simple_spmd, false, Void, IdentPtr, Int32)
 __OMP_RTL(__kmpc_barrier_simple_generic, false, Void, IdentPtr, Int32)
 
@@ -1104,6 +1105,7 @@ __OMP_RTL_ATTRS(__kmpc_shuffle_int64, AttributeSet(), AttributeSet(),
                 ParamAttrs(AttributeSet(), SExt, SExt))
 
 __OMP_RTL_ATTRS(__kmpc_is_spmd_exec_mode, AttributeSet(), SExt, ParamAttrs())
+__OMP_RTL_ATTRS(__kmpc_is_team_main_thread, AttributeSet(), SExt, ParamAttrs())
 
 #undef __OMP_RTL_ATTRS
 #undef OMP_RTL_ATTRS

diff  --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index fc3812502fb2e..a227397b06f4e 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -4650,6 +4650,16 @@ checkReductionInfos(ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos,
   }
 }
 
+// The atomic cross-team reduction fast path applies when every reduction in the
+// set can be represented by an atomicrmw. Clang only populates it for scalar
+// reductions with a supported atomic operator.
+static bool isAtomicableReductionSet(
+    ArrayRef<OpenMPIRBuilder::ReductionInfo> ReductionInfos) {
+  return all_of(ReductionInfos, [](const OpenMPIRBuilder::ReductionInfo &RI) {
+    return static_cast<bool>(RI.AtomicReductionGen);
+  });
+}
+
 OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createReductionsGPU(
     const LocationDescription &Loc, InsertPointTy AllocaIP,
     InsertPointTy CodeGenIP, ArrayRef<ReductionInfo> ReductionInfos,
@@ -4791,6 +4801,9 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createReductionsGPU(
   // copied back. (Basically RL, appropriately casted if necessary.)
   Value *RLForCopyBack = RL;
 
+  bool IsAtomicReduction =
+      IsTeamsReduction && isAtomicableReductionSet(ReductionInfos);
+
   if (!IsTeamsReduction) {
     Value *SarFuncCast =
         Builder.CreatePointerBitCastOrAddrSpaceCast(*SarFunc, FuncPtrTy);
@@ -4801,6 +4814,12 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createReductionsGPU(
     Function *Pv2Ptr = getOrCreateRuntimeFunctionPtr(
         RuntimeFunction::OMPRTL___kmpc_nvptx_parallel_reduce_nowait_v2);
     Res = createRuntimeFunctionCall(Pv2Ptr, Args);
+  } else if (IsAtomicReduction) {
+    // Atomic cross-team reduction fast path: determine the team's main thread
+    // that is later to fold its value atomically into the mapped variable.
+    Function *IsMainThreadFn = getOrCreateRuntimeFunctionPtr(
+        RuntimeFunction::OMPRTL___kmpc_is_team_main_thread);
+    Res = createRuntimeFunctionCall(IsMainThreadFn, {});
   } else {
     CodeGenIP = Builder.saveIP();
     StructType *ReductionsBufferTy = StructType::create(
@@ -4939,6 +4958,19 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::createReductionsGPU(
   // Add emission of __kmpc_end_reduce{_nowait}(<gtid>);
   for (auto En : enumerate(ReductionInfos)) {
     const ReductionInfo &RI = En.value();
+
+    // Atomic cross-team fast path: each team's main thread folds its
+    // team-reduced value directly into the mapped reduction variable with a
+    // single atomicrmw.
+    if (IsAtomicReduction) {
+      InsertPointOrErrorTy AfterIP = RI.AtomicReductionGen(
+          Builder.saveIP(), RI.ElementType, RI.Variable, RI.PrivateVariable);
+      if (!AfterIP)
+        return AfterIP.takeError();
+      Builder.restoreIP(*AfterIP);
+      continue;
+    }
+
     Type *ValueType = RI.ElementType;
     Value *RedValue = RI.Variable;
 

diff  --git a/offload/test/offloading/multiple_reductions.cpp b/offload/test/offloading/multiple_reductions.cpp
index 6c759416feaf5..9d993b5f48e00 100644
--- a/offload/test/offloading/multiple_reductions.cpp
+++ b/offload/test/offloading/multiple_reductions.cpp
@@ -1,5 +1,9 @@
 // RUN: %libomptarget-compilexx-run-and-check-generic
 // RUN: %libomptarget-compileoptxx-run-and-check-generic
+// RUN: %libomptarget-compilexx-generic -fopenmp-target-atomic-reduction && \
+// RUN:   %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compileoptxx-generic -fopenmp-target-atomic-reduction && \
+// RUN:   %libomptarget-run-generic | %fcheck-generic
 // UNSUPPORTED: intelgpu
 
 // See array_reductions.cpp for a clone of this file for array reductions.

diff  --git a/offload/test/offloading/xteam_atomic_reduction_usm.cpp b/offload/test/offloading/xteam_atomic_reduction_usm.cpp
new file mode 100644
index 0000000000000..ef00e0a2ee6df
--- /dev/null
+++ b/offload/test/offloading/xteam_atomic_reduction_usm.cpp
@@ -0,0 +1,64 @@
+// Validate the atomic cross-team reduction fast path under unified shared
+// memory.
+//
+// RUN: %libomptarget-compilexx-generic -fopenmp-force-usm \
+// RUN:   -fopenmp-target-atomic-reduction && env HSA_XNACK=1 \
+// RUN:   %libomptarget-run-generic | %fcheck-generic
+// RUN: %libomptarget-compileoptxx-generic -fopenmp-force-usm \
+// RUN:   -fopenmp-target-atomic-reduction && env HSA_XNACK=1 \
+// RUN:   %libomptarget-run-generic | %fcheck-generic
+//
+// REQUIRES: amdgpu, unified_shared_memory
+
+#include <cassert>
+#include <climits>
+#include <cstdio>
+#include <omp.h>
+
+int main() {
+  const int N = 1 << 16;
+
+  long *sum = (long *)omp_alloc(sizeof(long), llvm_omp_target_shared_mem_alloc);
+  *sum = 7;
+#pragma omp target teams distribute parallel for reduction(+ : sum[0])
+  for (int i = 0; i < N; ++i)
+    sum[0] += i;
+  assert(sum[0] == 7 + (long)(N - 1) * N / 2 && "atomic + reduction incorrect");
+  omp_free(sum, llvm_omp_target_shared_mem_alloc);
+
+  double *fsum =
+      (double *)omp_alloc(sizeof(double), llvm_omp_target_shared_mem_alloc);
+  *fsum = 0.5;
+#pragma omp target teams distribute parallel for reduction(+ : fsum[0])
+  for (int i = 0; i < N; ++i)
+    fsum[0] += 1.0;
+  assert(fsum[0] == 0.5 + (double)N && "atomic fp + reduction incorrect");
+  omp_free(fsum, llvm_omp_target_shared_mem_alloc);
+
+  int *mx = (int *)omp_alloc(sizeof(int), llvm_omp_target_shared_mem_alloc);
+  int *mn = (int *)omp_alloc(sizeof(int), llvm_omp_target_shared_mem_alloc);
+  *mx = INT_MIN;
+  *mn = INT_MAX;
+#pragma omp target teams distribute parallel for reduction(max : mx[0])        \
+    reduction(min : mn[0])
+  for (int i = 0; i < N; ++i) {
+    mx[0] = i > mx[0] ? i : mx[0];
+    mn[0] = i < mn[0] ? i : mn[0];
+  }
+  assert(mx[0] == N - 1 && mn[0] == 0 && "atomic min/max reduction incorrect");
+  omp_free(mx, llvm_omp_target_shared_mem_alloc);
+  omp_free(mn, llvm_omp_target_shared_mem_alloc);
+
+  unsigned *bits =
+      (unsigned *)omp_alloc(sizeof(unsigned), llvm_omp_target_shared_mem_alloc);
+  *bits = 0;
+#pragma omp target teams distribute parallel for reduction(| : bits[0])
+  for (int i = 0; i < N; ++i)
+    bits[0] |= (unsigned)i;
+  assert(bits[0] == (unsigned)(N - 1) && "atomic | reduction incorrect");
+  omp_free(bits, llvm_omp_target_shared_mem_alloc);
+
+  printf("SUCCESS\n");
+  // CHECK: SUCCESS
+  return 0;
+}

diff  --git a/openmp/device/include/Interface.h b/openmp/device/include/Interface.h
index cf455bf030270..601694871597b 100644
--- a/openmp/device/include/Interface.h
+++ b/openmp/device/include/Interface.h
@@ -225,6 +225,8 @@ struct KernelEnvironmentTy;
 
 int8_t __kmpc_is_spmd_exec_mode();
 
+int32_t __kmpc_is_team_main_thread();
+
 int32_t __kmpc_target_init(KernelEnvironmentTy &KernelEnvironment,
                            KernelLaunchEnvironmentTy *KernelLaunchEnvironment);
 

diff  --git a/openmp/device/src/Kernel.cpp b/openmp/device/src/Kernel.cpp
index 600aa0c8528b7..2f40e44496c5e 100644
--- a/openmp/device/src/Kernel.cpp
+++ b/openmp/device/src/Kernel.cpp
@@ -173,4 +173,9 @@ void __kmpc_target_deinit() {
 }
 
 int8_t __kmpc_is_spmd_exec_mode() { return mapping::isSPMDMode(); }
+
+/// Whether the calling thread is its team's main thread.
+int32_t __kmpc_is_team_main_thread() {
+  return mapping::isInitialThreadInLevel0(mapping::isSPMDMode());
+}
 }


        

From openmp-commits at lists.llvm.org  Sun Aug  2 05:01:40 2026
From: openmp-commits at lists.llvm.org (Robert Imschweiler via Openmp-commits)
Date: Sun, 02 Aug 2026 05:01:40 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [offload][OpenMP] Add
 atomic cross-team reductions (PR #209298)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/209298@github.com>
Message-ID: <6a6f31a4.2915925c.b6ff8.efa9@mx.google.com>


https://github.com/ro-i closed https://github.com/llvm/llvm-project/pull/209298

From openmp-commits at lists.llvm.org  Sun Aug  2 22:03:44 2026
From: openmp-commits at lists.llvm.org (Sairudra More via Openmp-commits)
Date: Sun, 02 Aug 2026 22:03:44 -0700 (PDT)
Subject: [Openmp-commits] [flang] [llvm] [openmp] [OpenMP][Flang] Add
 integer-kind wrappers to omp_lib (PR #213505)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213505@github.com>
Message-ID: <6a702130.af6ba3ed.191286.69e6@mx.google.com>


https://github.com/Saieiei commented:

Thanks for working on this. 

The new module currently fails to compile with gfortran 12 and 13 because nine added lines exceed the free-form line-length limit; details inline.

https://github.com/llvm/llvm-project/pull/213505

From openmp-commits at lists.llvm.org  Sun Aug  2 22:03:44 2026
From: openmp-commits at lists.llvm.org (Sairudra More via Openmp-commits)
Date: Sun, 02 Aug 2026 22:03:44 -0700 (PDT)
Subject: [Openmp-commits] [flang] [llvm] [openmp] [OpenMP][Flang] Add
 integer-kind wrappers to omp_lib (PR #213505)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213505@github.com>
Message-ID: <6a702130.2915925c.b6ff8.27ec@mx.google.com>


https://github.com/Saieiei edited https://github.com/llvm/llvm-project/pull/213505

From openmp-commits at lists.llvm.org  Sun Aug  2 22:03:45 2026
From: openmp-commits at lists.llvm.org (Sairudra More via Openmp-commits)
Date: Sun, 02 Aug 2026 22:03:45 -0700 (PDT)
Subject: [Openmp-commits] [flang] [llvm] [openmp] [OpenMP][Flang] Add
 integer-kind wrappers to omp_lib (PR #213505)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213505@github.com>
Message-ID: <6a702131.21a19891.251803.db8c@mx.google.com>


================
@@ -1150,6 +1171,69 @@
 
         end interface
 
+        ! Generic interfaces for multiple integer kinds
+        interface omp_set_num_threads
+          module procedure omp_set_num_threads_i1, omp_set_num_threads_i2, omp_set_num_threads_i4, omp_set_num_threads_i8
+        end interface omp_set_num_threads
+        interface omp_set_max_active_levels
+          module procedure omp_set_max_active_levels_i1, omp_set_max_active_levels_i2, omp_set_max_active_levels_i4, omp_set_max_active_levels_i8
----------------
Saieiei wrote:

This and eight other added `module procedure` lines exceed 132 columns. With the existing GNU build flags, the merge-base module compiles, but this head fails with nine line-truncation errors on gfortran 12 and 13. Could these lists be wrapped using `&` continuations?

https://github.com/llvm/llvm-project/pull/213505

From openmp-commits at lists.llvm.org  Sun Aug  2 22:10:20 2026
From: openmp-commits at lists.llvm.org (Sairudra More via Openmp-commits)
Date: Sun, 02 Aug 2026 22:10:20 -0700 (PDT)
Subject: [Openmp-commits] [flang] [llvm] [openmp] [OpenMP][Flang] Add
 integer-kind wrappers to omp_lib (PR #213505)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213505@github.com>
Message-ID: <6a7022bc.46c6f052.1e65a3.6fd7@mx.google.com>


https://github.com/Saieiei deleted https://github.com/llvm/llvm-project/pull/213505

From openmp-commits at lists.llvm.org  Sun Aug  2 22:11:05 2026
From: openmp-commits at lists.llvm.org (Sairudra More via Openmp-commits)
Date: Sun, 02 Aug 2026 22:11:05 -0700 (PDT)
Subject: [Openmp-commits] [flang] [llvm] [openmp] [OpenMP][Flang] Add
 integer-kind wrappers to omp_lib (PR #213505)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213505@github.com>
Message-ID: <6a7022e9.436e7ae1.315368.9a08@mx.google.com>


https://github.com/Saieiei requested changes to this pull request.


https://github.com/llvm/llvm-project/pull/213505

From openmp-commits at lists.llvm.org  Sun Aug  2 22:11:05 2026
From: openmp-commits at lists.llvm.org (Sairudra More via Openmp-commits)
Date: Sun, 02 Aug 2026 22:11:05 -0700 (PDT)
Subject: [Openmp-commits] [flang] [llvm] [openmp] [OpenMP][Flang] Add
 integer-kind wrappers to omp_lib (PR #213505)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213505@github.com>
Message-ID: <6a7022e9.7d702ed2.104899.1a16@mx.google.com>


================
@@ -1150,6 +1171,69 @@
 
         end interface
 
+        ! Generic interfaces for multiple integer kinds
+        interface omp_set_num_threads
+          module procedure omp_set_num_threads_i1, omp_set_num_threads_i2, omp_set_num_threads_i4, omp_set_num_threads_i8
+        end interface omp_set_num_threads
+        interface omp_set_max_active_levels
+          module procedure omp_set_max_active_levels_i1, omp_set_max_active_levels_i2, omp_set_max_active_levels_i4, omp_set_max_active_levels_i8
----------------
Saieiei wrote:

This and eight other added `module procedure` lines exceed 132 columns. With the existing GNU build flags, the merge-base module compiles, but this head fails with nine line-truncation errors on gfortran 12 and 13. Could these lists be wrapped using `&` continuations?

https://github.com/llvm/llvm-project/pull/213505

From openmp-commits at lists.llvm.org  Sun Aug  2 23:41:55 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Sun, 02 Aug 2026 23:41:55 -0700 (PDT)
Subject: [Openmp-commits] [flang] [llvm] [openmp] [OpenMP][Flang] Add
 integer-kind wrappers to omp_lib (PR #213505)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213505@github.com>
Message-ID: <6a703833.af6ba3ed.191286.9b29@mx.google.com>


https://github.com/shivaramaarao updated https://github.com/llvm/llvm-project/pull/213505

>From e50fd4249697dd08447deb4f1bc5ecf71b59214b Mon Sep 17 00:00:00 2001
From: Shivarama Rao <shivarama.rao at amd.com>
Date: Sun, 2 Aug 2026 10:20:39 +0530
Subject: [PATCH] [OpenMP][Flang] Add integer-kind wrappers to omp_lib

Many OpenMP routines in omp_lib only accept integer(kind=omp_integer_kind),
which is c_int. Calls with integer(1), integer(2), or integer(8) arguments
therefore fail Flang's type checking. These wrapper functions are added
to omp_lib module file to resolve this issue.

fixes issue #123948

Co-authored-by: Cursor <cursoragent at cursor.com>
---
 flang-rt/lib/runtime/CMakeLists.txt           |  12 +
 .../Lower/OpenMP/omp-lib-integer-wrappers.f90 | 237 +++++
 .../test/Lower/OpenMP/omp-lib-num-threads.f90 |   2 +-
 openmp/module/omp_lib.F90.var                 | 848 +++++++++++++++++-
 4 files changed, 1051 insertions(+), 48 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/omp-lib-integer-wrappers.f90

diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index ae9a6e7fb7188..4cd3090113af3 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -322,6 +322,13 @@ else ()
   set(sources ${supported_sources} ${host_sources} ${f128_sources})
 endif ()
 
+# Include the Fortran OpenMP module implementation (omp_lib.o) in
+# libflang_rt.runtime when the OpenMP runtime is also being built.
+# Skip for GPU-default targets: omp_lib is a host-side Fortran module.
+if (TARGET libomp-mod AND NOT "${LLVM_DEFAULT_TARGET_TRIPLE}" MATCHES "^amdgcn|^nvptx")
+  list(APPEND sources "$<TARGET_OBJECTS:libomp-mod>")
+endif ()
+
 
 if (NOT WIN32)
   add_flangrt_library(flang_rt.runtime STATIC SHARED
@@ -333,6 +340,11 @@ if (NOT WIN32)
 
   enable_cuda_compilation(flang_rt.runtime "${sources}")
 
+  if (TARGET libomp-mod AND TARGET flang_rt.runtime.shared AND TARGET omp)
+    # Resolve bind(c) references from omp_lib when building the shared runtime.
+    target_link_libraries(flang_rt.runtime.shared PUBLIC omp)
+  endif ()
+
   # Select a default runtime, which is used for unit and regression tests.
   if (TARGET flang_rt.runtime.default)
     get_target_property(default_target flang_rt.runtime.default ALIASED_TARGET)
diff --git a/flang/test/Lower/OpenMP/omp-lib-integer-wrappers.f90 b/flang/test/Lower/OpenMP/omp-lib-integer-wrappers.f90
new file mode 100644
index 0000000000000..9881fe56261fa
--- /dev/null
+++ b/flang/test/Lower/OpenMP/omp-lib-integer-wrappers.f90
@@ -0,0 +1,237 @@
+! REQUIRES: openmp_runtime
+
+! RUN: %flang_fc1 -emit-hlfir %openmp_flags %s -o - 2>&1 | FileCheck %s
+! RUN: bbc %openmp_flags -emit-hlfir -o - %s 2>&1 | FileCheck %s
+! RUN: %flang_fc1 -emit-fir %openmp_flags %s -o - 2>&1 | FileCheck %s
+! RUN: bbc -emit-fir %openmp_flags -o - %s 2>&1 | FileCheck %s
+!
+! Test that omp_lib integer-kind wrapper procedures lower to module
+! procedures instead of failing intrinsic resolution.
+
+program main
+  use omp_lib
+  integer(1) :: i1 = 1_1
+  integer(2) :: i2 = 2_2
+  integer(4) :: i4 = 4_4
+  integer(8) :: i8 = 8_8
+  integer(omp_integer_kind) :: ires
+  integer(omp_sched_kind) :: sched_kind
+  integer(kmp_affinity_mask_kind) :: mask
+
+  call omp_set_num_threads(i1)
+  call omp_set_num_threads(i2)
+  call omp_set_num_threads(i4)
+  call omp_set_num_threads(i8)
+
+  call omp_set_max_active_levels(i1)
+  call omp_set_max_active_levels(i2)
+  call omp_set_max_active_levels(i4)
+  call omp_set_max_active_levels(i8)
+
+  call omp_set_default_device(i1)
+  call omp_set_default_device(i2)
+  call omp_set_default_device(i4)
+  call omp_set_default_device(i8)
+
+  call omp_set_num_teams(i1)
+  call omp_set_num_teams(i2)
+  call omp_set_num_teams(i4)
+  call omp_set_num_teams(i8)
+
+  call omp_set_teams_thread_limit(i1)
+  call omp_set_teams_thread_limit(i2)
+  call omp_set_teams_thread_limit(i4)
+  call omp_set_teams_thread_limit(i8)
+
+  call kmp_set_stacksize(i1)
+  call kmp_set_stacksize(i2)
+  call kmp_set_stacksize(i4)
+  call kmp_set_stacksize(i8)
+
+  call kmp_set_blocktime(i1)
+  call kmp_set_blocktime(i2)
+  call kmp_set_blocktime(i4)
+  call kmp_set_blocktime(i8)
+
+  call kmp_set_library(i1)
+  call kmp_set_library(i2)
+  call kmp_set_library(i4)
+  call kmp_set_library(i8)
+
+  call kmp_set_disp_num_buffers(i1)
+  call kmp_set_disp_num_buffers(i2)
+  call kmp_set_disp_num_buffers(i4)
+  call kmp_set_disp_num_buffers(i8)
+
+  ires = omp_get_ancestor_thread_num(i1)
+  ires = omp_get_ancestor_thread_num(i2)
+  ires = omp_get_ancestor_thread_num(i4)
+  ires = omp_get_ancestor_thread_num(i8)
+
+  ires = omp_get_team_size(i1)
+  ires = omp_get_team_size(i2)
+  ires = omp_get_team_size(i4)
+  ires = omp_get_team_size(i8)
+
+  ires = omp_get_place_num_procs(i1)
+  ires = omp_get_place_num_procs(i2)
+  ires = omp_get_place_num_procs(i4)
+  ires = omp_get_place_num_procs(i8)
+
+  call omp_set_schedule(omp_sched_static, i1)
+  call omp_set_schedule(omp_sched_static, i2)
+  call omp_set_schedule(omp_sched_static, i4)
+  call omp_set_schedule(omp_sched_static, i8)
+
+  call omp_get_schedule(sched_kind, i1)
+  call omp_get_schedule(sched_kind, i2)
+  call omp_get_schedule(sched_kind, i4)
+  call omp_get_schedule(sched_kind, i8)
+
+  ires = omp_pause_resource(omp_pause_soft, i1)
+  ires = omp_pause_resource(omp_pause_soft, i2)
+  ires = omp_pause_resource(omp_pause_soft, i4)
+  ires = omp_pause_resource(omp_pause_soft, i8)
+
+  call kmp_create_affinity_mask(mask)
+  ires = kmp_set_affinity_mask_proc(i1, mask)
+  ires = kmp_set_affinity_mask_proc(i2, mask)
+  ires = kmp_set_affinity_mask_proc(i4, mask)
+  ires = kmp_set_affinity_mask_proc(i8, mask)
+
+  ires = kmp_unset_affinity_mask_proc(i1, mask)
+  ires = kmp_unset_affinity_mask_proc(i2, mask)
+  ires = kmp_unset_affinity_mask_proc(i4, mask)
+  ires = kmp_unset_affinity_mask_proc(i8, mask)
+
+  ires = kmp_get_affinity_mask_proc(i1, mask)
+  ires = kmp_get_affinity_mask_proc(i2, mask)
+  ires = kmp_get_affinity_mask_proc(i4, mask)
+  ires = kmp_get_affinity_mask_proc(i8, mask)
+
+  block
+    integer(4) :: ids4(1)
+    integer(8) :: ids8(1)
+    integer(4) :: parts4(1)
+    integer(8) :: parts8(1)
+    call omp_get_place_proc_ids(i4, ids4)
+    call omp_get_place_proc_ids(i8, ids8)
+    call omp_get_partition_place_nums(parts4)
+    call omp_get_partition_place_nums(parts8)
+  end block
+end program
+
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_max_active_levels
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_default_device
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_num_teams
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_teams_thread_limit
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_stacksize
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_blocktime
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_library
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_disp_num_buffers
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_ancestor_thread_num
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_team_size
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_place_num_procs
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_schedule
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_schedule
+!CHECK-NOT: not yet implemented: intrinsic: omp_pause_resource
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_affinity_mask_proc
+!CHECK-NOT: not yet implemented: intrinsic: kmp_unset_affinity_mask_proc
+!CHECK-NOT: not yet implemented: intrinsic: kmp_get_affinity_mask_proc
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_place_proc_ids
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_partition_place_nums
+
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i1
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i2
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i4
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_max_active_levels_i1
+!CHECK: fir.call @_QMomp_libPomp_set_max_active_levels_i2
+!CHECK: fir.call @_QMomp_libPomp_set_max_active_levels_i4
+!CHECK: fir.call @_QMomp_libPomp_set_max_active_levels_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_default_device_i1
+!CHECK: fir.call @_QMomp_libPomp_set_default_device_i2
+!CHECK: fir.call @_QMomp_libPomp_set_default_device_i4
+!CHECK: fir.call @_QMomp_libPomp_set_default_device_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_num_teams_i1
+!CHECK: fir.call @_QMomp_libPomp_set_num_teams_i2
+!CHECK: fir.call @_QMomp_libPomp_set_num_teams_i4
+!CHECK: fir.call @_QMomp_libPomp_set_num_teams_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_teams_thread_limit_i1
+!CHECK: fir.call @_QMomp_libPomp_set_teams_thread_limit_i2
+!CHECK: fir.call @_QMomp_libPomp_set_teams_thread_limit_i4
+!CHECK: fir.call @_QMomp_libPomp_set_teams_thread_limit_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_stacksize_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_stacksize_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_stacksize_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_stacksize_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_blocktime_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_blocktime_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_blocktime_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_blocktime_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_library_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_library_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_library_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_library_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_disp_num_buffers_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_disp_num_buffers_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_disp_num_buffers_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_disp_num_buffers_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_ancestor_thread_num_i1
+!CHECK: fir.call @_QMomp_libPomp_get_ancestor_thread_num_i2
+!CHECK: fir.call @_QMomp_libPomp_get_ancestor_thread_num_i4
+!CHECK: fir.call @_QMomp_libPomp_get_ancestor_thread_num_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_team_size_i1
+!CHECK: fir.call @_QMomp_libPomp_get_team_size_i2
+!CHECK: fir.call @_QMomp_libPomp_get_team_size_i4
+!CHECK: fir.call @_QMomp_libPomp_get_team_size_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_place_num_procs_i1
+!CHECK: fir.call @_QMomp_libPomp_get_place_num_procs_i2
+!CHECK: fir.call @_QMomp_libPomp_get_place_num_procs_i4
+!CHECK: fir.call @_QMomp_libPomp_get_place_num_procs_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_schedule_i1
+!CHECK: fir.call @_QMomp_libPomp_set_schedule_i2
+!CHECK: fir.call @_QMomp_libPomp_set_schedule_i4
+!CHECK: fir.call @_QMomp_libPomp_set_schedule_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_schedule_i1
+!CHECK: fir.call @_QMomp_libPomp_get_schedule_i2
+!CHECK: fir.call @_QMomp_libPomp_get_schedule_i4
+!CHECK: fir.call @_QMomp_libPomp_get_schedule_i8
+
+!CHECK: fir.call @_QMomp_libPomp_pause_resource_i1
+!CHECK: fir.call @_QMomp_libPomp_pause_resource_i2
+!CHECK: fir.call @_QMomp_libPomp_pause_resource_i4
+!CHECK: fir.call @_QMomp_libPomp_pause_resource_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_affinity_mask_proc_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_affinity_mask_proc_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_affinity_mask_proc_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_affinity_mask_proc_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_unset_affinity_mask_proc_i1
+!CHECK: fir.call @_QMomp_libPkmp_unset_affinity_mask_proc_i2
+!CHECK: fir.call @_QMomp_libPkmp_unset_affinity_mask_proc_i4
+!CHECK: fir.call @_QMomp_libPkmp_unset_affinity_mask_proc_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_get_affinity_mask_proc_i1
+!CHECK: fir.call @_QMomp_libPkmp_get_affinity_mask_proc_i2
+!CHECK: fir.call @_QMomp_libPkmp_get_affinity_mask_proc_i4
+!CHECK: fir.call @_QMomp_libPkmp_get_affinity_mask_proc_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_place_proc_ids_i4
+!CHECK: fir.call @_QMomp_libPomp_get_place_proc_ids_i8
+!CHECK: fir.call @_QMomp_libPomp_get_partition_place_nums_i4
+!CHECK: fir.call @_QMomp_libPomp_get_partition_place_nums_i8
diff --git a/flang/test/Lower/OpenMP/omp-lib-num-threads.f90 b/flang/test/Lower/OpenMP/omp-lib-num-threads.f90
index 6e76dce473057..a2ec031a62d88 100644
--- a/flang/test/Lower/OpenMP/omp-lib-num-threads.f90
+++ b/flang/test/Lower/OpenMP/omp-lib-num-threads.f90
@@ -21,5 +21,5 @@ program main
 
 !CHECK-NOT: not yet implemented: intrinsic: omp_set_num_threads
 !CHECK-NOT: not yet implemented: intrinsic: omp_get_num_threads
-!CHECK: fir.call @omp_set_num_threads
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i4
 !CHECK: fir.call @omp_get_num_threads
diff --git a/openmp/module/omp_lib.F90.var b/openmp/module/omp_lib.F90.var
index a41b2faad9d3d..c5b0b8222b109 100644
--- a/openmp/module/omp_lib.F90.var
+++ b/openmp/module/omp_lib.F90.var
@@ -63,6 +63,27 @@
         ! to be exported by this MODULE.
         private
 
+        private :: kmp_get_affinity_mask_proc_c
+        private :: kmp_set_affinity_mask_proc_c
+        private :: kmp_set_blocktime_c
+        private :: kmp_set_disp_num_buffers_c
+        private :: kmp_set_library_c
+        private :: kmp_set_stacksize_c
+        private :: kmp_unset_affinity_mask_proc_c
+        private :: omp_get_ancestor_thread_num_c
+        private :: omp_get_partition_place_nums_c
+        private :: omp_get_place_num_procs_c
+        private :: omp_get_place_proc_ids_c
+        private :: omp_get_schedule_c
+        private :: omp_get_team_size_c
+        private :: omp_pause_resource_c
+        private :: omp_set_default_device_c
+        private :: omp_set_max_active_levels_c
+        private :: omp_set_num_teams_c
+        private :: omp_set_num_threads_c
+        private :: omp_set_schedule_c
+        private :: omp_set_teams_thread_limit_c
+
         ! Re-export definitions in omp_lib_kinds
         public :: omp_integer_kind
         public :: omp_logical_kind
@@ -250,10 +271,11 @@
 !         *** omp_* entry points
 !         ***
 
-          subroutine omp_set_num_threads(num_threads) bind(c)
+          subroutine omp_set_num_threads_c(num_threads) bind(c,                &
+              name='omp_set_num_threads')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: num_threads
-          end subroutine omp_set_num_threads
+          end subroutine omp_set_num_threads_c
 
           subroutine omp_set_dynamic(dynamic_threads) bind(c)
             use omp_lib_kinds
@@ -310,10 +332,11 @@
             integer (kind=omp_integer_kind) omp_get_thread_limit
           end function omp_get_thread_limit
 
-          subroutine omp_set_max_active_levels(max_levels) bind(c)
+          subroutine omp_set_max_active_levels_c(max_levels) bind(c,           &
+              name='omp_set_max_active_levels')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: max_levels
-          end subroutine omp_set_max_active_levels
+          end subroutine omp_set_max_active_levels_c
 
           function omp_get_max_active_levels() bind(c)
             use omp_lib_kinds
@@ -330,29 +353,32 @@
             integer (kind=omp_integer_kind) omp_get_active_level
           end function omp_get_active_level
 
-          function omp_get_ancestor_thread_num(level) bind(c)
+          function omp_get_ancestor_thread_num_c(level) bind(c,                &
+              name='omp_get_ancestor_thread_num')
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) omp_get_ancestor_thread_num
+            integer (kind=omp_integer_kind) omp_get_ancestor_thread_num_c
             integer (kind=omp_integer_kind), value :: level
-          end function omp_get_ancestor_thread_num
+          end function omp_get_ancestor_thread_num_c
 
-          function omp_get_team_size(level) bind(c)
+          function omp_get_team_size_c(level) bind(c, name='omp_get_team_size')
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) omp_get_team_size
+            integer (kind=omp_integer_kind) omp_get_team_size_c
             integer (kind=omp_integer_kind), value :: level
-          end function omp_get_team_size
+          end function omp_get_team_size_c
 
-          subroutine omp_set_schedule(kind, chunk_size) bind(c)
+          subroutine omp_set_schedule_c(kind, chunk_size) bind(c,              &
+              name='omp_set_schedule')
             use omp_lib_kinds
             integer (kind=omp_sched_kind), value :: kind
             integer (kind=omp_integer_kind), value :: chunk_size
-          end subroutine omp_set_schedule
+          end subroutine omp_set_schedule_c
 
-          subroutine omp_get_schedule(kind, chunk_size) bind(c)
+          subroutine omp_get_schedule_c(kind, chunk_size) bind(c,              &
+              name='omp_get_schedule')
             use omp_lib_kinds
             integer (kind=omp_sched_kind) kind
             integer (kind=omp_integer_kind) chunk_size
-          end subroutine omp_get_schedule
+          end subroutine omp_get_schedule_c
 
           function omp_get_proc_bind() bind(c)
             use omp_lib_kinds
@@ -364,17 +390,19 @@
             integer (kind=omp_integer_kind) omp_get_num_places
           end function omp_get_num_places
 
-          function omp_get_place_num_procs(place_num) bind(c)
+          function omp_get_place_num_procs_c(place_num) bind(c,                &
+              name='omp_get_place_num_procs')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: place_num
-            integer (kind=omp_integer_kind) omp_get_place_num_procs
-          end function omp_get_place_num_procs
+            integer (kind=omp_integer_kind) omp_get_place_num_procs_c
+          end function omp_get_place_num_procs_c
 
-          subroutine omp_get_place_proc_ids(place_num, ids) bind(c)
+          subroutine omp_get_place_proc_ids_c(place_num, ids) bind(c,          &
+              name='omp_get_place_proc_ids')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: place_num
             integer (kind=omp_integer_kind) ids(*)
-          end subroutine omp_get_place_proc_ids
+          end subroutine omp_get_place_proc_ids_c
 
           function omp_get_place_num() bind(c)
             use omp_lib_kinds
@@ -386,10 +414,11 @@
             integer (kind=omp_integer_kind) omp_get_partition_num_places
           end function omp_get_partition_num_places
 
-          subroutine omp_get_partition_place_nums(place_nums) bind(c)
+          subroutine omp_get_partition_place_nums_c(place_nums) bind(c,        &
+              name='omp_get_partition_place_nums')
             use omp_lib_kinds
             integer (kind=omp_integer_kind) place_nums(*)
-          end subroutine omp_get_partition_place_nums
+          end subroutine omp_get_partition_place_nums_c
 
           function omp_get_wtime() bind(c)
             use omp_lib_kinds
@@ -406,10 +435,11 @@
             integer (kind=omp_integer_kind) omp_get_default_device
           end function omp_get_default_device
 
-          subroutine omp_set_default_device(device_num) bind(c)
+          subroutine omp_set_default_device_c(device_num) bind(c,              &
+              name='omp_set_default_device')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: device_num
-          end subroutine omp_set_default_device
+          end subroutine omp_set_default_device_c
 
           function omp_get_num_devices() bind(c)
             use omp_lib_kinds
@@ -446,12 +476,13 @@
             integer (kind=omp_integer_kind) omp_get_device_num
           end function omp_get_device_num
 
-          function omp_pause_resource(kind, device_num) bind(c)
+          function omp_pause_resource_c(kind, device_num) bind(c,              &
+              name='omp_pause_resource')
             use omp_lib_kinds
             integer (kind=omp_pause_resource_kind), value :: kind
             integer (kind=omp_integer_kind), value :: device_num
-            integer (kind=omp_integer_kind) omp_pause_resource
-          end function omp_pause_resource
+            integer (kind=omp_integer_kind) omp_pause_resource_c
+          end function omp_pause_resource_c
 
           function omp_pause_resource_all(kind) bind(c)
             use omp_lib_kinds
@@ -640,20 +671,22 @@
             integer (kind=kmp_size_t_kind) :: omp_capture_affinity
           end function omp_capture_affinity
 
-          subroutine omp_set_num_teams(num_teams) bind(c)
+          subroutine omp_set_num_teams_c(num_teams) bind(c,                    &
+              name='omp_set_num_teams')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: num_teams
-          end subroutine omp_set_num_teams
+          end subroutine omp_set_num_teams_c
 
           function omp_get_max_teams() bind(c)
             use omp_lib_kinds
             integer (kind=omp_integer_kind) omp_get_max_teams
           end function omp_get_max_teams
 
-          subroutine omp_set_teams_thread_limit(thread_limit) bind(c)
+          subroutine omp_set_teams_thread_limit_c(thread_limit) bind(c,        &
+              name='omp_set_teams_thread_limit')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: thread_limit
-          end subroutine omp_set_teams_thread_limit
+          end subroutine omp_set_teams_thread_limit_c
 
           function omp_get_teams_thread_limit() bind(c)
             use omp_lib_kinds
@@ -937,20 +970,20 @@
 !         *** kmp_* entry points
 !         ***
 
-          subroutine kmp_set_stacksize(size) bind(c)
+          subroutine kmp_set_stacksize_c(size) bind(c, name='kmp_set_stacksize')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: size
-          end subroutine kmp_set_stacksize
+          end subroutine kmp_set_stacksize_c
 
           subroutine kmp_set_stacksize_s(size) bind(c)
             use omp_lib_kinds
             integer (kind=kmp_size_t_kind), value :: size
           end subroutine kmp_set_stacksize_s
 
-          subroutine kmp_set_blocktime(msec) bind(c)
+          subroutine kmp_set_blocktime_c(msec) bind(c, name='kmp_set_blocktime')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: msec
-          end subroutine kmp_set_blocktime
+          end subroutine kmp_set_blocktime_c
 
           subroutine kmp_set_library_serial() bind(c)
           end subroutine kmp_set_library_serial
@@ -961,10 +994,10 @@
           subroutine kmp_set_library_throughput() bind(c)
           end subroutine kmp_set_library_throughput
 
-          subroutine kmp_set_library(libnum) bind(c)
+          subroutine kmp_set_library_c(libnum) bind(c, name='kmp_set_library')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: libnum
-          end subroutine kmp_set_library
+          end subroutine kmp_set_library_c
 
           subroutine kmp_set_defaults(string)
             character (len=*) :: string
@@ -990,10 +1023,11 @@
             integer (kind=omp_integer_kind) kmp_get_library
           end function kmp_get_library
 
-          subroutine kmp_set_disp_num_buffers(num) bind(c)
+          subroutine kmp_set_disp_num_buffers_c(num) bind(c,                   &
+              name='kmp_set_disp_num_buffers')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: num
-          end subroutine kmp_set_disp_num_buffers
+          end subroutine kmp_set_disp_num_buffers_c
 
           function kmp_set_affinity(mask) bind(c)
             use omp_lib_kinds
@@ -1022,26 +1056,29 @@
             integer (kind=kmp_affinity_mask_kind) mask
           end subroutine kmp_destroy_affinity_mask
 
-          function kmp_set_affinity_mask_proc(proc, mask) bind(c)
+          function kmp_set_affinity_mask_proc_c(proc, mask) bind(c,            &
+              name='kmp_set_affinity_mask_proc')
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) kmp_set_affinity_mask_proc
+            integer (kind=omp_integer_kind) kmp_set_affinity_mask_proc_c
             integer (kind=omp_integer_kind), value :: proc
             integer (kind=kmp_affinity_mask_kind) mask
-          end function kmp_set_affinity_mask_proc
+          end function kmp_set_affinity_mask_proc_c
 
-          function kmp_unset_affinity_mask_proc(proc, mask) bind(c)
+          function kmp_unset_affinity_mask_proc_c(proc, mask) bind(c,          &
+              name='kmp_unset_affinity_mask_proc')
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) kmp_unset_affinity_mask_proc
+            integer (kind=omp_integer_kind) kmp_unset_affinity_mask_proc_c
             integer (kind=omp_integer_kind), value :: proc
             integer (kind=kmp_affinity_mask_kind) mask
-          end function kmp_unset_affinity_mask_proc
+          end function kmp_unset_affinity_mask_proc_c
 
-          function kmp_get_affinity_mask_proc(proc, mask) bind(c)
+          function kmp_get_affinity_mask_proc_c(proc, mask) bind(c,            &
+              name='kmp_get_affinity_mask_proc')
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) kmp_get_affinity_mask_proc
+            integer (kind=omp_integer_kind) kmp_get_affinity_mask_proc_c
             integer (kind=omp_integer_kind), value :: proc
             integer (kind=kmp_affinity_mask_kind) mask
-          end function kmp_get_affinity_mask_proc
+          end function kmp_get_affinity_mask_proc_c
 
           function kmp_malloc(size) bind(c)
             use omp_lib_kinds
@@ -1150,6 +1187,98 @@
 
         end interface
 
+        ! Generic interfaces for multiple integer kinds
+        interface omp_set_num_threads
+          module procedure omp_set_num_threads_i1, omp_set_num_threads_i2,     &
+              omp_set_num_threads_i4, omp_set_num_threads_i8
+        end interface omp_set_num_threads
+        interface omp_set_max_active_levels
+          module procedure omp_set_max_active_levels_i1,                       &
+              omp_set_max_active_levels_i2, omp_set_max_active_levels_i4,      &
+              omp_set_max_active_levels_i8
+        end interface omp_set_max_active_levels
+        interface omp_set_default_device
+          module procedure omp_set_default_device_i1,                          &
+              omp_set_default_device_i2, omp_set_default_device_i4,            &
+              omp_set_default_device_i8
+        end interface omp_set_default_device
+        interface omp_set_num_teams
+          module procedure omp_set_num_teams_i1, omp_set_num_teams_i2,         &
+              omp_set_num_teams_i4, omp_set_num_teams_i8
+        end interface omp_set_num_teams
+        interface omp_set_teams_thread_limit
+          module procedure omp_set_teams_thread_limit_i1,                      &
+              omp_set_teams_thread_limit_i2, omp_set_teams_thread_limit_i4,    &
+              omp_set_teams_thread_limit_i8
+        end interface omp_set_teams_thread_limit
+        interface kmp_set_stacksize
+          module procedure kmp_set_stacksize_i1, kmp_set_stacksize_i2,         &
+              kmp_set_stacksize_i4, kmp_set_stacksize_i8
+        end interface kmp_set_stacksize
+        interface kmp_set_blocktime
+          module procedure kmp_set_blocktime_i1, kmp_set_blocktime_i2,         &
+              kmp_set_blocktime_i4, kmp_set_blocktime_i8
+        end interface kmp_set_blocktime
+        interface kmp_set_library
+          module procedure kmp_set_library_i1, kmp_set_library_i2,             &
+              kmp_set_library_i4, kmp_set_library_i8
+        end interface kmp_set_library
+        interface kmp_set_disp_num_buffers
+          module procedure kmp_set_disp_num_buffers_i1,                        &
+              kmp_set_disp_num_buffers_i2, kmp_set_disp_num_buffers_i4,        &
+              kmp_set_disp_num_buffers_i8
+        end interface kmp_set_disp_num_buffers
+        interface omp_get_ancestor_thread_num
+          module procedure omp_get_ancestor_thread_num_i1,                     &
+              omp_get_ancestor_thread_num_i2, omp_get_ancestor_thread_num_i4,  &
+              omp_get_ancestor_thread_num_i8
+        end interface omp_get_ancestor_thread_num
+        interface omp_get_team_size
+          module procedure omp_get_team_size_i1, omp_get_team_size_i2,         &
+              omp_get_team_size_i4, omp_get_team_size_i8
+        end interface omp_get_team_size
+        interface omp_get_place_num_procs
+          module procedure omp_get_place_num_procs_i1,                         &
+              omp_get_place_num_procs_i2, omp_get_place_num_procs_i4,          &
+              omp_get_place_num_procs_i8
+        end interface omp_get_place_num_procs
+        interface omp_set_schedule
+          module procedure omp_set_schedule_i1, omp_set_schedule_i2,           &
+              omp_set_schedule_i4, omp_set_schedule_i8
+        end interface omp_set_schedule
+        interface omp_get_schedule
+          module procedure omp_get_schedule_i1, omp_get_schedule_i2,           &
+              omp_get_schedule_i4, omp_get_schedule_i8
+        end interface omp_get_schedule
+        interface omp_pause_resource
+          module procedure omp_pause_resource_i1, omp_pause_resource_i2,       &
+              omp_pause_resource_i4, omp_pause_resource_i8
+        end interface omp_pause_resource
+        interface omp_get_place_proc_ids
+          module procedure omp_get_place_proc_ids_i4, omp_get_place_proc_ids_i8
+        end interface omp_get_place_proc_ids
+        interface omp_get_partition_place_nums
+          module procedure omp_get_partition_place_nums_i4,                    &
+              omp_get_partition_place_nums_i8
+        end interface omp_get_partition_place_nums
+        interface kmp_set_affinity_mask_proc
+          module procedure kmp_set_affinity_mask_proc_i1,                      &
+              kmp_set_affinity_mask_proc_i2, kmp_set_affinity_mask_proc_i4,    &
+              kmp_set_affinity_mask_proc_i8
+        end interface kmp_set_affinity_mask_proc
+        interface kmp_unset_affinity_mask_proc
+          module procedure kmp_unset_affinity_mask_proc_i1,                    &
+              kmp_unset_affinity_mask_proc_i2,                                 &
+              kmp_unset_affinity_mask_proc_i4,                                 &
+              kmp_unset_affinity_mask_proc_i8
+        end interface kmp_unset_affinity_mask_proc
+        interface kmp_get_affinity_mask_proc
+          module procedure kmp_get_affinity_mask_proc_i1,                      &
+              kmp_get_affinity_mask_proc_i2, kmp_get_affinity_mask_proc_i4,    &
+              kmp_get_affinity_mask_proc_i8
+        end interface kmp_get_affinity_mask_proc
+
+
         ! make the above routine definitions public
         public :: omp_set_num_threads
         public :: omp_set_dynamic
@@ -1277,4 +1406,629 @@
         public :: omp_get_interop_type_desc
         public :: omp_get_interop_rc_desc
 
+
+        contains
+
+!===----------------------------------------------------------------------===!
+! Integer-kind wrapper procedures for omp_lib (generated).
+!===----------------------------------------------------------------------===!
+
+
+        subroutine omp_set_num_threads_i1(num_threads)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: num_threads
+          call omp_set_num_threads_c(int(num_threads, kind=c_int))
+        end subroutine omp_set_num_threads_i1
+
+        subroutine omp_set_num_threads_i2(num_threads)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: num_threads
+          call omp_set_num_threads_c(int(num_threads, kind=c_int))
+        end subroutine omp_set_num_threads_i2
+
+        subroutine omp_set_num_threads_i4(num_threads)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: num_threads
+          call omp_set_num_threads_c(int(num_threads, kind=c_int))
+        end subroutine omp_set_num_threads_i4
+
+        subroutine omp_set_num_threads_i8(num_threads)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: num_threads
+          call omp_set_num_threads_c(int(num_threads, kind=c_int))
+        end subroutine omp_set_num_threads_i8
+
+        subroutine omp_set_max_active_levels_i1(max_levels)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: max_levels
+          call omp_set_max_active_levels_c(int(max_levels, kind=c_int))
+        end subroutine omp_set_max_active_levels_i1
+
+        subroutine omp_set_max_active_levels_i2(max_levels)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: max_levels
+          call omp_set_max_active_levels_c(int(max_levels, kind=c_int))
+        end subroutine omp_set_max_active_levels_i2
+
+        subroutine omp_set_max_active_levels_i4(max_levels)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: max_levels
+          call omp_set_max_active_levels_c(int(max_levels, kind=c_int))
+        end subroutine omp_set_max_active_levels_i4
+
+        subroutine omp_set_max_active_levels_i8(max_levels)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: max_levels
+          call omp_set_max_active_levels_c(int(max_levels, kind=c_int))
+        end subroutine omp_set_max_active_levels_i8
+
+        subroutine omp_set_default_device_i1(device_num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: device_num
+          call omp_set_default_device_c(int(device_num, kind=c_int))
+        end subroutine omp_set_default_device_i1
+
+        subroutine omp_set_default_device_i2(device_num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: device_num
+          call omp_set_default_device_c(int(device_num, kind=c_int))
+        end subroutine omp_set_default_device_i2
+
+        subroutine omp_set_default_device_i4(device_num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: device_num
+          call omp_set_default_device_c(int(device_num, kind=c_int))
+        end subroutine omp_set_default_device_i4
+
+        subroutine omp_set_default_device_i8(device_num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: device_num
+          call omp_set_default_device_c(int(device_num, kind=c_int))
+        end subroutine omp_set_default_device_i8
+
+        subroutine omp_set_num_teams_i1(num_teams)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: num_teams
+          call omp_set_num_teams_c(int(num_teams, kind=c_int))
+        end subroutine omp_set_num_teams_i1
+
+        subroutine omp_set_num_teams_i2(num_teams)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: num_teams
+          call omp_set_num_teams_c(int(num_teams, kind=c_int))
+        end subroutine omp_set_num_teams_i2
+
+        subroutine omp_set_num_teams_i4(num_teams)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: num_teams
+          call omp_set_num_teams_c(int(num_teams, kind=c_int))
+        end subroutine omp_set_num_teams_i4
+
+        subroutine omp_set_num_teams_i8(num_teams)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: num_teams
+          call omp_set_num_teams_c(int(num_teams, kind=c_int))
+        end subroutine omp_set_num_teams_i8
+
+        subroutine omp_set_teams_thread_limit_i1(thread_limit)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: thread_limit
+          call omp_set_teams_thread_limit_c(int(thread_limit, kind=c_int))
+        end subroutine omp_set_teams_thread_limit_i1
+
+        subroutine omp_set_teams_thread_limit_i2(thread_limit)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: thread_limit
+          call omp_set_teams_thread_limit_c(int(thread_limit, kind=c_int))
+        end subroutine omp_set_teams_thread_limit_i2
+
+        subroutine omp_set_teams_thread_limit_i4(thread_limit)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: thread_limit
+          call omp_set_teams_thread_limit_c(int(thread_limit, kind=c_int))
+        end subroutine omp_set_teams_thread_limit_i4
+
+        subroutine omp_set_teams_thread_limit_i8(thread_limit)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: thread_limit
+          call omp_set_teams_thread_limit_c(int(thread_limit, kind=c_int))
+        end subroutine omp_set_teams_thread_limit_i8
+
+        subroutine kmp_set_stacksize_i1(size)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: size
+          call kmp_set_stacksize_c(int(size, kind=c_int))
+        end subroutine kmp_set_stacksize_i1
+
+        subroutine kmp_set_stacksize_i2(size)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: size
+          call kmp_set_stacksize_c(int(size, kind=c_int))
+        end subroutine kmp_set_stacksize_i2
+
+        subroutine kmp_set_stacksize_i4(size)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: size
+          call kmp_set_stacksize_c(int(size, kind=c_int))
+        end subroutine kmp_set_stacksize_i4
+
+        subroutine kmp_set_stacksize_i8(size)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: size
+          call kmp_set_stacksize_c(int(size, kind=c_int))
+        end subroutine kmp_set_stacksize_i8
+
+        subroutine kmp_set_blocktime_i1(msec)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: msec
+          call kmp_set_blocktime_c(int(msec, kind=c_int))
+        end subroutine kmp_set_blocktime_i1
+
+        subroutine kmp_set_blocktime_i2(msec)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: msec
+          call kmp_set_blocktime_c(int(msec, kind=c_int))
+        end subroutine kmp_set_blocktime_i2
+
+        subroutine kmp_set_blocktime_i4(msec)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: msec
+          call kmp_set_blocktime_c(int(msec, kind=c_int))
+        end subroutine kmp_set_blocktime_i4
+
+        subroutine kmp_set_blocktime_i8(msec)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: msec
+          call kmp_set_blocktime_c(int(msec, kind=c_int))
+        end subroutine kmp_set_blocktime_i8
+
+        subroutine kmp_set_library_i1(libnum)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: libnum
+          call kmp_set_library_c(int(libnum, kind=c_int))
+        end subroutine kmp_set_library_i1
+
+        subroutine kmp_set_library_i2(libnum)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: libnum
+          call kmp_set_library_c(int(libnum, kind=c_int))
+        end subroutine kmp_set_library_i2
+
+        subroutine kmp_set_library_i4(libnum)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: libnum
+          call kmp_set_library_c(int(libnum, kind=c_int))
+        end subroutine kmp_set_library_i4
+
+        subroutine kmp_set_library_i8(libnum)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: libnum
+          call kmp_set_library_c(int(libnum, kind=c_int))
+        end subroutine kmp_set_library_i8
+
+        subroutine kmp_set_disp_num_buffers_i1(num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: num
+          call kmp_set_disp_num_buffers_c(int(num, kind=c_int))
+        end subroutine kmp_set_disp_num_buffers_i1
+
+        subroutine kmp_set_disp_num_buffers_i2(num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: num
+          call kmp_set_disp_num_buffers_c(int(num, kind=c_int))
+        end subroutine kmp_set_disp_num_buffers_i2
+
+        subroutine kmp_set_disp_num_buffers_i4(num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: num
+          call kmp_set_disp_num_buffers_c(int(num, kind=c_int))
+        end subroutine kmp_set_disp_num_buffers_i4
+
+        subroutine kmp_set_disp_num_buffers_i8(num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: num
+          call kmp_set_disp_num_buffers_c(int(num, kind=c_int))
+        end subroutine kmp_set_disp_num_buffers_i8
+
+        function omp_get_ancestor_thread_num_i1(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_ancestor_thread_num_i1
+          omp_get_ancestor_thread_num_i1 =                                     &
+              omp_get_ancestor_thread_num_c(int(level, kind=c_int))
+        end function omp_get_ancestor_thread_num_i1
+
+        function omp_get_ancestor_thread_num_i2(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_ancestor_thread_num_i2
+          omp_get_ancestor_thread_num_i2 =                                     &
+              omp_get_ancestor_thread_num_c(int(level, kind=c_int))
+        end function omp_get_ancestor_thread_num_i2
+
+        function omp_get_ancestor_thread_num_i4(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_ancestor_thread_num_i4
+          omp_get_ancestor_thread_num_i4 =                                     &
+              omp_get_ancestor_thread_num_c(int(level, kind=c_int))
+        end function omp_get_ancestor_thread_num_i4
+
+        function omp_get_ancestor_thread_num_i8(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_ancestor_thread_num_i8
+          omp_get_ancestor_thread_num_i8 =                                     &
+              omp_get_ancestor_thread_num_c(int(level, kind=c_int))
+        end function omp_get_ancestor_thread_num_i8
+
+        function omp_get_team_size_i1(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_team_size_i1
+          omp_get_team_size_i1 = omp_get_team_size_c(int(level, kind=c_int))
+        end function omp_get_team_size_i1
+
+        function omp_get_team_size_i2(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_team_size_i2
+          omp_get_team_size_i2 = omp_get_team_size_c(int(level, kind=c_int))
+        end function omp_get_team_size_i2
+
+        function omp_get_team_size_i4(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_team_size_i4
+          omp_get_team_size_i4 = omp_get_team_size_c(int(level, kind=c_int))
+        end function omp_get_team_size_i4
+
+        function omp_get_team_size_i8(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_team_size_i8
+          omp_get_team_size_i8 = omp_get_team_size_c(int(level, kind=c_int))
+        end function omp_get_team_size_i8
+
+        function omp_get_place_num_procs_i1(place_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: place_num
+          integer(kind=omp_integer_kind) :: omp_get_place_num_procs_i1
+          omp_get_place_num_procs_i1 =                                         &
+              omp_get_place_num_procs_c(int(place_num, kind=c_int))
+        end function omp_get_place_num_procs_i1
+
+        function omp_get_place_num_procs_i2(place_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: place_num
+          integer(kind=omp_integer_kind) :: omp_get_place_num_procs_i2
+          omp_get_place_num_procs_i2 =                                         &
+              omp_get_place_num_procs_c(int(place_num, kind=c_int))
+        end function omp_get_place_num_procs_i2
+
+        function omp_get_place_num_procs_i4(place_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: place_num
+          integer(kind=omp_integer_kind) :: omp_get_place_num_procs_i4
+          omp_get_place_num_procs_i4 =                                         &
+              omp_get_place_num_procs_c(int(place_num, kind=c_int))
+        end function omp_get_place_num_procs_i4
+
+        function omp_get_place_num_procs_i8(place_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: place_num
+          integer(kind=omp_integer_kind) :: omp_get_place_num_procs_i8
+          omp_get_place_num_procs_i8 =                                         &
+              omp_get_place_num_procs_c(int(place_num, kind=c_int))
+        end function omp_get_place_num_procs_i8
+
+        subroutine omp_set_schedule_i1(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(kind=omp_sched_kind), value :: kind
+          integer(c_int8_t), value :: chunk_size
+          call omp_set_schedule_c(kind, int(chunk_size, kind=c_int))
+        end subroutine omp_set_schedule_i1
+
+        subroutine omp_set_schedule_i2(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(kind=omp_sched_kind), value :: kind
+          integer(c_int16_t), value :: chunk_size
+          call omp_set_schedule_c(kind, int(chunk_size, kind=c_int))
+        end subroutine omp_set_schedule_i2
+
+        subroutine omp_set_schedule_i4(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(kind=omp_sched_kind), value :: kind
+          integer(c_int), value :: chunk_size
+          call omp_set_schedule_c(kind, int(chunk_size, kind=c_int))
+        end subroutine omp_set_schedule_i4
+
+        subroutine omp_set_schedule_i8(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(kind=omp_sched_kind), value :: kind
+          integer(c_int64_t), value :: chunk_size
+          call omp_set_schedule_c(kind, int(chunk_size, kind=c_int))
+        end subroutine omp_set_schedule_i8
+
+        subroutine omp_get_schedule_i1(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(kind=omp_sched_kind), intent(out) :: kind
+          integer(c_int8_t), intent(out) :: chunk_size
+          integer(c_int) :: kind_c, chunk_c
+          call omp_get_schedule_c(kind_c, chunk_c)
+          kind = kind_c
+          chunk_size = chunk_c
+        end subroutine omp_get_schedule_i1
+
+        subroutine omp_get_schedule_i2(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(kind=omp_sched_kind), intent(out) :: kind
+          integer(c_int16_t), intent(out) :: chunk_size
+          integer(c_int) :: kind_c, chunk_c
+          call omp_get_schedule_c(kind_c, chunk_c)
+          kind = kind_c
+          chunk_size = chunk_c
+        end subroutine omp_get_schedule_i2
+
+        subroutine omp_get_schedule_i4(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(kind=omp_sched_kind), intent(out) :: kind
+          integer(c_int), intent(out) :: chunk_size
+          integer(c_int) :: kind_c, chunk_c
+          call omp_get_schedule_c(kind_c, chunk_c)
+          kind = kind_c
+          chunk_size = chunk_c
+        end subroutine omp_get_schedule_i4
+
+        subroutine omp_get_schedule_i8(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(kind=omp_sched_kind), intent(out) :: kind
+          integer(c_int64_t), intent(out) :: chunk_size
+          integer(c_int) :: kind_c, chunk_c
+          call omp_get_schedule_c(kind_c, chunk_c)
+          kind = kind_c
+          chunk_size = chunk_c
+        end subroutine omp_get_schedule_i8
+
+        function omp_pause_resource_i1(kind, device_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(kind=omp_pause_resource_kind), value :: kind
+          integer(c_int8_t), value :: device_num
+          integer(kind=omp_integer_kind) :: omp_pause_resource_i1
+          omp_pause_resource_i1 =                                              &
+              omp_pause_resource_c(kind, int(device_num, kind=c_int))
+        end function omp_pause_resource_i1
+
+        function omp_pause_resource_i2(kind, device_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(kind=omp_pause_resource_kind), value :: kind
+          integer(c_int16_t), value :: device_num
+          integer(kind=omp_integer_kind) :: omp_pause_resource_i2
+          omp_pause_resource_i2 =                                              &
+              omp_pause_resource_c(kind, int(device_num, kind=c_int))
+        end function omp_pause_resource_i2
+
+        function omp_pause_resource_i4(kind, device_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(kind=omp_pause_resource_kind), value :: kind
+          integer(c_int), value :: device_num
+          integer(kind=omp_integer_kind) :: omp_pause_resource_i4
+          omp_pause_resource_i4 =                                              &
+              omp_pause_resource_c(kind, int(device_num, kind=c_int))
+        end function omp_pause_resource_i4
+
+        function omp_pause_resource_i8(kind, device_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(kind=omp_pause_resource_kind), value :: kind
+          integer(c_int64_t), value :: device_num
+          integer(kind=omp_integer_kind) :: omp_pause_resource_i8
+          omp_pause_resource_i8 =                                              &
+              omp_pause_resource_c(kind, int(device_num, kind=c_int))
+        end function omp_pause_resource_i8
+
+        function kmp_set_affinity_mask_proc_i1(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_set_affinity_mask_proc_i1
+          kmp_set_affinity_mask_proc_i1 =                                      &
+              kmp_set_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end function kmp_set_affinity_mask_proc_i1
+
+        function kmp_set_affinity_mask_proc_i2(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_set_affinity_mask_proc_i2
+          kmp_set_affinity_mask_proc_i2 =                                      &
+              kmp_set_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end function kmp_set_affinity_mask_proc_i2
+
+        function kmp_set_affinity_mask_proc_i4(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_set_affinity_mask_proc_i4
+          kmp_set_affinity_mask_proc_i4 =                                      &
+              kmp_set_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end function kmp_set_affinity_mask_proc_i4
+
+        function kmp_set_affinity_mask_proc_i8(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_set_affinity_mask_proc_i8
+          kmp_set_affinity_mask_proc_i8 =                                      &
+              kmp_set_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end function kmp_set_affinity_mask_proc_i8
+
+        function kmp_unset_affinity_mask_proc_i1(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_unset_affinity_mask_proc_i1
+          kmp_unset_affinity_mask_proc_i1 =                                    &
+              kmp_unset_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end function kmp_unset_affinity_mask_proc_i1
+
+        function kmp_unset_affinity_mask_proc_i2(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_unset_affinity_mask_proc_i2
+          kmp_unset_affinity_mask_proc_i2 =                                    &
+              kmp_unset_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end function kmp_unset_affinity_mask_proc_i2
+
+        function kmp_unset_affinity_mask_proc_i4(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_unset_affinity_mask_proc_i4
+          kmp_unset_affinity_mask_proc_i4 =                                    &
+              kmp_unset_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end function kmp_unset_affinity_mask_proc_i4
+
+        function kmp_unset_affinity_mask_proc_i8(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_unset_affinity_mask_proc_i8
+          kmp_unset_affinity_mask_proc_i8 =                                    &
+              kmp_unset_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end function kmp_unset_affinity_mask_proc_i8
+
+        function kmp_get_affinity_mask_proc_i1(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_get_affinity_mask_proc_i1
+          kmp_get_affinity_mask_proc_i1 =                                      &
+              kmp_get_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end function kmp_get_affinity_mask_proc_i1
+
+        function kmp_get_affinity_mask_proc_i2(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_get_affinity_mask_proc_i2
+          kmp_get_affinity_mask_proc_i2 =                                      &
+              kmp_get_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end function kmp_get_affinity_mask_proc_i2
+
+        function kmp_get_affinity_mask_proc_i4(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_get_affinity_mask_proc_i4
+          kmp_get_affinity_mask_proc_i4 =                                      &
+              kmp_get_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end function kmp_get_affinity_mask_proc_i4
+
+        function kmp_get_affinity_mask_proc_i8(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_get_affinity_mask_proc_i8
+          kmp_get_affinity_mask_proc_i8 =                                      &
+              kmp_get_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end function kmp_get_affinity_mask_proc_i8
+
+        subroutine omp_get_place_proc_ids_i4(place_num, ids)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: place_num
+          integer(c_int), intent(out) :: ids(*)
+          integer(c_int) :: n, i
+          integer(c_int), allocatable :: ids_c(:)
+          n = omp_get_place_num_procs_c(int(place_num, kind=c_int))
+          allocate(ids_c(n))
+          call omp_get_place_proc_ids_c(int(place_num, kind=c_int), ids_c)
+          do i = 1, n
+            ids(i) = ids_c(i)
+          end do
+          deallocate(ids_c)
+        end subroutine omp_get_place_proc_ids_i4
+
+        subroutine omp_get_place_proc_ids_i8(place_num, ids)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: place_num
+          integer(c_int64_t), intent(out) :: ids(*)
+          integer(c_int) :: n, i
+          integer(c_int), allocatable :: ids_c(:)
+          n = omp_get_place_num_procs_c(int(place_num, kind=c_int))
+          allocate(ids_c(n))
+          call omp_get_place_proc_ids_c(int(place_num, kind=c_int), ids_c)
+          do i = 1, n
+            ids(i) = ids_c(i)
+          end do
+          deallocate(ids_c)
+        end subroutine omp_get_place_proc_ids_i8
+
+        subroutine omp_get_partition_place_nums_i4(place_nums)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), intent(out) :: place_nums(*)
+          integer(c_int) :: n, i
+          integer(c_int), allocatable :: place_nums_c(:)
+          n = omp_get_partition_num_places()
+          allocate(place_nums_c(n))
+          call omp_get_partition_place_nums_c(place_nums_c)
+          do i = 1, n
+            place_nums(i) = place_nums_c(i)
+          end do
+          deallocate(place_nums_c)
+        end subroutine omp_get_partition_place_nums_i4
+
+        subroutine omp_get_partition_place_nums_i8(place_nums)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), intent(out) :: place_nums(*)
+          integer(c_int) :: n, i
+          integer(c_int), allocatable :: place_nums_c(:)
+          n = omp_get_partition_num_places()
+          allocate(place_nums_c(n))
+          call omp_get_partition_place_nums_c(place_nums_c)
+          do i = 1, n
+            place_nums(i) = place_nums_c(i)
+          end do
+          deallocate(place_nums_c)
+        end subroutine omp_get_partition_place_nums_i8
       end module omp_lib


From openmp-commits at lists.llvm.org  Mon Aug  3 00:27:23 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Mon, 03 Aug 2026 00:27:23 -0700 (PDT)
Subject: [Openmp-commits] [flang] [llvm] [openmp] [OpenMP][Flang] Add
 integer-kind wrappers to omp_lib (PR #213505)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213505@github.com>
Message-ID: <6a7042db.507fc713.27f122.baee@mx.google.com>


================
@@ -1150,6 +1171,69 @@
 
         end interface
 
+        ! Generic interfaces for multiple integer kinds
+        interface omp_set_num_threads
+          module procedure omp_set_num_threads_i1, omp_set_num_threads_i2, omp_set_num_threads_i4, omp_set_num_threads_i8
+        end interface omp_set_num_threads
+        interface omp_set_max_active_levels
+          module procedure omp_set_max_active_levels_i1, omp_set_max_active_levels_i2, omp_set_max_active_levels_i4, omp_set_max_active_levels_i8
----------------
shivaramaarao wrote:

done. thanks for your review.

https://github.com/llvm/llvm-project/pull/213505

From openmp-commits at lists.llvm.org  Mon Aug  3 02:38:19 2026
From: openmp-commits at lists.llvm.org (CHANDRA GHALE via Openmp-commits)
Date: Mon, 03 Aug 2026 02:38:19 -0700 (PDT)
Subject: [Openmp-commits] [flang] [llvm] [mlir] [openmp] [Flang][OpenMP]
 Lower scan directive and inscan reduction modifier (PR #206747)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206747@github.com>
Message-ID: <6a70618b.b4a331f0.8d65c.cd46@mx.google.com>


https://github.com/chandraghale updated https://github.com/llvm/llvm-project/pull/206747

>From 82416bc775c4befdbcce1f999a5f5b97ef954cd8 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Tue, 30 Jun 2026 10:00:56 -0500
Subject: [PATCH 1/9] Lower scan directive and inscan reduction modifier

---
 flang/lib/Lower/OpenMP/OpenMP.cpp             |  57 ++-
 .../Lower/OpenMP/Todo/nested-wsloop-scan.f90  |  34 ++
 .../OpenMP/Todo/wsloop-scan-collapse.f90      |  29 ++
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      | 476 +++++++++++++++---
 .../Target/LLVMIR/openmp-reduction-scan.mlir  | 123 +++++
 mlir/test/Target/LLVMIR/openmp-todo.mlir      |  41 +-
 openmp/runtime/test/scan/scan.f90             |  38 ++
 7 files changed, 726 insertions(+), 72 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/Todo/nested-wsloop-scan.f90
 create mode 100644 flang/test/Lower/OpenMP/Todo/wsloop-scan-collapse.f90
 create mode 100644 mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
 create mode 100644 openmp/runtime/test/scan/scan.f90

diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index ea8c279962508..e77584a142eea 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -2972,12 +2972,59 @@ genParallelOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
 
 static mlir::omp::ScanOp
 genScanOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
-          semantics::SemanticsContext &semaCtx, mlir::Location loc,
-          const ConstructQueue &queue, ConstructQueue::const_iterator item) {
+          semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+          mlir::Location loc, const ConstructQueue &queue,
+          ConstructQueue::const_iterator item) {
   mlir::omp::ScanOperands clauseOps;
   genScanClauses(converter, semaCtx, item->clauses, loc, clauseOps);
-  return mlir::omp::ScanOp::create(converter.getFirOpBuilder(),
-                                   converter.getCurrentLocation(), clauseOps);
+  mlir::omp::ScanOp scanOp = mlir::omp::ScanOp::create(
+      converter.getFirOpBuilder(), converter.getCurrentLocation(), clauseOps);
+
+  /// Scan reduction is not implemented with nested workshare loops, linear
+  /// clause, tiling
+  mlir::omp::LoopNestOp loopNestOp =
+      scanOp->getParentOfType<mlir::omp::LoopNestOp>();
+  llvm::SmallVector<mlir::omp::LoopWrapperInterface> loopWrappers;
+  loopNestOp.gatherWrappers(loopWrappers);
+  mlir::Operation *loopWrapperOp = loopWrappers.front().getOperation();
+  if (llvm::isa<mlir::omp::SimdOp>(loopWrapperOp))
+    TODO(loc, "unsupported simd");
+  if (loopWrappers.size() > 1)
+    TODO(loc, "unsupported composite");
+  mlir::omp::WsloopOp wsLoopOp = llvm::cast<mlir::omp::WsloopOp>(loopWrapperOp);
+  bool isNested =
+      (loopNestOp.getNumLoops() > 1) ||
+      (wsLoopOp && (wsLoopOp->getParentOfType<mlir::omp::WsloopOp>()));
+  if (isNested)
+    TODO(loc, "Scan directive inside nested workshare loops");
+  if (wsLoopOp && !wsLoopOp.getLinearVars().empty())
+    TODO(loc, "Scan directive with linear clause");
+  if (loopNestOp.getTileSizes())
+    TODO(loc, "Scan directive with loop tiling");
+
+  // All loop indices should be loaded after the scan construct as otherwise,
+  // it would result in using the index variable across scan directive.
+  // (`Intra-iteration dependences from a statement in the structured
+  // block sequence that precede a scan directive to a statement in the
+  // structured block sequence that follows a scan directive must not exist,
+  // except for dependences for the list items specified in an inclusive or
+  // exclusive clause.`).
+  // TODO: Nested loops are not handled.
+  mlir::Region &region = loopNestOp->getRegion(0);
+  mlir::Value indexVal = fir::getBase(region.getArgument(0));
+  lower::pft::Evaluation *doConstructEval = eval.parentConstruct;
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+  lower::pft::Evaluation *doLoop = &doConstructEval->getFirstNestedEvaluation();
+  auto *doStmt = doLoop->getIf<parser::NonLabelDoStmt>();
+  assert(doStmt && "Expected do loop to be in the nested evaluation");
+  const auto &loopControl =
+      std::get<std::optional<parser::LoopControl>>(doStmt->t);
+  const parser::LoopControl::Bounds *bounds =
+      std::get_if<parser::LoopControl::Bounds>(&loopControl->u);
+  mlir::Operation *storeOp =
+      setLoopVar(converter, loc, indexVal, bounds->Name().thing.symbol);
+  firOpBuilder.setInsertionPointAfter(storeOp);
+  return scanOp;
 }
 
 static mlir::omp::SectionsOp
@@ -4305,7 +4352,7 @@ static void genOMPDispatch(lower::AbstractConverter &converter,
                                   loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_scan:
-    newOp = genScanOp(converter, symTable, semaCtx, loc, queue, item);
+    newOp = genScanOp(converter, symTable, semaCtx, eval, loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_section:
     llvm_unreachable("genOMPDispatch: OMPD_section");
diff --git a/flang/test/Lower/OpenMP/Todo/nested-wsloop-scan.f90 b/flang/test/Lower/OpenMP/Todo/nested-wsloop-scan.f90
new file mode 100644
index 0000000000000..414e2ef2d5aed
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/nested-wsloop-scan.f90
@@ -0,0 +1,34 @@
+! Tests scan reduction behavior when used in nested workshare loops
+
+! RUN: %not_todo_cmd %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+program nested_scan_example
+  implicit none
+  integer, parameter :: n = 4, m = 5
+  integer :: a(n, m), b(n, m)
+  integer :: i, j
+  integer :: row_sum, col_sum
+
+  do i = 1, n
+     do j = 1, m
+        a(i, j) = i + j
+     end do
+  end do
+
+  !$omp parallel do reduction(inscan, +: row_sum) private(col_sum, j)
+  do i = 1, n
+     row_sum = row_sum + i
+     !$omp scan inclusive(row_sum)
+
+     col_sum = 0
+     !$omp parallel do reduction(inscan, +: col_sum)
+     do j = 1, m
+        col_sum = col_sum + a(i, j)
+        !CHECK: not yet implemented: Scan directive inside nested workshare loops
+        !$omp scan inclusive(col_sum)
+        b(i, j) = col_sum + row_sum 
+     end do
+     !$omp end parallel do
+  end do
+  !$omp end parallel do
+end program nested_scan_example
diff --git a/flang/test/Lower/OpenMP/Todo/wsloop-scan-collapse.f90 b/flang/test/Lower/OpenMP/Todo/wsloop-scan-collapse.f90
new file mode 100644
index 0000000000000..b8e6e831884ab
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/wsloop-scan-collapse.f90
@@ -0,0 +1,29 @@
+! Tests scan reduction behavior when used in nested workshare loops
+
+! RUN: %not_todo_cmd %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+program nested_loop_example
+  implicit none
+  integer :: i, j, x
+  integer, parameter :: N = 100, M = 200
+  real :: A(N, M), B(N, M)
+  x = 0
+
+  do i = 1, N
+    do j = 1, M
+      A(i, j) = i * j
+    end do
+  end do
+  
+  !$omp parallel do collapse(2) reduction(inscan, +:x)
+  do i = 1, N
+    do j = 1, M
+      x = x + A(i,j)
+      !CHECK: not yet implemented: Scan directive inside nested workshare loops
+      !$omp scan inclusive(x)
+      B(i, j) = x
+    end do
+  end do
+  !$omp end parallel do
+
+end program nested_loop_example
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index d860c408e0fd2..297ebac51de88 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -41,6 +41,8 @@
 #include "llvm/TargetParser/Triple.h"
 #include "llvm/Transforms/Utils/ModuleUtils.h"
 
+#include <cassert>
+#include <cstddef>
 #include <cstdint>
 #include <iterator>
 #include <numeric>
@@ -84,6 +86,10 @@ class OpenMPAllocStackFrame
       : allocInsertPoint(allocaIP), deallocBlocks(deallocBlocks) {}
   llvm::OpenMPIRBuilder::InsertPointTy allocInsertPoint;
   llvm::SmallVector<llvm::BasicBlock *> deallocBlocks;
+  /// Set to true when this alloca frame encloses an omp.parallel operation.
+  /// The alloca insertion point of a function in which a parallel op is
+  /// defined may be used to allocate the temporary buffer for scan reductions.
+  bool containsParallelOp = false;
 };
 
 /// Stack frame to hold a \see llvm::CanonicalLoopInfo representing the
@@ -93,7 +99,17 @@ class OpenMPLoopInfoStackFrame
     : public StateStackFrameBase<OpenMPLoopInfoStackFrame> {
 public:
   MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(OpenMPLoopInfoStackFrame)
+  /// For constructs like `scan`, a single `omp.loop_nest` is split into an
+  /// input loop and a scan loop. In that case `loopInfo` holds the input loop
+  /// info and `scanloopInfo` holds the scan loop info.
   llvm::CanonicalLoopInfo *loopInfo = nullptr;
+  llvm::CanonicalLoopInfo *scanloopInfo = nullptr;
+  llvm::ScanInfo *scanInfo = nullptr;
+  /// Map reduction variables to their LLVM types. Populated when the reduction
+  /// clause is processed and used when a `scan` directive is encountered in the
+  /// loop body.
+  std::unique_ptr<llvm::DenseMap<llvm::Value *, llvm::Type *>>
+      reductionVarToType = nullptr;
 };
 
 /// Custom error class to signal translation errors that don't need reporting,
@@ -359,6 +375,10 @@ static LogicalResult checkImplementationStatus(Operation &op) {
     if (!op.getDependVars().empty() || op.getDependKinds())
       result = todo("depend");
   };
+  auto checkExclusive = [&todo](auto op, LogicalResult &result) {
+    if (!op.getExclusiveVars().empty())
+      result = todo("exclusive");
+  };
   auto checkHint = [](auto op, LogicalResult &) {
     if (op.getHint())
       op.emitWarning("hint clause discarded");
@@ -389,21 +409,27 @@ static LogicalResult checkImplementationStatus(Operation &op) {
         op.getReductionMod().value() != omp::ReductionModifier::defaultmod) {
       omp::ReductionModifier mod = op.getReductionMod().value();
       // The `task` reduction modifier is supported on the parallel and
-      // worksharing (do/for and sections) constructs. Other modifiers, and the
-      // `task` modifier on other constructs, are not yet implemented.
+      // worksharing (do/for and sections) constructs. The `inscan` modifier is
+      // supported on the worksharing-loop construct (it is translated as a scan
+      // reduction). Other modifiers, and these modifiers on other constructs,
+      // are not yet implemented.
       bool taskModifierSupported =
           mod == omp::ReductionModifier::task &&
           isa<omp::ParallelOp, omp::WsloopOp, omp::SectionsOp>(op);
-      if (!taskModifierSupported) {
+      bool inscanModifierSupported =
+          mod == omp::ReductionModifier::inscan && isa<omp::WsloopOp>(op);
+      if (!taskModifierSupported && !inscanModifierSupported) {
         result = todo("reduction with modifier");
-      } else if (auto byref = op.getReductionByref()) {
-        // The task reduction modifier lowering only handles non-byref
-        // reductions for now.
-        for (bool isByRef : *byref)
-          if (isByRef) {
-            result = todo("task reduction modifier with by-ref reduction");
-            break;
-          }
+      } else if (taskModifierSupported) {
+        if (auto byref = op.getReductionByref()) {
+          // The task reduction modifier lowering only handles non-byref
+          // reductions for now.
+          for (bool isByRef : *byref)
+            if (isByRef) {
+              result = todo("task reduction modifier with by-ref reduction");
+              break;
+            }
+        }
       }
     }
   };
@@ -460,6 +486,7 @@ static LogicalResult checkImplementationStatus(Operation &op) {
         checkAllocate(op, result);
         checkOrder(op, result);
       })
+      .Case([&](omp::ScanOp op) { checkExclusive(op, result); })
       .Case([&](omp::SectionsOp op) {
         checkAllocate(op, result);
         checkPrivate(op, result);
@@ -650,6 +677,97 @@ findCurrentLoopInfo(LLVM::ModuleTranslation &moduleTranslation) {
   return loopInfo;
 }
 
+/// Find the scan loop information structure for the scan loop nest being
+/// translated. It will return a `null` value unless called from the
+/// translation function for a loop wrapper operation after successfully
+/// translating its body.
+static llvm::CanonicalLoopInfo *
+findCurrentScanLoopInfo(LLVM::ModuleTranslation &moduleTranslation) {
+  llvm::CanonicalLoopInfo *scanLoopInfo = nullptr;
+  moduleTranslation.stackWalk<OpenMPLoopInfoStackFrame>(
+      [&](OpenMPLoopInfoStackFrame &frame) {
+        scanLoopInfo = frame.scanloopInfo;
+        return WalkResult::interrupt();
+      });
+  return scanLoopInfo;
+}
+
+/// Find the `ScanInfo` stored on the loop stack frame. Upon encountering an
+/// `inscan` reduction modifier, `scanInfoInitialize` initializes the
+/// `ScanInfo`, which is then used when a `scan` directive is encountered in the
+/// body of the loop nest.
+static llvm::ScanInfo *
+findScanInfo(LLVM::ModuleTranslation &moduleTranslation) {
+  llvm::ScanInfo *scanInfo = nullptr;
+  moduleTranslation.stackWalk<OpenMPLoopInfoStackFrame>(
+      [&](OpenMPLoopInfoStackFrame &frame) {
+        scanInfo = frame.scanInfo;
+        return WalkResult::interrupt();
+      });
+  return scanInfo;
+}
+
+/// The types of reduction variables are used for lowering a `scan` directive
+/// that appears in the body of the loop. The types are stored in the loop frame
+/// when the reduction clause is encountered and used when the `scan` directive
+/// is encountered.
+static llvm::DenseMap<llvm::Value *, llvm::Type *> *
+findReductionVarTypes(LLVM::ModuleTranslation &moduleTranslation) {
+  llvm::DenseMap<llvm::Value *, llvm::Type *> *reductionVarToType = nullptr;
+  moduleTranslation.stackWalk<OpenMPLoopInfoStackFrame>(
+      [&](OpenMPLoopInfoStackFrame &frame) {
+        if (!frame.reductionVarToType)
+          frame.reductionVarToType =
+              std::make_unique<llvm::DenseMap<llvm::Value *, llvm::Type *>>();
+        reductionVarToType = frame.reductionVarToType.get();
+        return WalkResult::interrupt();
+      });
+  return reductionVarToType;
+}
+
+/// Scan reduction requires a shared buffer to be allocated to perform the
+/// reduction. The allocation needs to be done outside the parallel region in
+/// which the scan operation is used.
+static llvm::OpenMPIRBuilder::InsertPointTy
+findParallelAllocaIP(llvm::IRBuilderBase &builder,
+                     LLVM::ModuleTranslation &moduleTranslation) {
+  // If there is an alloca insertion point on the stack belonging to a frame
+  // that encloses a parallel op, use it.
+  llvm::OpenMPIRBuilder::InsertPointTy allocaInsertPoint;
+  WalkResult walkResult = moduleTranslation.stackWalk<OpenMPAllocStackFrame>(
+      [&](OpenMPAllocStackFrame &frame) {
+        if (frame.containsParallelOp) {
+          allocaInsertPoint = frame.allocInsertPoint;
+          return WalkResult::interrupt();
+        }
+        return WalkResult::skip();
+      });
+  if (walkResult.wasInterrupted())
+    return allocaInsertPoint;
+  // Otherwise, insert into the entry block of the surrounding function.
+  // If the current IRBuilder InsertPoint is the function's entry, it cannot
+  // also be used for alloca insertion which would result in insertion order
+  // confusion. Create a new BasicBlock for the Builder and use the entry block
+  // for the allocs.
+  // TODO: Create a dedicated alloca BasicBlock at function creation such that
+  // we do not need to move the current InsertPoint here.
+  if (builder.GetInsertBlock() ==
+      &builder.GetInsertBlock()->getParent()->getEntryBlock()) {
+    assert(builder.GetInsertPoint() == builder.GetInsertBlock()->end() &&
+           "Assuming end of basic block");
+    llvm::BasicBlock *entryBB = llvm::BasicBlock::Create(
+        builder.getContext(), "entry", builder.GetInsertBlock()->getParent(),
+        builder.GetInsertBlock()->getNextNode());
+    builder.CreateBr(entryBB);
+    builder.SetInsertPoint(entryBB);
+  }
+
+  llvm::BasicBlock &funcEntryBlock =
+      builder.GetInsertBlock()->getParent()->getEntryBlock();
+  return llvm::OpenMPIRBuilder::InsertPointTy(
+      &funcEntryBlock, funcEntryBlock.getFirstInsertionPt());
+}
+
 /// Converts the given region that appears within an OpenMP dialect operation to
 /// LLVM IR, creating a branch from the `sourceBlock` to the entry block of the
 /// region, and a branch from any block with an successor-less OpenMP terminator
@@ -1414,7 +1532,8 @@ initReductionVars(OP op, ArrayRef<BlockArgument> reductionArgs,
                   SmallVectorImpl<llvm::Value *> &privateReductionVariables,
                   DenseMap<Value, llvm::Value *> &reductionVariableMap,
                   llvm::ArrayRef<bool> isByRef,
-                  SmallVectorImpl<DeferredStore> &deferredStores) {
+                  SmallVectorImpl<DeferredStore> &deferredStores,
+                  bool isInScanRegion = false) {
   if (op.getNumReductionVars() == 0)
     return success();
 
@@ -1451,11 +1570,17 @@ initReductionVars(OP op, ArrayRef<BlockArgument> reductionArgs,
   for (auto [data, addr] : deferredStores)
     builder.CreateStore(data, addr);
 
+  llvm::DenseMap<llvm::Value *, llvm::Type *> *reductionVarToType =
+      findReductionVarTypes(moduleTranslation);
   // Before the loop, store the initial values of reductions into reduction
   // variables. Although this could be done after allocas, we don't want to mess
   // up with the alloca insertion point.
   for (unsigned i = 0; i < op.getNumReductionVars(); ++i) {
     SmallVector<llvm::Value *, 1> phis;
+    llvm::Type *reductionType =
+        moduleTranslation.convertType(reductionDecls[i].getType());
+    if (isInScanRegion && reductionVarToType != nullptr)
+      (*reductionVarToType)[privateReductionVariables[i]] = reductionType;
 
     // map block argument to initializer region
     mapInitializationArgs(op, moduleTranslation, builder, reductionDecls,
@@ -4406,11 +4531,15 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
     return failure();
 
   assert(afterAllocas.get()->getSinglePredecessor());
+  bool isInScanRegion =
+      wsloopOp.getReductionMod() && (wsloopOp.getReductionMod().value() ==
+                                     mlir::omp::ReductionModifier::inscan);
   if (failed(initReductionVars(wsloopOp, reductionArgs, builder,
                                moduleTranslation,
                                afterAllocas.get()->getSinglePredecessor(),
                                reductionDecls, privateReductionVariables,
-                               reductionVariableMap, isByRef, deferredStores)))
+                               reductionVariableMap, isByRef, deferredStores,
+                               isInScanRegion)))
     return failure();
 
   // For `reduction(task, ...)` open a task-reduction scope for the worksharing
@@ -4432,6 +4561,10 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
   std::optional<omp::ScheduleModifier> scheduleMod = wsloopOp.getScheduleMod();
   bool isSimd = wsloopOp.getScheduleSimd();
   bool loopNeedsBarrier = !wsloopOp.getNowait();
+  // TODO: Linear clause support needs to be enabled for scan reduction.
+  if (isInScanRegion)
+    assert(wsloopOp.getLinearVars().empty() &&
+           "Linear clause support is not enabled with scan reduction");
 
   // The only legal way for the direct parent to be omp.distribute is that this
   // represents 'distribute parallel do'. Otherwise, this is a regular
@@ -4469,21 +4602,93 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
   if (failed(handleError(regionBlock, opInst)))
     return failure();
 
+  // Generates the loop body for a worksharing loop, including linear-variable
+  // handling and the call into the OpenMPIRBuilder's worksharing-loop helper.
+  // For scan reductions this lambda is invoked twice: once for the input loop
+  // and once for the scan loop.
+  const auto &&wsloopCodeGen = [&](llvm::CanonicalLoopInfo *loopInfo,
+                                   bool noLoopMode,
+                                   bool inputScanLoop) -> LogicalResult {
+    // Emit Initialization and Update IR for linear variables
+    if (!wsloopOp.getLinearVars().empty()) {
+      linearClauseProcessor.initLinearVar(builder, moduleTranslation,
+                                          loopInfo->getPreheader());
+      llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP =
+          moduleTranslation.getOpenMPBuilder()->createBarrier(
+              builder.saveIP(), llvm::omp::OMPD_barrier);
+      if (failed(handleError(afterBarrierIP, *loopOp)))
+        return failure();
+      builder.restoreIP(*afterBarrierIP);
+      linearClauseProcessor.updateLinearVar(builder, loopInfo->getBody(),
+                                            loopInfo->getIndVar());
+      linearClauseProcessor.splitLinearFiniBB(builder, loopInfo->getExit());
+    }
+
+    builder.SetInsertPoint(*regionBlock, (*regionBlock)->begin());
+
+    for (size_t index = 0; index < wsloopOp.getLinearVars().size(); index++)
+      linearClauseProcessor.rewriteInPlace(builder, loopInfo->getBody(),
+                                           loopInfo->getLatch(), index);
+
+    llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP =
+        ompBuilder->applyWorkshareLoop(
+            ompLoc.DL, loopInfo, allocaIP, loopNeedsBarrier,
+            convertToScheduleKind(schedule), chunk, isSimd,
+            scheduleMod == omp::ScheduleModifier::monotonic,
+            scheduleMod == omp::ScheduleModifier::nonmonotonic, isOrdered,
+            workshareLoopType, noLoopMode, hasDistSchedule, distScheduleChunk);
+
+    if (failed(handleError(wsloopIP, opInst)))
+      return failure();
+
+    // Emit finalization for linear vars.
+    if (!wsloopOp.getLinearVars().empty()) {
+      llvm::OpenMPIRBuilder::InsertPointTy oldIP = builder.saveIP();
+      assert(loopInfo->getLastIter() &&
+             "`lastiter` in CanonicalLoopInfo is nullptr");
+      llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP =
+          linearClauseProcessor.finalizeLinearVar(builder, moduleTranslation,
+                                                  loopInfo->getLastIter());
+      if (failed(handleError(afterBarrierIP, *loopOp)))
+        return failure();
+
+      builder.restoreIP(oldIP);
+    }
+
+    // For scan reductions, the cancellation finalization callback is popped
+    // only after the scan (second) loop has been generated.
+    if (!inputScanLoop || !isInScanRegion)
+      popCancelFinalizationCB(cancelTerminators, *ompBuilder, wsloopIP.get());
+
+    return success();
+  };
+
   llvm::CanonicalLoopInfo *loopInfo = findCurrentLoopInfo(moduleTranslation);
 
-  // Emit Initialization and Update IR for linear variables
-  if (!wsloopOp.getLinearVars().empty()) {
-    linearClauseProcessor.initLinearVar(builder, moduleTranslation,
-                                        loopInfo->getPreheader());
-    llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP =
-        moduleTranslation.getOpenMPBuilder()->createBarrier(
-            builder.saveIP(), llvm::omp::OMPD_barrier);
-    if (failed(handleError(afterBarrierIP, *loopOp)))
+  if (isInScanRegion) {
+    // Emit the scan reduction combiner between the input loop and the scan
+    // loop, operating on the per-iteration partial values stored in the shared
+    // buffer.
+    auto inputLoopFinishIp = loopInfo->getAfterIP();
+    builder.restoreIP(inputLoopFinishIp);
+    SmallVector<OwningReductionGen> owningReductionGens;
+    SmallVector<OwningAtomicReductionGen> owningAtomicReductionGens;
+    SmallVector<llvm::OpenMPIRBuilder::ReductionInfo, 2> reductionInfos;
+    SmallVector<OwningDataPtrPtrReductionGen> owningReductionGenRefDataPtrGens;
+    collectReductionInfo(wsloopOp, builder, moduleTranslation, reductionDecls,
+                         owningReductionGens, owningAtomicReductionGens,
+                         owningReductionGenRefDataPtrGens,
+                         privateReductionVariables, reductionInfos, isByRef);
+    llvm::BasicBlock *cont = splitBB(builder, false, "omp.scan.loop.cont");
+    llvm::ScanInfo *scanInfo = findScanInfo(moduleTranslation);
+    llvm::OpenMPIRBuilder::InsertPointOrErrorTy redIP =
+        ompBuilder->emitScanReduction(builder.saveIP(), reductionInfos,
+                                      scanInfo);
+    if (failed(handleError(redIP, opInst)))
       return failure();
-    builder.restoreIP(*afterBarrierIP);
-    linearClauseProcessor.updateLinearVar(builder, loopInfo->getBody(),
-                                          loopInfo->getIndVar());
-    linearClauseProcessor.splitLinearFiniBB(builder, loopInfo->getExit());
+
+    builder.restoreIP(*redIP);
+    builder.CreateBr(cont);
   }
 
   builder.SetInsertPoint(*regionBlock, (*regionBlock)->begin());
@@ -4502,50 +4707,41 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
       noLoopMode = true;
   }
 
-  for (size_t index = 0; index < wsloopOp.getLinearVars().size(); index++)
-    linearClauseProcessor.rewriteInPlace(builder, loopInfo->getBody(),
-                                         loopInfo->getLatch(), index);
-
-  llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP =
-      ompBuilder->applyWorkshareLoop(
-          ompLoc.DL, loopInfo, allocaIP, loopNeedsBarrier,
-          convertToScheduleKind(schedule), chunk, isSimd,
-          scheduleMod == omp::ScheduleModifier::monotonic,
-          scheduleMod == omp::ScheduleModifier::nonmonotonic, isOrdered,
-          workshareLoopType, noLoopMode, hasDistSchedule, distScheduleChunk);
-
-  if (failed(handleError(wsloopIP, opInst)))
+  // For scan loops, the input loop does not pop the cancellation finalization
+  // callback; that happens after the scan loop is generated below.
+  bool inputScanLoop = isInScanRegion;
+  if (failed(wsloopCodeGen(loopInfo, noLoopMode, inputScanLoop)))
     return failure();
+  inputScanLoop = false;
 
-  // Emit finalization and in-place rewrites for linear vars.
-  if (!wsloopOp.getLinearVars().empty()) {
-    llvm::OpenMPIRBuilder::InsertPointTy oldIP = builder.saveIP();
-    assert(loopInfo->getLastIter() &&
-           "`lastiter` in CanonicalLoopInfo is nullptr");
-    llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP =
-        linearClauseProcessor.finalizeLinearVar(builder, moduleTranslation,
-                                                loopInfo->getLastIter());
-    if (failed(handleError(afterBarrierIP, *loopOp)))
+  if (isInScanRegion) {
+    llvm::CanonicalLoopInfo *scanLoopInfo =
+        findCurrentScanLoopInfo(moduleTranslation);
+    if (failed(wsloopCodeGen(scanLoopInfo, noLoopMode, inputScanLoop)))
+      return failure();
+    SmallVector<Region *> reductionRegions;
+    llvm::transform(reductionDecls, std::back_inserter(reductionRegions),
+                    [](omp::DeclareReductionOp reductionDecl) {
+                      return &reductionDecl.getCleanupRegion();
+                    });
+    if (failed(inlineOmpRegionCleanup(
+            reductionRegions, privateReductionVariables, moduleTranslation,
+            builder, "omp.reduction.cleanup")))
       return failure();
+  } else {
+    // Close the task-reduction scope before the worksharing reduction combine.
+    if (isTaskReductionMod)
+      emitTaskReductionModifierFini(/*isWorksharing=*/true, builder,
+                                    moduleTranslation);
 
-    builder.restoreIP(oldIP);
+    // Process the reductions if required.
+    if (failed(createReductionsAndCleanup(
+            wsloopOp, builder, moduleTranslation, allocaIP, reductionDecls,
+            privateReductionVariables, isByRef, wsloopOp.getNowait(),
+            /*isTeamsReduction=*/false)))
+      return failure();
   }
 
-  // Set the correct branch target for task cancellation
-  popCancelFinalizationCB(cancelTerminators, *ompBuilder, wsloopIP.get());
-
-  // Close the task-reduction scope before the worksharing reduction combine.
-  if (isTaskReductionMod)
-    emitTaskReductionModifierFini(/*isWorksharing=*/true, builder,
-                                  moduleTranslation);
-
-  // Process the reductions if required.
-  if (failed(createReductionsAndCleanup(
-          wsloopOp, builder, moduleTranslation, allocaIP, reductionDecls,
-          privateReductionVariables, isByRef, wsloopOp.getNowait(),
-          /*isTeamsReduction=*/false)))
-    return failure();
-
   return cleanupPrivateVars(wsloopOp, builder, moduleTranslation,
                             wsloopOp.getLoc(), privateVarsInfo);
 }
@@ -4579,6 +4775,20 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
       opInst.getReductionMod() == omp::ReductionModifier::task &&
       opInst.getNumReductionVars() > 0;
 
+  // Mark the enclosing alloca stack frame as containing a parallel op. Scan
+  // reductions use the alloca insertion point of the function enclosing the
+  // parallel region to allocate their shared temporary buffer.
+  bool foundParallelOp = false;
+  moduleTranslation.stackWalk<OpenMPAllocStackFrame>(
+      [&](OpenMPAllocStackFrame &frame) {
+        if (foundParallelOp) {
+          frame.containsParallelOp = true;
+          return WalkResult::interrupt();
+        }
+        foundParallelOp = true;
+        return WalkResult::skip();
+      });
+
   auto bodyGenCB =
       [&](InsertPointTy allocaIP, InsertPointTy codeGenIP,
           llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) -> llvm::Error {
@@ -4972,6 +5182,99 @@ convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder,
                             privateVarsInfo);
 }
 
+static LogicalResult
+convertOmpScan(Operation &opInst, llvm::IRBuilderBase &builder,
+               LLVM::ModuleTranslation &moduleTranslation) {
+  if (failed(checkImplementationStatus(opInst)))
+    return failure();
+  auto scanOp = cast<omp::ScanOp>(opInst);
+  bool isInclusive = scanOp.hasInclusiveVars();
+  SmallVector<llvm::Value *> llvmScanVars;
+  SmallVector<llvm::Type *> llvmScanVarsType;
+  mlir::OperandRange mlirScanVars = scanOp.getInclusiveVars();
+  if (!isInclusive)
+    mlirScanVars = scanOp.getExclusiveVars();
+
+  llvm::DenseMap<llvm::Value *, llvm::Type *> *reductionVarToType =
+      findReductionVarTypes(moduleTranslation);
+  for (auto val : mlirScanVars) {
+    llvm::Value *llvmVal = moduleTranslation.lookupValue(val);
+    llvmScanVars.push_back(llvmVal);
+    llvmScanVarsType.push_back((*reductionVarToType)[llvmVal]);
+  }
+  llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
+      findParallelAllocaIP(builder, moduleTranslation);
+  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
+  llvm::ScanInfo *scanInfo = findScanInfo(moduleTranslation);
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
+      moduleTranslation.getOpenMPBuilder()->createScan(
+          ompLoc, allocaIP, llvmScanVars, llvmScanVarsType, isInclusive,
+          scanInfo);
+  if (failed(handleError(afterIP, opInst)))
+    return failure();
+  builder.restoreIP(*afterIP);
+  return success();
+}
+
+/// Re-initialize the scan reduction variables to their identity at the start of
+/// each input-loop iteration of a scan reduction. This is required because the
+/// OpenMPIRBuilder stores the per-iteration reduction value into the scan
+/// buffer (`buffer[i] = red`) and `emitScanReduction` then computes the prefix
+/// sum across the buffer. Without resetting `red` to the reduction identity
+/// each iteration, the buffer would hold a running sum and the prefix sum would
+/// be computed twice. This reset only applies to the input loop; in the scan
+/// loop the reduction variable is loaded back from the buffer.
+static LogicalResult
+initScanReductionVars(omp::LoopNestOp loopOp, llvm::IRBuilderBase &builder,
+                      LLVM::ModuleTranslation &moduleTranslation) {
+  auto wsloopOp = loopOp->getParentOfType<omp::WsloopOp>();
+  if (!wsloopOp || !wsloopOp.getReductionMod() ||
+      wsloopOp.getReductionMod().value() != omp::ReductionModifier::inscan)
+    return success();
+
+  // Only reset in the input loop; in the scan loop the reduction variable holds
+  // the value loaded from the buffer and must not be overwritten.
+  llvm::ScanInfo *scanInfo = findScanInfo(moduleTranslation);
+  if (!scanInfo || !scanInfo->OMPFirstScanLoop)
+    return success();
+
+  unsigned numReductions = wsloopOp.getNumReductionVars();
+  if (numReductions == 0)
+    return success();
+
+  SmallVector<omp::DeclareReductionOp> reductionDecls;
+  collectReductionDecls(wsloopOp, reductionDecls);
+  ArrayRef<bool> isByRef = getIsByRef(wsloopOp.getReductionByref());
+  MutableArrayRef<BlockArgument> reductionArgs =
+      cast<omp::BlockArgOpenMPOpInterface>(wsloopOp.getOperation())
+          .getReductionBlockArgs();
+
+  DenseMap<Value, llvm::Value *> reductionVariableMap;
+  for (unsigned i = 0; i < numReductions; ++i)
+    reductionVariableMap.try_emplace(
+        wsloopOp.getReductionVars()[i],
+        moduleTranslation.lookupValue(reductionArgs[i]));
+
+  for (unsigned i = 0; i < numReductions; ++i) {
+    // Scan reductions only support by-value (scalar) reductions.
+    if (isByRef[i])
+      continue;
+    SmallVector<llvm::Value *, 1> phis;
+    mapInitializationArgs(wsloopOp, moduleTranslation, builder, reductionDecls,
+                          reductionVariableMap, i);
+    if (failed(inlineConvertOmpRegions(
+            reductionDecls[i].getInitializerRegion(),
+            "omp.scan.reduction.init", builder, moduleTranslation, &phis)))
+      return failure();
+    assert(phis.size() == 1 &&
+           "expected one value to be yielded from the reduction init region");
+    setInsertPointForPossiblyEmptyBlock(builder);
+    builder.CreateStore(phis[0],
+                        moduleTranslation.lookupValue(reductionArgs[i]));
+  }
+  return success();
+}
+
 /// Converts an OpenMP loop nest into LLVM IR using OpenMPIRBuilder.
 static LogicalResult
 convertOmpLoopNest(Operation &opInst, llvm::IRBuilderBase &builder,
@@ -5004,6 +5307,10 @@ convertOmpLoopNest(Operation &opInst, llvm::IRBuilderBase &builder,
 
     // Convert the body of the loop.
     builder.restoreIP(ip);
+    // For scan reductions, reset the reduction variables to their identity at
+    // the start of each input-loop iteration, before the input phase runs.
+    if (failed(initScanReductionVars(loopOp, builder, moduleTranslation)))
+      return llvm::make_error<PreviouslyReportedError>();
     llvm::Expected<llvm::BasicBlock *> regionBlock = convertOmpOpRegions(
         loopOp.getRegion(), "omp.loop_nest.region", builder, moduleTranslation);
     if (!regionBlock)
@@ -5036,6 +5343,46 @@ convertOmpLoopNest(Operation &opInst, llvm::IRBuilderBase &builder,
       computeIP = loopInfos.front()->getPreheaderIP();
     }
 
+    // If this loop is the worksharing loop of a scan reduction, generate a
+    // pair of canonical loops (an input loop and a scan loop) instead of a
+    // single loop. The `scan` directive in the body is translated against the
+    // `ScanInfo` recorded here.
+    bool isInScanRegion = false;
+    if (auto wsloopOp = loopOp->getParentOfType<omp::WsloopOp>())
+      isInScanRegion =
+          wsloopOp.getReductionMod() && (wsloopOp.getReductionMod().value() ==
+                                         mlir::omp::ReductionModifier::inscan);
+    if (isInScanRegion) {
+      llvm::Expected<llvm::ScanInfo *> res = ompBuilder->scanInfoInitialize();
+      if (failed(handleError(res, *loopOp)))
+        return failure();
+      llvm::ScanInfo *scanInfo = res.get();
+      moduleTranslation.stackWalk<OpenMPLoopInfoStackFrame>(
+          [&](OpenMPLoopInfoStackFrame &frame) {
+            frame.scanInfo = scanInfo;
+            return WalkResult::interrupt();
+          });
+      llvm::Expected<llvm::SmallVector<llvm::CanonicalLoopInfo *>> loopResults =
+          ompBuilder->createCanonicalScanLoops(
+              loc, bodyGen, lowerBound, upperBound, step,
+              /*IsSigned=*/true, loopOp.getLoopInclusive(), computeIP, "loop",
+              scanInfo);
+
+      if (failed(handleError(loopResults, *loopOp)))
+        return failure();
+      llvm::CanonicalLoopInfo *inputLoop = loopResults.get().front();
+      llvm::CanonicalLoopInfo *scanLoop = loopResults.get().back();
+      moduleTranslation.stackWalk<OpenMPLoopInfoStackFrame>(
+          [&](OpenMPLoopInfoStackFrame &frame) {
+            frame.loopInfo = inputLoop;
+            frame.scanloopInfo = scanLoop;
+            return WalkResult::interrupt();
+          });
+      builder.restoreIP(scanLoop->getAfterIP());
+      // TODO: tiling and collapse are not yet implemented for scan reduction.
+      return success();
+    }
+
     llvm::Expected<llvm::CanonicalLoopInfo *> loopResult =
         ompBuilder->createCanonicalLoop(
             loc, bodyGen, lowerBound, upperBound, step,
@@ -9454,6 +9801,9 @@ LogicalResult OpenMPDialectLLVMIRTranslationInterface::convertOperation(
           .Case([&](omp::WsloopOp) {
             return convertOmpWsloop(*op, builder, moduleTranslation);
           })
+          .Case([&](omp::ScanOp) {
+            return convertOmpScan(*op, builder, moduleTranslation);
+          })
           .Case([&](omp::SimdOp) {
             return convertOmpSimd(*op, builder, moduleTranslation);
           })
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
new file mode 100644
index 0000000000000..c6b8daccad313
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
@@ -0,0 +1,123 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+omp.declare_reduction @add_reduction_i32 : i32 init {
+^bb0(%arg0: i32):
+  %0 = llvm.mlir.constant(0 : i32) : i32
+  omp.yield(%0 : i32)
+} combiner {
+^bb0(%arg0: i32, %arg1: i32):
+  %0 = llvm.add %arg0, %arg1 : i32
+  omp.yield(%0 : i32)
+}
+// CHECK-LABEL: @scan_reduction
+llvm.func @scan_reduction() {
+  %0 = llvm.mlir.constant(1 : i64) : i64
+  %1 = llvm.alloca %0 x i32 {bindc_name = "z"} : (i64) -> !llvm.ptr
+  %3 = llvm.alloca %0 x i32 {bindc_name = "y"} : (i64) -> !llvm.ptr
+  %5 = llvm.alloca %0 x i32 {bindc_name = "x"} : (i64) -> !llvm.ptr
+  %7 = llvm.alloca %0 x i32 {bindc_name = "k"} : (i64) -> !llvm.ptr
+  %10 = llvm.mlir.constant(100 : i32) : i32
+  %11 = llvm.mlir.constant(1 : i32) : i32
+  %12 = llvm.mlir.constant(0 : i32) : i32
+  %13 = llvm.mlir.constant(100 : index) : i64
+  %14 = llvm.mlir.addressof @_QFEa : !llvm.ptr
+  %15 = llvm.mlir.addressof @_QFEb : !llvm.ptr
+  omp.parallel {
+    %37 = llvm.mlir.constant(1 : i64) : i64
+    %38 = llvm.alloca %37 x i32 {bindc_name = "k", pinned} : (i64) -> !llvm.ptr
+    %39 = llvm.mlir.constant(1 : i64) : i64
+    omp.wsloop reduction(mod: inscan, @add_reduction_i32 %5 -> %arg0 : !llvm.ptr) {
+      omp.loop_nest (%arg1) : i32 = (%11) to (%10) inclusive step (%11) {
+        llvm.store %arg1, %38 : i32, !llvm.ptr
+        %40 = llvm.load %arg0 : !llvm.ptr -> i32
+        %41 = llvm.load %38 : !llvm.ptr -> i32
+        %42 = llvm.sext %41 : i32 to i64
+        %50 = llvm.getelementptr %14[%42] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+        %51 = llvm.load %50 : !llvm.ptr -> i32
+        %52 = llvm.add %40, %51 : i32
+        llvm.store %52, %arg0 : i32, !llvm.ptr
+        omp.scan inclusive(%arg0 : !llvm.ptr)
+        llvm.store %arg1, %38 : i32, !llvm.ptr
+        %53 = llvm.load %arg0 : !llvm.ptr -> i32
+        %54 = llvm.load %38 : !llvm.ptr -> i32
+        %55 = llvm.sext %54 : i32 to i64
+        %63 = llvm.getelementptr %15[%55] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+        llvm.store %53, %63 : i32, !llvm.ptr
+        omp.yield
+      }
+    }
+    omp.terminator
+  }
+  llvm.return
+}
+llvm.mlir.global internal @_QFEa() {addr_space = 0 : i32} : !llvm.array<100 x i32> {
+  %0 = llvm.mlir.zero : !llvm.array<100 x i32>
+  llvm.return %0 : !llvm.array<100 x i32>
+}
+llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i32> {
+  %0 = llvm.mlir.zero : !llvm.array<100 x i32>
+  llvm.return %0 : !llvm.array<100 x i32>
+}
+//CHECK: %vla = alloca ptr, align 8
+//CHECK: omp_parallel
+//CHECK: store ptr %vla, ptr %gep_vla, align 8
+//CHECK: @__kmpc_fork_call
+//CHECK: void @scan_reduction..omp_par
+//CHECK: %[[BUFF_PTR:.+]] = load ptr, ptr %gep_vla
+//CHECK: @__kmpc_masked
+//CHECK: @__kmpc_barrier
+//CHECK: @__kmpc_masked
+//CHECK: @__kmpc_barrier
+//CHECK: omp.scan.loop.cont:
+//CHECK: @__kmpc_masked
+//CHECK: @__kmpc_barrier
+//CHECK: %[[FREE_VAR:.+]] = load ptr, ptr %[[BUFF_PTR]], align 8
+//CHECK:  %[[ARRLAST:.+]] = getelementptr inbounds i32, ptr %[[FREE_VAR]], i32 100
+//CHECK:  %[[RES:.+]] = load i32, ptr %[[ARRLAST]], align 4
+//CHECK:  store i32 %[[RES]], ptr %loadgep{{.*}}, align 4
+//CHECK: tail call void @free(ptr %[[FREE_VAR]])
+//CHECK: @__kmpc_end_masked
+//CHECK: omp.inscan.dispatch{{.*}}:                            ; preds = %omp_loop.body{{.*}}
+//CHECK:   %[[BUFFVAR:.+]] = load ptr, ptr %[[BUFF_PTR]], align 8
+//CHECK:   %[[arrayOffset1:.+]] = getelementptr inbounds i32, ptr %[[BUFFVAR]], i32 %{{.*}}
+//CHECK:   %[[BUFFVAL1:.+]] = load i32, ptr %[[arrayOffset1]], align 4
+//CHECK:   store i32 %[[BUFFVAL1]], ptr %{{.*}}, align 4
+//CHECK:   %[[LOG:.+]] = call double @llvm.log2.f64(double 1.000000e+02) #0
+//CHECK:   %[[CEIL:.+]] = call double @llvm.ceil.f64(double %[[LOG]]) #0
+//CHECK:   %[[UB:.+]] = fptoui double %[[CEIL]] to i32
+//CHECK:   br label %omp.outer.log.scan.body
+//CHECK: omp.outer.log.scan.body:
+//CHECK:   %[[K:.+]] = phi i32 [ 0, %{{.*}} ], [ %[[NEXTK:.+]], %omp.inner.log.scan.exit ]
+//CHECK:   %[[I:.+]] = phi i32 [ 1, %{{.*}} ], [ %[[NEXTI:.+]], %omp.inner.log.scan.exit ]
+//CHECK:   %[[CMP1:.+]] = icmp uge i32 99, %[[I]]
+//CHECK:   br i1 %[[CMP1]], label %omp.inner.log.scan.body, label %omp.inner.log.scan.exit
+//CHECK: omp.inner.log.scan.exit:                          ; preds = %omp.inner.log.scan.body, %omp.outer.log.scan.body
+//CHECK:   %[[NEXTK]] = add nuw i32 %[[K]], 1
+//CHECK:   %[[NEXTI]] = shl nuw i32 %[[I]], 1
+//CHECK:   %[[CMP2:.+]] = icmp ne i32 %[[NEXTK]], %[[UB]]
+//CHECK:   br i1 %[[CMP2]], label %omp.outer.log.scan.body, label %omp.outer.log.scan.exit
+//CHECK: omp.outer.log.scan.exit:                          ; preds = %omp.inner.log.scan.exit
+//CHECK: @__kmpc_end_masked
+//CHECK: omp.inner.log.scan.body:                          ; preds = %omp.inner.log.scan.body, %omp.outer.log.scan.body
+//CHECK:   %[[CNT:.+]] = phi i32 [ 99, %omp.outer.log.scan.body ], [ %[[CNTNXT:.+]], %omp.inner.log.scan.body ]
+//CHECK:   %[[BUFF:.+]] = load ptr, ptr %[[BUFF_PTR]]
+//CHECK:   %[[IND1:.+]] = add i32 %[[CNT]], 1
+//CHECK:   %[[IND1PTR:.+]] = getelementptr inbounds i32, ptr %[[BUFF]], i32 %[[IND1]]
+//CHECK:   %[[IND2:.+]] = sub nuw i32 %[[IND1]], %[[I]]
+//CHECK:   %[[IND2PTR:.+]] = getelementptr inbounds i32, ptr %[[BUFF]], i32 %[[IND2]]
+//CHECK:   %[[IND1VAL:.+]] = load i32, ptr %[[IND1PTR]], align 4
+//CHECK:   %[[IND2VAL:.+]] = load i32, ptr %[[IND2PTR]], align 4
+//CHECK:   %[[REDVAL:.+]] = add i32 %[[IND1VAL]], %[[IND2VAL]]
+//CHECK:   store i32 %[[REDVAL]], ptr %[[IND1PTR]], align 4
+//CHECK:   %[[CNTNXT]] = sub nuw i32 %[[CNT]], 1
+//CHECK:   %[[CMP3:.+]] = icmp uge i32 %[[CNTNXT]], %[[I]]
+//CHECK:   br i1 %[[CMP3]], label %omp.inner.log.scan.body, label %omp.inner.log.scan.exit
+//CHECK: omp.inscan.dispatch:                              ; preds = %omp_loop.body
+//CHECK:   br i1 true, label %omp.before.scan.bb, label %omp.after.scan.bb
+//CHECK: omp.before.scan.bb:
+//CHECK:   store i32 0, ptr %[[REDPRIV:.+]], align 4
+//CHECK: omp.loop_nest.region:                             ; preds = %omp.before.scan.bb
+//CHECK:   %[[BUFFER:.+]] = load ptr, ptr %loadgep_vla, align 8
+//CHECK:   %[[ARRAYOFFSET2:.+]] = getelementptr inbounds i32, ptr %[[BUFFER]], i32 %{{.*}}
+//CHECK-NEXT:   %[[REDPRIVVAL:.+]] = load i32, ptr %{{.*}}, align 4
+//CHECK:   store i32 %[[REDPRIVVAL]], ptr %[[ARRAYOFFSET2]], align 4
+//CHECK:   br label %omp.scan.loop.exit
diff --git a/mlir/test/Target/LLVMIR/openmp-todo.mlir b/mlir/test/Target/LLVMIR/openmp-todo.mlir
index 71e8628595aab..7edf88b5ea559 100644
--- a/mlir/test/Target/LLVMIR/openmp-todo.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-todo.mlir
@@ -120,10 +120,10 @@ atomic {
   llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32
   omp.yield
 }
-llvm.func @scan_reduction(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
-  // expected-error at below {{not yet implemented: Unhandled clause reduction with modifier in omp.wsloop operation}}
-  // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
-  omp.wsloop reduction(mod:inscan, @add_f32 %x -> %prv : !llvm.ptr) {
+llvm.func @simd_reduction(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
+  // expected-error at below {{not yet implemented: Unhandled clause reduction with modifier in omp.simd operation}}
+  // expected-error at below {{LLVM Translation failed for operation: omp.simd}}
+  omp.simd reduction(mod:inscan, @add_f32 %x -> %prv : !llvm.ptr) {
     omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
       omp.scan inclusive(%prv : !llvm.ptr)
       omp.yield
@@ -162,6 +162,39 @@ llvm.func @parallel_task_reduction_modifier_byref(%x : !llvm.ptr) {
 
 // -----
 
+omp.declare_reduction @add_f32 : f32
+init {
+^bb0(%arg: f32):
+  %0 = llvm.mlir.constant(0.0 : f32) : f32
+  omp.yield (%0 : f32)
+}
+combiner {
+^bb1(%arg0: f32, %arg1: f32):
+  %1 = llvm.fadd %arg0, %arg1 : f32
+  omp.yield (%1 : f32)
+}
+atomic {
+^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr):
+  %2 = llvm.load %arg3 : !llvm.ptr -> f32
+  llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32
+  omp.yield
+}
+llvm.func @scan_reduction(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
+  // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
+  omp.wsloop reduction(mod:inscan, @add_f32 %x -> %prv : !llvm.ptr) {
+    // expected-error at below {{LLVM Translation failed for operation: omp.loop_nest}}
+    omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
+      // expected-error at below {{not yet implemented: Unhandled clause exclusive in omp.scan operation}}
+      // expected-error at below {{LLVM Translation failed for operation: omp.scan}}
+      omp.scan exclusive(%prv : !llvm.ptr)
+      omp.yield
+    }
+  }
+  llvm.return
+}
+
+// -----
+
 llvm.func @single_allocate(%x : !llvm.ptr) {
   // expected-error at below {{not yet implemented: Unhandled clause allocate in omp.single operation}}
   // expected-error at below {{LLVM Translation failed for operation: omp.single}}
diff --git a/openmp/runtime/test/scan/scan.f90 b/openmp/runtime/test/scan/scan.f90
new file mode 100644
index 0000000000000..76263508105d0
--- /dev/null
+++ b/openmp/runtime/test/scan/scan.f90
@@ -0,0 +1,38 @@
+! RUN: %flang %flags %openmp_flags -fopenmp-version=51 %s -o %t.exe
+! RUN: %t.exe | FileCheck %s --match-full-lines
+program inclusive_scan
+ implicit none
+ integer, parameter :: n = 100
+ integer a(n), b(n)
+ integer x, k, y, z
+
+ ! initialization
+ x = 0
+ do k = 1, n
+  a(k) = k
+ end do
+
+ ! a(k) is included in the computation of producing results in b(k)
+ !$omp parallel do reduction(inscan, +: x)
+ do k = 1, n
+   x = x + a(k)
+   !$omp scan inclusive(x)
+   b(k) = x
+ end do
+
+ print *,'x =', x
+ do k = 1, 10
+  print *, 'b(', k, ') =', b(k)
+ end do
+end program
+!CHECK: x = 5050
+!CHECK: b( 1 ) = 1
+!CHECK: b( 2 ) = 3
+!CHECK: b( 3 ) = 6
+!CHECK: b( 4 ) = 10
+!CHECK: b( 5 ) = 15
+!CHECK: b( 6 ) = 21
+!CHECK: b( 7 ) = 28
+!CHECK: b( 8 ) = 36
+!CHECK: b( 9 ) = 45
+!CHECK: b( 10 ) = 55

>From 4011dd537d9830ee4a0379cba80413d1e4cdf9c3 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Wed, 1 Jul 2026 02:44:02 -0500
Subject: [PATCH 2/9] formatting correction

---
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp        | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 297ebac51de88..e18cc0df79860 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -4534,12 +4534,11 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
   bool isInScanRegion =
       wsloopOp.getReductionMod() && (wsloopOp.getReductionMod().value() ==
                                      mlir::omp::ReductionModifier::inscan);
-  if (failed(initReductionVars(wsloopOp, reductionArgs, builder,
-                               moduleTranslation,
-                               afterAllocas.get()->getSinglePredecessor(),
-                               reductionDecls, privateReductionVariables,
-                               reductionVariableMap, isByRef, deferredStores,
-                               isInScanRegion)))
+  if (failed(initReductionVars(
+          wsloopOp, reductionArgs, builder, moduleTranslation,
+          afterAllocas.get()->getSinglePredecessor(), reductionDecls,
+          privateReductionVariables, reductionVariableMap, isByRef,
+          deferredStores, isInScanRegion)))
     return failure();
 
   // For `reduction(task, ...)` open a task-reduction scope for the worksharing
@@ -5262,9 +5261,9 @@ initScanReductionVars(omp::LoopNestOp loopOp, llvm::IRBuilderBase &builder,
     SmallVector<llvm::Value *, 1> phis;
     mapInitializationArgs(wsloopOp, moduleTranslation, builder, reductionDecls,
                           reductionVariableMap, i);
-    if (failed(inlineConvertOmpRegions(
-            reductionDecls[i].getInitializerRegion(),
-            "omp.scan.reduction.init", builder, moduleTranslation, &phis)))
+    if (failed(inlineConvertOmpRegions(reductionDecls[i].getInitializerRegion(),
+                                       "omp.scan.reduction.init", builder,
+                                       moduleTranslation, &phis)))
       return failure();
     assert(phis.size() == 1 &&
            "expected one value to be yielded from the reduction init region");

>From 64b530d3f37e0cd08ca622b538d9bec1eb348b50 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Wed, 22 Jul 2026 05:09:34 -0500
Subject: [PATCH 3/9] Address review comments

---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     | 46 +++++++++++---
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      | 26 +++++++-
 .../Target/LLVMIR/openmp-reduction-scan.mlir  | 14 +++--
 mlir/test/Target/LLVMIR/openmp-todo.mlir      | 31 ++++++++++
 openmp/runtime/test/scan/scan-bounds.f90      | 61 +++++++++++++++++++
 openmp/runtime/test/scan/scan-edge.f90        | 61 +++++++++++++++++++
 openmp/runtime/test/scan/scan-nonidentity.f90 | 28 +++++++++
 7 files changed, 251 insertions(+), 16 deletions(-)
 create mode 100644 openmp/runtime/test/scan/scan-bounds.f90
 create mode 100644 openmp/runtime/test/scan/scan-edge.f90
 create mode 100644 openmp/runtime/test/scan/scan-nonidentity.f90

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index f8c1999fe1b89..f62ea4cf2f9da 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5307,7 +5307,17 @@ Error OpenMPIRBuilder::emitScanBasedDirectiveFinalsIR(
                                              "arrayOffset");
       Value *Src = Builder.CreateLoad(SrcTy, Val);
 
-      Builder.CreateStore(Src, OrigVar);
+      // For a zero-trip loop the buffer slot is uninitialized; keep the
+      // original variable's incoming value instead of storing garbage. The
+      // load above stays in bounds because the buffer was allocated with
+      // `Span + 1` elements, so index `Span` is always valid.
+      Value *Cur = Builder.CreateLoad(SrcTy, OrigVar);
+      Value *HasIters = Builder.CreateICmpUGT(
+          ScanRedInfo->Span,
+          llvm::ConstantInt::get(ScanRedInfo->Span->getType(), 0));
+      Value *Final = Builder.CreateSelect(HasIters, Src, Cur);
+
+      Builder.CreateStore(Final, OrigVar);
       Builder.CreateFree(Buff);
     }
     return Error::success();
@@ -5370,7 +5380,15 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
         ScanRedInfo->Span,
         llvm::ConstantInt::get(ScanRedInfo->Span->getType(), 1));
     Builder.SetInsertPoint(InputBB);
-    Builder.CreateBr(LoopBB);
+    // The log-scan prefix computation is only well-defined when there are at
+    // least two elements: `log2(Span)` is invalid for Span == 0 and the loop
+    // trip computation underflows/loops forever for Span <= 1. For Span <= 1
+    // the buffer already holds the final value(s), so skip straight to the
+    // exit.
+    llvm::Value *SpanGuard = Builder.CreateICmpUGT(
+        ScanRedInfo->Span,
+        llvm::ConstantInt::get(ScanRedInfo->Span->getType(), 1));
+    Builder.CreateCondBr(SpanGuard, LoopBB, ExitBB);
     emitBlock(LoopBB, CurFn);
     Builder.SetInsertPoint(LoopBB);
 
@@ -5617,7 +5635,6 @@ OpenMPIRBuilder::createCanonicalScanLoops(
 
   auto BodyGen = [=](InsertPointTy CodeGenIP, Value *IV) {
     Builder.restoreIP(CodeGenIP);
-    ScanRedInfo->IV = IV;
     createScanBBs(ScanRedInfo);
     BasicBlock *InputBlock = Builder.GetInsertBlock();
     Instruction *Terminator = InputBlock->getTerminator();
@@ -5636,9 +5653,13 @@ OpenMPIRBuilder::createCanonicalScanLoops(
   };
 
   const auto &&InputLoopGen = [&]() -> Error {
+    // Pass an empty ComputeIP: the trip count must be (re)computed at the
+    // loop's own location. Reusing the outer ComputeIP is invalid here because
+    // it was invalidated by the `scan.init` split above and would emit the
+    // recomputed trip count after a block terminator for runtime bounds.
     Expected<CanonicalLoopInfo *> LoopInfo = createCanonicalLoop(
         Builder.saveIP(), BodyGen, Start, Stop, Step, IsSigned, InclusiveStop,
-        ComputeIP, Name, true, ScanRedInfo);
+        /*ComputeIP=*/InsertPointTy(), Name, true, ScanRedInfo);
     if (!LoopInfo)
       return LoopInfo.takeError();
     Result.push_back(*LoopInfo);
@@ -5646,9 +5667,9 @@ OpenMPIRBuilder::createCanonicalScanLoops(
     return Error::success();
   };
   const auto &&ScanLoopGen = [&](LocationDescription Loc) -> Error {
-    Expected<CanonicalLoopInfo *> LoopInfo =
-        createCanonicalLoop(Loc, BodyGen, Start, Stop, Step, IsSigned,
-                            InclusiveStop, ComputeIP, Name, true, ScanRedInfo);
+    Expected<CanonicalLoopInfo *> LoopInfo = createCanonicalLoop(
+        Loc, BodyGen, Start, Stop, Step, IsSigned, InclusiveStop,
+        /*ComputeIP=*/InsertPointTy(), Name, true, ScanRedInfo);
     if (!LoopInfo)
       return LoopInfo.takeError();
     Result.push_back(*LoopInfo);
@@ -5737,8 +5758,15 @@ Expected<CanonicalLoopInfo *> OpenMPIRBuilder::createCanonicalLoop(
     Builder.restoreIP(CodeGenIP);
     Value *Span = Builder.CreateMul(IV, Step);
     Value *IndVar = Builder.CreateAdd(Span, Start);
-    if (InScan)
-      ScanRedInfo->IV = IndVar;
+    if (InScan) {
+      // The scan buffer is 1-indexed by iteration number (buffer[1..TripCount])
+      // rather than by the source induction variable. Using the source value
+      // would index out of bounds for loops that do not start at 1 or that use
+      // a non-unit stride. `IV` here is the 0-based canonical loop counter, so
+      // the 1-based buffer index is `IV + 1`.
+      ScanRedInfo->IV =
+          Builder.CreateAdd(IV, ConstantInt::get(IV->getType(), 1));
+    }
     return BodyGenCB(Builder.saveIP(), IndVar);
   };
   LocationDescription LoopLoc =
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index e18cc0df79860..5af1a384d3edb 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -430,6 +430,16 @@ static LogicalResult checkImplementationStatus(Operation &op) {
               break;
             }
         }
+      } else if (inscanModifierSupported) {
+        if (auto byref = op.getReductionByref()) {
+          // Scan reductions (inscan modifier) only support by-value (scalar)
+          // reductions for now.
+          for (bool isByRef : *byref)
+            if (isByRef) {
+              result = todo("inscan reduction modifier with by-ref reduction");
+              break;
+            }
+        }
       }
     }
   };
@@ -5268,8 +5278,20 @@ initScanReductionVars(omp::LoopNestOp loopOp, llvm::IRBuilderBase &builder,
     assert(phis.size() == 1 &&
            "expected one value to be yielded from the reduction init region");
     setInsertPointForPossiblyEmptyBlock(builder);
-    builder.CreateStore(phis[0],
-                        moduleTranslation.lookupValue(reductionArgs[i]));
+
+    // On the first iteration, seed the reduction variable with the original
+    // variable's incoming value so that a non-identity initial value
+    // participates in the scan (e.g. `x = 10` before an inclusive-scan loop of
+    // `+1` must yield 11, 12, ...). On subsequent iterations, reset to the
+    // reduction identity. `scanInfo->IV` is the 1-based iteration index.
+    llvm::Value *identity = phis[0];
+    llvm::Value *origVar =
+        moduleTranslation.lookupValue(wsloopOp.getReductionVars()[i]);
+    llvm::Value *origVal = builder.CreateLoad(identity->getType(), origVar);
+    llvm::Value *isFirstIter = builder.CreateICmpEQ(
+        scanInfo->IV, llvm::ConstantInt::get(scanInfo->IV->getType(), 1));
+    llvm::Value *seed = builder.CreateSelect(isFirstIter, origVal, identity);
+    builder.CreateStore(seed, moduleTranslation.lookupValue(reductionArgs[i]));
   }
   return success();
 }
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
index c6b8daccad313..6ef58b5013e08 100644
--- a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
@@ -73,7 +73,9 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK: %[[FREE_VAR:.+]] = load ptr, ptr %[[BUFF_PTR]], align 8
 //CHECK:  %[[ARRLAST:.+]] = getelementptr inbounds i32, ptr %[[FREE_VAR]], i32 100
 //CHECK:  %[[RES:.+]] = load i32, ptr %[[ARRLAST]], align 4
-//CHECK:  store i32 %[[RES]], ptr %loadgep{{.*}}, align 4
+//CHECK:  %[[CUR:.+]] = load i32, ptr %loadgep{{.*}}, align 4
+//CHECK:  %[[FIN:.+]] = select i1 {{.*}}, i32 %[[RES]], i32 %[[CUR]]
+//CHECK:  store i32 %[[FIN]], ptr %loadgep{{.*}}, align 4
 //CHECK: tail call void @free(ptr %[[FREE_VAR]])
 //CHECK: @__kmpc_end_masked
 //CHECK: omp.inscan.dispatch{{.*}}:                            ; preds = %omp_loop.body{{.*}}
@@ -84,7 +86,9 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK:   %[[LOG:.+]] = call double @llvm.log2.f64(double 1.000000e+02) #0
 //CHECK:   %[[CEIL:.+]] = call double @llvm.ceil.f64(double %[[LOG]]) #0
 //CHECK:   %[[UB:.+]] = fptoui double %[[CEIL]] to i32
-//CHECK:   br label %omp.outer.log.scan.body
+//CHECK:   br i1 {{.*}}, label %omp.outer.log.scan.body, label %omp.outer.log.scan.exit
+//CHECK: omp.outer.log.scan.exit:                          ; preds = %omp.inner.log.scan.exit, %omp_region.body{{.*}}
+//CHECK: @__kmpc_end_masked
 //CHECK: omp.outer.log.scan.body:
 //CHECK:   %[[K:.+]] = phi i32 [ 0, %{{.*}} ], [ %[[NEXTK:.+]], %omp.inner.log.scan.exit ]
 //CHECK:   %[[I:.+]] = phi i32 [ 1, %{{.*}} ], [ %[[NEXTI:.+]], %omp.inner.log.scan.exit ]
@@ -95,8 +99,6 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK:   %[[NEXTI]] = shl nuw i32 %[[I]], 1
 //CHECK:   %[[CMP2:.+]] = icmp ne i32 %[[NEXTK]], %[[UB]]
 //CHECK:   br i1 %[[CMP2]], label %omp.outer.log.scan.body, label %omp.outer.log.scan.exit
-//CHECK: omp.outer.log.scan.exit:                          ; preds = %omp.inner.log.scan.exit
-//CHECK: @__kmpc_end_masked
 //CHECK: omp.inner.log.scan.body:                          ; preds = %omp.inner.log.scan.body, %omp.outer.log.scan.body
 //CHECK:   %[[CNT:.+]] = phi i32 [ 99, %omp.outer.log.scan.body ], [ %[[CNTNXT:.+]], %omp.inner.log.scan.body ]
 //CHECK:   %[[BUFF:.+]] = load ptr, ptr %[[BUFF_PTR]]
@@ -114,7 +116,9 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK: omp.inscan.dispatch:                              ; preds = %omp_loop.body
 //CHECK:   br i1 true, label %omp.before.scan.bb, label %omp.after.scan.bb
 //CHECK: omp.before.scan.bb:
-//CHECK:   store i32 0, ptr %[[REDPRIV:.+]], align 4
+//CHECK:   %[[ISFIRST:.+]] = icmp eq i32 %{{.*}}, 1
+//CHECK:   %[[SEED:.+]] = select i1 %[[ISFIRST]], i32 %{{.*}}, i32 0
+//CHECK:   store i32 %[[SEED]], ptr %[[REDPRIV:.+]], align 4
 //CHECK: omp.loop_nest.region:                             ; preds = %omp.before.scan.bb
 //CHECK:   %[[BUFFER:.+]] = load ptr, ptr %loadgep_vla, align 8
 //CHECK:   %[[ARRAYOFFSET2:.+]] = getelementptr inbounds i32, ptr %[[BUFFER]], i32 %{{.*}}
diff --git a/mlir/test/Target/LLVMIR/openmp-todo.mlir b/mlir/test/Target/LLVMIR/openmp-todo.mlir
index 7edf88b5ea559..fe3c4259e75e0 100644
--- a/mlir/test/Target/LLVMIR/openmp-todo.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-todo.mlir
@@ -195,6 +195,37 @@ llvm.func @scan_reduction(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
 
 // -----
 
+omp.declare_reduction @add_f32 : f32
+init {
+^bb0(%arg: f32):
+  %0 = llvm.mlir.constant(0.0 : f32) : f32
+  omp.yield (%0 : f32)
+}
+combiner {
+^bb1(%arg0: f32, %arg1: f32):
+  %1 = llvm.fadd %arg0, %arg1 : f32
+  omp.yield (%1 : f32)
+}
+atomic {
+^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr):
+  %2 = llvm.load %arg3 : !llvm.ptr -> f32
+  llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32
+  omp.yield
+}
+llvm.func @scan_reduction_byref(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
+  // expected-error at below {{not yet implemented: Unhandled clause inscan reduction modifier with by-ref reduction in omp.wsloop operation}}
+  // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
+  omp.wsloop reduction(mod:inscan, byref @add_f32 %x -> %prv : !llvm.ptr) {
+    omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
+      omp.scan inclusive(%prv : !llvm.ptr)
+      omp.yield
+    }
+  }
+  llvm.return
+}
+
+// -----
+
 llvm.func @single_allocate(%x : !llvm.ptr) {
   // expected-error at below {{not yet implemented: Unhandled clause allocate in omp.single operation}}
   // expected-error at below {{LLVM Translation failed for operation: omp.single}}
diff --git a/openmp/runtime/test/scan/scan-bounds.f90 b/openmp/runtime/test/scan/scan-bounds.f90
new file mode 100644
index 0000000000000..9d624954568ff
--- /dev/null
+++ b/openmp/runtime/test/scan/scan-bounds.f90
@@ -0,0 +1,61 @@
+! RUN: %flang %flags %openmp_flags -fopenmp-version=51 %s -o %t.exe
+! RUN: %t.exe | FileCheck %s --match-full-lines
+
+! Arbitrary loop bounds and a non-unit stride must index the scan buffer by the
+! logical iteration number, not by the induction-variable value.
+program scan_bounds
+ implicit none
+ integer, parameter :: n = 30
+ integer :: a(n), b(n)
+ integer :: x, k
+
+ do k = 1, n
+   a(k) = k
+ end do
+
+ ! Arbitrary bounds: iterate from 10 to 20.
+ b = -1
+ x = 0
+ !$omp parallel do reduction(inscan, +: x)
+ do k = 10, 20
+   x = x + a(k)
+   !$omp scan inclusive(x)
+   b(k) = x
+ end do
+ print *, 'bounds x =', x
+ do k = 10, 20
+   print *, 'bb(', k, ') =', b(k)
+ end do
+
+ ! Non-unit stride: iterate from 1 to 10 step 2.
+ b = -1
+ x = 0
+ !$omp parallel do reduction(inscan, +: x)
+ do k = 1, 10, 2
+   x = x + a(k)
+   !$omp scan inclusive(x)
+   b(k) = x
+ end do
+ print *, 'stride x =', x
+ do k = 1, 10, 2
+   print *, 'sb(', k, ') =', b(k)
+ end do
+end program
+!CHECK: bounds x = 165
+!CHECK: bb( 10 ) = 10
+!CHECK: bb( 11 ) = 21
+!CHECK: bb( 12 ) = 33
+!CHECK: bb( 13 ) = 46
+!CHECK: bb( 14 ) = 60
+!CHECK: bb( 15 ) = 75
+!CHECK: bb( 16 ) = 91
+!CHECK: bb( 17 ) = 108
+!CHECK: bb( 18 ) = 126
+!CHECK: bb( 19 ) = 145
+!CHECK: bb( 20 ) = 165
+!CHECK: stride x = 25
+!CHECK: sb( 1 ) = 1
+!CHECK: sb( 3 ) = 4
+!CHECK: sb( 5 ) = 9
+!CHECK: sb( 7 ) = 16
+!CHECK: sb( 9 ) = 25
diff --git a/openmp/runtime/test/scan/scan-edge.f90 b/openmp/runtime/test/scan/scan-edge.f90
new file mode 100644
index 0000000000000..7cc00da958e6e
--- /dev/null
+++ b/openmp/runtime/test/scan/scan-edge.f90
@@ -0,0 +1,61 @@
+! RUN: %flang %flags %openmp_flags -fopenmp-version=51 %s -o %t.exe
+! RUN: %t.exe | FileCheck %s --match-full-lines
+
+! Zero-trip loops must not crash and must leave the reduction variable
+! unchanged, and runtime (non-constant) trip counts must work correctly.
+module scan_edge_mod
+contains
+ subroutine run_scan(n, a, b)
+   implicit none
+   integer :: n, k
+   integer :: a(n), b(n)
+   integer :: x
+   x = 0
+   !$omp parallel do reduction(inscan, +: x)
+   do k = 1, n
+     x = x + a(k)
+     !$omp scan inclusive(x)
+     b(k) = x
+   end do
+ end subroutine
+end module
+
+program scan_edge
+ use scan_edge_mod
+ implicit none
+ integer :: a(6), b(6), k
+ integer :: z, zk
+ integer :: zb(1)
+
+ ! Runtime bounds: the trip count of run_scan is not a compile-time constant.
+ do k = 1, 6
+   a(k) = k
+ end do
+ b = -1
+ call run_scan(6, a, b)
+ do k = 1, 6
+   print *, 'rb(', k, ') =', b(k)
+ end do
+
+ ! Runtime zero-trip: n = 0 must not crash.
+ call run_scan(0, a, b)
+ print *, 'runtime zero-trip ok'
+
+ ! Constant zero-trip with a non-identity start: z must be unchanged.
+ z = 42
+ !$omp parallel do reduction(inscan, +: z)
+ do zk = 1, 0
+   z = z + 1
+   !$omp scan inclusive(z)
+   zb(zk) = z
+ end do
+ print *, 'z =', z
+end program
+!CHECK: rb( 1 ) = 1
+!CHECK: rb( 2 ) = 3
+!CHECK: rb( 3 ) = 6
+!CHECK: rb( 4 ) = 10
+!CHECK: rb( 5 ) = 15
+!CHECK: rb( 6 ) = 21
+!CHECK: runtime zero-trip ok
+!CHECK: z = 42
diff --git a/openmp/runtime/test/scan/scan-nonidentity.f90 b/openmp/runtime/test/scan/scan-nonidentity.f90
new file mode 100644
index 0000000000000..064b0d757af6e
--- /dev/null
+++ b/openmp/runtime/test/scan/scan-nonidentity.f90
@@ -0,0 +1,28 @@
+! RUN: %flang %flags %openmp_flags -fopenmp-version=51 %s -o %t.exe
+! RUN: %t.exe | FileCheck %s --match-full-lines
+
+! The reduction variable's non-identity incoming value must be included in the
+! first partial result of an inclusive scan.
+program scan_nonidentity
+ implicit none
+ integer :: x, k
+ integer :: b(4)
+
+ x = 10
+ !$omp parallel do reduction(inscan, +: x)
+ do k = 1, 4
+   x = x + 1
+   !$omp scan inclusive(x)
+   b(k) = x
+ end do
+
+ print *, 'x =', x
+ do k = 1, 4
+   print *, 'b(', k, ') =', b(k)
+ end do
+end program
+!CHECK: x = 14
+!CHECK: b( 1 ) = 11
+!CHECK: b( 2 ) = 12
+!CHECK: b( 3 ) = 13
+!CHECK: b( 4 ) = 14

>From 5506275c5cec52b0991c9f85bddeb1c8ebabf451 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Mon, 27 Jul 2026 10:07:57 -0500
Subject: [PATCH 4/9] updated with review comments fixes

---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     | 23 +++--
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      | 25 ++++-
 mlir/test/Target/LLVMIR/openmp-todo.mlir      | 94 +++++++++++++++++--
 openmp/runtime/test/scan/scan-i64.f90         | 37 ++++++++
 openmp/runtime/test/scan/scan-nowait.f90      | 38 ++++++++
 5 files changed, 198 insertions(+), 19 deletions(-)
 create mode 100644 openmp/runtime/test/scan/scan-i64.f90
 create mode 100644 openmp/runtime/test/scan/scan-nowait.f90

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 90db03da7380c..0ed215b3e2ea6 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5350,10 +5350,13 @@ Error OpenMPIRBuilder::emitScanBasedDirectiveDeclsIR(
   auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
                        ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
     Builder.restoreIP(CodeGenIP);
-    Value *AllocSpan =
-        Builder.CreateAdd(ScanRedInfo->Span, Builder.getInt32(1));
+    // Use the loop's own index type (matching `Span`) so the buffer allocation
+    // works for both i32 and i64 (e.g. `integer(kind=8)`) induction variables.
+    Type *IndexTy = ScanRedInfo->Span->getType();
+    Value *AllocSpan = Builder.CreateAdd(ScanRedInfo->Span,
+                                         ConstantInt::get(IndexTy, 1));
     for (size_t i = 0; i < ScanVars.size(); i++) {
-      Type *IntPtrTy = Builder.getInt32Ty();
+      Type *IntPtrTy = IndexTy;
       Constant *Allocsize = ConstantExpr::getSizeOf(ScanVarsType[i]);
       Allocsize = ConstantExpr::getTruncOrBitCast(Allocsize, IntPtrTy);
       Value *Buff = Builder.CreateMalloc(IntPtrTy, ScanVarsType[i], Allocsize,
@@ -5453,6 +5456,10 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
                        ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
     Builder.restoreIP(CodeGenIP);
     Function *CurFn = Builder.GetInsertBlock()->getParent();
+    // Index/counter values that participate in buffer-offset arithmetic must
+    // use the loop's own index type (`Span`'s type) so scan works for both i32
+    // and i64 (e.g. `integer(kind=8)`) induction variables.
+    Type *IndexTy = ScanRedInfo->Span->getType();
     // for (int k = 0; k <= ceil(log2(n)); ++k)
     llvm::BasicBlock *LoopBB =
         BasicBlock::Create(CurFn->getContext(), "omp.outer.log.scan.body");
@@ -5488,10 +5495,10 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
 
     PHINode *Counter = Builder.CreatePHI(Builder.getInt32Ty(), 2);
     // size pow2k = 1;
-    PHINode *Pow2K = Builder.CreatePHI(Builder.getInt32Ty(), 2);
+    PHINode *Pow2K = Builder.CreatePHI(IndexTy, 2);
     Counter->addIncoming(llvm::ConstantInt::get(Builder.getInt32Ty(), 0),
                          InputBB);
-    Pow2K->addIncoming(llvm::ConstantInt::get(Builder.getInt32Ty(), 1),
+    Pow2K->addIncoming(llvm::ConstantInt::get(IndexTy, 1),
                        InputBB);
     // for (size i = n - 1; i >= 2 ^ k; --i)
     //   tmp[i] op= tmp[i-pow2k];
@@ -5503,14 +5510,14 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
     Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
     emitBlock(InnerLoopBB, CurFn);
     Builder.SetInsertPoint(InnerLoopBB);
-    PHINode *IVal = Builder.CreatePHI(Builder.getInt32Ty(), 2);
+    PHINode *IVal = Builder.CreatePHI(IndexTy, 2);
     IVal->addIncoming(NMin1, LoopBB);
     for (ReductionInfo RedInfo : ReductionInfos) {
       Value *ReductionVal = RedInfo.PrivateVariable;
       Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ReductionVal];
       Value *Buff = Builder.CreateLoad(Builder.getPtrTy(), BuffPtr);
       Type *DestTy = RedInfo.ElementType;
-      Value *IV = Builder.CreateAdd(IVal, Builder.getInt32(1));
+      Value *IV = Builder.CreateAdd(IVal, ConstantInt::get(IndexTy, 1));
       Value *LHSPtr =
           Builder.CreateInBoundsGEP(DestTy, Buff, IV, "arrayOffset");
       Value *OffsetIval = Builder.CreateNUWSub(IV, Pow2K);
@@ -5526,7 +5533,7 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
       Builder.CreateStore(Result, LHSPtr);
     }
     llvm::Value *NextIVal = Builder.CreateNUWSub(
-        IVal, llvm::ConstantInt::get(Builder.getInt32Ty(), 1));
+        IVal, llvm::ConstantInt::get(IndexTy, 1));
     IVal->addIncoming(NextIVal, Builder.GetInsertBlock());
     CmpI = Builder.CreateICmpUGE(NextIVal, Pow2K);
     Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 7b8be030da4dc..b7381244a50b0 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -471,6 +471,22 @@ static LogicalResult checkImplementationStatus(Operation &op) {
               break;
             }
         }
+        // Scan reductions allocate a shared temporary buffer in the alloca
+        // frame of the enclosing parallel region. An orphaned worksharing loop
+        // (e.g. an `omp do` in a subroutine that is called from within a
+        // parallel region) has no such frame in its own function, so the buffer
+        // would be allocated per-thread and produce wrong results.
+        if (!op->template getParentOfType<omp::ParallelOp>())
+          result = todo("inscan reduction on an orphaned worksharing loop "
+                        "(no enclosing parallel region)");
+        // Scan reductions are only implemented for a single loop; a collapsed
+        // (multi-dimensional) loop nest is not yet supported.
+        if (auto wsloopOp = dyn_cast<omp::WsloopOp>(op.getOperation()))
+          if (auto loopNest = dyn_cast_or_null<omp::LoopNestOp>(
+                  wsloopOp.getWrappedLoop()))
+            if (loopNest.getNumLoops() > 1)
+              result = todo("inscan reduction on a collapsed "
+                            "(multi-dimensional) loop nest");
       }
     }
   };
@@ -4699,9 +4715,16 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
       linearClauseProcessor.rewriteInPlace(builder, loopInfo->getBody(),
                                            loopInfo->getLatch(), index);
 
+    // Scan reductions need a barrier at the end of the input (first) loop so
+    // that every thread has finished writing the temporary buffer before the
+    // masked prefix-sum reads it. A source-level `nowait` only elides the final
+    // barrier after the scan (second) loop, so force the barrier for the input
+    // loop regardless of `nowait` to avoid a data race.
+    bool needsBarrier =
+        loopNeedsBarrier || (isInScanRegion && inputScanLoop);
     llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP =
         ompBuilder->applyWorkshareLoop(
-            ompLoc.DL, loopInfo, allocaIP, loopNeedsBarrier,
+            ompLoc.DL, loopInfo, allocaIP, needsBarrier,
             convertToScheduleKind(schedule), chunk, isSimd,
             scheduleMod == omp::ScheduleModifier::monotonic,
             scheduleMod == omp::ScheduleModifier::nonmonotonic, isOrdered,
diff --git a/mlir/test/Target/LLVMIR/openmp-todo.mlir b/mlir/test/Target/LLVMIR/openmp-todo.mlir
index 44817110d8b20..befa9bba2680e 100644
--- a/mlir/test/Target/LLVMIR/openmp-todo.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-todo.mlir
@@ -180,15 +180,19 @@ atomic {
   omp.yield
 }
 llvm.func @scan_reduction(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
-  // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
-  omp.wsloop reduction(mod:inscan, @add_f32 %x -> %prv : !llvm.ptr) {
-    // expected-error at below {{LLVM Translation failed for operation: omp.loop_nest}}
-    omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
-      // expected-error at below {{not yet implemented: Unhandled clause exclusive in omp.scan operation}}
-      // expected-error at below {{LLVM Translation failed for operation: omp.scan}}
-      omp.scan exclusive(%prv : !llvm.ptr)
-      omp.yield
+  // expected-error at below {{LLVM Translation failed for operation: omp.parallel}}
+  omp.parallel {
+    // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
+    omp.wsloop reduction(mod:inscan, @add_f32 %x -> %prv : !llvm.ptr) {
+      // expected-error at below {{LLVM Translation failed for operation: omp.loop_nest}}
+      omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
+        // expected-error at below {{not yet implemented: Unhandled clause exclusive in omp.scan operation}}
+        // expected-error at below {{LLVM Translation failed for operation: omp.scan}}
+        omp.scan exclusive(%prv : !llvm.ptr)
+        omp.yield
+      }
     }
+    omp.terminator
   }
   llvm.return
 }
@@ -213,9 +217,44 @@ atomic {
   omp.yield
 }
 llvm.func @scan_reduction_byref(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
-  // expected-error at below {{not yet implemented: Unhandled clause inscan reduction modifier with by-ref reduction in omp.wsloop operation}}
+  // expected-error at below {{LLVM Translation failed for operation: omp.parallel}}
+  omp.parallel {
+    // expected-error at below {{not yet implemented: Unhandled clause inscan reduction modifier with by-ref reduction in omp.wsloop operation}}
+    // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
+    omp.wsloop reduction(mod:inscan, byref @add_f32 %x -> %prv : !llvm.ptr) {
+      omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
+        omp.scan inclusive(%prv : !llvm.ptr)
+        omp.yield
+      }
+    }
+    omp.terminator
+  }
+  llvm.return
+}
+
+// -----
+
+omp.declare_reduction @add_f32 : f32
+init {
+^bb0(%arg: f32):
+  %0 = llvm.mlir.constant(0.0 : f32) : f32
+  omp.yield (%0 : f32)
+}
+combiner {
+^bb1(%arg0: f32, %arg1: f32):
+  %1 = llvm.fadd %arg0, %arg1 : f32
+  omp.yield (%1 : f32)
+}
+atomic {
+^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr):
+  %2 = llvm.load %arg3 : !llvm.ptr -> f32
+  llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32
+  omp.yield
+}
+llvm.func @scan_reduction_orphaned(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
+  // expected-error at below {{not yet implemented: Unhandled clause inscan reduction on an orphaned worksharing loop (no enclosing parallel region) in omp.wsloop operation}}
   // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
-  omp.wsloop reduction(mod:inscan, byref @add_f32 %x -> %prv : !llvm.ptr) {
+  omp.wsloop reduction(mod:inscan, @add_f32 %x -> %prv : !llvm.ptr) {
     omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
       omp.scan inclusive(%prv : !llvm.ptr)
       omp.yield
@@ -226,6 +265,41 @@ llvm.func @scan_reduction_byref(%lb : i32, %ub : i32, %step : i32, %x : !llvm.pt
 
 // -----
 
+omp.declare_reduction @add_f32 : f32
+init {
+^bb0(%arg: f32):
+  %0 = llvm.mlir.constant(0.0 : f32) : f32
+  omp.yield (%0 : f32)
+}
+combiner {
+^bb1(%arg0: f32, %arg1: f32):
+  %1 = llvm.fadd %arg0, %arg1 : f32
+  omp.yield (%1 : f32)
+}
+atomic {
+^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr):
+  %2 = llvm.load %arg3 : !llvm.ptr -> f32
+  llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32
+  omp.yield
+}
+llvm.func @scan_reduction_multidim(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
+  // expected-error at below {{LLVM Translation failed for operation: omp.parallel}}
+  omp.parallel {
+    // expected-error at below {{not yet implemented: Unhandled clause inscan reduction on a collapsed (multi-dimensional) loop nest in omp.wsloop operation}}
+    // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
+    omp.wsloop reduction(mod:inscan, @add_f32 %x -> %prv : !llvm.ptr) {
+      omp.loop_nest (%iv, %iv2) : i32 = (%lb, %lb) to (%ub, %ub) step (%step, %step) {
+        omp.scan inclusive(%prv : !llvm.ptr)
+        omp.yield
+      }
+    }
+    omp.terminator
+  }
+  llvm.return
+}
+
+// -----
+
 llvm.func @single_allocate(%x : !llvm.ptr) {
   // expected-error at below {{not yet implemented: Unhandled clause allocate in omp.single operation}}
   // expected-error at below {{LLVM Translation failed for operation: omp.single}}
diff --git a/openmp/runtime/test/scan/scan-i64.f90 b/openmp/runtime/test/scan/scan-i64.f90
new file mode 100644
index 0000000000000..c2ef3e6e0ac76
--- /dev/null
+++ b/openmp/runtime/test/scan/scan-i64.f90
@@ -0,0 +1,37 @@
+! RUN: %flang %flags %openmp_flags -fopenmp-version=51 %s -o %t.exe
+! RUN: %t.exe | FileCheck %s --match-full-lines
+
+! An `integer(kind=8)` (i64) loop induction variable must work with an inscan
+! reduction. The scan temporary buffer is indexed using the loop's own index
+! type, so an i64 induction variable must not trip the i32/i64 type mismatch in
+! the scan buffer allocation or the prefix-sum computation.
+program scan_i64
+ implicit none
+ integer(kind=8) :: i, n
+ integer :: k
+ integer :: x
+ integer :: b(8)
+
+ n = 8
+ x = 0
+ !$omp parallel do reduction(inscan, +: x)
+ do i = 1_8, n
+   x = x + 1
+   !$omp scan inclusive(x)
+   b(i) = x
+ end do
+
+ print *, 'x =', x
+ do k = 1, 8
+  print *, 'b(', k, ') =', b(k)
+ end do
+end program
+!CHECK: x = 8
+!CHECK: b( 1 ) = 1
+!CHECK: b( 2 ) = 2
+!CHECK: b( 3 ) = 3
+!CHECK: b( 4 ) = 4
+!CHECK: b( 5 ) = 5
+!CHECK: b( 6 ) = 6
+!CHECK: b( 7 ) = 7
+!CHECK: b( 8 ) = 8
diff --git a/openmp/runtime/test/scan/scan-nowait.f90 b/openmp/runtime/test/scan/scan-nowait.f90
new file mode 100644
index 0000000000000..bb17aeeedf9d9
--- /dev/null
+++ b/openmp/runtime/test/scan/scan-nowait.f90
@@ -0,0 +1,38 @@
+! RUN: %flang %flags %openmp_flags -fopenmp-version=51 %s -o %t.exe
+! RUN: env OMP_NUM_THREADS=4 %t.exe | FileCheck %s --match-full-lines
+
+! A source-level `nowait` on a scan worksharing loop must only remove the final
+! barrier after the scan (second) loop. The barrier at the end of the input
+! (first) loop is required so that every thread finishes writing the temporary
+! buffer before the masked prefix-sum reads it. Forcing multiple threads
+! exercises the data race that occurred when `nowait` incorrectly removed that
+! internal barrier.
+program scan_nowait
+ implicit none
+ integer :: x, k
+ integer :: b(8)
+
+ b = -1
+ x = 0
+ !$omp parallel
+ !$omp do reduction(inscan, +: x)
+ do k = 1, 8
+   x = x + 1
+   !$omp scan inclusive(x)
+   b(k) = x
+ end do
+ !$omp end do nowait
+ !$omp end parallel
+
+ do k = 1, 8
+  print *, 'b(', k, ') =', b(k)
+ end do
+end program
+!CHECK: b( 1 ) = 1
+!CHECK: b( 2 ) = 2
+!CHECK: b( 3 ) = 3
+!CHECK: b( 4 ) = 4
+!CHECK: b( 5 ) = 5
+!CHECK: b( 6 ) = 6
+!CHECK: b( 7 ) = 7
+!CHECK: b( 8 ) = 8

>From c5283fcf901ea9d20f64a59ac15a2795c19abac0 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Mon, 27 Jul 2026 11:37:51 -0500
Subject: [PATCH 5/9] git formatting

---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp             | 11 +++++------
 .../Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp      |  7 +++----
 2 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 9f82ba18ded81..ff5d325075b93 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5378,8 +5378,8 @@ Error OpenMPIRBuilder::emitScanBasedDirectiveDeclsIR(
     // Use the loop's own index type (matching `Span`) so the buffer allocation
     // works for both i32 and i64 (e.g. `integer(kind=8)`) induction variables.
     Type *IndexTy = ScanRedInfo->Span->getType();
-    Value *AllocSpan = Builder.CreateAdd(ScanRedInfo->Span,
-                                         ConstantInt::get(IndexTy, 1));
+    Value *AllocSpan =
+        Builder.CreateAdd(ScanRedInfo->Span, ConstantInt::get(IndexTy, 1));
     for (size_t i = 0; i < ScanVars.size(); i++) {
       Type *IntPtrTy = IndexTy;
       Constant *Allocsize = ConstantExpr::getSizeOf(ScanVarsType[i]);
@@ -5523,8 +5523,7 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
     PHINode *Pow2K = Builder.CreatePHI(IndexTy, 2);
     Counter->addIncoming(llvm::ConstantInt::get(Builder.getInt32Ty(), 0),
                          InputBB);
-    Pow2K->addIncoming(llvm::ConstantInt::get(IndexTy, 1),
-                       InputBB);
+    Pow2K->addIncoming(llvm::ConstantInt::get(IndexTy, 1), InputBB);
     // for (size i = n - 1; i >= 2 ^ k; --i)
     //   tmp[i] op= tmp[i-pow2k];
     llvm::BasicBlock *InnerLoopBB =
@@ -5557,8 +5556,8 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
         return AfterIP.takeError();
       Builder.CreateStore(Result, LHSPtr);
     }
-    llvm::Value *NextIVal = Builder.CreateNUWSub(
-        IVal, llvm::ConstantInt::get(IndexTy, 1));
+    llvm::Value *NextIVal =
+        Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(IndexTy, 1));
     IVal->addIncoming(NextIVal, Builder.GetInsertBlock());
     CmpI = Builder.CreateICmpUGE(NextIVal, Pow2K);
     Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index d1a31a6870ced..7c27a18d285fb 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -540,8 +540,8 @@ static LogicalResult checkImplementationStatus(Operation &op) {
         // Scan reductions are only implemented for a single loop; a collapsed
         // (multi-dimensional) loop nest is not yet supported.
         if (auto wsloopOp = dyn_cast<omp::WsloopOp>(op.getOperation()))
-          if (auto loopNest = dyn_cast_or_null<omp::LoopNestOp>(
-                  wsloopOp.getWrappedLoop()))
+          if (auto loopNest =
+                  dyn_cast_or_null<omp::LoopNestOp>(wsloopOp.getWrappedLoop()))
             if (loopNest.getNumLoops() > 1)
               result = todo("inscan reduction on a collapsed "
                             "(multi-dimensional) loop nest");
@@ -4876,8 +4876,7 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
     // masked prefix-sum reads it. A source-level `nowait` only elides the final
     // barrier after the scan (second) loop, so force the barrier for the input
     // loop regardless of `nowait` to avoid a data race.
-    bool needsBarrier =
-        loopNeedsBarrier || (isInScanRegion && inputScanLoop);
+    bool needsBarrier = loopNeedsBarrier || (isInScanRegion && inputScanLoop);
     llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP =
         ompBuilder->applyWorkshareLoop(
             ompLoc.DL, loopInfo, allocaIP, needsBarrier,

>From 42d781704befcc65923ff419c6ba461d4796afbb Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Tue, 28 Jul 2026 11:01:16 -0500
Subject: [PATCH 6/9] rework on second race cond

---
 .../llvm/Frontend/OpenMP/OMPIRBuilder.h       | 14 +++-
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     | 40 +++++++---
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      | 19 +++--
 .../LLVMIR/openmp-reduction-scan-nowait.mlir  | 73 +++++++++++++++++++
 4 files changed, 127 insertions(+), 19 deletions(-)
 create mode 100644 mlir/test/Target/LLVMIR/openmp-reduction-scan-nowait.mlir

diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index 05307414c94d0..87dc1f067463e 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -1940,11 +1940,17 @@ class OpenMPIRBuilder {
   /// \param ReductionInfos Array type containing the ReductionOps.
   /// \param ScanRedInfo    Pointer to the ScanInfo objected created using
   ///                       `ScanInfoInitialize`.
+  /// \param NoWait         Whether the enclosing worksharing construct has a
+  ///                       `nowait` clause. The barrier that keeps the shared
+  ///                       scan buffer alive until every thread has finished
+  ///                       reading it is always emitted; only the
+  ///                       end-of-construct barrier after the masked region is
+  ///                       skipped when \p NoWait is set.
   ///
   /// \return error if any produced, else return success.
   Error emitScanBasedDirectiveFinalsIR(
       ArrayRef<llvm::OpenMPIRBuilder::ReductionInfo> ReductionInfos,
-      ScanInfo *ScanInfo);
+      ScanInfo *ScanInfo, bool NoWait = false);
 
   /// This function emits a helper that gathers Reduce lists from the first
   /// lane of every active warp to lanes in the first warp.
@@ -3173,12 +3179,16 @@ class OpenMPIRBuilder {
   /// \param ReductionInfos Array type containing the ReductionOps.
   /// \param ScanRedInfo    Pointer to the ScanInfo objected created using
   ///                       `ScanInfoInitialize`.
+  /// \param NoWait         Whether the enclosing worksharing construct has a
+  ///                       `nowait` clause. Only the end-of-construct barrier
+  ///                       emitted after freeing the shared scan buffer is
+  ///                       elided; barriers required for correctness are kept.
   ///
   /// \returns The insertion position *after* the masked.
   LLVM_ABI InsertPointOrErrorTy emitScanReduction(
       const LocationDescription &Loc,
       ArrayRef<llvm::OpenMPIRBuilder::ReductionInfo> ReductionInfos,
-      ScanInfo *ScanRedInfo);
+      ScanInfo *ScanRedInfo, bool NoWait = false);
 
   /// This directive split and directs the control flow to input phase
   ///  blocks or scan phase blocks based on 1. whether input loop or scan loop
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index ff5d325075b93..e81768e884922 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5414,7 +5414,8 @@ Error OpenMPIRBuilder::emitScanBasedDirectiveDeclsIR(
 }
 
 Error OpenMPIRBuilder::emitScanBasedDirectiveFinalsIR(
-    ArrayRef<ReductionInfo> ReductionInfos, ScanInfo *ScanRedInfo) {
+    ArrayRef<ReductionInfo> ReductionInfos, ScanInfo *ScanRedInfo,
+    bool NoWait) {
   auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
                        ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
     Builder.restoreIP(CodeGenIP);
@@ -5453,27 +5454,45 @@ Error OpenMPIRBuilder::emitScanBasedDirectiveFinalsIR(
   else
     Builder.SetInsertPoint(ScanRedInfo->OMPScanFinish);
 
-  llvm::Value *FilterVal = Builder.getInt32(0);
+  // Synchronize before the masked region frees the shared scan buffer. The
+  // scan (second) loop loads each thread's prefix value from the buffer, so
+  // every thread must have finished that loop before the single masked thread
+  // frees it; otherwise a thread still executing the loop could load from
+  // freed memory. A `masked` region is not a barrier, so this synchronization
+  // is required for correctness and is emitted even under `nowait`.
   llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
-      createMasked(Builder.saveIP(), BodyGenCB, FiniCB, FilterVal);
-
+      createBarrier(Builder.saveIP(), llvm::omp::OMPD_barrier);
   if (!AfterIP)
     return AfterIP.takeError();
   Builder.restoreIP(*AfterIP);
-  BasicBlock *InputBB = Builder.GetInsertBlock();
-  if (InputBB->hasTerminator())
-    Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
-  AfterIP = createBarrier(Builder.saveIP(), llvm::omp::OMPD_barrier);
+
+  llvm::Value *FilterVal = Builder.getInt32(0);
+  AfterIP = createMasked(Builder.saveIP(), BodyGenCB, FiniCB, FilterVal);
+
   if (!AfterIP)
     return AfterIP.takeError();
   Builder.restoreIP(*AfterIP);
+
+  // The barrier after the masked region provides the construct's
+  // end-of-construct synchronization and publishes the final reduction value
+  // stored into the original variable inside the masked region. Skip it when
+  // the construct has `nowait`, matching a normal worksharing loop.
+  if (!NoWait) {
+    BasicBlock *InputBB = Builder.GetInsertBlock();
+    if (InputBB->hasTerminator())
+      Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
+    AfterIP = createBarrier(Builder.saveIP(), llvm::omp::OMPD_barrier);
+    if (!AfterIP)
+      return AfterIP.takeError();
+    Builder.restoreIP(*AfterIP);
+  }
   return Error::success();
 }
 
 OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
     const LocationDescription &Loc,
     ArrayRef<llvm::OpenMPIRBuilder::ReductionInfo> ReductionInfos,
-    ScanInfo *ScanRedInfo) {
+    ScanInfo *ScanRedInfo, bool NoWait) {
 
   if (!updateToLocation(Loc))
     return Loc.IP;
@@ -5590,7 +5609,8 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
   if (!AfterIP)
     return AfterIP.takeError();
   Builder.restoreIP(*AfterIP);
-  Error Err = emitScanBasedDirectiveFinalsIR(ReductionInfos, ScanRedInfo);
+  Error Err =
+      emitScanBasedDirectiveFinalsIR(ReductionInfos, ScanRedInfo, NoWait);
   if (Err)
     return Err;
 
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 7c27a18d285fb..e5e0acf008116 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -4871,12 +4871,17 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
       linearClauseProcessor.rewriteInPlace(builder, loopInfo->getBody(),
                                            loopInfo->getLatch(), index);
 
-    // Scan reductions need a barrier at the end of the input (first) loop so
-    // that every thread has finished writing the temporary buffer before the
-    // masked prefix-sum reads it. A source-level `nowait` only elides the final
-    // barrier after the scan (second) loop, so force the barrier for the input
-    // loop regardless of `nowait` to avoid a data race.
-    bool needsBarrier = loopNeedsBarrier || (isInScanRegion && inputScanLoop);
+    // Scan reductions manage the barriers around the shared temporary buffer
+    // explicitly:
+    //  * The input (first) loop always needs its trailing barrier so that every
+    //    thread has finished writing the buffer before the masked prefix-sum
+    //    reads it; this is required even under `nowait`.
+    //  * The scan (second) loop must not emit its own trailing barrier here.
+    //    `emitScanReduction` emits a barrier *before* the masked region frees
+    //    the buffer (so no thread frees it while another is still reading it)
+    //    and, unless `nowait` is present, a barrier *after* it for
+    //    end-of-construct synchronization.
+    bool needsBarrier = isInScanRegion ? inputScanLoop : loopNeedsBarrier;
     llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP =
         ompBuilder->applyWorkshareLoop(
             ompLoc.DL, loopInfo, allocaIP, needsBarrier,
@@ -4930,7 +4935,7 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
     llvm::ScanInfo *scanInfo = findScanInfo(moduleTranslation);
     llvm::OpenMPIRBuilder::InsertPointOrErrorTy redIP =
         ompBuilder->emitScanReduction(builder.saveIP(), reductionInfos,
-                                      scanInfo);
+                                      scanInfo, wsloopOp.getNowait());
     if (failed(handleError(redIP, opInst)))
       return failure();
 
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan-nowait.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan-nowait.mlir
new file mode 100644
index 0000000000000..95306535c92cd
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan-nowait.mlir
@@ -0,0 +1,73 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// Verify buffer-lifetime handling for an inclusive scan reduction on a
+// worksharing loop with `nowait`. The scan (second) loop reads each thread's
+// prefix value from the shared scan buffer, and a masked region then frees
+// that buffer. A barrier must be emitted *before* the masked region so the
+// buffer cannot be freed while another thread is still reading it (a masked
+// region is not a barrier). Because of `nowait`, no end-of-construct barrier
+// is emitted *after* the masked region.
+
+omp.declare_reduction @add_reduction_i32 : i32 init {
+^bb0(%arg0: i32):
+  %0 = llvm.mlir.constant(0 : i32) : i32
+  omp.yield(%0 : i32)
+} combiner {
+^bb0(%arg0: i32, %arg1: i32):
+  %0 = llvm.add %arg0, %arg1 : i32
+  omp.yield(%0 : i32)
+}
+llvm.func @scan_reduction_nowait() {
+  %0 = llvm.mlir.constant(1 : i64) : i64
+  %5 = llvm.alloca %0 x i32 {bindc_name = "x"} : (i64) -> !llvm.ptr
+  %10 = llvm.mlir.constant(100 : i32) : i32
+  %11 = llvm.mlir.constant(1 : i32) : i32
+  %14 = llvm.mlir.addressof @_QFEa : !llvm.ptr
+  %15 = llvm.mlir.addressof @_QFEb : !llvm.ptr
+  omp.parallel {
+    %37 = llvm.mlir.constant(1 : i64) : i64
+    %38 = llvm.alloca %37 x i32 {bindc_name = "k", pinned} : (i64) -> !llvm.ptr
+    omp.wsloop nowait reduction(mod: inscan, @add_reduction_i32 %5 -> %arg0 : !llvm.ptr) {
+      omp.loop_nest (%arg1) : i32 = (%11) to (%10) inclusive step (%11) {
+        llvm.store %arg1, %38 : i32, !llvm.ptr
+        %40 = llvm.load %arg0 : !llvm.ptr -> i32
+        %41 = llvm.load %38 : !llvm.ptr -> i32
+        %42 = llvm.sext %41 : i32 to i64
+        %50 = llvm.getelementptr %14[%42] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+        %51 = llvm.load %50 : !llvm.ptr -> i32
+        %52 = llvm.add %40, %51 : i32
+        llvm.store %52, %arg0 : i32, !llvm.ptr
+        omp.scan inclusive(%arg0 : !llvm.ptr)
+        llvm.store %arg1, %38 : i32, !llvm.ptr
+        %53 = llvm.load %arg0 : !llvm.ptr -> i32
+        %54 = llvm.load %38 : !llvm.ptr -> i32
+        %55 = llvm.sext %54 : i32 to i64
+        %63 = llvm.getelementptr %15[%55] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+        llvm.store %53, %63 : i32, !llvm.ptr
+        omp.yield
+      }
+    }
+    omp.terminator
+  }
+  llvm.return
+}
+llvm.mlir.global internal @_QFEa() {addr_space = 0 : i32} : !llvm.array<100 x i32> {
+  %0 = llvm.mlir.zero : !llvm.array<100 x i32>
+  llvm.return %0 : !llvm.array<100 x i32>
+}
+llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i32> {
+  %0 = llvm.mlir.zero : !llvm.array<100 x i32>
+  llvm.return %0 : !llvm.array<100 x i32>
+}
+
+// CHECK-LABEL: define internal void @scan_reduction_nowait..omp_par
+// After the scan (second) loop, the mandatory buffer-lifetime barrier is
+// emitted before the masked region that frees the shared scan buffer.
+// CHECK: omp.scan.loop.cont:
+// CHECK: call void @__kmpc_barrier
+// CHECK: call i32 @__kmpc_masked
+// CHECK: call void @free(ptr
+// CHECK: call void @__kmpc_end_masked
+// With `nowait`, there is no end-of-construct barrier after the buffer is freed.
+// CHECK-NOT: call void @__kmpc_barrier
+// CHECK: ret void

>From 46cbc9122d8fa036d02c61ea3d740030701eb824 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Tue, 28 Jul 2026 12:18:10 -0500
Subject: [PATCH 7/9] fixex iteration input now  starts from reduction identity

---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     | 32 +++++++++++++++++++
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      | 21 ++++++------
 .../Target/LLVMIR/openmp-reduction-scan.mlir  | 16 ++++++++--
 3 files changed, 54 insertions(+), 15 deletions(-)

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index e81768e884922..a75c39be196c1 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5525,6 +5525,38 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
         ScanRedInfo->Span,
         llvm::ConstantInt::get(ScanRedInfo->Span->getType(), 1));
     Builder.SetInsertPoint(InputBB);
+    // Combine the original variable's incoming value (orig-val) into the first
+    // buffer element before computing the prefix sum. Per the OpenMP scan
+    // semantics orig-val is a single prefix element that must be reflected in
+    // every inclusive/exclusive scan result and in the final reduction value.
+    // Folding it in here lets the log-scan prefix computation propagate it into
+    // every element (and into the finals' read of `buffer[Span]`), regardless
+    // of whether an iteration's input phase accumulates into or overwrites the
+    // reduction variable. `select(Span > 0, 1, 0)` keeps the access in bounds
+    // when Span == 0: buffer[0] is otherwise unused and the finals' zero-trip
+    // guard discards it.
+    llvm::Value *HasElem = Builder.CreateICmpUGT(
+        ScanRedInfo->Span, llvm::ConstantInt::get(IndexTy, 0));
+    llvm::Value *FirstIdx =
+        Builder.CreateSelect(HasElem, llvm::ConstantInt::get(IndexTy, 1),
+                             llvm::ConstantInt::get(IndexTy, 0));
+    for (ReductionInfo RedInfo : ReductionInfos) {
+      Value *ReductionVal = RedInfo.PrivateVariable;
+      Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ReductionVal];
+      Value *Buff = Builder.CreateLoad(Builder.getPtrTy(), BuffPtr);
+      Type *DestTy = RedInfo.ElementType;
+      Value *ElemPtr =
+          Builder.CreateInBoundsGEP(DestTy, Buff, FirstIdx, "arrayOffset");
+      Value *OrigVal = Builder.CreateLoad(DestTy, RedInfo.Variable);
+      Value *Elem = Builder.CreateLoad(DestTy, ElemPtr);
+      llvm::Value *Combined;
+      InsertPointOrErrorTy AfterIP =
+          RedInfo.ReductionGen(Builder.saveIP(), Elem, OrigVal, Combined);
+      if (!AfterIP)
+        return AfterIP.takeError();
+      Builder.restoreIP(*AfterIP);
+      Builder.CreateStore(Combined, ElemPtr);
+    }
     // The log-scan prefix computation is only well-defined when there are at
     // least two elements: `log2(Span)` is invalid for Span == 0 and the loop
     // trip computation underflows/loops forever for Span <= 1. For Span <= 1
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index e5e0acf008116..23f2d06fdabe2 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -5526,19 +5526,16 @@ initScanReductionVars(omp::LoopNestOp loopOp, llvm::IRBuilderBase &builder,
            "expected one value to be yielded from the reduction init region");
     setInsertPointForPossiblyEmptyBlock(builder);
 
-    // On the first iteration, seed the reduction variable with the original
-    // variable's incoming value so that a non-identity initial value
-    // participates in the scan (e.g. `x = 10` before an inclusive-scan loop of
-    // `+1` must yield 11, 12, ...). On subsequent iterations, reset to the
-    // reduction identity. `scanInfo->IV` is the 1-based iteration index.
+    // Reset the private reduction variable to the reduction identity at the
+    // start of every input-loop iteration. The original variable's incoming
+    // value (orig-val) must not be folded into any single iteration here: per
+    // the OpenMP scan semantics it is a separate prefix element that is
+    // combined once into the scan buffer (see `emitScanReduction`). Seeding a
+    // particular iteration would be lost whenever the input phase overwrites
+    // the reduction variable (e.g. `x = i`), producing incorrect results.
     llvm::Value *identity = phis[0];
-    llvm::Value *origVar =
-        moduleTranslation.lookupValue(wsloopOp.getReductionVars()[i]);
-    llvm::Value *origVal = builder.CreateLoad(identity->getType(), origVar);
-    llvm::Value *isFirstIter = builder.CreateICmpEQ(
-        scanInfo->IV, llvm::ConstantInt::get(scanInfo->IV->getType(), 1));
-    llvm::Value *seed = builder.CreateSelect(isFirstIter, origVal, identity);
-    builder.CreateStore(seed, moduleTranslation.lookupValue(reductionArgs[i]));
+    builder.CreateStore(identity,
+                        moduleTranslation.lookupValue(reductionArgs[i]));
   }
   return success();
 }
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
index 6ef58b5013e08..2aa100d686bc1 100644
--- a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
@@ -86,6 +86,15 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK:   %[[LOG:.+]] = call double @llvm.log2.f64(double 1.000000e+02) #0
 //CHECK:   %[[CEIL:.+]] = call double @llvm.ceil.f64(double %[[LOG]]) #0
 //CHECK:   %[[UB:.+]] = fptoui double %[[CEIL]] to i32
+// Combine the original variable's incoming value (orig-val) into the first
+// buffer element once, before the prefix sum, so it is reflected in every
+// scan result and in the final reduction value.
+//CHECK:   %[[OBUFF:.+]] = load ptr, ptr %{{.*}}, align 8
+//CHECK:   %[[OELEMPTR:.+]] = getelementptr inbounds i32, ptr %[[OBUFF]], i32 1
+//CHECK:   %[[OORIG:.+]] = load i32, ptr %{{.*}}, align 4
+//CHECK:   %[[OELEM:.+]] = load i32, ptr %[[OELEMPTR]], align 4
+//CHECK:   %[[OCOMB:.+]] = add i32 %[[OELEM]], %[[OORIG]]
+//CHECK:   store i32 %[[OCOMB]], ptr %[[OELEMPTR]], align 4
 //CHECK:   br i1 {{.*}}, label %omp.outer.log.scan.body, label %omp.outer.log.scan.exit
 //CHECK: omp.outer.log.scan.exit:                          ; preds = %omp.inner.log.scan.exit, %omp_region.body{{.*}}
 //CHECK: @__kmpc_end_masked
@@ -116,9 +125,10 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK: omp.inscan.dispatch:                              ; preds = %omp_loop.body
 //CHECK:   br i1 true, label %omp.before.scan.bb, label %omp.after.scan.bb
 //CHECK: omp.before.scan.bb:
-//CHECK:   %[[ISFIRST:.+]] = icmp eq i32 %{{.*}}, 1
-//CHECK:   %[[SEED:.+]] = select i1 %[[ISFIRST]], i32 %{{.*}}, i32 0
-//CHECK:   store i32 %[[SEED]], ptr %[[REDPRIV:.+]], align 4
+// Each input-loop iteration resets the private reduction variable to the
+// reduction identity (0 for `+`). The original variable's incoming value
+// (orig-val) is NOT seeded here; it is combined once in the masked region.
+//CHECK:   store i32 0, ptr %{{.*}}, align 4
 //CHECK: omp.loop_nest.region:                             ; preds = %omp.before.scan.bb
 //CHECK:   %[[BUFFER:.+]] = load ptr, ptr %loadgep_vla, align 8
 //CHECK:   %[[ARRAYOFFSET2:.+]] = getelementptr inbounds i32, ptr %[[BUFFER]], i32 %{{.*}}

>From cbfd4440aa06d37f99c1bcd8e0e8649199b3d9f0 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Wed, 29 Jul 2026 04:52:51 -0500
Subject: [PATCH 8/9] inscan reduction orig-val handling and guard the scan
 seed for zero-trip loops

---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     | 37 +++++++++++--------
 .../Target/LLVMIR/openmp-reduction-scan.mlir  | 12 ++++--
 2 files changed, 30 insertions(+), 19 deletions(-)

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index a75c39be196c1..fd9b857d88b9b 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5525,6 +5525,18 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
         ScanRedInfo->Span,
         llvm::ConstantInt::get(ScanRedInfo->Span->getType(), 1));
     Builder.SetInsertPoint(InputBB);
+    // Branch around the orig-val seed load and combine when the loop has zero
+    // logical iterations. When Span == 0 no scan-buffer entry is populated, so
+    // loading buffer[0] and feeding it to a (possibly input-sensitive
+    // user-defined) reduction combiner would be undefined behavior. In that
+    // case skip straight to the exit; the finals' zero-trip guard preserves
+    // orig-val as the result.
+    llvm::Value *HasElem = Builder.CreateICmpUGT(
+        ScanRedInfo->Span, llvm::ConstantInt::get(IndexTy, 0));
+    llvm::BasicBlock *SeedBB =
+        BasicBlock::Create(CurFn->getContext(), "omp.scan.seed", CurFn, ExitBB);
+    Builder.CreateCondBr(HasElem, SeedBB, ExitBB);
+    Builder.SetInsertPoint(SeedBB);
     // Combine the original variable's incoming value (orig-val) into the first
     // buffer element before computing the prefix sum. Per the OpenMP scan
     // semantics orig-val is a single prefix element that must be reflected in
@@ -5532,21 +5544,14 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
     // Folding it in here lets the log-scan prefix computation propagate it into
     // every element (and into the finals' read of `buffer[Span]`), regardless
     // of whether an iteration's input phase accumulates into or overwrites the
-    // reduction variable. `select(Span > 0, 1, 0)` keeps the access in bounds
-    // when Span == 0: buffer[0] is otherwise unused and the finals' zero-trip
-    // guard discards it.
-    llvm::Value *HasElem = Builder.CreateICmpUGT(
-        ScanRedInfo->Span, llvm::ConstantInt::get(IndexTy, 0));
-    llvm::Value *FirstIdx =
-        Builder.CreateSelect(HasElem, llvm::ConstantInt::get(IndexTy, 1),
-                             llvm::ConstantInt::get(IndexTy, 0));
+    // reduction variable. Span > 0 is guaranteed here, so buffer[1] is valid.
     for (ReductionInfo RedInfo : ReductionInfos) {
       Value *ReductionVal = RedInfo.PrivateVariable;
       Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ReductionVal];
       Value *Buff = Builder.CreateLoad(Builder.getPtrTy(), BuffPtr);
       Type *DestTy = RedInfo.ElementType;
-      Value *ElemPtr =
-          Builder.CreateInBoundsGEP(DestTy, Buff, FirstIdx, "arrayOffset");
+      Value *ElemPtr = Builder.CreateInBoundsGEP(
+          DestTy, Buff, llvm::ConstantInt::get(IndexTy, 1), "arrayOffset");
       Value *OrigVal = Builder.CreateLoad(DestTy, RedInfo.Variable);
       Value *Elem = Builder.CreateLoad(DestTy, ElemPtr);
       llvm::Value *Combined;
@@ -5559,9 +5564,9 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
     }
     // The log-scan prefix computation is only well-defined when there are at
     // least two elements: `log2(Span)` is invalid for Span == 0 and the loop
-    // trip computation underflows/loops forever for Span <= 1. For Span <= 1
-    // the buffer already holds the final value(s), so skip straight to the
-    // exit.
+    // trip computation underflows/loops forever for Span <= 1. For Span == 1
+    // the buffer already holds the final (seeded) value, so skip straight to
+    // the exit.
     llvm::Value *SpanGuard = Builder.CreateICmpUGT(
         ScanRedInfo->Span,
         llvm::ConstantInt::get(ScanRedInfo->Span->getType(), 1));
@@ -5572,9 +5577,11 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
     PHINode *Counter = Builder.CreatePHI(Builder.getInt32Ty(), 2);
     // size pow2k = 1;
     PHINode *Pow2K = Builder.CreatePHI(IndexTy, 2);
+    // The loop is now entered from the seed block (guarded by Span > 0), not
+    // directly from InputBB.
     Counter->addIncoming(llvm::ConstantInt::get(Builder.getInt32Ty(), 0),
-                         InputBB);
-    Pow2K->addIncoming(llvm::ConstantInt::get(IndexTy, 1), InputBB);
+                         SeedBB);
+    Pow2K->addIncoming(llvm::ConstantInt::get(IndexTy, 1), SeedBB);
     // for (size i = n - 1; i >= 2 ^ k; --i)
     //   tmp[i] op= tmp[i-pow2k];
     llvm::BasicBlock *InnerLoopBB =
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
index 2aa100d686bc1..38e413a504315 100644
--- a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
@@ -86,9 +86,15 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK:   %[[LOG:.+]] = call double @llvm.log2.f64(double 1.000000e+02) #0
 //CHECK:   %[[CEIL:.+]] = call double @llvm.ceil.f64(double %[[LOG]]) #0
 //CHECK:   %[[UB:.+]] = fptoui double %[[CEIL]] to i32
+// Zero-trip guard: skip the orig-val seed load/combine when Span == 0 so
+// buffer[0] is never loaded or passed to the reduction combiner.
+//CHECK:   br i1 {{.*}}, label %omp.scan.seed, label %omp.outer.log.scan.exit
+//CHECK: omp.outer.log.scan.exit:
+//CHECK: @__kmpc_end_masked
+//CHECK: omp.scan.seed:
 // Combine the original variable's incoming value (orig-val) into the first
-// buffer element once, before the prefix sum, so it is reflected in every
-// scan result and in the final reduction value.
+// buffer element once (Span > 0 is guaranteed here), before the prefix sum, so
+// it is reflected in every scan result and in the final reduction value.
 //CHECK:   %[[OBUFF:.+]] = load ptr, ptr %{{.*}}, align 8
 //CHECK:   %[[OELEMPTR:.+]] = getelementptr inbounds i32, ptr %[[OBUFF]], i32 1
 //CHECK:   %[[OORIG:.+]] = load i32, ptr %{{.*}}, align 4
@@ -96,8 +102,6 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK:   %[[OCOMB:.+]] = add i32 %[[OELEM]], %[[OORIG]]
 //CHECK:   store i32 %[[OCOMB]], ptr %[[OELEMPTR]], align 4
 //CHECK:   br i1 {{.*}}, label %omp.outer.log.scan.body, label %omp.outer.log.scan.exit
-//CHECK: omp.outer.log.scan.exit:                          ; preds = %omp.inner.log.scan.exit, %omp_region.body{{.*}}
-//CHECK: @__kmpc_end_masked
 //CHECK: omp.outer.log.scan.body:
 //CHECK:   %[[K:.+]] = phi i32 [ 0, %{{.*}} ], [ %[[NEXTK:.+]], %omp.inner.log.scan.exit ]
 //CHECK:   %[[I:.+]] = phi i32 [ 1, %{{.*}} ], [ %[[NEXTI:.+]], %omp.inner.log.scan.exit ]

>From a51ea62ab9586e3219388f07e8df525af46733d9 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Mon, 3 Aug 2026 04:37:41 -0500
Subject: [PATCH 9/9] fix operand order in reduction seeding

---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     | 14 ++-
 .../openmp-reduction-scan-noncommutative.mlir | 95 +++++++++++++++++++
 .../Target/LLVMIR/openmp-reduction-scan.mlir  |  4 +-
 3 files changed, 109 insertions(+), 4 deletions(-)
 create mode 100644 mlir/test/Target/LLVMIR/openmp-reduction-scan-noncommutative.mlir

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index fd9b857d88b9b..961ec39af4efc 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5555,8 +5555,12 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
       Value *OrigVal = Builder.CreateLoad(DestTy, RedInfo.Variable);
       Value *Elem = Builder.CreateLoad(DestTy, ElemPtr);
       llvm::Value *Combined;
+      // orig-val is the leftmost/earliest element of the prefix, so it must be
+      // the accumulator (`omp_out`, the combiner's first/lhs operand) while the
+      // first buffer element is the incoming value (`omp_in`). The combiner is
+      // associative but not necessarily commutative, so this order matters.
       InsertPointOrErrorTy AfterIP =
-          RedInfo.ReductionGen(Builder.saveIP(), Elem, OrigVal, Combined);
+          RedInfo.ReductionGen(Builder.saveIP(), OrigVal, Elem, Combined);
       if (!AfterIP)
         return AfterIP.takeError();
       Builder.restoreIP(*AfterIP);
@@ -5608,8 +5612,14 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
       Value *LHS = Builder.CreateLoad(DestTy, LHSPtr);
       Value *RHS = Builder.CreateLoad(DestTy, RHSPtr);
       llvm::Value *Result;
+      // buffer[IV-pow2k] holds the earlier partial prefix and buffer[IV] the
+      // later one. The earlier element must be the accumulator (`omp_out`, the
+      // combiner's first/lhs operand) and the later element the incoming value
+      // (`omp_in`); the combiner is associative but not necessarily
+      // commutative, so passing them in prefix order is required. The result
+      // is written back into the later slot (LHSPtr = buffer[IV]).
       InsertPointOrErrorTy AfterIP =
-          RedInfo.ReductionGen(Builder.saveIP(), LHS, RHS, Result);
+          RedInfo.ReductionGen(Builder.saveIP(), RHS, LHS, Result);
       if (!AfterIP)
         return AfterIP.takeError();
       Builder.CreateStore(Result, LHSPtr);
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan-noncommutative.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan-noncommutative.mlir
new file mode 100644
index 0000000000000..0c595de9e80c1
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan-noncommutative.mlir
@@ -0,0 +1,95 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// Regression test for inscan reduction combiner operand ordering. The combiner
+// here is intentionally NONcommutative (subtraction) so that reversing the two
+// operands passed to the reduction combiner changes the emitted IR. This pins
+// the operand order at the two combiner call sites of the log-scan lowering:
+// the orig-val seed and the prefix computation. A commutative combiner such as
+// `+` produces identical IR regardless of operand order and would not catch a
+// reversal, so this complements openmp-reduction-scan.mlir. The scan combines
+// elements left-to-right, so the earlier prefix element must be the
+// accumulator (`omp_out`, the combiner's first operand) and the later element
+// the incoming value (`omp_in`, the second operand).
+
+omp.declare_reduction @sub_reduction_i32 : i32 init {
+^bb0(%arg0: i32):
+  %0 = llvm.mlir.constant(0 : i32) : i32
+  omp.yield(%0 : i32)
+} combiner {
+^bb0(%arg0: i32, %arg1: i32):
+  %0 = llvm.sub %arg0, %arg1 : i32
+  omp.yield(%0 : i32)
+}
+// CHECK-LABEL: @scan_reduction_noncommutative
+llvm.func @scan_reduction_noncommutative() {
+  %0 = llvm.mlir.constant(1 : i64) : i64
+  %1 = llvm.alloca %0 x i32 {bindc_name = "z"} : (i64) -> !llvm.ptr
+  %3 = llvm.alloca %0 x i32 {bindc_name = "y"} : (i64) -> !llvm.ptr
+  %5 = llvm.alloca %0 x i32 {bindc_name = "x"} : (i64) -> !llvm.ptr
+  %7 = llvm.alloca %0 x i32 {bindc_name = "k"} : (i64) -> !llvm.ptr
+  %10 = llvm.mlir.constant(100 : i32) : i32
+  %11 = llvm.mlir.constant(1 : i32) : i32
+  %12 = llvm.mlir.constant(0 : i32) : i32
+  %13 = llvm.mlir.constant(100 : index) : i64
+  %14 = llvm.mlir.addressof @_QFEa : !llvm.ptr
+  %15 = llvm.mlir.addressof @_QFEb : !llvm.ptr
+  omp.parallel {
+    %37 = llvm.mlir.constant(1 : i64) : i64
+    %38 = llvm.alloca %37 x i32 {bindc_name = "k", pinned} : (i64) -> !llvm.ptr
+    %39 = llvm.mlir.constant(1 : i64) : i64
+    omp.wsloop reduction(mod: inscan, @sub_reduction_i32 %5 -> %arg0 : !llvm.ptr) {
+      omp.loop_nest (%arg1) : i32 = (%11) to (%10) inclusive step (%11) {
+        llvm.store %arg1, %38 : i32, !llvm.ptr
+        %40 = llvm.load %arg0 : !llvm.ptr -> i32
+        %41 = llvm.load %38 : !llvm.ptr -> i32
+        %42 = llvm.sext %41 : i32 to i64
+        %50 = llvm.getelementptr %14[%42] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+        %51 = llvm.load %50 : !llvm.ptr -> i32
+        %52 = llvm.add %40, %51 : i32
+        llvm.store %52, %arg0 : i32, !llvm.ptr
+        omp.scan inclusive(%arg0 : !llvm.ptr)
+        llvm.store %arg1, %38 : i32, !llvm.ptr
+        %53 = llvm.load %arg0 : !llvm.ptr -> i32
+        %54 = llvm.load %38 : !llvm.ptr -> i32
+        %55 = llvm.sext %54 : i32 to i64
+        %63 = llvm.getelementptr %15[%55] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+        llvm.store %53, %63 : i32, !llvm.ptr
+        omp.yield
+      }
+    }
+    omp.terminator
+  }
+  llvm.return
+}
+llvm.mlir.global internal @_QFEa() {addr_space = 0 : i32} : !llvm.array<100 x i32> {
+  %0 = llvm.mlir.zero : !llvm.array<100 x i32>
+  llvm.return %0 : !llvm.array<100 x i32>
+}
+llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i32> {
+  %0 = llvm.mlir.zero : !llvm.array<100 x i32>
+  llvm.return %0 : !llvm.array<100 x i32>
+}
+
+// The orig-val seed must combine as sub(orig-val, buffer[1]): the original
+// variable's incoming value (the leftmost prefix element) is the accumulator
+// (first operand) and buffer[1] is the incoming value (second operand).
+//CHECK: omp.scan.seed:
+//CHECK:   %[[OBUFF:.+]] = load ptr, ptr %{{.*}}, align 8
+//CHECK:   %[[OELEMPTR:.+]] = getelementptr inbounds i32, ptr %[[OBUFF]], i32 1
+//CHECK:   %[[OORIG:.+]] = load i32, ptr %{{.*}}, align 4
+//CHECK:   %[[OELEM:.+]] = load i32, ptr %[[OELEMPTR]], align 4
+//CHECK:   %[[OCOMB:.+]] = sub i32 %[[OORIG]], %[[OELEM]]
+//CHECK:   store i32 %[[OCOMB]], ptr %[[OELEMPTR]], align 4
+
+// The prefix computation must combine as sub(buffer[i-pow2k], buffer[i]): the
+// earlier partial prefix is the accumulator (first operand) and the later one
+// the incoming value (second operand). buffer[i] (the later slot) is updated.
+//CHECK: omp.inner.log.scan.body:
+//CHECK:   %[[IND1:.+]] = add i32 %{{.*}}, 1
+//CHECK:   %[[IND1PTR:.+]] = getelementptr inbounds i32, ptr %{{.*}}, i32 %[[IND1]]
+//CHECK:   %[[IND2:.+]] = sub nuw i32 %[[IND1]], %{{.*}}
+//CHECK:   %[[IND2PTR:.+]] = getelementptr inbounds i32, ptr %{{.*}}, i32 %[[IND2]]
+//CHECK:   %[[IND1VAL:.+]] = load i32, ptr %[[IND1PTR]], align 4
+//CHECK:   %[[IND2VAL:.+]] = load i32, ptr %[[IND2PTR]], align 4
+//CHECK:   %[[REDVAL:.+]] = sub i32 %[[IND2VAL]], %[[IND1VAL]]
+//CHECK:   store i32 %[[REDVAL]], ptr %[[IND1PTR]], align 4
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
index 38e413a504315..0006434167c7c 100644
--- a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
@@ -99,7 +99,7 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK:   %[[OELEMPTR:.+]] = getelementptr inbounds i32, ptr %[[OBUFF]], i32 1
 //CHECK:   %[[OORIG:.+]] = load i32, ptr %{{.*}}, align 4
 //CHECK:   %[[OELEM:.+]] = load i32, ptr %[[OELEMPTR]], align 4
-//CHECK:   %[[OCOMB:.+]] = add i32 %[[OELEM]], %[[OORIG]]
+//CHECK:   %[[OCOMB:.+]] = add i32 %[[OORIG]], %[[OELEM]]
 //CHECK:   store i32 %[[OCOMB]], ptr %[[OELEMPTR]], align 4
 //CHECK:   br i1 {{.*}}, label %omp.outer.log.scan.body, label %omp.outer.log.scan.exit
 //CHECK: omp.outer.log.scan.body:
@@ -121,7 +121,7 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK:   %[[IND2PTR:.+]] = getelementptr inbounds i32, ptr %[[BUFF]], i32 %[[IND2]]
 //CHECK:   %[[IND1VAL:.+]] = load i32, ptr %[[IND1PTR]], align 4
 //CHECK:   %[[IND2VAL:.+]] = load i32, ptr %[[IND2PTR]], align 4
-//CHECK:   %[[REDVAL:.+]] = add i32 %[[IND1VAL]], %[[IND2VAL]]
+//CHECK:   %[[REDVAL:.+]] = add i32 %[[IND2VAL]], %[[IND1VAL]]
 //CHECK:   store i32 %[[REDVAL]], ptr %[[IND1PTR]], align 4
 //CHECK:   %[[CNTNXT]] = sub nuw i32 %[[CNT]], 1
 //CHECK:   %[[CMP3:.+]] = icmp uge i32 %[[CNTNXT]], %[[I]]


From openmp-commits at lists.llvm.org  Mon Aug  3 04:05:23 2026
From: openmp-commits at lists.llvm.org (CHANDRA GHALE via Openmp-commits)
Date: Mon, 03 Aug 2026 04:05:23 -0700 (PDT)
Subject: [Openmp-commits] [flang] [llvm] [mlir] [openmp] [Flang][OpenMP]
 Lower scan directive and inscan reduction modifier (PR #206747)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206747@github.com>
Message-ID: <6a7075f3.436e7ae1.315368.63ea@mx.google.com>


================
@@ -5489,17 +5525,63 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
         ScanRedInfo->Span,
         llvm::ConstantInt::get(ScanRedInfo->Span->getType(), 1));
     Builder.SetInsertPoint(InputBB);
-    Builder.CreateBr(LoopBB);
+    // Branch around the orig-val seed load and combine when the loop has zero
+    // logical iterations. When Span == 0 no scan-buffer entry is populated, so
+    // loading buffer[0] and feeding it to a (possibly input-sensitive
+    // user-defined) reduction combiner would be undefined behavior. In that
+    // case skip straight to the exit; the finals' zero-trip guard preserves
+    // orig-val as the result.
+    llvm::Value *HasElem = Builder.CreateICmpUGT(
+        ScanRedInfo->Span, llvm::ConstantInt::get(IndexTy, 0));
+    llvm::BasicBlock *SeedBB =
+        BasicBlock::Create(CurFn->getContext(), "omp.scan.seed", CurFn, ExitBB);
+    Builder.CreateCondBr(HasElem, SeedBB, ExitBB);
+    Builder.SetInsertPoint(SeedBB);
+    // Combine the original variable's incoming value (orig-val) into the first
+    // buffer element before computing the prefix sum. Per the OpenMP scan
+    // semantics orig-val is a single prefix element that must be reflected in
+    // every inclusive/exclusive scan result and in the final reduction value.
+    // Folding it in here lets the log-scan prefix computation propagate it into
+    // every element (and into the finals' read of `buffer[Span]`), regardless
+    // of whether an iteration's input phase accumulates into or overwrites the
+    // reduction variable. Span > 0 is guaranteed here, so buffer[1] is valid.
+    for (ReductionInfo RedInfo : ReductionInfos) {
+      Value *ReductionVal = RedInfo.PrivateVariable;
+      Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ReductionVal];
+      Value *Buff = Builder.CreateLoad(Builder.getPtrTy(), BuffPtr);
+      Type *DestTy = RedInfo.ElementType;
+      Value *ElemPtr = Builder.CreateInBoundsGEP(
----------------
chandraghale wrote:

You're right — the combiner operands were reversed at both sites. ReductionGen's first operand is omp_out (the accumulator), so the earlier prefix element must go first; I've swapped them in the orig-val seed (OrigVal, Elem) and the prefix computation (RHS, LHS). Verified with your left reducer (now b=10 10 10, x=10) and added a noncommutative sub-combiner lit test so +'s commutativity can't mask this again


https://github.com/llvm/llvm-project/pull/206747

From openmp-commits at lists.llvm.org  Mon Aug  3 06:26:13 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Mon, 03 Aug 2026 06:26:13 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [openmp][WebAssembly] add
 support for wasm64 (PR #181669)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/181669@github.com>
Message-ID: <6a7096f5.e5907290.26ab43.496e@mx.google.com>


https://github.com/arsnyder16 updated https://github.com/llvm/llvm-project/pull/181669

>From f56e99c0c4a1b878dbc4f55c300c18b76c49c5c4 Mon Sep 17 00:00:00 2001
From: Andrew Snyder <asnyder at minitab.com>
Date: Mon, 16 Feb 2026 09:02:43 -0500
Subject: [PATCH 1/5] [openmp][WebAssembly] add support for wasm64

---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     |  2 +-
 openmp/CMakeLists.txt                         |  2 +-
 openmp/cmake/modules/LibompUtils.cmake        |  4 ++++
 openmp/runtime/CMakeLists.txt                 | 19 ++++++++++++-------
 .../runtime/cmake/LibompGetArchitecture.cmake |  2 ++
 openmp/runtime/cmake/config-ix.cmake          |  2 +-
 openmp/runtime/src/kmp_gsupport.cpp           |  2 +-
 openmp/runtime/src/kmp_os.h                   | 11 ++++++-----
 openmp/runtime/src/kmp_platform.h             | 12 ++++++++----
 openmp/runtime/src/kmp_runtime.cpp            |  4 ++--
 openmp/runtime/src/z_Linux_asm.S              |  2 +-
 openmp/runtime/src/z_Linux_util.cpp           |  5 +++--
 12 files changed, 42 insertions(+), 25 deletions(-)

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 464ec5b5a2ece..25285c5fefa31 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -9285,7 +9285,7 @@ GlobalVariable *OpenMPIRBuilder::getOrCreateInternalVariable(
                                : M.getTargetTriple().isAMDGPU()
                                    ? 0
                                    : DL.getDefaultGlobalsAddressSpace();
-    auto Linkage = this->M.getTargetTriple().getArch() == Triple::wasm32
+    auto Linkage = this->M.getTargetTriple().isWasm()
                        ? GlobalValue::InternalLinkage
                        : GlobalValue::CommonLinkage;
     auto *GV = new GlobalVariable(M, Ty, /*IsConstant=*/false, Linkage,
diff --git a/openmp/CMakeLists.txt b/openmp/CMakeLists.txt
index 355c64d28d296..4b7e4dccfac0a 100644
--- a/openmp/CMakeLists.txt
+++ b/openmp/CMakeLists.txt
@@ -140,7 +140,7 @@ set(ENABLE_LIBOMPTARGET ON)
 # Since the device plugins are only supported on Linux anyway,
 # there is no point in trying to compile libomptarget on other OSes.
 # 32-bit systems are not supported either.
-if (APPLE OR WIN32 OR WASM OR NOT "cxx_std_17" IN_LIST CMAKE_CXX_COMPILE_FEATURES
+if (APPLE OR WIN32 OR WASM32 OR WASM64 OR NOT "cxx_std_17" IN_LIST CMAKE_CXX_COMPILE_FEATURES
     OR NOT CMAKE_SIZEOF_VOID_P EQUAL 8 OR "${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
   set(ENABLE_LIBOMPTARGET OFF)
 endif()
diff --git a/openmp/cmake/modules/LibompUtils.cmake b/openmp/cmake/modules/LibompUtils.cmake
index a810881c2544e..1c60ffb242727 100644
--- a/openmp/cmake/modules/LibompUtils.cmake
+++ b/openmp/cmake/modules/LibompUtils.cmake
@@ -121,6 +121,10 @@ function(libomp_get_legal_arch return_arch_string)
     set(${return_arch_string} "SPARC" PARENT_SCOPE)
   elseif(${SPARCV9})
     set(${return_arch_string} "SPARCV9" PARENT_SCOPE)
+  elseif(${WASM32})
+    set(${return_arch_string} "WASM32" PARENT_SCOPE)
+  elseif(${WASM64})
+    set(${return_arch_string} "WASM64" PARENT_SCOPE)
   else()
     set(${return_arch_string} "${LIBOMP_ARCH}" PARENT_SCOPE)
     libomp_warning_say("libomp_get_legal_arch(): Warning: Unknown architecture: Using ${LIBOMP_ARCH}")
diff --git a/openmp/runtime/CMakeLists.txt b/openmp/runtime/CMakeLists.txt
index dc2f2be079bf3..58f8646c2d73d 100644
--- a/openmp/runtime/CMakeLists.txt
+++ b/openmp/runtime/CMakeLists.txt
@@ -24,7 +24,7 @@ if(${OPENMP_STANDALONE_BUILD})
   # If adding a new architecture, take a look at cmake/LibompGetArchitecture.cmake
   libomp_get_architecture(LIBOMP_DETECTED_ARCH)
   set(LIBOMP_ARCH ${LIBOMP_DETECTED_ARCH} CACHE STRING
-    "The architecture to build for (x86_64/i386/arm/ppc/ppc64/ppc64le/aarch64/aarch64_32/mic/mips/mips64/riscv64/loongarch64/ve/s390x/sparc/sparcv9/wasm32).")
+    "The architecture to build for (x86_64/i386/arm/ppc/ppc64/ppc64le/aarch64/aarch64_32/mic/mips/mips64/riscv64/loongarch64/ve/s390x/sparc/sparcv9/wasm32/wasm64).")
   # Should assertions be enabled?  They are on by default.
   set(LIBOMP_ENABLE_ASSERTIONS TRUE CACHE BOOL
     "enable assertions?")
@@ -69,8 +69,10 @@ else() # Part of LLVM build
     set(LIBOMP_ARCH sparcv9)
   elseif(LIBOMP_NATIVE_ARCH MATCHES "sparc")
     set(LIBOMP_ARCH sparc)
-  elseif(LIBOMP_NATIVE_ARCH MATCHES "wasm")
+  elseif(LIBOMP_NATIVE_ARCH MATCHES "wasm32")
     set(LIBOMP_ARCH wasm32)
+  elseif(LIBOMP_NATIVE_ARCH MATCHES "wasm64")
+    set(LIBOMP_ARCH wasm64)
   else()
     # last ditch effort
     libomp_get_architecture(LIBOMP_ARCH)
@@ -91,7 +93,7 @@ if(LIBOMP_ARCH STREQUAL "aarch64")
   endif()
 endif()
 
-libomp_check_variable(LIBOMP_ARCH 32e x86_64 32 i386 arm ppc ppc64 ppc64le aarch64 aarch64_32 aarch64_a64fx mic mips mips64 riscv64 loongarch64 ve s390x sparc sparcv9 wasm32)
+libomp_check_variable(LIBOMP_ARCH 32e x86_64 32 i386 arm ppc ppc64 ppc64le aarch64 aarch64_32 aarch64_a64fx mic mips mips64 riscv64 loongarch64 ve s390x sparc sparcv9 wasm32 wasm64)
 
 set(LIBOMP_LIB_TYPE normal CACHE STRING
   "Performance,Profiling,Stubs library (normal/profile/stubs)")
@@ -186,7 +188,8 @@ set(RISCV64 FALSE)
 set(LOONGARCH64 FALSE)
 set(VE FALSE)
 set(S390X FALSE)
-set(WASM FALSE)
+set(WASM32 FALSE)
+set(WASM64 FALSE)
 set(PPC FALSE)
 set(SPARC FALSE)
 set(SPARCV9 FALSE)
@@ -224,8 +227,10 @@ elseif("${LIBOMP_ARCH}" STREQUAL "ve") # VE architecture
     set(VE TRUE)
 elseif("${LIBOMP_ARCH}" STREQUAL "s390x") # S390x (Z) architecture
     set(S390X TRUE)
-elseif("${LIBOMP_ARCH}" STREQUAL "wasm32") # WebAssembly architecture
-    set(WASM TRUE)
+elseif("${LIBOMP_ARCH}" STREQUAL "wasm32") # WebAssembly 32 bit architecture
+    set(WASM32 TRUE)
+elseif("${LIBOMP_ARCH}" STREQUAL "wasm64") # WebAssembly 64 bit architecture
+    set(WASM64 TRUE)
 elseif("${LIBOMP_ARCH}" STREQUAL "sparc") # SPARC architecture
     set(SPARC TRUE)
 elseif("${LIBOMP_ARCH}" STREQUAL "sparcv9") # SPARC V9 architecture
@@ -334,7 +339,7 @@ endif()
 set(LIBOMP_ENABLE_SHARED TRUE CACHE BOOL
   "Shared library instead of static library?")
 
-if(WASM)
+if(WASM32 OR WASM64)
   libomp_warning_say("The WebAssembly build currently only supports static libraries; forcing LIBOMP_ENABLE_SHARED to false")
   set(LIBOMP_ENABLE_SHARED FALSE)
 endif()
diff --git a/openmp/runtime/cmake/LibompGetArchitecture.cmake b/openmp/runtime/cmake/LibompGetArchitecture.cmake
index 81aa700e3b6db..b69b129aa693f 100644
--- a/openmp/runtime/cmake/LibompGetArchitecture.cmake
+++ b/openmp/runtime/cmake/LibompGetArchitecture.cmake
@@ -59,6 +59,8 @@ function(libomp_get_architecture return_arch)
       #error ARCHITECTURE=s390x
     #elif defined(__wasm32__)
       #error ARCHITECTURE=wasm32
+    #elif defined(__wasm64__)
+      #error ARCHITECTURE=wasm64
     #elif defined(__sparcv9)
       #error ARCHITECTURE=sparcv9
     #elif defined(__sparc)
diff --git a/openmp/runtime/cmake/config-ix.cmake b/openmp/runtime/cmake/config-ix.cmake
index 150a0c3c7217b..fcd02c578465c 100644
--- a/openmp/runtime/cmake/config-ix.cmake
+++ b/openmp/runtime/cmake/config-ix.cmake
@@ -154,7 +154,7 @@ endif()
 # with either the Emscripten or wasi-threads flavor ends up using the pthreads
 # interface in a WebAssembly-compiled libc; CMake does not yet know how to
 # detect this.
-if (NOT WASM)
+if (NOT WASM32 AND NOT WASM64)
   find_package(Threads REQUIRED)
   if(WIN32)
     if(NOT CMAKE_USE_WIN32_THREADS_INIT)
diff --git a/openmp/runtime/src/kmp_gsupport.cpp b/openmp/runtime/src/kmp_gsupport.cpp
index 0d04045f7b165..be95e0f3de6e2 100644
--- a/openmp/runtime/src/kmp_gsupport.cpp
+++ b/openmp/runtime/src/kmp_gsupport.cpp
@@ -357,7 +357,7 @@ void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_ORDERED_END)(void) {
 // They come in two flavors: 64-bit unsigned, and either 32-bit signed
 // (IA-32 architecture) or 64-bit signed (Intel(R) 64).
 
-#if KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_MIPS || KMP_ARCH_WASM ||          \
+#if KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_MIPS || KMP_ARCH_WASM32 ||        \
     KMP_ARCH_PPC || KMP_ARCH_AARCH64_32 || KMP_ARCH_SPARC32
 #define KMP_DISPATCH_INIT __kmp_aux_dispatch_init_4
 #define KMP_DISPATCH_FINI_CHUNK __kmp_aux_dispatch_fini_chunk_4
diff --git a/openmp/runtime/src/kmp_os.h b/openmp/runtime/src/kmp_os.h
index e8ad2a6fdb78e..f74fec5a1f82e 100644
--- a/openmp/runtime/src/kmp_os.h
+++ b/openmp/runtime/src/kmp_os.h
@@ -178,18 +178,18 @@ typedef unsigned long long kmp_uint64;
 #define KMP_UINT64_SPEC "llu"
 #endif /* KMP_OS_UNIX */
 
-#if KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_MIPS || KMP_ARCH_WASM ||          \
+#if KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_MIPS || KMP_ARCH_WASM32 ||        \
     KMP_ARCH_PPC || KMP_ARCH_AARCH64_32 || KMP_ARCH_SPARC32
 #define KMP_SIZE_T_SPEC KMP_UINT32_SPEC
 #elif KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 ||                 \
     KMP_ARCH_MIPS64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 ||             \
-    KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_SPARC64
+    KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_SPARC64 || KMP_ARCH_WASM64
 #define KMP_SIZE_T_SPEC KMP_UINT64_SPEC
 #else
 #error "Can't determine size_t printf format specifier."
 #endif
 
-#if KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_WASM || KMP_ARCH_PPC
+#if KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_WASM32 || KMP_ARCH_PPC
 #define KMP_SIZE_T_MAX (0xFFFFFFFF)
 #else
 #define KMP_SIZE_T_MAX (0xFFFFFFFFFFFFFFFF)
@@ -218,7 +218,8 @@ typedef kmp_uint32 kmp_uint;
 #define KMP_INT_MIN ((kmp_int32)0x80000000)
 
 // stdarg handling
-#if (KMP_ARCH_ARM || KMP_ARCH_X86_64 || KMP_ARCH_AARCH64 || KMP_ARCH_WASM) &&  \
+#if (KMP_ARCH_ARM || KMP_ARCH_X86_64 || KMP_ARCH_AARCH64 || KMP_ARCH_WASM32 || \
+     KMP_ARCH_WASM64) &&                                                       \
     (KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_OPENBSD || KMP_OS_DRAGONFLY ||  \
      KMP_OS_LINUX || KMP_OS_WASI)
 typedef va_list *kmp_va_list;
@@ -1153,7 +1154,7 @@ extern kmp_real64 __kmp_xchg_real64(volatile kmp_real64 *p, kmp_real64 v);
   KMP_COMPARE_AND_STORE_REL64((volatile kmp_int64 *)(volatile void *)&(a),     \
                               (kmp_int64)(b), (kmp_int64)(c))
 
-#if KMP_ARCH_X86 || KMP_ARCH_MIPS || KMP_ARCH_WASM || KMP_ARCH_PPC
+#if KMP_ARCH_X86 || KMP_ARCH_MIPS || KMP_ARCH_WASM32 || KMP_ARCH_PPC
 // What about ARM?
 #define TCR_PTR(a) ((void *)TCR_4(a))
 #define TCW_PTR(a, b) TCW_4((a), (b))
diff --git a/openmp/runtime/src/kmp_platform.h b/openmp/runtime/src/kmp_platform.h
index 609b7c4688842..2d789e95f29bb 100644
--- a/openmp/runtime/src/kmp_platform.h
+++ b/openmp/runtime/src/kmp_platform.h
@@ -245,7 +245,11 @@
 #endif
 
 #if defined(__wasm32__)
-#define KMP_ARCH_WASM 1
+#define KMP_ARCH_WASM32 1
+#endif
+
+#if defined(__wasm64__)
+#define KMP_ARCH_WASM64 1
 #endif
 
 #define KMP_ARCH_PPC64                                                         \
@@ -279,7 +283,7 @@
 
 /* Specify 32 bit architectures here */
 #define KMP_32_BIT_ARCH                                                        \
-  (KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_MIPS || KMP_ARCH_WASM ||           \
+  (KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_MIPS || KMP_ARCH_WASM32 ||         \
    KMP_ARCH_PPC || KMP_ARCH_AARCH64_32 || KMP_ARCH_SPARC32)
 
 // Platforms which support Intel(R) Many Integrated Core Architecture
@@ -290,8 +294,8 @@
 #if (1 != KMP_ARCH_X86 + KMP_ARCH_X86_64 + KMP_ARCH_ARM + KMP_ARCH_PPC64 +     \
               KMP_ARCH_AARCH64 + KMP_ARCH_MIPS + KMP_ARCH_MIPS64 +             \
               KMP_ARCH_RISCV64 + KMP_ARCH_LOONGARCH64 + KMP_ARCH_VE +          \
-              KMP_ARCH_S390X + KMP_ARCH_WASM + KMP_ARCH_PPC +                  \
-              KMP_ARCH_AARCH64_32 + KMP_ARCH_SPARC)
+              KMP_ARCH_S390X + KMP_ARCH_WASM32 + KMP_ARCH_WASM64 +             \
+              KMP_ARCH_PPC + KMP_ARCH_AARCH64_32 + KMP_ARCH_SPARC)
 #error Unknown or unsupported architecture
 #endif
 
diff --git a/openmp/runtime/src/kmp_runtime.cpp b/openmp/runtime/src/kmp_runtime.cpp
index 3277a09c6497f..861c030db3fb1 100644
--- a/openmp/runtime/src/kmp_runtime.cpp
+++ b/openmp/runtime/src/kmp_runtime.cpp
@@ -8924,7 +8924,7 @@ __kmp_determine_reduction_method(
 
 #if KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 ||                   \
     KMP_ARCH_MIPS64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 ||             \
-    KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_WASM
+    KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_WASM64
 
 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD ||     \
     KMP_OS_OPENBSD || KMP_OS_WINDOWS || KMP_OS_DARWIN || KMP_OS_HAIKU ||       \
@@ -8956,7 +8956,7 @@ __kmp_determine_reduction_method(
        // KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_WASI || KMP_OS_AIX
 
 #elif KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_AARCH || KMP_ARCH_MIPS ||       \
-    KMP_ARCH_WASM || KMP_ARCH_PPC || KMP_ARCH_AARCH64_32 || KMP_ARCH_SPARC
+    KMP_ARCH_WASM32 || KMP_ARCH_PPC || KMP_ARCH_AARCH64_32 || KMP_ARCH_SPARC
 
 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD ||     \
     KMP_OS_OPENBSD || KMP_OS_WINDOWS || KMP_OS_HAIKU || KMP_OS_HURD ||         \
diff --git a/openmp/runtime/src/z_Linux_asm.S b/openmp/runtime/src/z_Linux_asm.S
index 12fea67e000e8..bfc8aff325f2a 100644
--- a/openmp/runtime/src/z_Linux_asm.S
+++ b/openmp/runtime/src/z_Linux_asm.S
@@ -2512,7 +2512,7 @@ KMP_PREFIX_UNDERSCORE(__kmp_unnamed_critical_addr):
 #if KMP_OS_LINUX
 # if KMP_ARCH_ARM || KMP_ARCH_AARCH64
 .section .note.GNU-stack,"",%progbits
-# elif !KMP_ARCH_WASM
+# elif !KMP_ARCH_WASM32 && !KMP_ARCH_WASM64 
 .section .note.GNU-stack,"", at progbits
 # endif
 #endif
diff --git a/openmp/runtime/src/z_Linux_util.cpp b/openmp/runtime/src/z_Linux_util.cpp
index c7fe0642cea63..18ffedadee505 100644
--- a/openmp/runtime/src/z_Linux_util.cpp
+++ b/openmp/runtime/src/z_Linux_util.cpp
@@ -321,7 +321,8 @@ int __kmp_futex_determine_capable() {
 
 #endif // KMP_USE_FUTEX
 
-#if (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_WASM) && (!KMP_ASM_INTRINS)
+#if (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_WASM32 || KMP_ARCH_WASM64) && \
+    (!KMP_ASM_INTRINS)
 /* Only 32-bit "add-exchange" instruction on IA-32 architecture causes us to
    use compare_and_store for these routines */
 
@@ -381,7 +382,7 @@ kmp_uint32 __kmp_test_then_and32(volatile kmp_uint32 *p, kmp_uint32 d) {
   return old_value;
 }
 
-#if KMP_ARCH_X86 || KMP_ARCH_WASM
+#if KMP_ARCH_X86 || KMP_ARCH_WASM32
 kmp_int8 __kmp_test_then_add8(volatile kmp_int8 *p, kmp_int8 d) {
   kmp_int8 old_value, new_value;
 

>From 8d5304b51a977b81f63e47491114592b5ee9016f Mon Sep 17 00:00:00 2001
From: Andrew Snyder <asnyder at minitab.com>
Date: Tue, 28 Jul 2026 09:04:30 -0400
Subject: [PATCH 2/5] formatting

---
 openmp/runtime/src/kmp_os.h        | 3 ++-
 openmp/runtime/src/kmp_platform.h  | 5 +++--
 openmp/runtime/src/kmp_runtime.cpp | 6 +-----
 3 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/openmp/runtime/src/kmp_os.h b/openmp/runtime/src/kmp_os.h
index c742e8272999f..be8c121e18736 100644
--- a/openmp/runtime/src/kmp_os.h
+++ b/openmp/runtime/src/kmp_os.h
@@ -183,7 +183,8 @@ typedef unsigned long long kmp_uint64;
 #define KMP_SIZE_T_SPEC KMP_UINT32_SPEC
 #elif KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 ||                 \
     KMP_ARCH_MIPS64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 ||             \
-    KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_SPARC64 || KMP_ARCH_WASM64 || KMP_ARCH_ARM64EC
+    KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_SPARC64 || KMP_ARCH_WASM64 ||    \
+    KMP_ARCH_ARM64EC
 #define KMP_SIZE_T_SPEC KMP_UINT64_SPEC
 #else
 #error "Can't determine size_t printf format specifier."
diff --git a/openmp/runtime/src/kmp_platform.h b/openmp/runtime/src/kmp_platform.h
index 6370047260681..3cafd90225614 100644
--- a/openmp/runtime/src/kmp_platform.h
+++ b/openmp/runtime/src/kmp_platform.h
@@ -298,8 +298,9 @@
 #if (1 != KMP_ARCH_X86 + KMP_ARCH_X86_64 + KMP_ARCH_ARM + KMP_ARCH_PPC64 +     \
               KMP_ARCH_AARCH64 + KMP_ARCH_MIPS + KMP_ARCH_MIPS64 +             \
               KMP_ARCH_RISCV64 + KMP_ARCH_LOONGARCH64 + KMP_ARCH_VE +          \
-              KMP_ARCH_S390X + KMP_ARCH_WASM32 + KMP_ARCH_WASM64 + KMP_ARCH_PPC +                  \
-              KMP_ARCH_AARCH64_32 + KMP_ARCH_SPARC + KMP_ARCH_ARM64EC)
+              KMP_ARCH_S390X + KMP_ARCH_WASM32 + KMP_ARCH_WASM64 +             \
+              KMP_ARCH_PPC + KMP_ARCH_AARCH64_32 + KMP_ARCH_SPARC +            \
+              KMP_ARCH_ARM64EC)
 #error Unknown or unsupported architecture
 #endif
 
diff --git a/openmp/runtime/src/kmp_runtime.cpp b/openmp/runtime/src/kmp_runtime.cpp
index b752991780678..8f7f5c8c7bfe5 100644
--- a/openmp/runtime/src/kmp_runtime.cpp
+++ b/openmp/runtime/src/kmp_runtime.cpp
@@ -8924,11 +8924,7 @@ __kmp_determine_reduction_method(
 
 #if KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 ||                   \
     KMP_ARCH_MIPS64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 ||             \
-<<<<<<< HEAD
-    KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_WASM64
-=======
-    KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_WASM || KMP_ARCH_ARM64EC
->>>>>>> origin/main
+    KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_WASM64 || KMP_ARCH_ARM64EC
 
 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD ||     \
     KMP_OS_OPENBSD || KMP_OS_WINDOWS || KMP_OS_DARWIN || KMP_OS_HAIKU ||       \

>From 1f22451463e5d5ca1317e91b79dce728c590a735 Mon Sep 17 00:00:00 2001
From: Andrew Snyder <asnyder at minitab.com>
Date: Tue, 28 Jul 2026 10:25:15 -0400
Subject: [PATCH 3/5] cleanup

---
 openmp/CMakeLists.txt              | 5 +++--
 openmp/runtime/CMakeLists.txt      | 4 ++++
 openmp/runtime/src/kmp_runtime.cpp | 5 +++--
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/openmp/CMakeLists.txt b/openmp/CMakeLists.txt
index d5d4f79d4a6d4..f9a89859be7af 100644
--- a/openmp/CMakeLists.txt
+++ b/openmp/CMakeLists.txt
@@ -134,11 +134,12 @@ set(OPENMP_TEST_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" CACHE STRING
   "Additional compiler flags to use for testing Fortran programs (e.g. additional module search paths via -fintrinsic-modules-path )")
 
 set(ENABLE_LIBOMPTARGET ON)
-# Currently libomptarget cannot be compiled on Windows or MacOS X.
+# Currently libomptarget cannot be compiled on Windows, MacOS X, or WebAssembly.
 # Since the device plugins are only supported on Linux anyway,
 # there is no point in trying to compile libomptarget on other OSes.
 # 32-bit systems are not supported either.
-if (APPLE OR WIN32 OR WASM32 OR WASM64 OR NOT "cxx_std_17" IN_LIST CMAKE_CXX_COMPILE_FEATURES
+if (APPLE OR WIN32 OR "${CMAKE_SYSTEM_NAME}" MATCHES "^(Emscripten|WASI)$" OR
+    NOT "cxx_std_17" IN_LIST CMAKE_CXX_COMPILE_FEATURES
     OR NOT CMAKE_SIZEOF_VOID_P EQUAL 8 OR "${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
   set(ENABLE_LIBOMPTARGET OFF)
 endif()
diff --git a/openmp/runtime/CMakeLists.txt b/openmp/runtime/CMakeLists.txt
index 61feef08c574d..0256bfaf4a892 100644
--- a/openmp/runtime/CMakeLists.txt
+++ b/openmp/runtime/CMakeLists.txt
@@ -140,12 +140,16 @@ set(LIBOMP_LDFLAGS "" CACHE STRING
 if("${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
   set(LIBOMP_LIBFLAGS "-lperfstat" CACHE STRING
     "Appended user specified linked libs flags. (e.g., -lm)")
+  if("${LIBOMP_ARCH}" STREQUAL "ppc")
+    # PPC (32-bit) on AIX needs libcompiler_rt for __atomic_load_8, etc.
+    set(LIBOMP_LIBFLAGS "${LIBOMP_LIBFLAGS} -lcompiler_rt")
   endif()
 else()
   set(LIBOMP_LIBFLAGS "" CACHE STRING
     "Appended user specified linked libs flags. (e.g., -lm)")
 endif()
 
+if("${LIBOMP_ARCH}" STREQUAL "mips" OR "${LIBOMP_ARCH}" STREQUAL "mips64")
   set(LIBOMP_LIBFLAGS "${LIBOMP_LIBFLAGS} -latomic")
 endif()
 
diff --git a/openmp/runtime/src/kmp_runtime.cpp b/openmp/runtime/src/kmp_runtime.cpp
index 8f7f5c8c7bfe5..234a211bcfba3 100644
--- a/openmp/runtime/src/kmp_runtime.cpp
+++ b/openmp/runtime/src/kmp_runtime.cpp
@@ -8924,7 +8924,8 @@ __kmp_determine_reduction_method(
 
 #if KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 ||                   \
     KMP_ARCH_MIPS64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 ||             \
-    KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_WASM64 || KMP_ARCH_ARM64EC
+    KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_WASM32 || KMP_ARCH_WASM64 ||     \
+    KMP_ARCH_ARM64EC
 
 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD ||     \
     KMP_OS_OPENBSD || KMP_OS_WINDOWS || KMP_OS_DARWIN || KMP_OS_HAIKU ||       \
@@ -8956,7 +8957,7 @@ __kmp_determine_reduction_method(
        // KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_WASI || KMP_OS_AIX
 
 #elif KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_AARCH || KMP_ARCH_MIPS ||       \
-    KMP_ARCH_WASM32 || KMP_ARCH_PPC || KMP_ARCH_AARCH64_32 || KMP_ARCH_SPARC
+    KMP_ARCH_PPC || KMP_ARCH_AARCH64_32 || KMP_ARCH_SPARC
 
 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD ||     \
     KMP_OS_OPENBSD || KMP_OS_WINDOWS || KMP_OS_HAIKU || KMP_OS_HURD ||         \

>From d57aad137058f122e8283cd329b4a6cb4721fd9d Mon Sep 17 00:00:00 2001
From: Andrew Snyder <asnyder at minitab.com>
Date: Tue, 28 Jul 2026 11:02:04 -0400
Subject: [PATCH 4/5] add a code gen test for wasm

---
 clang/test/OpenMP/wasm_codegen.c | 58 ++++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)
 create mode 100644 clang/test/OpenMP/wasm_codegen.c

diff --git a/clang/test/OpenMP/wasm_codegen.c b/clang/test/OpenMP/wasm_codegen.c
new file mode 100644
index 0000000000000..0d40cd68fb710
--- /dev/null
+++ b/clang/test/OpenMP/wasm_codegen.c
@@ -0,0 +1,58 @@
+// REQUIRES: webassembly-registered-target
+// RUN: %clang_cc1 -triple wasm32-unknown-emscripten -fopenmp -fnoopenmp-use-tls -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,WASM32
+// RUN: %clang_cc1 -triple wasm64-unknown-emscripten -fopenmp -fnoopenmp-use-tls -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,WASM64
+// RUN: %clang_cc1 -triple wasm32-unknown-emscripten -fopenmp -fnoopenmp-use-tls -emit-obj -o %t.wasm32.o %s
+// RUN: %clang_cc1 -triple wasm64-unknown-emscripten -fopenmp -fnoopenmp-use-tls -emit-obj -o %t.wasm64.o %s
+
+// Exercise representative host OpenMP lowering for both WebAssembly pointer
+// models. WebAssembly does not support common linkage, so compiler-generated
+// locks and caches must have internal linkage.
+// CHECK-DAG: @.gomp_critical_user_.var = internal global [8 x i32] zeroinitializer
+// CHECK-DAG: @.gomp_critical_user_named.var = internal global [8 x i32] zeroinitializer
+// CHECK-DAG: @.gomp_critical_user_.reduction.var = internal global [8 x i32] zeroinitializer
+// CHECK-DAG: @threadprivate_var.cache. = internal global ptr null
+
+// The dependency record and size_t parameters must follow the target pointer
+// width rather than assuming wasm32.
+// WASM32-DAG: %struct.kmp_depend_info = type { i32, i32, i8 }
+// WASM64-DAG: %struct.kmp_depend_info = type { i64, i64, i8 }
+// WASM32-DAG: call ptr @__kmpc_omp_task_alloc({{.*}}i32 {{[0-9]+}}, i32 {{[0-9]+}}, ptr
+// WASM64-DAG: call ptr @__kmpc_omp_task_alloc({{.*}}i64 {{[0-9]+}}, i64 {{[0-9]+}}, ptr
+// WASM32-DAG: call ptr @__kmpc_threadprivate_cached({{.*}}i32 4, ptr @threadprivate_var.cache.)
+// WASM64-DAG: call ptr @__kmpc_threadprivate_cached({{.*}}i64 4, ptr @threadprivate_var.cache.)
+// WASM32-DAG: call i32 @__kmpc_reduce_nowait({{.*}}i32 4, ptr
+// WASM64-DAG: call i32 @__kmpc_reduce_nowait({{.*}}i64 8, ptr
+
+// CHECK-DAG: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(
+// CHECK-DAG: call void @__kmpc_for_static_init_4(
+// CHECK-DAG: call i32 @__kmpc_omp_task_with_deps(
+// CHECK-DAG: call i32 @__kmpc_omp_taskwait(
+// CHECK-DAG: call void @__kmpc_critical(
+
+void critical_regions(void) {
+#pragma omp critical
+  {}
+
+#pragma omp critical(named)
+  {}
+}
+
+int threadprivate_var;
+#pragma omp threadprivate(threadprivate_var)
+
+int openmp_constructs(int *values, int count) {
+  int sum = 0;
+#pragma omp parallel for reduction(+ : sum)
+  for (int i = 0; i < count; ++i)
+    sum += values[i];
+
+#pragma omp task shared(values) depend(inout : values[0 : count])
+  values[0] = sum;
+
+#pragma omp taskwait
+
+#pragma omp critical(named)
+  threadprivate_var += sum;
+
+  return sum;
+}

>From 3e64bf4050c054842de5b4584971d93a0c7bf726 Mon Sep 17 00:00:00 2001
From: Andrew Snyder <asnyder at minitab.com>
Date: Tue, 28 Jul 2026 11:20:26 -0400
Subject: [PATCH 5/5] consistency

---
 openmp/cmake/modules/LibompUtils.cmake | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/openmp/cmake/modules/LibompUtils.cmake b/openmp/cmake/modules/LibompUtils.cmake
index 2a639bccc47b6..9dd80c28dcaf0 100644
--- a/openmp/cmake/modules/LibompUtils.cmake
+++ b/openmp/cmake/modules/LibompUtils.cmake
@@ -125,9 +125,9 @@ function(libomp_get_legal_arch return_arch_string)
     set(${return_arch_string} "SPARC" PARENT_SCOPE)
   elseif(SPARCV9)
     set(${return_arch_string} "SPARCV9" PARENT_SCOPE)
-  elseif(${WASM32})
+  elseif(WASM32)
     set(${return_arch_string} "WASM32" PARENT_SCOPE)
-  elseif(${WASM64})
+  elseif(WASM64)
     set(${return_arch_string} "WASM64" PARENT_SCOPE)
   else()
     set(${return_arch_string} "${LIBOMP_ARCH}" PARENT_SCOPE)


From openmp-commits at lists.llvm.org  Mon Aug  3 06:26:14 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Mon, 03 Aug 2026 06:26:14 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP] Support up to 32 arguments in
 the generic microtask dispatcher (PR #211071)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/211071@github.com>
Message-ID: <6a7096f6.f06f4994.2376c6.ebdb@mx.google.com>


https://github.com/arsnyder16 updated https://github.com/llvm/llvm-project/pull/211071

>From 7b76acc6c9fa6222b4360dd90ce465df1acaff2a Mon Sep 17 00:00:00 2001
From: Andrew Snyder <asnyder at minitab.com>
Date: Tue, 21 Jul 2026 13:39:00 -0400
Subject: [PATCH 1/3] [OpenMP] Support up to 32 arguments in the generic
 microtask dispatcher

---
 openmp/runtime/src/kmp_invoke_microtask.cpp   | 177 +++++++-----------
 .../test/misc_bugs/microtask-32-args.c        |  56 ++++++
 .../test/misc_bugs/wasm-microtask-33-args.c   |  60 ++++++
 3 files changed, 183 insertions(+), 110 deletions(-)
 create mode 100644 openmp/runtime/test/misc_bugs/microtask-32-args.c
 create mode 100644 openmp/runtime/test/misc_bugs/wasm-microtask-33-args.c

diff --git a/openmp/runtime/src/kmp_invoke_microtask.cpp b/openmp/runtime/src/kmp_invoke_microtask.cpp
index f14f27cc06ee3..474423b62188e 100644
--- a/openmp/runtime/src/kmp_invoke_microtask.cpp
+++ b/openmp/runtime/src/kmp_invoke_microtask.cpp
@@ -5,45 +5,33 @@
       KMP_ARCH_ARM || KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_PPC_XCOFF ||   \
       KMP_ARCH_AARCH64_32)
 
-// Because WebAssembly will use `call_indirect` to invoke the microtask and
-// WebAssembly indirect calls check that the called signature is a precise
-// match, we need to cast each microtask function pointer back from `void *` to
-// its original type.
-typedef void (*microtask_t0)(int *, int *);
-typedef void (*microtask_t1)(int *, int *, void *);
-typedef void (*microtask_t2)(int *, int *, void *, void *);
-typedef void (*microtask_t3)(int *, int *, void *, void *, void *);
-typedef void (*microtask_t4)(int *, int *, void *, void *, void *, void *);
-typedef void (*microtask_t5)(int *, int *, void *, void *, void *, void *,
-                             void *);
-typedef void (*microtask_t6)(int *, int *, void *, void *, void *, void *,
-                             void *, void *);
-typedef void (*microtask_t7)(int *, int *, void *, void *, void *, void *,
-                             void *, void *, void *);
-typedef void (*microtask_t8)(int *, int *, void *, void *, void *, void *,
-                             void *, void *, void *, void *);
-typedef void (*microtask_t9)(int *, int *, void *, void *, void *, void *,
-                             void *, void *, void *, void *, void *);
-typedef void (*microtask_t10)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *);
-typedef void (*microtask_t11)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *);
-typedef void (*microtask_t12)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *);
-typedef void (*microtask_t13)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *, void *);
-typedef void (*microtask_t14)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *, void *, void *);
-typedef void (*microtask_t15)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *);
+template <size_t... Indices> struct microtask_index_sequence {};
 
-// we really only need the case with 1 argument, because CLANG always build
-// a struct of pointers to shared variables referenced in the outlined function
+template <size_t N, size_t... Indices>
+struct make_microtask_index_sequence
+    : make_microtask_index_sequence<N - 1, N - 1, Indices...> {};
+
+template <size_t... Indices>
+struct make_microtask_index_sequence<0, Indices...> {
+  using type = microtask_index_sequence<Indices...>;
+};
+
+template <size_t> using microtask_argument_t = void *;
+
+template <size_t... Indices>
+static void invokeMicrotask(microtask_t pkfn, int *gtid, int *tid,
+                            void *p_argv[],
+                            microtask_index_sequence<Indices...>) {
+  // WebAssembly's `call_indirect` requires the callee type to exactly match the
+  // call site. Cast the variadic microtask_t to the fixed-arity signature that
+  // matches argc before invoking it.
+  using typed_microtask_t =
+      void (*)(int *, int *, microtask_argument_t<Indices>...);
+  (*(typed_microtask_t)pkfn)(gtid, tid, p_argv[Indices]...);
+}
+
+// Keep a bounded set of exact signatures for targets that cannot dynamically
+// construct a variable-argument microtask call.
 int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,
                            void *p_argv[]
 #if OMPT_SUPPORT
@@ -55,85 +43,54 @@ int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,
   *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
 #endif
 
+#define KMP_INVOKE_MICROTASK_CASE(N)                                           \
+  case N:                                                                      \
+    invokeMicrotask(pkfn, &gtid, &tid, p_argv,                                 \
+                    make_microtask_index_sequence<N>::type{});                 \
+    break
+
   switch (argc) {
+    KMP_INVOKE_MICROTASK_CASE(0);
+    KMP_INVOKE_MICROTASK_CASE(1);
+    KMP_INVOKE_MICROTASK_CASE(2);
+    KMP_INVOKE_MICROTASK_CASE(3);
+    KMP_INVOKE_MICROTASK_CASE(4);
+    KMP_INVOKE_MICROTASK_CASE(5);
+    KMP_INVOKE_MICROTASK_CASE(6);
+    KMP_INVOKE_MICROTASK_CASE(7);
+    KMP_INVOKE_MICROTASK_CASE(8);
+    KMP_INVOKE_MICROTASK_CASE(9);
+    KMP_INVOKE_MICROTASK_CASE(10);
+    KMP_INVOKE_MICROTASK_CASE(11);
+    KMP_INVOKE_MICROTASK_CASE(12);
+    KMP_INVOKE_MICROTASK_CASE(13);
+    KMP_INVOKE_MICROTASK_CASE(14);
+    KMP_INVOKE_MICROTASK_CASE(15);
+    KMP_INVOKE_MICROTASK_CASE(16);
+    KMP_INVOKE_MICROTASK_CASE(17);
+    KMP_INVOKE_MICROTASK_CASE(18);
+    KMP_INVOKE_MICROTASK_CASE(19);
+    KMP_INVOKE_MICROTASK_CASE(20);
+    KMP_INVOKE_MICROTASK_CASE(21);
+    KMP_INVOKE_MICROTASK_CASE(22);
+    KMP_INVOKE_MICROTASK_CASE(23);
+    KMP_INVOKE_MICROTASK_CASE(24);
+    KMP_INVOKE_MICROTASK_CASE(25);
+    KMP_INVOKE_MICROTASK_CASE(26);
+    KMP_INVOKE_MICROTASK_CASE(27);
+    KMP_INVOKE_MICROTASK_CASE(28);
+    KMP_INVOKE_MICROTASK_CASE(29);
+    KMP_INVOKE_MICROTASK_CASE(30);
+    KMP_INVOKE_MICROTASK_CASE(31);
+    KMP_INVOKE_MICROTASK_CASE(32);
   default:
     fprintf(stderr, "Too many args to microtask: %d!\n", argc);
     fflush(stderr);
     exit(-1);
-  case 0:
-    (*(microtask_t0)pkfn)(&gtid, &tid);
-    break;
-  case 1:
-    (*(microtask_t1)pkfn)(&gtid, &tid, p_argv[0]);
-    break;
-  case 2:
-    (*(microtask_t2)pkfn)(&gtid, &tid, p_argv[0], p_argv[1]);
-    break;
-  case 3:
-    (*(microtask_t3)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2]);
-    break;
-  case 4:
-    (*(microtask_t4)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3]);
-    break;
-  case 5:
-    (*(microtask_t5)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3], p_argv[4]);
-    break;
-  case 6:
-    (*(microtask_t6)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3], p_argv[4], p_argv[5]);
-    break;
-  case 7:
-    (*(microtask_t7)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3], p_argv[4], p_argv[5], p_argv[6]);
-    break;
-  case 8:
-    (*(microtask_t8)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                          p_argv[7]);
-    break;
-  case 9:
-    (*(microtask_t9)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3], p_argv[4], p_argv[5], p_argv[6], p_argv[7],
-                          p_argv[8]);
-    break;
-  case 10:
-    (*(microtask_t10)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9]);
-    break;
-  case 11:
-    (*(microtask_t11)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9], p_argv[10]);
-    break;
-  case 12:
-    (*(microtask_t12)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9], p_argv[10],
-                           p_argv[11]);
-    break;
-  case 13:
-    (*(microtask_t13)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9], p_argv[10],
-                           p_argv[11], p_argv[12]);
-    break;
-  case 14:
-    (*(microtask_t14)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9], p_argv[10],
-                           p_argv[11], p_argv[12], p_argv[13]);
-    break;
-  case 15:
-    (*(microtask_t15)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9], p_argv[10],
-                           p_argv[11], p_argv[12], p_argv[13], p_argv[14]);
-    break;
   }
 
+#undef KMP_INVOKE_MICROTASK_CASE
+
   return 1;
 }
 
diff --git a/openmp/runtime/test/misc_bugs/microtask-32-args.c b/openmp/runtime/test/misc_bugs/microtask-32-args.c
new file mode 100644
index 0000000000000..848ab7c368481
--- /dev/null
+++ b/openmp/runtime/test/misc_bugs/microtask-32-args.c
@@ -0,0 +1,56 @@
+// RUN: %libomp-compile-and-run
+
+// Exercise the upper bound supported by the generic microtask dispatcher.
+
+int main(void) {
+  volatile int value0 = 0;
+  volatile int value1 = 1;
+  volatile int value2 = 2;
+  volatile int value3 = 3;
+  volatile int value4 = 4;
+  volatile int value5 = 5;
+  volatile int value6 = 6;
+  volatile int value7 = 7;
+  volatile int value8 = 8;
+  volatile int value9 = 9;
+  volatile int value10 = 10;
+  volatile int value11 = 11;
+  volatile int value12 = 12;
+  volatile int value13 = 13;
+  volatile int value14 = 14;
+  volatile int value15 = 15;
+  volatile int value16 = 16;
+  volatile int value17 = 17;
+  volatile int value18 = 18;
+  volatile int value19 = 19;
+  volatile int value20 = 20;
+  volatile int value21 = 21;
+  volatile int value22 = 22;
+  volatile int value23 = 23;
+  volatile int value24 = 24;
+  volatile int value25 = 25;
+  volatile int value26 = 26;
+  volatile int value27 = 27;
+  volatile int value28 = 28;
+  volatile int value29 = 29;
+  volatile int value30 = 30;
+  volatile int value31 = 31;
+
+#pragma omp parallel num_threads(2)                                            \
+    shared(value0, value1, value2, value3, value4, value5, value6, value7,     \
+               value8, value9, value10, value11, value12, value13, value14,    \
+               value15, value16, value17, value18, value19, value20, value21,  \
+               value22, value23, value24, value25, value26, value27, value28,  \
+               value29, value30, value31)
+  {
+    int sum = value0 + value1 + value2 + value3 + value4 + value5 + value6 +
+              value7 + value8 + value9 + value10 + value11 + value12 + value13 +
+              value14 + value15 + value16 + value17 + value18 + value19 +
+              value20 + value21 + value22 + value23 + value24 + value25 +
+              value26 + value27 + value28 + value29 + value30 + value31;
+    if (sum != 496)
+      __builtin_trap();
+  }
+
+  return 0;
+}
diff --git a/openmp/runtime/test/misc_bugs/wasm-microtask-33-args.c b/openmp/runtime/test/misc_bugs/wasm-microtask-33-args.c
new file mode 100644
index 0000000000000..bb88fcb239777
--- /dev/null
+++ b/openmp/runtime/test/misc_bugs/wasm-microtask-33-args.c
@@ -0,0 +1,60 @@
+// REQUIRES: wasm32-target-arch
+// RUN: %libomp-compile
+// RUN: %not %libomp-run 2>&1 | FileCheck %s
+
+// CHECK: Too many args to microtask: 33!
+
+int main(void) {
+  volatile int value0 = 0;
+  volatile int value1 = 1;
+  volatile int value2 = 2;
+  volatile int value3 = 3;
+  volatile int value4 = 4;
+  volatile int value5 = 5;
+  volatile int value6 = 6;
+  volatile int value7 = 7;
+  volatile int value8 = 8;
+  volatile int value9 = 9;
+  volatile int value10 = 10;
+  volatile int value11 = 11;
+  volatile int value12 = 12;
+  volatile int value13 = 13;
+  volatile int value14 = 14;
+  volatile int value15 = 15;
+  volatile int value16 = 16;
+  volatile int value17 = 17;
+  volatile int value18 = 18;
+  volatile int value19 = 19;
+  volatile int value20 = 20;
+  volatile int value21 = 21;
+  volatile int value22 = 22;
+  volatile int value23 = 23;
+  volatile int value24 = 24;
+  volatile int value25 = 25;
+  volatile int value26 = 26;
+  volatile int value27 = 27;
+  volatile int value28 = 28;
+  volatile int value29 = 29;
+  volatile int value30 = 30;
+  volatile int value31 = 31;
+  volatile int value32 = 32;
+
+#pragma omp parallel num_threads(2)                                            \
+    shared(value0, value1, value2, value3, value4, value5, value6, value7,     \
+               value8, value9, value10, value11, value12, value13, value14,    \
+               value15, value16, value17, value18, value19, value20, value21,  \
+               value22, value23, value24, value25, value26, value27, value28,  \
+               value29, value30, value31, value32)
+  {
+    int sum = value0 + value1 + value2 + value3 + value4 + value5 + value6 +
+              value7 + value8 + value9 + value10 + value11 + value12 + value13 +
+              value14 + value15 + value16 + value17 + value18 + value19 +
+              value20 + value21 + value22 + value23 + value24 + value25 +
+              value26 + value27 + value28 + value29 + value30 + value31 +
+              value32;
+    if (sum != 528)
+      __builtin_trap();
+  }
+
+  return 0;
+}

>From 0d4675ceb222fac240640591567daf089a9f6f87 Mon Sep 17 00:00:00 2001
From: Andrew Snyder <asnyder at minitab.com>
Date: Tue, 21 Jul 2026 14:00:54 -0400
Subject: [PATCH 2/3] use make_index_sequence

---
 openmp/runtime/src/kmp_invoke_microtask.cpp | 19 ++++---------------
 1 file changed, 4 insertions(+), 15 deletions(-)

diff --git a/openmp/runtime/src/kmp_invoke_microtask.cpp b/openmp/runtime/src/kmp_invoke_microtask.cpp
index 474423b62188e..ec13eb8af7e97 100644
--- a/openmp/runtime/src/kmp_invoke_microtask.cpp
+++ b/openmp/runtime/src/kmp_invoke_microtask.cpp
@@ -1,27 +1,17 @@
 #include "kmp.h"
 
+#include <utility>
+
 #if !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_MIC || KMP_ARCH_AARCH64 ||        \
       KMP_ARCH_PPC64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 ||            \
       KMP_ARCH_ARM || KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_PPC_XCOFF ||   \
       KMP_ARCH_AARCH64_32)
 
-template <size_t... Indices> struct microtask_index_sequence {};
-
-template <size_t N, size_t... Indices>
-struct make_microtask_index_sequence
-    : make_microtask_index_sequence<N - 1, N - 1, Indices...> {};
-
-template <size_t... Indices>
-struct make_microtask_index_sequence<0, Indices...> {
-  using type = microtask_index_sequence<Indices...>;
-};
-
 template <size_t> using microtask_argument_t = void *;
 
 template <size_t... Indices>
 static void invokeMicrotask(microtask_t pkfn, int *gtid, int *tid,
-                            void *p_argv[],
-                            microtask_index_sequence<Indices...>) {
+                            void *p_argv[], std::index_sequence<Indices...>) {
   // WebAssembly's `call_indirect` requires the callee type to exactly match the
   // call site. Cast the variadic microtask_t to the fixed-arity signature that
   // matches argc before invoking it.
@@ -45,8 +35,7 @@ int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,
 
 #define KMP_INVOKE_MICROTASK_CASE(N)                                           \
   case N:                                                                      \
-    invokeMicrotask(pkfn, &gtid, &tid, p_argv,                                 \
-                    make_microtask_index_sequence<N>::type{});                 \
+    invokeMicrotask(pkfn, &gtid, &tid, p_argv, std::make_index_sequence<N>{}); \
     break
 
   switch (argc) {

>From 33ed4a3e001d2b52a24d678870ce6722f17b6a61 Mon Sep 17 00:00:00 2001
From: Andrew Snyder <asnyder at minitab.com>
Date: Tue, 21 Jul 2026 14:12:13 -0400
Subject: [PATCH 3/3] should work on arm64ec now

---
 openmp/runtime/test/misc_bugs/many-microtask-args.c | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/openmp/runtime/test/misc_bugs/many-microtask-args.c b/openmp/runtime/test/misc_bugs/many-microtask-args.c
index 48ef2a0222d17..b91cfe542079f 100644
--- a/openmp/runtime/test/misc_bugs/many-microtask-args.c
+++ b/openmp/runtime/test/misc_bugs/many-microtask-args.c
@@ -1,10 +1,6 @@
 // RUN: %libomp-compile-and-run
 #include <stdio.h>
 
-// This test fails on Arm64EC as __kmp_invoke_microtask doesn't support more
-// than 16 arguments on that target.
-// XFAIL: arm64ec
-
 int main()
 {
 
@@ -40,4 +36,3 @@ int main()
 
   return 0;
 }
-


From openmp-commits at lists.llvm.org  Mon Aug  3 06:27:00 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Mon, 03 Aug 2026 06:27:00 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP] Support up to 32 arguments in
 the generic microtask dispatcher (PR #211071)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/211071@github.com>
Message-ID: <6a709724.436e7ae1.315368.39ff@mx.google.com>


https://github.com/arsnyder16 updated https://github.com/llvm/llvm-project/pull/211071

>From 7b76acc6c9fa6222b4360dd90ce465df1acaff2a Mon Sep 17 00:00:00 2001
From: Andrew Snyder <asnyder at minitab.com>
Date: Tue, 21 Jul 2026 13:39:00 -0400
Subject: [PATCH 1/3] [OpenMP] Support up to 32 arguments in the generic
 microtask dispatcher

---
 openmp/runtime/src/kmp_invoke_microtask.cpp   | 177 +++++++-----------
 .../test/misc_bugs/microtask-32-args.c        |  56 ++++++
 .../test/misc_bugs/wasm-microtask-33-args.c   |  60 ++++++
 3 files changed, 183 insertions(+), 110 deletions(-)
 create mode 100644 openmp/runtime/test/misc_bugs/microtask-32-args.c
 create mode 100644 openmp/runtime/test/misc_bugs/wasm-microtask-33-args.c

diff --git a/openmp/runtime/src/kmp_invoke_microtask.cpp b/openmp/runtime/src/kmp_invoke_microtask.cpp
index f14f27cc06ee3..474423b62188e 100644
--- a/openmp/runtime/src/kmp_invoke_microtask.cpp
+++ b/openmp/runtime/src/kmp_invoke_microtask.cpp
@@ -5,45 +5,33 @@
       KMP_ARCH_ARM || KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_PPC_XCOFF ||   \
       KMP_ARCH_AARCH64_32)
 
-// Because WebAssembly will use `call_indirect` to invoke the microtask and
-// WebAssembly indirect calls check that the called signature is a precise
-// match, we need to cast each microtask function pointer back from `void *` to
-// its original type.
-typedef void (*microtask_t0)(int *, int *);
-typedef void (*microtask_t1)(int *, int *, void *);
-typedef void (*microtask_t2)(int *, int *, void *, void *);
-typedef void (*microtask_t3)(int *, int *, void *, void *, void *);
-typedef void (*microtask_t4)(int *, int *, void *, void *, void *, void *);
-typedef void (*microtask_t5)(int *, int *, void *, void *, void *, void *,
-                             void *);
-typedef void (*microtask_t6)(int *, int *, void *, void *, void *, void *,
-                             void *, void *);
-typedef void (*microtask_t7)(int *, int *, void *, void *, void *, void *,
-                             void *, void *, void *);
-typedef void (*microtask_t8)(int *, int *, void *, void *, void *, void *,
-                             void *, void *, void *, void *);
-typedef void (*microtask_t9)(int *, int *, void *, void *, void *, void *,
-                             void *, void *, void *, void *, void *);
-typedef void (*microtask_t10)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *);
-typedef void (*microtask_t11)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *);
-typedef void (*microtask_t12)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *);
-typedef void (*microtask_t13)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *, void *);
-typedef void (*microtask_t14)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *, void *, void *);
-typedef void (*microtask_t15)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *);
+template <size_t... Indices> struct microtask_index_sequence {};
 
-// we really only need the case with 1 argument, because CLANG always build
-// a struct of pointers to shared variables referenced in the outlined function
+template <size_t N, size_t... Indices>
+struct make_microtask_index_sequence
+    : make_microtask_index_sequence<N - 1, N - 1, Indices...> {};
+
+template <size_t... Indices>
+struct make_microtask_index_sequence<0, Indices...> {
+  using type = microtask_index_sequence<Indices...>;
+};
+
+template <size_t> using microtask_argument_t = void *;
+
+template <size_t... Indices>
+static void invokeMicrotask(microtask_t pkfn, int *gtid, int *tid,
+                            void *p_argv[],
+                            microtask_index_sequence<Indices...>) {
+  // WebAssembly's `call_indirect` requires the callee type to exactly match the
+  // call site. Cast the variadic microtask_t to the fixed-arity signature that
+  // matches argc before invoking it.
+  using typed_microtask_t =
+      void (*)(int *, int *, microtask_argument_t<Indices>...);
+  (*(typed_microtask_t)pkfn)(gtid, tid, p_argv[Indices]...);
+}
+
+// Keep a bounded set of exact signatures for targets that cannot dynamically
+// construct a variable-argument microtask call.
 int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,
                            void *p_argv[]
 #if OMPT_SUPPORT
@@ -55,85 +43,54 @@ int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,
   *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
 #endif
 
+#define KMP_INVOKE_MICROTASK_CASE(N)                                           \
+  case N:                                                                      \
+    invokeMicrotask(pkfn, &gtid, &tid, p_argv,                                 \
+                    make_microtask_index_sequence<N>::type{});                 \
+    break
+
   switch (argc) {
+    KMP_INVOKE_MICROTASK_CASE(0);
+    KMP_INVOKE_MICROTASK_CASE(1);
+    KMP_INVOKE_MICROTASK_CASE(2);
+    KMP_INVOKE_MICROTASK_CASE(3);
+    KMP_INVOKE_MICROTASK_CASE(4);
+    KMP_INVOKE_MICROTASK_CASE(5);
+    KMP_INVOKE_MICROTASK_CASE(6);
+    KMP_INVOKE_MICROTASK_CASE(7);
+    KMP_INVOKE_MICROTASK_CASE(8);
+    KMP_INVOKE_MICROTASK_CASE(9);
+    KMP_INVOKE_MICROTASK_CASE(10);
+    KMP_INVOKE_MICROTASK_CASE(11);
+    KMP_INVOKE_MICROTASK_CASE(12);
+    KMP_INVOKE_MICROTASK_CASE(13);
+    KMP_INVOKE_MICROTASK_CASE(14);
+    KMP_INVOKE_MICROTASK_CASE(15);
+    KMP_INVOKE_MICROTASK_CASE(16);
+    KMP_INVOKE_MICROTASK_CASE(17);
+    KMP_INVOKE_MICROTASK_CASE(18);
+    KMP_INVOKE_MICROTASK_CASE(19);
+    KMP_INVOKE_MICROTASK_CASE(20);
+    KMP_INVOKE_MICROTASK_CASE(21);
+    KMP_INVOKE_MICROTASK_CASE(22);
+    KMP_INVOKE_MICROTASK_CASE(23);
+    KMP_INVOKE_MICROTASK_CASE(24);
+    KMP_INVOKE_MICROTASK_CASE(25);
+    KMP_INVOKE_MICROTASK_CASE(26);
+    KMP_INVOKE_MICROTASK_CASE(27);
+    KMP_INVOKE_MICROTASK_CASE(28);
+    KMP_INVOKE_MICROTASK_CASE(29);
+    KMP_INVOKE_MICROTASK_CASE(30);
+    KMP_INVOKE_MICROTASK_CASE(31);
+    KMP_INVOKE_MICROTASK_CASE(32);
   default:
     fprintf(stderr, "Too many args to microtask: %d!\n", argc);
     fflush(stderr);
     exit(-1);
-  case 0:
-    (*(microtask_t0)pkfn)(&gtid, &tid);
-    break;
-  case 1:
-    (*(microtask_t1)pkfn)(&gtid, &tid, p_argv[0]);
-    break;
-  case 2:
-    (*(microtask_t2)pkfn)(&gtid, &tid, p_argv[0], p_argv[1]);
-    break;
-  case 3:
-    (*(microtask_t3)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2]);
-    break;
-  case 4:
-    (*(microtask_t4)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3]);
-    break;
-  case 5:
-    (*(microtask_t5)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3], p_argv[4]);
-    break;
-  case 6:
-    (*(microtask_t6)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3], p_argv[4], p_argv[5]);
-    break;
-  case 7:
-    (*(microtask_t7)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3], p_argv[4], p_argv[5], p_argv[6]);
-    break;
-  case 8:
-    (*(microtask_t8)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                          p_argv[7]);
-    break;
-  case 9:
-    (*(microtask_t9)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3], p_argv[4], p_argv[5], p_argv[6], p_argv[7],
-                          p_argv[8]);
-    break;
-  case 10:
-    (*(microtask_t10)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9]);
-    break;
-  case 11:
-    (*(microtask_t11)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9], p_argv[10]);
-    break;
-  case 12:
-    (*(microtask_t12)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9], p_argv[10],
-                           p_argv[11]);
-    break;
-  case 13:
-    (*(microtask_t13)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9], p_argv[10],
-                           p_argv[11], p_argv[12]);
-    break;
-  case 14:
-    (*(microtask_t14)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9], p_argv[10],
-                           p_argv[11], p_argv[12], p_argv[13]);
-    break;
-  case 15:
-    (*(microtask_t15)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9], p_argv[10],
-                           p_argv[11], p_argv[12], p_argv[13], p_argv[14]);
-    break;
   }
 
+#undef KMP_INVOKE_MICROTASK_CASE
+
   return 1;
 }
 
diff --git a/openmp/runtime/test/misc_bugs/microtask-32-args.c b/openmp/runtime/test/misc_bugs/microtask-32-args.c
new file mode 100644
index 0000000000000..848ab7c368481
--- /dev/null
+++ b/openmp/runtime/test/misc_bugs/microtask-32-args.c
@@ -0,0 +1,56 @@
+// RUN: %libomp-compile-and-run
+
+// Exercise the upper bound supported by the generic microtask dispatcher.
+
+int main(void) {
+  volatile int value0 = 0;
+  volatile int value1 = 1;
+  volatile int value2 = 2;
+  volatile int value3 = 3;
+  volatile int value4 = 4;
+  volatile int value5 = 5;
+  volatile int value6 = 6;
+  volatile int value7 = 7;
+  volatile int value8 = 8;
+  volatile int value9 = 9;
+  volatile int value10 = 10;
+  volatile int value11 = 11;
+  volatile int value12 = 12;
+  volatile int value13 = 13;
+  volatile int value14 = 14;
+  volatile int value15 = 15;
+  volatile int value16 = 16;
+  volatile int value17 = 17;
+  volatile int value18 = 18;
+  volatile int value19 = 19;
+  volatile int value20 = 20;
+  volatile int value21 = 21;
+  volatile int value22 = 22;
+  volatile int value23 = 23;
+  volatile int value24 = 24;
+  volatile int value25 = 25;
+  volatile int value26 = 26;
+  volatile int value27 = 27;
+  volatile int value28 = 28;
+  volatile int value29 = 29;
+  volatile int value30 = 30;
+  volatile int value31 = 31;
+
+#pragma omp parallel num_threads(2)                                            \
+    shared(value0, value1, value2, value3, value4, value5, value6, value7,     \
+               value8, value9, value10, value11, value12, value13, value14,    \
+               value15, value16, value17, value18, value19, value20, value21,  \
+               value22, value23, value24, value25, value26, value27, value28,  \
+               value29, value30, value31)
+  {
+    int sum = value0 + value1 + value2 + value3 + value4 + value5 + value6 +
+              value7 + value8 + value9 + value10 + value11 + value12 + value13 +
+              value14 + value15 + value16 + value17 + value18 + value19 +
+              value20 + value21 + value22 + value23 + value24 + value25 +
+              value26 + value27 + value28 + value29 + value30 + value31;
+    if (sum != 496)
+      __builtin_trap();
+  }
+
+  return 0;
+}
diff --git a/openmp/runtime/test/misc_bugs/wasm-microtask-33-args.c b/openmp/runtime/test/misc_bugs/wasm-microtask-33-args.c
new file mode 100644
index 0000000000000..bb88fcb239777
--- /dev/null
+++ b/openmp/runtime/test/misc_bugs/wasm-microtask-33-args.c
@@ -0,0 +1,60 @@
+// REQUIRES: wasm32-target-arch
+// RUN: %libomp-compile
+// RUN: %not %libomp-run 2>&1 | FileCheck %s
+
+// CHECK: Too many args to microtask: 33!
+
+int main(void) {
+  volatile int value0 = 0;
+  volatile int value1 = 1;
+  volatile int value2 = 2;
+  volatile int value3 = 3;
+  volatile int value4 = 4;
+  volatile int value5 = 5;
+  volatile int value6 = 6;
+  volatile int value7 = 7;
+  volatile int value8 = 8;
+  volatile int value9 = 9;
+  volatile int value10 = 10;
+  volatile int value11 = 11;
+  volatile int value12 = 12;
+  volatile int value13 = 13;
+  volatile int value14 = 14;
+  volatile int value15 = 15;
+  volatile int value16 = 16;
+  volatile int value17 = 17;
+  volatile int value18 = 18;
+  volatile int value19 = 19;
+  volatile int value20 = 20;
+  volatile int value21 = 21;
+  volatile int value22 = 22;
+  volatile int value23 = 23;
+  volatile int value24 = 24;
+  volatile int value25 = 25;
+  volatile int value26 = 26;
+  volatile int value27 = 27;
+  volatile int value28 = 28;
+  volatile int value29 = 29;
+  volatile int value30 = 30;
+  volatile int value31 = 31;
+  volatile int value32 = 32;
+
+#pragma omp parallel num_threads(2)                                            \
+    shared(value0, value1, value2, value3, value4, value5, value6, value7,     \
+               value8, value9, value10, value11, value12, value13, value14,    \
+               value15, value16, value17, value18, value19, value20, value21,  \
+               value22, value23, value24, value25, value26, value27, value28,  \
+               value29, value30, value31, value32)
+  {
+    int sum = value0 + value1 + value2 + value3 + value4 + value5 + value6 +
+              value7 + value8 + value9 + value10 + value11 + value12 + value13 +
+              value14 + value15 + value16 + value17 + value18 + value19 +
+              value20 + value21 + value22 + value23 + value24 + value25 +
+              value26 + value27 + value28 + value29 + value30 + value31 +
+              value32;
+    if (sum != 528)
+      __builtin_trap();
+  }
+
+  return 0;
+}

>From 0d4675ceb222fac240640591567daf089a9f6f87 Mon Sep 17 00:00:00 2001
From: Andrew Snyder <asnyder at minitab.com>
Date: Tue, 21 Jul 2026 14:00:54 -0400
Subject: [PATCH 2/3] use make_index_sequence

---
 openmp/runtime/src/kmp_invoke_microtask.cpp | 19 ++++---------------
 1 file changed, 4 insertions(+), 15 deletions(-)

diff --git a/openmp/runtime/src/kmp_invoke_microtask.cpp b/openmp/runtime/src/kmp_invoke_microtask.cpp
index 474423b62188e..ec13eb8af7e97 100644
--- a/openmp/runtime/src/kmp_invoke_microtask.cpp
+++ b/openmp/runtime/src/kmp_invoke_microtask.cpp
@@ -1,27 +1,17 @@
 #include "kmp.h"
 
+#include <utility>
+
 #if !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_MIC || KMP_ARCH_AARCH64 ||        \
       KMP_ARCH_PPC64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 ||            \
       KMP_ARCH_ARM || KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_PPC_XCOFF ||   \
       KMP_ARCH_AARCH64_32)
 
-template <size_t... Indices> struct microtask_index_sequence {};
-
-template <size_t N, size_t... Indices>
-struct make_microtask_index_sequence
-    : make_microtask_index_sequence<N - 1, N - 1, Indices...> {};
-
-template <size_t... Indices>
-struct make_microtask_index_sequence<0, Indices...> {
-  using type = microtask_index_sequence<Indices...>;
-};
-
 template <size_t> using microtask_argument_t = void *;
 
 template <size_t... Indices>
 static void invokeMicrotask(microtask_t pkfn, int *gtid, int *tid,
-                            void *p_argv[],
-                            microtask_index_sequence<Indices...>) {
+                            void *p_argv[], std::index_sequence<Indices...>) {
   // WebAssembly's `call_indirect` requires the callee type to exactly match the
   // call site. Cast the variadic microtask_t to the fixed-arity signature that
   // matches argc before invoking it.
@@ -45,8 +35,7 @@ int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,
 
 #define KMP_INVOKE_MICROTASK_CASE(N)                                           \
   case N:                                                                      \
-    invokeMicrotask(pkfn, &gtid, &tid, p_argv,                                 \
-                    make_microtask_index_sequence<N>::type{});                 \
+    invokeMicrotask(pkfn, &gtid, &tid, p_argv, std::make_index_sequence<N>{}); \
     break
 
   switch (argc) {

>From 33ed4a3e001d2b52a24d678870ce6722f17b6a61 Mon Sep 17 00:00:00 2001
From: Andrew Snyder <asnyder at minitab.com>
Date: Tue, 21 Jul 2026 14:12:13 -0400
Subject: [PATCH 3/3] should work on arm64ec now

---
 openmp/runtime/test/misc_bugs/many-microtask-args.c | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/openmp/runtime/test/misc_bugs/many-microtask-args.c b/openmp/runtime/test/misc_bugs/many-microtask-args.c
index 48ef2a0222d17..b91cfe542079f 100644
--- a/openmp/runtime/test/misc_bugs/many-microtask-args.c
+++ b/openmp/runtime/test/misc_bugs/many-microtask-args.c
@@ -1,10 +1,6 @@
 // RUN: %libomp-compile-and-run
 #include <stdio.h>
 
-// This test fails on Arm64EC as __kmp_invoke_microtask doesn't support more
-// than 16 arguments on that target.
-// XFAIL: arm64ec
-
 int main()
 {
 
@@ -40,4 +36,3 @@ int main()
 
   return 0;
 }
-


From openmp-commits at lists.llvm.org  Mon Aug  3 06:27:04 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Mon, 03 Aug 2026 06:27:04 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [openmp][WebAssembly] add
 support for wasm64 (PR #181669)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/181669@github.com>
Message-ID: <6a709728.0f490953.5a095.af4c@mx.google.com>


https://github.com/arsnyder16 updated https://github.com/llvm/llvm-project/pull/181669

>From f56e99c0c4a1b878dbc4f55c300c18b76c49c5c4 Mon Sep 17 00:00:00 2001
From: Andrew Snyder <asnyder at minitab.com>
Date: Mon, 16 Feb 2026 09:02:43 -0500
Subject: [PATCH 1/5] [openmp][WebAssembly] add support for wasm64

---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     |  2 +-
 openmp/CMakeLists.txt                         |  2 +-
 openmp/cmake/modules/LibompUtils.cmake        |  4 ++++
 openmp/runtime/CMakeLists.txt                 | 19 ++++++++++++-------
 .../runtime/cmake/LibompGetArchitecture.cmake |  2 ++
 openmp/runtime/cmake/config-ix.cmake          |  2 +-
 openmp/runtime/src/kmp_gsupport.cpp           |  2 +-
 openmp/runtime/src/kmp_os.h                   | 11 ++++++-----
 openmp/runtime/src/kmp_platform.h             | 12 ++++++++----
 openmp/runtime/src/kmp_runtime.cpp            |  4 ++--
 openmp/runtime/src/z_Linux_asm.S              |  2 +-
 openmp/runtime/src/z_Linux_util.cpp           |  5 +++--
 12 files changed, 42 insertions(+), 25 deletions(-)

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 464ec5b5a2ece..25285c5fefa31 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -9285,7 +9285,7 @@ GlobalVariable *OpenMPIRBuilder::getOrCreateInternalVariable(
                                : M.getTargetTriple().isAMDGPU()
                                    ? 0
                                    : DL.getDefaultGlobalsAddressSpace();
-    auto Linkage = this->M.getTargetTriple().getArch() == Triple::wasm32
+    auto Linkage = this->M.getTargetTriple().isWasm()
                        ? GlobalValue::InternalLinkage
                        : GlobalValue::CommonLinkage;
     auto *GV = new GlobalVariable(M, Ty, /*IsConstant=*/false, Linkage,
diff --git a/openmp/CMakeLists.txt b/openmp/CMakeLists.txt
index 355c64d28d296..4b7e4dccfac0a 100644
--- a/openmp/CMakeLists.txt
+++ b/openmp/CMakeLists.txt
@@ -140,7 +140,7 @@ set(ENABLE_LIBOMPTARGET ON)
 # Since the device plugins are only supported on Linux anyway,
 # there is no point in trying to compile libomptarget on other OSes.
 # 32-bit systems are not supported either.
-if (APPLE OR WIN32 OR WASM OR NOT "cxx_std_17" IN_LIST CMAKE_CXX_COMPILE_FEATURES
+if (APPLE OR WIN32 OR WASM32 OR WASM64 OR NOT "cxx_std_17" IN_LIST CMAKE_CXX_COMPILE_FEATURES
     OR NOT CMAKE_SIZEOF_VOID_P EQUAL 8 OR "${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
   set(ENABLE_LIBOMPTARGET OFF)
 endif()
diff --git a/openmp/cmake/modules/LibompUtils.cmake b/openmp/cmake/modules/LibompUtils.cmake
index a810881c2544e..1c60ffb242727 100644
--- a/openmp/cmake/modules/LibompUtils.cmake
+++ b/openmp/cmake/modules/LibompUtils.cmake
@@ -121,6 +121,10 @@ function(libomp_get_legal_arch return_arch_string)
     set(${return_arch_string} "SPARC" PARENT_SCOPE)
   elseif(${SPARCV9})
     set(${return_arch_string} "SPARCV9" PARENT_SCOPE)
+  elseif(${WASM32})
+    set(${return_arch_string} "WASM32" PARENT_SCOPE)
+  elseif(${WASM64})
+    set(${return_arch_string} "WASM64" PARENT_SCOPE)
   else()
     set(${return_arch_string} "${LIBOMP_ARCH}" PARENT_SCOPE)
     libomp_warning_say("libomp_get_legal_arch(): Warning: Unknown architecture: Using ${LIBOMP_ARCH}")
diff --git a/openmp/runtime/CMakeLists.txt b/openmp/runtime/CMakeLists.txt
index dc2f2be079bf3..58f8646c2d73d 100644
--- a/openmp/runtime/CMakeLists.txt
+++ b/openmp/runtime/CMakeLists.txt
@@ -24,7 +24,7 @@ if(${OPENMP_STANDALONE_BUILD})
   # If adding a new architecture, take a look at cmake/LibompGetArchitecture.cmake
   libomp_get_architecture(LIBOMP_DETECTED_ARCH)
   set(LIBOMP_ARCH ${LIBOMP_DETECTED_ARCH} CACHE STRING
-    "The architecture to build for (x86_64/i386/arm/ppc/ppc64/ppc64le/aarch64/aarch64_32/mic/mips/mips64/riscv64/loongarch64/ve/s390x/sparc/sparcv9/wasm32).")
+    "The architecture to build for (x86_64/i386/arm/ppc/ppc64/ppc64le/aarch64/aarch64_32/mic/mips/mips64/riscv64/loongarch64/ve/s390x/sparc/sparcv9/wasm32/wasm64).")
   # Should assertions be enabled?  They are on by default.
   set(LIBOMP_ENABLE_ASSERTIONS TRUE CACHE BOOL
     "enable assertions?")
@@ -69,8 +69,10 @@ else() # Part of LLVM build
     set(LIBOMP_ARCH sparcv9)
   elseif(LIBOMP_NATIVE_ARCH MATCHES "sparc")
     set(LIBOMP_ARCH sparc)
-  elseif(LIBOMP_NATIVE_ARCH MATCHES "wasm")
+  elseif(LIBOMP_NATIVE_ARCH MATCHES "wasm32")
     set(LIBOMP_ARCH wasm32)
+  elseif(LIBOMP_NATIVE_ARCH MATCHES "wasm64")
+    set(LIBOMP_ARCH wasm64)
   else()
     # last ditch effort
     libomp_get_architecture(LIBOMP_ARCH)
@@ -91,7 +93,7 @@ if(LIBOMP_ARCH STREQUAL "aarch64")
   endif()
 endif()
 
-libomp_check_variable(LIBOMP_ARCH 32e x86_64 32 i386 arm ppc ppc64 ppc64le aarch64 aarch64_32 aarch64_a64fx mic mips mips64 riscv64 loongarch64 ve s390x sparc sparcv9 wasm32)
+libomp_check_variable(LIBOMP_ARCH 32e x86_64 32 i386 arm ppc ppc64 ppc64le aarch64 aarch64_32 aarch64_a64fx mic mips mips64 riscv64 loongarch64 ve s390x sparc sparcv9 wasm32 wasm64)
 
 set(LIBOMP_LIB_TYPE normal CACHE STRING
   "Performance,Profiling,Stubs library (normal/profile/stubs)")
@@ -186,7 +188,8 @@ set(RISCV64 FALSE)
 set(LOONGARCH64 FALSE)
 set(VE FALSE)
 set(S390X FALSE)
-set(WASM FALSE)
+set(WASM32 FALSE)
+set(WASM64 FALSE)
 set(PPC FALSE)
 set(SPARC FALSE)
 set(SPARCV9 FALSE)
@@ -224,8 +227,10 @@ elseif("${LIBOMP_ARCH}" STREQUAL "ve") # VE architecture
     set(VE TRUE)
 elseif("${LIBOMP_ARCH}" STREQUAL "s390x") # S390x (Z) architecture
     set(S390X TRUE)
-elseif("${LIBOMP_ARCH}" STREQUAL "wasm32") # WebAssembly architecture
-    set(WASM TRUE)
+elseif("${LIBOMP_ARCH}" STREQUAL "wasm32") # WebAssembly 32 bit architecture
+    set(WASM32 TRUE)
+elseif("${LIBOMP_ARCH}" STREQUAL "wasm64") # WebAssembly 64 bit architecture
+    set(WASM64 TRUE)
 elseif("${LIBOMP_ARCH}" STREQUAL "sparc") # SPARC architecture
     set(SPARC TRUE)
 elseif("${LIBOMP_ARCH}" STREQUAL "sparcv9") # SPARC V9 architecture
@@ -334,7 +339,7 @@ endif()
 set(LIBOMP_ENABLE_SHARED TRUE CACHE BOOL
   "Shared library instead of static library?")
 
-if(WASM)
+if(WASM32 OR WASM64)
   libomp_warning_say("The WebAssembly build currently only supports static libraries; forcing LIBOMP_ENABLE_SHARED to false")
   set(LIBOMP_ENABLE_SHARED FALSE)
 endif()
diff --git a/openmp/runtime/cmake/LibompGetArchitecture.cmake b/openmp/runtime/cmake/LibompGetArchitecture.cmake
index 81aa700e3b6db..b69b129aa693f 100644
--- a/openmp/runtime/cmake/LibompGetArchitecture.cmake
+++ b/openmp/runtime/cmake/LibompGetArchitecture.cmake
@@ -59,6 +59,8 @@ function(libomp_get_architecture return_arch)
       #error ARCHITECTURE=s390x
     #elif defined(__wasm32__)
       #error ARCHITECTURE=wasm32
+    #elif defined(__wasm64__)
+      #error ARCHITECTURE=wasm64
     #elif defined(__sparcv9)
       #error ARCHITECTURE=sparcv9
     #elif defined(__sparc)
diff --git a/openmp/runtime/cmake/config-ix.cmake b/openmp/runtime/cmake/config-ix.cmake
index 150a0c3c7217b..fcd02c578465c 100644
--- a/openmp/runtime/cmake/config-ix.cmake
+++ b/openmp/runtime/cmake/config-ix.cmake
@@ -154,7 +154,7 @@ endif()
 # with either the Emscripten or wasi-threads flavor ends up using the pthreads
 # interface in a WebAssembly-compiled libc; CMake does not yet know how to
 # detect this.
-if (NOT WASM)
+if (NOT WASM32 AND NOT WASM64)
   find_package(Threads REQUIRED)
   if(WIN32)
     if(NOT CMAKE_USE_WIN32_THREADS_INIT)
diff --git a/openmp/runtime/src/kmp_gsupport.cpp b/openmp/runtime/src/kmp_gsupport.cpp
index 0d04045f7b165..be95e0f3de6e2 100644
--- a/openmp/runtime/src/kmp_gsupport.cpp
+++ b/openmp/runtime/src/kmp_gsupport.cpp
@@ -357,7 +357,7 @@ void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_ORDERED_END)(void) {
 // They come in two flavors: 64-bit unsigned, and either 32-bit signed
 // (IA-32 architecture) or 64-bit signed (Intel(R) 64).
 
-#if KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_MIPS || KMP_ARCH_WASM ||          \
+#if KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_MIPS || KMP_ARCH_WASM32 ||        \
     KMP_ARCH_PPC || KMP_ARCH_AARCH64_32 || KMP_ARCH_SPARC32
 #define KMP_DISPATCH_INIT __kmp_aux_dispatch_init_4
 #define KMP_DISPATCH_FINI_CHUNK __kmp_aux_dispatch_fini_chunk_4
diff --git a/openmp/runtime/src/kmp_os.h b/openmp/runtime/src/kmp_os.h
index e8ad2a6fdb78e..f74fec5a1f82e 100644
--- a/openmp/runtime/src/kmp_os.h
+++ b/openmp/runtime/src/kmp_os.h
@@ -178,18 +178,18 @@ typedef unsigned long long kmp_uint64;
 #define KMP_UINT64_SPEC "llu"
 #endif /* KMP_OS_UNIX */
 
-#if KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_MIPS || KMP_ARCH_WASM ||          \
+#if KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_MIPS || KMP_ARCH_WASM32 ||        \
     KMP_ARCH_PPC || KMP_ARCH_AARCH64_32 || KMP_ARCH_SPARC32
 #define KMP_SIZE_T_SPEC KMP_UINT32_SPEC
 #elif KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 ||                 \
     KMP_ARCH_MIPS64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 ||             \
-    KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_SPARC64
+    KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_SPARC64 || KMP_ARCH_WASM64
 #define KMP_SIZE_T_SPEC KMP_UINT64_SPEC
 #else
 #error "Can't determine size_t printf format specifier."
 #endif
 
-#if KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_WASM || KMP_ARCH_PPC
+#if KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_WASM32 || KMP_ARCH_PPC
 #define KMP_SIZE_T_MAX (0xFFFFFFFF)
 #else
 #define KMP_SIZE_T_MAX (0xFFFFFFFFFFFFFFFF)
@@ -218,7 +218,8 @@ typedef kmp_uint32 kmp_uint;
 #define KMP_INT_MIN ((kmp_int32)0x80000000)
 
 // stdarg handling
-#if (KMP_ARCH_ARM || KMP_ARCH_X86_64 || KMP_ARCH_AARCH64 || KMP_ARCH_WASM) &&  \
+#if (KMP_ARCH_ARM || KMP_ARCH_X86_64 || KMP_ARCH_AARCH64 || KMP_ARCH_WASM32 || \
+     KMP_ARCH_WASM64) &&                                                       \
     (KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_OPENBSD || KMP_OS_DRAGONFLY ||  \
      KMP_OS_LINUX || KMP_OS_WASI)
 typedef va_list *kmp_va_list;
@@ -1153,7 +1154,7 @@ extern kmp_real64 __kmp_xchg_real64(volatile kmp_real64 *p, kmp_real64 v);
   KMP_COMPARE_AND_STORE_REL64((volatile kmp_int64 *)(volatile void *)&(a),     \
                               (kmp_int64)(b), (kmp_int64)(c))
 
-#if KMP_ARCH_X86 || KMP_ARCH_MIPS || KMP_ARCH_WASM || KMP_ARCH_PPC
+#if KMP_ARCH_X86 || KMP_ARCH_MIPS || KMP_ARCH_WASM32 || KMP_ARCH_PPC
 // What about ARM?
 #define TCR_PTR(a) ((void *)TCR_4(a))
 #define TCW_PTR(a, b) TCW_4((a), (b))
diff --git a/openmp/runtime/src/kmp_platform.h b/openmp/runtime/src/kmp_platform.h
index 609b7c4688842..2d789e95f29bb 100644
--- a/openmp/runtime/src/kmp_platform.h
+++ b/openmp/runtime/src/kmp_platform.h
@@ -245,7 +245,11 @@
 #endif
 
 #if defined(__wasm32__)
-#define KMP_ARCH_WASM 1
+#define KMP_ARCH_WASM32 1
+#endif
+
+#if defined(__wasm64__)
+#define KMP_ARCH_WASM64 1
 #endif
 
 #define KMP_ARCH_PPC64                                                         \
@@ -279,7 +283,7 @@
 
 /* Specify 32 bit architectures here */
 #define KMP_32_BIT_ARCH                                                        \
-  (KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_MIPS || KMP_ARCH_WASM ||           \
+  (KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_MIPS || KMP_ARCH_WASM32 ||         \
    KMP_ARCH_PPC || KMP_ARCH_AARCH64_32 || KMP_ARCH_SPARC32)
 
 // Platforms which support Intel(R) Many Integrated Core Architecture
@@ -290,8 +294,8 @@
 #if (1 != KMP_ARCH_X86 + KMP_ARCH_X86_64 + KMP_ARCH_ARM + KMP_ARCH_PPC64 +     \
               KMP_ARCH_AARCH64 + KMP_ARCH_MIPS + KMP_ARCH_MIPS64 +             \
               KMP_ARCH_RISCV64 + KMP_ARCH_LOONGARCH64 + KMP_ARCH_VE +          \
-              KMP_ARCH_S390X + KMP_ARCH_WASM + KMP_ARCH_PPC +                  \
-              KMP_ARCH_AARCH64_32 + KMP_ARCH_SPARC)
+              KMP_ARCH_S390X + KMP_ARCH_WASM32 + KMP_ARCH_WASM64 +             \
+              KMP_ARCH_PPC + KMP_ARCH_AARCH64_32 + KMP_ARCH_SPARC)
 #error Unknown or unsupported architecture
 #endif
 
diff --git a/openmp/runtime/src/kmp_runtime.cpp b/openmp/runtime/src/kmp_runtime.cpp
index 3277a09c6497f..861c030db3fb1 100644
--- a/openmp/runtime/src/kmp_runtime.cpp
+++ b/openmp/runtime/src/kmp_runtime.cpp
@@ -8924,7 +8924,7 @@ __kmp_determine_reduction_method(
 
 #if KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 ||                   \
     KMP_ARCH_MIPS64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 ||             \
-    KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_WASM
+    KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_WASM64
 
 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD ||     \
     KMP_OS_OPENBSD || KMP_OS_WINDOWS || KMP_OS_DARWIN || KMP_OS_HAIKU ||       \
@@ -8956,7 +8956,7 @@ __kmp_determine_reduction_method(
        // KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_WASI || KMP_OS_AIX
 
 #elif KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_AARCH || KMP_ARCH_MIPS ||       \
-    KMP_ARCH_WASM || KMP_ARCH_PPC || KMP_ARCH_AARCH64_32 || KMP_ARCH_SPARC
+    KMP_ARCH_WASM32 || KMP_ARCH_PPC || KMP_ARCH_AARCH64_32 || KMP_ARCH_SPARC
 
 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD ||     \
     KMP_OS_OPENBSD || KMP_OS_WINDOWS || KMP_OS_HAIKU || KMP_OS_HURD ||         \
diff --git a/openmp/runtime/src/z_Linux_asm.S b/openmp/runtime/src/z_Linux_asm.S
index 12fea67e000e8..bfc8aff325f2a 100644
--- a/openmp/runtime/src/z_Linux_asm.S
+++ b/openmp/runtime/src/z_Linux_asm.S
@@ -2512,7 +2512,7 @@ KMP_PREFIX_UNDERSCORE(__kmp_unnamed_critical_addr):
 #if KMP_OS_LINUX
 # if KMP_ARCH_ARM || KMP_ARCH_AARCH64
 .section .note.GNU-stack,"",%progbits
-# elif !KMP_ARCH_WASM
+# elif !KMP_ARCH_WASM32 && !KMP_ARCH_WASM64 
 .section .note.GNU-stack,"", at progbits
 # endif
 #endif
diff --git a/openmp/runtime/src/z_Linux_util.cpp b/openmp/runtime/src/z_Linux_util.cpp
index c7fe0642cea63..18ffedadee505 100644
--- a/openmp/runtime/src/z_Linux_util.cpp
+++ b/openmp/runtime/src/z_Linux_util.cpp
@@ -321,7 +321,8 @@ int __kmp_futex_determine_capable() {
 
 #endif // KMP_USE_FUTEX
 
-#if (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_WASM) && (!KMP_ASM_INTRINS)
+#if (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_WASM32 || KMP_ARCH_WASM64) && \
+    (!KMP_ASM_INTRINS)
 /* Only 32-bit "add-exchange" instruction on IA-32 architecture causes us to
    use compare_and_store for these routines */
 
@@ -381,7 +382,7 @@ kmp_uint32 __kmp_test_then_and32(volatile kmp_uint32 *p, kmp_uint32 d) {
   return old_value;
 }
 
-#if KMP_ARCH_X86 || KMP_ARCH_WASM
+#if KMP_ARCH_X86 || KMP_ARCH_WASM32
 kmp_int8 __kmp_test_then_add8(volatile kmp_int8 *p, kmp_int8 d) {
   kmp_int8 old_value, new_value;
 

>From 8d5304b51a977b81f63e47491114592b5ee9016f Mon Sep 17 00:00:00 2001
From: Andrew Snyder <asnyder at minitab.com>
Date: Tue, 28 Jul 2026 09:04:30 -0400
Subject: [PATCH 2/5] formatting

---
 openmp/runtime/src/kmp_os.h        | 3 ++-
 openmp/runtime/src/kmp_platform.h  | 5 +++--
 openmp/runtime/src/kmp_runtime.cpp | 6 +-----
 3 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/openmp/runtime/src/kmp_os.h b/openmp/runtime/src/kmp_os.h
index c742e8272999f..be8c121e18736 100644
--- a/openmp/runtime/src/kmp_os.h
+++ b/openmp/runtime/src/kmp_os.h
@@ -183,7 +183,8 @@ typedef unsigned long long kmp_uint64;
 #define KMP_SIZE_T_SPEC KMP_UINT32_SPEC
 #elif KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 ||                 \
     KMP_ARCH_MIPS64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 ||             \
-    KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_SPARC64 || KMP_ARCH_WASM64 || KMP_ARCH_ARM64EC
+    KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_SPARC64 || KMP_ARCH_WASM64 ||    \
+    KMP_ARCH_ARM64EC
 #define KMP_SIZE_T_SPEC KMP_UINT64_SPEC
 #else
 #error "Can't determine size_t printf format specifier."
diff --git a/openmp/runtime/src/kmp_platform.h b/openmp/runtime/src/kmp_platform.h
index 6370047260681..3cafd90225614 100644
--- a/openmp/runtime/src/kmp_platform.h
+++ b/openmp/runtime/src/kmp_platform.h
@@ -298,8 +298,9 @@
 #if (1 != KMP_ARCH_X86 + KMP_ARCH_X86_64 + KMP_ARCH_ARM + KMP_ARCH_PPC64 +     \
               KMP_ARCH_AARCH64 + KMP_ARCH_MIPS + KMP_ARCH_MIPS64 +             \
               KMP_ARCH_RISCV64 + KMP_ARCH_LOONGARCH64 + KMP_ARCH_VE +          \
-              KMP_ARCH_S390X + KMP_ARCH_WASM32 + KMP_ARCH_WASM64 + KMP_ARCH_PPC +                  \
-              KMP_ARCH_AARCH64_32 + KMP_ARCH_SPARC + KMP_ARCH_ARM64EC)
+              KMP_ARCH_S390X + KMP_ARCH_WASM32 + KMP_ARCH_WASM64 +             \
+              KMP_ARCH_PPC + KMP_ARCH_AARCH64_32 + KMP_ARCH_SPARC +            \
+              KMP_ARCH_ARM64EC)
 #error Unknown or unsupported architecture
 #endif
 
diff --git a/openmp/runtime/src/kmp_runtime.cpp b/openmp/runtime/src/kmp_runtime.cpp
index b752991780678..8f7f5c8c7bfe5 100644
--- a/openmp/runtime/src/kmp_runtime.cpp
+++ b/openmp/runtime/src/kmp_runtime.cpp
@@ -8924,11 +8924,7 @@ __kmp_determine_reduction_method(
 
 #if KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 ||                   \
     KMP_ARCH_MIPS64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 ||             \
-<<<<<<< HEAD
-    KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_WASM64
-=======
-    KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_WASM || KMP_ARCH_ARM64EC
->>>>>>> origin/main
+    KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_WASM64 || KMP_ARCH_ARM64EC
 
 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD ||     \
     KMP_OS_OPENBSD || KMP_OS_WINDOWS || KMP_OS_DARWIN || KMP_OS_HAIKU ||       \

>From 1f22451463e5d5ca1317e91b79dce728c590a735 Mon Sep 17 00:00:00 2001
From: Andrew Snyder <asnyder at minitab.com>
Date: Tue, 28 Jul 2026 10:25:15 -0400
Subject: [PATCH 3/5] cleanup

---
 openmp/CMakeLists.txt              | 5 +++--
 openmp/runtime/CMakeLists.txt      | 4 ++++
 openmp/runtime/src/kmp_runtime.cpp | 5 +++--
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/openmp/CMakeLists.txt b/openmp/CMakeLists.txt
index d5d4f79d4a6d4..f9a89859be7af 100644
--- a/openmp/CMakeLists.txt
+++ b/openmp/CMakeLists.txt
@@ -134,11 +134,12 @@ set(OPENMP_TEST_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" CACHE STRING
   "Additional compiler flags to use for testing Fortran programs (e.g. additional module search paths via -fintrinsic-modules-path )")
 
 set(ENABLE_LIBOMPTARGET ON)
-# Currently libomptarget cannot be compiled on Windows or MacOS X.
+# Currently libomptarget cannot be compiled on Windows, MacOS X, or WebAssembly.
 # Since the device plugins are only supported on Linux anyway,
 # there is no point in trying to compile libomptarget on other OSes.
 # 32-bit systems are not supported either.
-if (APPLE OR WIN32 OR WASM32 OR WASM64 OR NOT "cxx_std_17" IN_LIST CMAKE_CXX_COMPILE_FEATURES
+if (APPLE OR WIN32 OR "${CMAKE_SYSTEM_NAME}" MATCHES "^(Emscripten|WASI)$" OR
+    NOT "cxx_std_17" IN_LIST CMAKE_CXX_COMPILE_FEATURES
     OR NOT CMAKE_SIZEOF_VOID_P EQUAL 8 OR "${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
   set(ENABLE_LIBOMPTARGET OFF)
 endif()
diff --git a/openmp/runtime/CMakeLists.txt b/openmp/runtime/CMakeLists.txt
index 61feef08c574d..0256bfaf4a892 100644
--- a/openmp/runtime/CMakeLists.txt
+++ b/openmp/runtime/CMakeLists.txt
@@ -140,12 +140,16 @@ set(LIBOMP_LDFLAGS "" CACHE STRING
 if("${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
   set(LIBOMP_LIBFLAGS "-lperfstat" CACHE STRING
     "Appended user specified linked libs flags. (e.g., -lm)")
+  if("${LIBOMP_ARCH}" STREQUAL "ppc")
+    # PPC (32-bit) on AIX needs libcompiler_rt for __atomic_load_8, etc.
+    set(LIBOMP_LIBFLAGS "${LIBOMP_LIBFLAGS} -lcompiler_rt")
   endif()
 else()
   set(LIBOMP_LIBFLAGS "" CACHE STRING
     "Appended user specified linked libs flags. (e.g., -lm)")
 endif()
 
+if("${LIBOMP_ARCH}" STREQUAL "mips" OR "${LIBOMP_ARCH}" STREQUAL "mips64")
   set(LIBOMP_LIBFLAGS "${LIBOMP_LIBFLAGS} -latomic")
 endif()
 
diff --git a/openmp/runtime/src/kmp_runtime.cpp b/openmp/runtime/src/kmp_runtime.cpp
index 8f7f5c8c7bfe5..234a211bcfba3 100644
--- a/openmp/runtime/src/kmp_runtime.cpp
+++ b/openmp/runtime/src/kmp_runtime.cpp
@@ -8924,7 +8924,8 @@ __kmp_determine_reduction_method(
 
 #if KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 ||                   \
     KMP_ARCH_MIPS64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 ||             \
-    KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_WASM64 || KMP_ARCH_ARM64EC
+    KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_WASM32 || KMP_ARCH_WASM64 ||     \
+    KMP_ARCH_ARM64EC
 
 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD ||     \
     KMP_OS_OPENBSD || KMP_OS_WINDOWS || KMP_OS_DARWIN || KMP_OS_HAIKU ||       \
@@ -8956,7 +8957,7 @@ __kmp_determine_reduction_method(
        // KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_WASI || KMP_OS_AIX
 
 #elif KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_AARCH || KMP_ARCH_MIPS ||       \
-    KMP_ARCH_WASM32 || KMP_ARCH_PPC || KMP_ARCH_AARCH64_32 || KMP_ARCH_SPARC
+    KMP_ARCH_PPC || KMP_ARCH_AARCH64_32 || KMP_ARCH_SPARC
 
 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD ||     \
     KMP_OS_OPENBSD || KMP_OS_WINDOWS || KMP_OS_HAIKU || KMP_OS_HURD ||         \

>From d57aad137058f122e8283cd329b4a6cb4721fd9d Mon Sep 17 00:00:00 2001
From: Andrew Snyder <asnyder at minitab.com>
Date: Tue, 28 Jul 2026 11:02:04 -0400
Subject: [PATCH 4/5] add a code gen test for wasm

---
 clang/test/OpenMP/wasm_codegen.c | 58 ++++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)
 create mode 100644 clang/test/OpenMP/wasm_codegen.c

diff --git a/clang/test/OpenMP/wasm_codegen.c b/clang/test/OpenMP/wasm_codegen.c
new file mode 100644
index 0000000000000..0d40cd68fb710
--- /dev/null
+++ b/clang/test/OpenMP/wasm_codegen.c
@@ -0,0 +1,58 @@
+// REQUIRES: webassembly-registered-target
+// RUN: %clang_cc1 -triple wasm32-unknown-emscripten -fopenmp -fnoopenmp-use-tls -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,WASM32
+// RUN: %clang_cc1 -triple wasm64-unknown-emscripten -fopenmp -fnoopenmp-use-tls -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,WASM64
+// RUN: %clang_cc1 -triple wasm32-unknown-emscripten -fopenmp -fnoopenmp-use-tls -emit-obj -o %t.wasm32.o %s
+// RUN: %clang_cc1 -triple wasm64-unknown-emscripten -fopenmp -fnoopenmp-use-tls -emit-obj -o %t.wasm64.o %s
+
+// Exercise representative host OpenMP lowering for both WebAssembly pointer
+// models. WebAssembly does not support common linkage, so compiler-generated
+// locks and caches must have internal linkage.
+// CHECK-DAG: @.gomp_critical_user_.var = internal global [8 x i32] zeroinitializer
+// CHECK-DAG: @.gomp_critical_user_named.var = internal global [8 x i32] zeroinitializer
+// CHECK-DAG: @.gomp_critical_user_.reduction.var = internal global [8 x i32] zeroinitializer
+// CHECK-DAG: @threadprivate_var.cache. = internal global ptr null
+
+// The dependency record and size_t parameters must follow the target pointer
+// width rather than assuming wasm32.
+// WASM32-DAG: %struct.kmp_depend_info = type { i32, i32, i8 }
+// WASM64-DAG: %struct.kmp_depend_info = type { i64, i64, i8 }
+// WASM32-DAG: call ptr @__kmpc_omp_task_alloc({{.*}}i32 {{[0-9]+}}, i32 {{[0-9]+}}, ptr
+// WASM64-DAG: call ptr @__kmpc_omp_task_alloc({{.*}}i64 {{[0-9]+}}, i64 {{[0-9]+}}, ptr
+// WASM32-DAG: call ptr @__kmpc_threadprivate_cached({{.*}}i32 4, ptr @threadprivate_var.cache.)
+// WASM64-DAG: call ptr @__kmpc_threadprivate_cached({{.*}}i64 4, ptr @threadprivate_var.cache.)
+// WASM32-DAG: call i32 @__kmpc_reduce_nowait({{.*}}i32 4, ptr
+// WASM64-DAG: call i32 @__kmpc_reduce_nowait({{.*}}i64 8, ptr
+
+// CHECK-DAG: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(
+// CHECK-DAG: call void @__kmpc_for_static_init_4(
+// CHECK-DAG: call i32 @__kmpc_omp_task_with_deps(
+// CHECK-DAG: call i32 @__kmpc_omp_taskwait(
+// CHECK-DAG: call void @__kmpc_critical(
+
+void critical_regions(void) {
+#pragma omp critical
+  {}
+
+#pragma omp critical(named)
+  {}
+}
+
+int threadprivate_var;
+#pragma omp threadprivate(threadprivate_var)
+
+int openmp_constructs(int *values, int count) {
+  int sum = 0;
+#pragma omp parallel for reduction(+ : sum)
+  for (int i = 0; i < count; ++i)
+    sum += values[i];
+
+#pragma omp task shared(values) depend(inout : values[0 : count])
+  values[0] = sum;
+
+#pragma omp taskwait
+
+#pragma omp critical(named)
+  threadprivate_var += sum;
+
+  return sum;
+}

>From 3e64bf4050c054842de5b4584971d93a0c7bf726 Mon Sep 17 00:00:00 2001
From: Andrew Snyder <asnyder at minitab.com>
Date: Tue, 28 Jul 2026 11:20:26 -0400
Subject: [PATCH 5/5] consistency

---
 openmp/cmake/modules/LibompUtils.cmake | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/openmp/cmake/modules/LibompUtils.cmake b/openmp/cmake/modules/LibompUtils.cmake
index 2a639bccc47b6..9dd80c28dcaf0 100644
--- a/openmp/cmake/modules/LibompUtils.cmake
+++ b/openmp/cmake/modules/LibompUtils.cmake
@@ -125,9 +125,9 @@ function(libomp_get_legal_arch return_arch_string)
     set(${return_arch_string} "SPARC" PARENT_SCOPE)
   elseif(SPARCV9)
     set(${return_arch_string} "SPARCV9" PARENT_SCOPE)
-  elseif(${WASM32})
+  elseif(WASM32)
     set(${return_arch_string} "WASM32" PARENT_SCOPE)
-  elseif(${WASM64})
+  elseif(WASM64)
     set(${return_arch_string} "WASM64" PARENT_SCOPE)
   else()
     set(${return_arch_string} "${LIBOMP_ARCH}" PARENT_SCOPE)


From openmp-commits at lists.llvm.org  Mon Aug  3 06:32:04 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Mon, 03 Aug 2026 06:32:04 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP] Support up to 32 arguments in
 the generic microtask dispatcher (PR #211071)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/211071@github.com>
Message-ID: <6a709854.d1496011.a1901.9fb9@mx.google.com>


https://github.com/arsnyder16 updated https://github.com/llvm/llvm-project/pull/211071

>From 7b76acc6c9fa6222b4360dd90ce465df1acaff2a Mon Sep 17 00:00:00 2001
From: Andrew Snyder <asnyder at minitab.com>
Date: Tue, 21 Jul 2026 13:39:00 -0400
Subject: [PATCH 1/3] [OpenMP] Support up to 32 arguments in the generic
 microtask dispatcher

---
 openmp/runtime/src/kmp_invoke_microtask.cpp   | 177 +++++++-----------
 .../test/misc_bugs/microtask-32-args.c        |  56 ++++++
 .../test/misc_bugs/wasm-microtask-33-args.c   |  60 ++++++
 3 files changed, 183 insertions(+), 110 deletions(-)
 create mode 100644 openmp/runtime/test/misc_bugs/microtask-32-args.c
 create mode 100644 openmp/runtime/test/misc_bugs/wasm-microtask-33-args.c

diff --git a/openmp/runtime/src/kmp_invoke_microtask.cpp b/openmp/runtime/src/kmp_invoke_microtask.cpp
index f14f27cc06ee3..474423b62188e 100644
--- a/openmp/runtime/src/kmp_invoke_microtask.cpp
+++ b/openmp/runtime/src/kmp_invoke_microtask.cpp
@@ -5,45 +5,33 @@
       KMP_ARCH_ARM || KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_PPC_XCOFF ||   \
       KMP_ARCH_AARCH64_32)
 
-// Because WebAssembly will use `call_indirect` to invoke the microtask and
-// WebAssembly indirect calls check that the called signature is a precise
-// match, we need to cast each microtask function pointer back from `void *` to
-// its original type.
-typedef void (*microtask_t0)(int *, int *);
-typedef void (*microtask_t1)(int *, int *, void *);
-typedef void (*microtask_t2)(int *, int *, void *, void *);
-typedef void (*microtask_t3)(int *, int *, void *, void *, void *);
-typedef void (*microtask_t4)(int *, int *, void *, void *, void *, void *);
-typedef void (*microtask_t5)(int *, int *, void *, void *, void *, void *,
-                             void *);
-typedef void (*microtask_t6)(int *, int *, void *, void *, void *, void *,
-                             void *, void *);
-typedef void (*microtask_t7)(int *, int *, void *, void *, void *, void *,
-                             void *, void *, void *);
-typedef void (*microtask_t8)(int *, int *, void *, void *, void *, void *,
-                             void *, void *, void *, void *);
-typedef void (*microtask_t9)(int *, int *, void *, void *, void *, void *,
-                             void *, void *, void *, void *, void *);
-typedef void (*microtask_t10)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *);
-typedef void (*microtask_t11)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *);
-typedef void (*microtask_t12)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *);
-typedef void (*microtask_t13)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *, void *);
-typedef void (*microtask_t14)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *, void *, void *);
-typedef void (*microtask_t15)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *);
+template <size_t... Indices> struct microtask_index_sequence {};
 
-// we really only need the case with 1 argument, because CLANG always build
-// a struct of pointers to shared variables referenced in the outlined function
+template <size_t N, size_t... Indices>
+struct make_microtask_index_sequence
+    : make_microtask_index_sequence<N - 1, N - 1, Indices...> {};
+
+template <size_t... Indices>
+struct make_microtask_index_sequence<0, Indices...> {
+  using type = microtask_index_sequence<Indices...>;
+};
+
+template <size_t> using microtask_argument_t = void *;
+
+template <size_t... Indices>
+static void invokeMicrotask(microtask_t pkfn, int *gtid, int *tid,
+                            void *p_argv[],
+                            microtask_index_sequence<Indices...>) {
+  // WebAssembly's `call_indirect` requires the callee type to exactly match the
+  // call site. Cast the variadic microtask_t to the fixed-arity signature that
+  // matches argc before invoking it.
+  using typed_microtask_t =
+      void (*)(int *, int *, microtask_argument_t<Indices>...);
+  (*(typed_microtask_t)pkfn)(gtid, tid, p_argv[Indices]...);
+}
+
+// Keep a bounded set of exact signatures for targets that cannot dynamically
+// construct a variable-argument microtask call.
 int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,
                            void *p_argv[]
 #if OMPT_SUPPORT
@@ -55,85 +43,54 @@ int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,
   *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
 #endif
 
+#define KMP_INVOKE_MICROTASK_CASE(N)                                           \
+  case N:                                                                      \
+    invokeMicrotask(pkfn, &gtid, &tid, p_argv,                                 \
+                    make_microtask_index_sequence<N>::type{});                 \
+    break
+
   switch (argc) {
+    KMP_INVOKE_MICROTASK_CASE(0);
+    KMP_INVOKE_MICROTASK_CASE(1);
+    KMP_INVOKE_MICROTASK_CASE(2);
+    KMP_INVOKE_MICROTASK_CASE(3);
+    KMP_INVOKE_MICROTASK_CASE(4);
+    KMP_INVOKE_MICROTASK_CASE(5);
+    KMP_INVOKE_MICROTASK_CASE(6);
+    KMP_INVOKE_MICROTASK_CASE(7);
+    KMP_INVOKE_MICROTASK_CASE(8);
+    KMP_INVOKE_MICROTASK_CASE(9);
+    KMP_INVOKE_MICROTASK_CASE(10);
+    KMP_INVOKE_MICROTASK_CASE(11);
+    KMP_INVOKE_MICROTASK_CASE(12);
+    KMP_INVOKE_MICROTASK_CASE(13);
+    KMP_INVOKE_MICROTASK_CASE(14);
+    KMP_INVOKE_MICROTASK_CASE(15);
+    KMP_INVOKE_MICROTASK_CASE(16);
+    KMP_INVOKE_MICROTASK_CASE(17);
+    KMP_INVOKE_MICROTASK_CASE(18);
+    KMP_INVOKE_MICROTASK_CASE(19);
+    KMP_INVOKE_MICROTASK_CASE(20);
+    KMP_INVOKE_MICROTASK_CASE(21);
+    KMP_INVOKE_MICROTASK_CASE(22);
+    KMP_INVOKE_MICROTASK_CASE(23);
+    KMP_INVOKE_MICROTASK_CASE(24);
+    KMP_INVOKE_MICROTASK_CASE(25);
+    KMP_INVOKE_MICROTASK_CASE(26);
+    KMP_INVOKE_MICROTASK_CASE(27);
+    KMP_INVOKE_MICROTASK_CASE(28);
+    KMP_INVOKE_MICROTASK_CASE(29);
+    KMP_INVOKE_MICROTASK_CASE(30);
+    KMP_INVOKE_MICROTASK_CASE(31);
+    KMP_INVOKE_MICROTASK_CASE(32);
   default:
     fprintf(stderr, "Too many args to microtask: %d!\n", argc);
     fflush(stderr);
     exit(-1);
-  case 0:
-    (*(microtask_t0)pkfn)(&gtid, &tid);
-    break;
-  case 1:
-    (*(microtask_t1)pkfn)(&gtid, &tid, p_argv[0]);
-    break;
-  case 2:
-    (*(microtask_t2)pkfn)(&gtid, &tid, p_argv[0], p_argv[1]);
-    break;
-  case 3:
-    (*(microtask_t3)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2]);
-    break;
-  case 4:
-    (*(microtask_t4)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3]);
-    break;
-  case 5:
-    (*(microtask_t5)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3], p_argv[4]);
-    break;
-  case 6:
-    (*(microtask_t6)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3], p_argv[4], p_argv[5]);
-    break;
-  case 7:
-    (*(microtask_t7)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3], p_argv[4], p_argv[5], p_argv[6]);
-    break;
-  case 8:
-    (*(microtask_t8)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                          p_argv[7]);
-    break;
-  case 9:
-    (*(microtask_t9)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3], p_argv[4], p_argv[5], p_argv[6], p_argv[7],
-                          p_argv[8]);
-    break;
-  case 10:
-    (*(microtask_t10)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9]);
-    break;
-  case 11:
-    (*(microtask_t11)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9], p_argv[10]);
-    break;
-  case 12:
-    (*(microtask_t12)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9], p_argv[10],
-                           p_argv[11]);
-    break;
-  case 13:
-    (*(microtask_t13)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9], p_argv[10],
-                           p_argv[11], p_argv[12]);
-    break;
-  case 14:
-    (*(microtask_t14)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9], p_argv[10],
-                           p_argv[11], p_argv[12], p_argv[13]);
-    break;
-  case 15:
-    (*(microtask_t15)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9], p_argv[10],
-                           p_argv[11], p_argv[12], p_argv[13], p_argv[14]);
-    break;
   }
 
+#undef KMP_INVOKE_MICROTASK_CASE
+
   return 1;
 }
 
diff --git a/openmp/runtime/test/misc_bugs/microtask-32-args.c b/openmp/runtime/test/misc_bugs/microtask-32-args.c
new file mode 100644
index 0000000000000..848ab7c368481
--- /dev/null
+++ b/openmp/runtime/test/misc_bugs/microtask-32-args.c
@@ -0,0 +1,56 @@
+// RUN: %libomp-compile-and-run
+
+// Exercise the upper bound supported by the generic microtask dispatcher.
+
+int main(void) {
+  volatile int value0 = 0;
+  volatile int value1 = 1;
+  volatile int value2 = 2;
+  volatile int value3 = 3;
+  volatile int value4 = 4;
+  volatile int value5 = 5;
+  volatile int value6 = 6;
+  volatile int value7 = 7;
+  volatile int value8 = 8;
+  volatile int value9 = 9;
+  volatile int value10 = 10;
+  volatile int value11 = 11;
+  volatile int value12 = 12;
+  volatile int value13 = 13;
+  volatile int value14 = 14;
+  volatile int value15 = 15;
+  volatile int value16 = 16;
+  volatile int value17 = 17;
+  volatile int value18 = 18;
+  volatile int value19 = 19;
+  volatile int value20 = 20;
+  volatile int value21 = 21;
+  volatile int value22 = 22;
+  volatile int value23 = 23;
+  volatile int value24 = 24;
+  volatile int value25 = 25;
+  volatile int value26 = 26;
+  volatile int value27 = 27;
+  volatile int value28 = 28;
+  volatile int value29 = 29;
+  volatile int value30 = 30;
+  volatile int value31 = 31;
+
+#pragma omp parallel num_threads(2)                                            \
+    shared(value0, value1, value2, value3, value4, value5, value6, value7,     \
+               value8, value9, value10, value11, value12, value13, value14,    \
+               value15, value16, value17, value18, value19, value20, value21,  \
+               value22, value23, value24, value25, value26, value27, value28,  \
+               value29, value30, value31)
+  {
+    int sum = value0 + value1 + value2 + value3 + value4 + value5 + value6 +
+              value7 + value8 + value9 + value10 + value11 + value12 + value13 +
+              value14 + value15 + value16 + value17 + value18 + value19 +
+              value20 + value21 + value22 + value23 + value24 + value25 +
+              value26 + value27 + value28 + value29 + value30 + value31;
+    if (sum != 496)
+      __builtin_trap();
+  }
+
+  return 0;
+}
diff --git a/openmp/runtime/test/misc_bugs/wasm-microtask-33-args.c b/openmp/runtime/test/misc_bugs/wasm-microtask-33-args.c
new file mode 100644
index 0000000000000..bb88fcb239777
--- /dev/null
+++ b/openmp/runtime/test/misc_bugs/wasm-microtask-33-args.c
@@ -0,0 +1,60 @@
+// REQUIRES: wasm32-target-arch
+// RUN: %libomp-compile
+// RUN: %not %libomp-run 2>&1 | FileCheck %s
+
+// CHECK: Too many args to microtask: 33!
+
+int main(void) {
+  volatile int value0 = 0;
+  volatile int value1 = 1;
+  volatile int value2 = 2;
+  volatile int value3 = 3;
+  volatile int value4 = 4;
+  volatile int value5 = 5;
+  volatile int value6 = 6;
+  volatile int value7 = 7;
+  volatile int value8 = 8;
+  volatile int value9 = 9;
+  volatile int value10 = 10;
+  volatile int value11 = 11;
+  volatile int value12 = 12;
+  volatile int value13 = 13;
+  volatile int value14 = 14;
+  volatile int value15 = 15;
+  volatile int value16 = 16;
+  volatile int value17 = 17;
+  volatile int value18 = 18;
+  volatile int value19 = 19;
+  volatile int value20 = 20;
+  volatile int value21 = 21;
+  volatile int value22 = 22;
+  volatile int value23 = 23;
+  volatile int value24 = 24;
+  volatile int value25 = 25;
+  volatile int value26 = 26;
+  volatile int value27 = 27;
+  volatile int value28 = 28;
+  volatile int value29 = 29;
+  volatile int value30 = 30;
+  volatile int value31 = 31;
+  volatile int value32 = 32;
+
+#pragma omp parallel num_threads(2)                                            \
+    shared(value0, value1, value2, value3, value4, value5, value6, value7,     \
+               value8, value9, value10, value11, value12, value13, value14,    \
+               value15, value16, value17, value18, value19, value20, value21,  \
+               value22, value23, value24, value25, value26, value27, value28,  \
+               value29, value30, value31, value32)
+  {
+    int sum = value0 + value1 + value2 + value3 + value4 + value5 + value6 +
+              value7 + value8 + value9 + value10 + value11 + value12 + value13 +
+              value14 + value15 + value16 + value17 + value18 + value19 +
+              value20 + value21 + value22 + value23 + value24 + value25 +
+              value26 + value27 + value28 + value29 + value30 + value31 +
+              value32;
+    if (sum != 528)
+      __builtin_trap();
+  }
+
+  return 0;
+}

>From 0d4675ceb222fac240640591567daf089a9f6f87 Mon Sep 17 00:00:00 2001
From: Andrew Snyder <asnyder at minitab.com>
Date: Tue, 21 Jul 2026 14:00:54 -0400
Subject: [PATCH 2/3] use make_index_sequence

---
 openmp/runtime/src/kmp_invoke_microtask.cpp | 19 ++++---------------
 1 file changed, 4 insertions(+), 15 deletions(-)

diff --git a/openmp/runtime/src/kmp_invoke_microtask.cpp b/openmp/runtime/src/kmp_invoke_microtask.cpp
index 474423b62188e..ec13eb8af7e97 100644
--- a/openmp/runtime/src/kmp_invoke_microtask.cpp
+++ b/openmp/runtime/src/kmp_invoke_microtask.cpp
@@ -1,27 +1,17 @@
 #include "kmp.h"
 
+#include <utility>
+
 #if !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_MIC || KMP_ARCH_AARCH64 ||        \
       KMP_ARCH_PPC64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 ||            \
       KMP_ARCH_ARM || KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_PPC_XCOFF ||   \
       KMP_ARCH_AARCH64_32)
 
-template <size_t... Indices> struct microtask_index_sequence {};
-
-template <size_t N, size_t... Indices>
-struct make_microtask_index_sequence
-    : make_microtask_index_sequence<N - 1, N - 1, Indices...> {};
-
-template <size_t... Indices>
-struct make_microtask_index_sequence<0, Indices...> {
-  using type = microtask_index_sequence<Indices...>;
-};
-
 template <size_t> using microtask_argument_t = void *;
 
 template <size_t... Indices>
 static void invokeMicrotask(microtask_t pkfn, int *gtid, int *tid,
-                            void *p_argv[],
-                            microtask_index_sequence<Indices...>) {
+                            void *p_argv[], std::index_sequence<Indices...>) {
   // WebAssembly's `call_indirect` requires the callee type to exactly match the
   // call site. Cast the variadic microtask_t to the fixed-arity signature that
   // matches argc before invoking it.
@@ -45,8 +35,7 @@ int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,
 
 #define KMP_INVOKE_MICROTASK_CASE(N)                                           \
   case N:                                                                      \
-    invokeMicrotask(pkfn, &gtid, &tid, p_argv,                                 \
-                    make_microtask_index_sequence<N>::type{});                 \
+    invokeMicrotask(pkfn, &gtid, &tid, p_argv, std::make_index_sequence<N>{}); \
     break
 
   switch (argc) {

>From 33ed4a3e001d2b52a24d678870ce6722f17b6a61 Mon Sep 17 00:00:00 2001
From: Andrew Snyder <asnyder at minitab.com>
Date: Tue, 21 Jul 2026 14:12:13 -0400
Subject: [PATCH 3/3] should work on arm64ec now

---
 openmp/runtime/test/misc_bugs/many-microtask-args.c | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/openmp/runtime/test/misc_bugs/many-microtask-args.c b/openmp/runtime/test/misc_bugs/many-microtask-args.c
index 48ef2a0222d17..b91cfe542079f 100644
--- a/openmp/runtime/test/misc_bugs/many-microtask-args.c
+++ b/openmp/runtime/test/misc_bugs/many-microtask-args.c
@@ -1,10 +1,6 @@
 // RUN: %libomp-compile-and-run
 #include <stdio.h>
 
-// This test fails on Arm64EC as __kmp_invoke_microtask doesn't support more
-// than 16 arguments on that target.
-// XFAIL: arm64ec
-
 int main()
 {
 
@@ -40,4 +36,3 @@ int main()
 
   return 0;
 }
-


From openmp-commits at lists.llvm.org  Mon Aug  3 08:29:57 2026
From: openmp-commits at lists.llvm.org (=?UTF-8?Q?Jan_Andr=C3=A9_Reuter?= via Openmp-commits)
Date: Mon, 03 Aug 2026 08:29:57 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMP][OMPT] Error when
 registering EMI and non-EMI callbacks (PR #213697)
Message-ID: <llvm.org/llvm/llvm-project/pull/213697@github.com>


https://github.com/Thyre created https://github.com/llvm/llvm-project/pull/213697

The OpenMP specification includes a restriction for registering both EMI and non-EMI callbacks (e.g. `ompt_callback_target`, OpenMP v5.2, p. 503, l. 18-20):

  These callbacks must not be registered at the same time.

However, this was not respected in the runtime. A tool was able to register both, receiving `ompt_set_always` for each `ompt_set_callback` call. When an event associated with the EMI or non-EMI callbacks was dispatched and both callbacks were registered, only the EMI version was dispatched.

To align the runtime behavior with the OpenMP specification, return `ompt_set_error` when a tool tries to register both the EMI and non-EMI variant for a callback. With this, only the tool's first registration is used.

Update the offload `veccopy_disallow_both.c` test, which already tested the prior, non-compliant, behavior.

>From dbd22185eb20279c40e21ed391f3993cbc917d5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Andr=C3=A9=20Reuter?= <jan at zyten.de>
Date: Mon, 3 Aug 2026 17:29:16 +0200
Subject: [PATCH] [OpenMP][OMPT] Error when registering EMI and non-EMI
 callbacks
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The OpenMP specification includes a restriction for registering both EMI and
non-EMI callbacks (e.g. `ompt_callback_target`, OpenMP v5.2, p. 503, l. 18-20):

  These callbacks must not be registered at the same time.

However, this was not respected in the runtime. A tool was able to register
both, receiving `ompt_set_always` for each `ompt_set_callback` call.
When an event associated with the EMI or non-EMI callbacks was dispatched
and both callbacks were registered, only the EMI version was dispatched.

To align the runtime behavior with the OpenMP specification, return
`ompt_set_error` when a tool tries to register both the EMI and non-EMI variant
for a callback. With this, only the tool's first registration is used.

Update the offload `veccopy_disallow_both.c` test, which already tested the
prior, non-compliant, behavior.

Signed-off-by: Jan André Reuter <jan at zyten.de>
---
 offload/test/ompt/register_both.h          |  4 +++-
 offload/test/ompt/veccopy_disallow_both.c  | 19 ++++++++-------
 openmp/runtime/src/include/omp-tools.h.var | 13 +++++++++++
 openmp/runtime/src/ompt-general.cpp        | 27 +++++++++++++++++++++-
 4 files changed, 53 insertions(+), 10 deletions(-)

diff --git a/offload/test/ompt/register_both.h b/offload/test/ompt/register_both.h
index 77cb430aa1f97..4e9157c2b51ed 100644
--- a/offload/test/ompt/register_both.h
+++ b/offload/test/ompt/register_both.h
@@ -4,7 +4,9 @@
 #define register_ompt_callback_t(name, type)                                   \
   do {                                                                         \
     type f_##name = &on_##name;                                                \
-    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+    ompt_set_result_t result =                                                 \
+      ompt_set_callback(name, (ompt_callback_t)f_##name);                      \
+    if (result == ompt_set_never || result == ompt_set_error)                  \
       printf("0: Could not register callback '" #name "'\n");                  \
   } while (0)
 
diff --git a/offload/test/ompt/veccopy_disallow_both.c b/offload/test/ompt/veccopy_disallow_both.c
index bfc67c5f4d274..2e24529042b37 100644
--- a/offload/test/ompt/veccopy_disallow_both.c
+++ b/offload/test/ompt/veccopy_disallow_both.c
@@ -7,8 +7,9 @@
 /*
  * Example OpenMP program that shows that both EMI and non-EMI
  * callbacks cannot be registered for the same type. In the
- * current implementation, the EMI callback overrides the non-EMI
- * callback.
+ * current implementation, the first registered callback takes
+ * precedence. Trying to register the other callback results in
+ * ompt_set_error for this callback.
  */
 
 #include <omp.h>
@@ -57,9 +58,11 @@ int main() {
 }
 
 // clang-format off
+/// CHECK: [[THREAD_ID:[0-9]+]]: Could not register callback 'ompt_callback_target_data_op'
+/// CHECK: [[THREAD_ID]]: Could not register callback 'ompt_callback_target_emi'
 /// CHECK: Callback Init:
 /// CHECK: Callback Load:
-/// CHECK: Callback Target EMI: kind=ompt_target endpoint=ompt_scope_begin
+/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=ompt_target endpoint=ompt_scope_begin
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_alloc
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_alloc
 /// CHECK-NOT: dest=(nil)
@@ -70,7 +73,7 @@ int main() {
 /// CHECK-NOT: dest=(nil)
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_transfer_to_device
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_transfer_to_device
-/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1
+/// CHECK: Callback Submit: target_id=[[TARGET_ID]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_transfer_from_device
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_transfer_from_device
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_transfer_from_device
@@ -79,8 +82,8 @@ int main() {
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_delete
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_delete
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_delete
-/// CHECK: Callback Target EMI: kind=ompt_target endpoint=ompt_scope_end
-/// CHECK: Callback Target EMI: kind=ompt_target endpoint=ompt_scope_begin
+/// CHECK: Callback Target: target_id=[[TARGET_ID]] kind=ompt_target endpoint=ompt_scope_end
+/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=ompt_target endpoint=ompt_scope_begin
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_alloc
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_alloc
 /// CHECK-NOT: dest=(nil)
@@ -91,7 +94,7 @@ int main() {
 /// CHECK-NOT: dest=(nil)
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_transfer_to_device
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_transfer_to_device
-/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0
+/// CHECK: Callback Submit: target_id=[[TARGET_ID]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_transfer_from_device
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_transfer_from_device
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_transfer_from_device
@@ -100,5 +103,5 @@ int main() {
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_delete
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_delete
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_delete
-/// CHECK: Callback Target EMI: kind=ompt_target endpoint=ompt_scope_end
+/// CHECK: Callback Target: target_id=[[TARGET_ID]] kind=ompt_target endpoint=ompt_scope_end
 /// CHECK: Callback Fini:
diff --git a/openmp/runtime/src/include/omp-tools.h.var b/openmp/runtime/src/include/omp-tools.h.var
index af9213a7290e7..46643dff5feab 100644
--- a/openmp/runtime/src/include/omp-tools.h.var
+++ b/openmp/runtime/src/include/omp-tools.h.var
@@ -185,6 +185,19 @@
     /* Optional Events */                                                                                                \
     macro (ompt_callback_target_map_emi,    ompt_callback_target_map_emi_t,    36) /* target map                      */
 
+#define FOREACH_MAPPED_OMPT_TARGET_EVENT(macro)                                                                          \
+    /* macro including correlated callback enum (e.g. target -- target_emi) for comparisons between both */              \
+    /*--- Mandatory Events ---*/                                                                                         \
+    macro (ompt_callback_target,             ompt_callback_target_emi,         ompt_callback_target_t,              8)   \
+    macro (ompt_callback_target_data_op,     ompt_callback_target_data_op_emi, ompt_callback_target_data_op_t,      9)   \
+    macro (ompt_callback_target_submit,      ompt_callback_target_submit_emi,  ompt_callback_target_submit_t,      10)   \
+    macro (ompt_callback_target_emi,         ompt_callback_target,             ompt_callback_target_emi_t,         33)   \
+    macro (ompt_callback_target_data_op_emi, ompt_callback_target_data_op,     ompt_callback_target_data_op_emi_t, 34)   \
+    macro (ompt_callback_target_submit_emi,  ompt_callback_target_submit,      ompt_callback_target_submit_emi_t,  35)   \
+    /* Optional Events */                                                                                                \
+    macro (ompt_callback_target_map,         ompt_callback_target_map_emi,     ompt_callback_target_map_t,         22)   \
+    macro (ompt_callback_target_map_emi,     ompt_callback_target_map,         ompt_callback_target_map_emi_t,     36)
+
 #define FOREACH_OMPT_50_TARGET_EVENT(macro)                                                                              \
     FOREACH_OMPT_DEVICE_EVENT(macro)                                                                                     \
     FOREACH_OMPT_NOEMI_EVENT(macro) 
diff --git a/openmp/runtime/src/ompt-general.cpp b/openmp/runtime/src/ompt-general.cpp
index 1682f7e74bd14..1cb2fb2693a2c 100644
--- a/openmp/runtime/src/ompt-general.cpp
+++ b/openmp/runtime/src/ompt-general.cpp
@@ -597,10 +597,35 @@ OMPT_API_ROUTINE ompt_set_result_t ompt_set_callback(ompt_callbacks_t which,
     else                                                                       \
       return ompt_set_always;
 
-    FOREACH_OMPT_EVENT(ompt_event_macro)
+    FOREACH_OMPT_HOST_EVENT(ompt_event_macro)
+    FOREACH_OMPT_DEVICE_EVENT(ompt_event_macro)
 
 #undef ompt_event_macro
 
+    // OpenMP v5.2, p. 503, l. 18-20:
+    // Restrictions to the ompt_callback_target_emi and ompt_callback_target
+    // callbacks are as follows:
+    //   These callbacks must not be registered at the same time.
+    //
+    // Similar restrictions apply to target_data_op, target_submit,
+    // and target_map. Hence, handle registration of them separately to ensure
+    // only one can be registered at the same time. */
+#define ompt_target_event_macro(event_name, comparison_name, callback_type,    \
+                                event_id)                                      \
+  case event_name:                                                             \
+    if (ompt_callbacks.ompt_callback(comparison_name))                         \
+      return ompt_set_error;                                                   \
+    ompt_callbacks.ompt_callback(event_name) = (callback_type)callback;        \
+    ompt_enabled.event_name = (callback != 0);                                 \
+    if (callback)                                                              \
+      return ompt_event_implementation_status(event_name);                     \
+    else                                                                       \
+      return ompt_set_always;
+
+    FOREACH_MAPPED_OMPT_TARGET_EVENT(ompt_target_event_macro);
+
+#undef ompt_target_event_macro
+
   default:
     return ompt_set_error;
   }


From openmp-commits at lists.llvm.org  Mon Aug  3 08:30:01 2026
From: openmp-commits at lists.llvm.org (Sam Clegg via Openmp-commits)
Date: Mon, 03 Aug 2026 08:30:01 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP] Support up to 32 arguments in
 the generic microtask dispatcher (PR #211071)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/211071@github.com>
Message-ID: <6a70b3f9.1728eaf3.f34fc.6dd3@mx.google.com>


https://github.com/sbc100 edited https://github.com/llvm/llvm-project/pull/211071

From openmp-commits at lists.llvm.org  Mon Aug  3 08:30:01 2026
From: openmp-commits at lists.llvm.org (Sam Clegg via Openmp-commits)
Date: Mon, 03 Aug 2026 08:30:01 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP] Support up to 32 arguments in
 the generic microtask dispatcher (PR #211071)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/211071@github.com>
Message-ID: <6a70b3f9.3cce8983.2002c6.1eb9@mx.google.com>


================
@@ -1,49 +1,27 @@
 #include "kmp.h"
 
+#include <utility>
+
 #if !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_MIC || KMP_ARCH_AARCH64 ||        \
       KMP_ARCH_PPC64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 ||            \
       KMP_ARCH_ARM || KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_PPC_XCOFF ||   \
       KMP_ARCH_AARCH64_32)
 
-// Because WebAssembly will use `call_indirect` to invoke the microtask and
-// WebAssembly indirect calls check that the called signature is a precise
-// match, we need to cast each microtask function pointer back from `void *` to
-// its original type.
-typedef void (*microtask_t0)(int *, int *);
-typedef void (*microtask_t1)(int *, int *, void *);
-typedef void (*microtask_t2)(int *, int *, void *, void *);
-typedef void (*microtask_t3)(int *, int *, void *, void *, void *);
-typedef void (*microtask_t4)(int *, int *, void *, void *, void *, void *);
-typedef void (*microtask_t5)(int *, int *, void *, void *, void *, void *,
-                             void *);
-typedef void (*microtask_t6)(int *, int *, void *, void *, void *, void *,
-                             void *, void *);
-typedef void (*microtask_t7)(int *, int *, void *, void *, void *, void *,
-                             void *, void *, void *);
-typedef void (*microtask_t8)(int *, int *, void *, void *, void *, void *,
-                             void *, void *, void *, void *);
-typedef void (*microtask_t9)(int *, int *, void *, void *, void *, void *,
-                             void *, void *, void *, void *, void *);
-typedef void (*microtask_t10)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *);
-typedef void (*microtask_t11)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *);
-typedef void (*microtask_t12)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *);
-typedef void (*microtask_t13)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *, void *);
-typedef void (*microtask_t14)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *, void *, void *);
-typedef void (*microtask_t15)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *);
+template <size_t> using microtask_argument_t = void *;
+
+template <size_t... Indices>
+static void invokeMicrotask(microtask_t pkfn, int *gtid, int *tid,
+                            void *p_argv[], std::index_sequence<Indices...>) {
+  // WebAssembly's `call_indirect` requires the callee type to exactly match the
+  // call site. Cast the variadic microtask_t to the fixed-arity signature that
+  // matches argc before invoking it.
+  using typed_microtask_t =
+      void (*)(int *, int *, microtask_argument_t<Indices>...);
+  (*(typed_microtask_t)pkfn)(gtid, tid, p_argv[Indices]...);
+}
----------------
sbc100 wrote:

You could land this templatization as a separate cleanup PR first?

https://github.com/llvm/llvm-project/pull/211071

From openmp-commits at lists.llvm.org  Mon Aug  3 08:30:02 2026
From: openmp-commits at lists.llvm.org (Sam Clegg via Openmp-commits)
Date: Mon, 03 Aug 2026 08:30:02 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP] Support up to 32 arguments in
 the generic microtask dispatcher (PR #211071)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/211071@github.com>
Message-ID: <6a70b3fa.c5504292.253bd2.9992@mx.google.com>


https://github.com/sbc100 commented:

I'm not an openmp maintainer so I don't know how much it would coast to instead convert the dispatcher to use varargs after a certain point?   Or if that is even desirable?  Certainly having a fixed upper limit seems sad

https://github.com/llvm/llvm-project/pull/211071

From openmp-commits at lists.llvm.org  Mon Aug  3 08:30:02 2026
From: openmp-commits at lists.llvm.org (Sam Clegg via Openmp-commits)
Date: Mon, 03 Aug 2026 08:30:02 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP] Support up to 32 arguments in
 the generic microtask dispatcher (PR #211071)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/211071@github.com>
Message-ID: <6a70b3fa.c56fb102.15e4aa.4221@mx.google.com>


================
@@ -1,10 +1,6 @@
 // RUN: %libomp-compile-and-run
 #include <stdio.h>
 
-// This test fails on Arm64EC as __kmp_invoke_microtask doesn't support more
-// than 16 arguments on that target.
-// XFAIL: arm64ec
----------------
sbc100 wrote:

Should this test instead be extended to that it exceeds the fixed internal limit?

https://github.com/llvm/llvm-project/pull/211071

From openmp-commits at lists.llvm.org  Mon Aug  3 08:31:56 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Mon, 03 Aug 2026 08:31:56 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMP][OMPT] Error when
 registering EMI and non-EMI callbacks (PR #213697)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213697@github.com>
Message-ID: <6a70b46c.160324b0.f2e18.7584@mx.google.com>


github-actions[bot] wrote:

<!--LLVM CODE FORMAT COMMENT: {clang-format}-->


:warning: C/C++ code formatter, clang-format found issues in your code. :warning:

<details>
<summary>
You can test this locally with the following command:
</summary>

``````````bash
git-clang-format --diff origin/main HEAD --extensions h,c,cpp -- offload/test/ompt/register_both.h offload/test/ompt/veccopy_disallow_both.c openmp/runtime/src/ompt-general.cpp --diff_from_common_commit
``````````

:warning:
The reproduction instructions above might return results for more than one PR
in a stack if you are using a stacked PR workflow. You can limit the results by
changing `origin/main` to the base branch/commit you want to compare against.
:warning:

</details>

<details>
<summary>
View the diff from clang-format here.
</summary>

``````````diff
diff --git a/offload/test/ompt/register_both.h b/offload/test/ompt/register_both.h
index 4e9157c2b..a20b42bad 100644
--- a/offload/test/ompt/register_both.h
+++ b/offload/test/ompt/register_both.h
@@ -5,7 +5,7 @@
   do {                                                                         \
     type f_##name = &on_##name;                                                \
     ompt_set_result_t result =                                                 \
-      ompt_set_callback(name, (ompt_callback_t)f_##name);                      \
+        ompt_set_callback(name, (ompt_callback_t)f_##name);                    \
     if (result == ompt_set_never || result == ompt_set_error)                  \
       printf("0: Could not register callback '" #name "'\n");                  \
   } while (0)

``````````

</details>


https://github.com/llvm/llvm-project/pull/213697

From openmp-commits at lists.llvm.org  Mon Aug  3 08:32:46 2026
From: openmp-commits at lists.llvm.org (Sam Clegg via Openmp-commits)
Date: Mon, 03 Aug 2026 08:32:46 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [openmp][WebAssembly] add
 support for wasm64 (PR #181669)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/181669@github.com>
Message-ID: <6a70b49e.eb165e5c.68c24.1269@mx.google.com>


================
@@ -155,7 +155,7 @@ endif()
 # with either the Emscripten or wasi-threads flavor ends up using the pthreads
 # interface in a WebAssembly-compiled libc; CMake does not yet know how to
 # detect this.
----------------
sbc100 wrote:

Hmm, we should probably fix this.

Actually I think this is maybe just not true.  We have test for this in Emscripten: https://github.com/emscripten-core/emscripten/blob/2073d7dd3176253344561b2bef84073baa568298/test/cmake/threads/CMakeLists.txt#L5-L9.

Perhaps we can clean this up separately, no need to make it part of this PR

https://github.com/llvm/llvm-project/pull/181669

From openmp-commits at lists.llvm.org  Mon Aug  3 08:32:51 2026
From: openmp-commits at lists.llvm.org (=?UTF-8?Q?Jan_Andr=C3=A9_Reuter?= via Openmp-commits)
Date: Mon, 03 Aug 2026 08:32:51 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMP][OMPT] Error when
 registering EMI and non-EMI callbacks (PR #213697)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213697@github.com>
Message-ID: <6a70b4a3.4a342cbc.2cb7cf.4e60@mx.google.com>


https://github.com/Thyre updated https://github.com/llvm/llvm-project/pull/213697

>From f9ca5b7ba2145548cf458c7c3956bac3e1f28871 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Andr=C3=A9=20Reuter?= <jan at zyten.de>
Date: Mon, 3 Aug 2026 17:29:16 +0200
Subject: [PATCH] [OpenMP][OMPT] Error when registering EMI and non-EMI
 callbacks
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The OpenMP specification includes a restriction for registering both EMI and
non-EMI callbacks (e.g. `ompt_callback_target`, OpenMP v5.2, p. 503, l. 18-20):

  These callbacks must not be registered at the same time.

However, this was not respected in the runtime. A tool was able to register
both, receiving `ompt_set_always` for each `ompt_set_callback` call.
When an event associated with the EMI or non-EMI callbacks was dispatched
and both callbacks were registered, only the EMI version was dispatched.

To align the runtime behavior with the OpenMP specification, return
`ompt_set_error` when a tool tries to register both the EMI and non-EMI variant
for a callback. With this, only the tool's first registration is used.

Update the offload `veccopy_disallow_both.c` test, which already tested the
prior, non-compliant, behavior.

Signed-off-by: Jan André Reuter <jan at zyten.de>
---
 offload/test/ompt/register_both.h          |  4 +++-
 offload/test/ompt/veccopy_disallow_both.c  | 19 ++++++++-------
 openmp/runtime/src/include/omp-tools.h.var | 13 +++++++++++
 openmp/runtime/src/ompt-general.cpp        | 27 +++++++++++++++++++++-
 4 files changed, 53 insertions(+), 10 deletions(-)

diff --git a/offload/test/ompt/register_both.h b/offload/test/ompt/register_both.h
index 77cb430aa1f97..a20b42bad2196 100644
--- a/offload/test/ompt/register_both.h
+++ b/offload/test/ompt/register_both.h
@@ -4,7 +4,9 @@
 #define register_ompt_callback_t(name, type)                                   \
   do {                                                                         \
     type f_##name = &on_##name;                                                \
-    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+    ompt_set_result_t result =                                                 \
+        ompt_set_callback(name, (ompt_callback_t)f_##name);                    \
+    if (result == ompt_set_never || result == ompt_set_error)                  \
       printf("0: Could not register callback '" #name "'\n");                  \
   } while (0)
 
diff --git a/offload/test/ompt/veccopy_disallow_both.c b/offload/test/ompt/veccopy_disallow_both.c
index bfc67c5f4d274..2e24529042b37 100644
--- a/offload/test/ompt/veccopy_disallow_both.c
+++ b/offload/test/ompt/veccopy_disallow_both.c
@@ -7,8 +7,9 @@
 /*
  * Example OpenMP program that shows that both EMI and non-EMI
  * callbacks cannot be registered for the same type. In the
- * current implementation, the EMI callback overrides the non-EMI
- * callback.
+ * current implementation, the first registered callback takes
+ * precedence. Trying to register the other callback results in
+ * ompt_set_error for this callback.
  */
 
 #include <omp.h>
@@ -57,9 +58,11 @@ int main() {
 }
 
 // clang-format off
+/// CHECK: [[THREAD_ID:[0-9]+]]: Could not register callback 'ompt_callback_target_data_op'
+/// CHECK: [[THREAD_ID]]: Could not register callback 'ompt_callback_target_emi'
 /// CHECK: Callback Init:
 /// CHECK: Callback Load:
-/// CHECK: Callback Target EMI: kind=ompt_target endpoint=ompt_scope_begin
+/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=ompt_target endpoint=ompt_scope_begin
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_alloc
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_alloc
 /// CHECK-NOT: dest=(nil)
@@ -70,7 +73,7 @@ int main() {
 /// CHECK-NOT: dest=(nil)
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_transfer_to_device
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_transfer_to_device
-/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1
+/// CHECK: Callback Submit: target_id=[[TARGET_ID]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_transfer_from_device
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_transfer_from_device
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_transfer_from_device
@@ -79,8 +82,8 @@ int main() {
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_delete
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_delete
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_delete
-/// CHECK: Callback Target EMI: kind=ompt_target endpoint=ompt_scope_end
-/// CHECK: Callback Target EMI: kind=ompt_target endpoint=ompt_scope_begin
+/// CHECK: Callback Target: target_id=[[TARGET_ID]] kind=ompt_target endpoint=ompt_scope_end
+/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=ompt_target endpoint=ompt_scope_begin
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_alloc
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_alloc
 /// CHECK-NOT: dest=(nil)
@@ -91,7 +94,7 @@ int main() {
 /// CHECK-NOT: dest=(nil)
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_transfer_to_device
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_transfer_to_device
-/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0
+/// CHECK: Callback Submit: target_id=[[TARGET_ID]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_transfer_from_device
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_transfer_from_device
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_transfer_from_device
@@ -100,5 +103,5 @@ int main() {
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_delete
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_delete
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_delete
-/// CHECK: Callback Target EMI: kind=ompt_target endpoint=ompt_scope_end
+/// CHECK: Callback Target: target_id=[[TARGET_ID]] kind=ompt_target endpoint=ompt_scope_end
 /// CHECK: Callback Fini:
diff --git a/openmp/runtime/src/include/omp-tools.h.var b/openmp/runtime/src/include/omp-tools.h.var
index af9213a7290e7..46643dff5feab 100644
--- a/openmp/runtime/src/include/omp-tools.h.var
+++ b/openmp/runtime/src/include/omp-tools.h.var
@@ -185,6 +185,19 @@
     /* Optional Events */                                                                                                \
     macro (ompt_callback_target_map_emi,    ompt_callback_target_map_emi_t,    36) /* target map                      */
 
+#define FOREACH_MAPPED_OMPT_TARGET_EVENT(macro)                                                                          \
+    /* macro including correlated callback enum (e.g. target -- target_emi) for comparisons between both */              \
+    /*--- Mandatory Events ---*/                                                                                         \
+    macro (ompt_callback_target,             ompt_callback_target_emi,         ompt_callback_target_t,              8)   \
+    macro (ompt_callback_target_data_op,     ompt_callback_target_data_op_emi, ompt_callback_target_data_op_t,      9)   \
+    macro (ompt_callback_target_submit,      ompt_callback_target_submit_emi,  ompt_callback_target_submit_t,      10)   \
+    macro (ompt_callback_target_emi,         ompt_callback_target,             ompt_callback_target_emi_t,         33)   \
+    macro (ompt_callback_target_data_op_emi, ompt_callback_target_data_op,     ompt_callback_target_data_op_emi_t, 34)   \
+    macro (ompt_callback_target_submit_emi,  ompt_callback_target_submit,      ompt_callback_target_submit_emi_t,  35)   \
+    /* Optional Events */                                                                                                \
+    macro (ompt_callback_target_map,         ompt_callback_target_map_emi,     ompt_callback_target_map_t,         22)   \
+    macro (ompt_callback_target_map_emi,     ompt_callback_target_map,         ompt_callback_target_map_emi_t,     36)
+
 #define FOREACH_OMPT_50_TARGET_EVENT(macro)                                                                              \
     FOREACH_OMPT_DEVICE_EVENT(macro)                                                                                     \
     FOREACH_OMPT_NOEMI_EVENT(macro) 
diff --git a/openmp/runtime/src/ompt-general.cpp b/openmp/runtime/src/ompt-general.cpp
index 1682f7e74bd14..1cb2fb2693a2c 100644
--- a/openmp/runtime/src/ompt-general.cpp
+++ b/openmp/runtime/src/ompt-general.cpp
@@ -597,10 +597,35 @@ OMPT_API_ROUTINE ompt_set_result_t ompt_set_callback(ompt_callbacks_t which,
     else                                                                       \
       return ompt_set_always;
 
-    FOREACH_OMPT_EVENT(ompt_event_macro)
+    FOREACH_OMPT_HOST_EVENT(ompt_event_macro)
+    FOREACH_OMPT_DEVICE_EVENT(ompt_event_macro)
 
 #undef ompt_event_macro
 
+    // OpenMP v5.2, p. 503, l. 18-20:
+    // Restrictions to the ompt_callback_target_emi and ompt_callback_target
+    // callbacks are as follows:
+    //   These callbacks must not be registered at the same time.
+    //
+    // Similar restrictions apply to target_data_op, target_submit,
+    // and target_map. Hence, handle registration of them separately to ensure
+    // only one can be registered at the same time. */
+#define ompt_target_event_macro(event_name, comparison_name, callback_type,    \
+                                event_id)                                      \
+  case event_name:                                                             \
+    if (ompt_callbacks.ompt_callback(comparison_name))                         \
+      return ompt_set_error;                                                   \
+    ompt_callbacks.ompt_callback(event_name) = (callback_type)callback;        \
+    ompt_enabled.event_name = (callback != 0);                                 \
+    if (callback)                                                              \
+      return ompt_event_implementation_status(event_name);                     \
+    else                                                                       \
+      return ompt_set_always;
+
+    FOREACH_MAPPED_OMPT_TARGET_EVENT(ompt_target_event_macro);
+
+#undef ompt_target_event_macro
+
   default:
     return ompt_set_error;
   }


From openmp-commits at lists.llvm.org  Mon Aug  3 08:55:45 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Mon, 03 Aug 2026 08:55:45 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP] Support up to 32 arguments in
 the generic microtask dispatcher (PR #211071)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/211071@github.com>
Message-ID: <6a70ba01.510f576d.2d4e39.1b3c@mx.google.com>


arsnyder16 wrote:

>convert the dispatcher to use varargs after a certain point? 

That’s a fair point. My understanding is that there are two calls involved: the call into __kmp_invoke_microtask, and then its call to the compiler-generated microtask. Making the first one variadic would not necessarily help with the second, because the generated microtask has a specific signature and C++ cannot dynamically expand the argument array into that call.

The second call becomes a WebAssembly call_indirect because the runtime only has a function pointer to the generated microtask. WebAssembly checks that an indirect call’s complete signature matches the target function, including the number of parameters, so calling it with a different fixed or variadic signature can trap.

Changing the generated microtask to accept packed arguments might remove the limit, but I believe that would also require a Clang change and could affect the existing compiler/runtime convention. I may be missing another way to use varargs here, though. Would you be willing to sketch the approach you had in mind?



https://github.com/llvm/llvm-project/pull/211071

From openmp-commits at lists.llvm.org  Mon Aug  3 12:03:59 2026
From: openmp-commits at lists.llvm.org (Andrew Brown via Openmp-commits)
Date: Mon, 03 Aug 2026 12:03:59 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [openmp][WebAssembly] add
 support for wasm64 (PR #181669)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/181669@github.com>
Message-ID: <6a70e61f.160324b0.f2e18.0cdc@mx.google.com>


================
@@ -155,7 +155,7 @@ endif()
 # with either the Emscripten or wasi-threads flavor ends up using the pthreads
 # interface in a WebAssembly-compiled libc; CMake does not yet know how to
 # detect this.
----------------
abrown wrote:

Glad to see you are picking this up! It's been a long while since I thought about this, so I may not be of too much help here: I would say (and you may already know) that eons ago wasi-sdk relied on wasi-threads, which matche the "instance per thread" approach in Emscripten. But wasi-sdk over time moved to using components and wasi-threads was abandoned. Perhaps when I added this it was protecting against some incompatibility related to this? But it's been too long so I would just say go with whatever makes sense today.

https://github.com/llvm/llvm-project/pull/181669

From openmp-commits at lists.llvm.org  Mon Aug  3 12:04:17 2026
From: openmp-commits at lists.llvm.org (Andrew Brown via Openmp-commits)
Date: Mon, 03 Aug 2026 12:04:17 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [openmp][WebAssembly] add
 support for wasm64 (PR #181669)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/181669@github.com>
Message-ID: <6a70e631.21a19891.251803.3174@mx.google.com>


https://github.com/abrown edited https://github.com/llvm/llvm-project/pull/181669

From openmp-commits at lists.llvm.org  Mon Aug  3 21:34:01 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Mon, 03 Aug 2026 21:34:01 -0700 (PDT)
Subject: [Openmp-commits] [openmp] 513a02e - [Offload] Save jit image
	(#212384)
Message-ID: <6a716bb9.0318b02e.2a198c.06d7@mx.google.com>


Author: jandrovins
Date: 2026-08-03T21:33:56-07:00
New Revision: 513a02ee150335a84c39ab2e2d0f965f1b0e260e

URL: https://github.com/llvm/llvm-project/commit/513a02ee150335a84c39ab2e2d0f965f1b0e260e
DIFF: https://github.com/llvm/llvm-project/commit/513a02ee150335a84c39ab2e2d0f965f1b0e260e.diff

LOG: [Offload] Save jit image (#212384)

This PR adds the envar LIBOMPTARGET_JIT_SAVE_IMAGE_FILENAME
to indicate that the device image generated after JIT compilation
should be saved to a file.

Added: 
    offload/test/jit/save_image.c

Modified: 
    offload/plugins-nextgen/common/include/JIT.h
    offload/plugins-nextgen/common/src/JIT.cpp
    openmp/docs/design/Runtimes.rst

Removed: 
    


################################################################################
diff  --git a/offload/plugins-nextgen/common/include/JIT.h b/offload/plugins-nextgen/common/include/JIT.h
index b4e3712d9c980..96dea06335730 100644
--- a/offload/plugins-nextgen/common/include/JIT.h
+++ b/offload/plugins-nextgen/common/include/JIT.h
@@ -103,6 +103,8 @@ struct JITEngine {
       StringEnvar("LIBOMPTARGET_JIT_PRE_OPT_IR_MODULE");
   StringEnvar PostOptIRModuleFileName =
       StringEnvar("LIBOMPTARGET_JIT_POST_OPT_IR_MODULE");
+  StringEnvar SaveImageFileName =
+      StringEnvar("LIBOMPTARGET_JIT_SAVE_IMAGE_FILENAME");
   UInt32Envar JITOptLevel = UInt32Envar("LIBOMPTARGET_JIT_OPT_LEVEL", 3);
   BoolEnvar JITSkipOpt = BoolEnvar("LIBOMPTARGET_JIT_SKIP_OPT", false);
 };

diff  --git a/offload/plugins-nextgen/common/src/JIT.cpp b/offload/plugins-nextgen/common/src/JIT.cpp
index 9153059887757..b78c9b1768139 100644
--- a/offload/plugins-nextgen/common/src/JIT.cpp
+++ b/offload/plugins-nextgen/common/src/JIT.cpp
@@ -300,5 +300,16 @@ JITEngine::process(StringRef Image, target::plugin::GenericDeviceTy &Device) {
     return Device.doJITPostProcessing(std::move(MB));
   };
 
-  return compile(Image, ComputeUnitKind, PostProcessing);
+  auto ImageOrError = compile(Image, ComputeUnitKind, PostProcessing);
+
+  if (SaveImageFileName.isPresent() && ImageOrError) {
+    std::error_code EC;
+    raw_fd_ostream OS(SaveImageFileName.get(), EC);
+    if (EC)
+      return createStringError(error::ErrorCode::HOST_IO,
+                               "saving JIT image file\n");
+    OS << ImageOrError.get()->getBuffer();
+    OS.close();
+  }
+  return ImageOrError;
 }

diff  --git a/offload/test/jit/save_image.c b/offload/test/jit/save_image.c
new file mode 100644
index 0000000000000..66936e196c528
--- /dev/null
+++ b/offload/test/jit/save_image.c
@@ -0,0 +1,20 @@
+// clang-format off
+// RUN: %libomptarget-compileopt-generic -fopenmp-target-jit
+// RUN: rm -f %t.image
+// RUN: env LIBOMPTARGET_JIT_SAVE_IMAGE_FILENAME=%t.image %libomptarget-run-generic
+// RUN: test -s %t.image
+// clang-format on
+
+// REQUIRES: gpu
+// XFAIL: intelgpu
+
+int main() {
+  int X = 0;
+
+#pragma omp target map(tofrom : X)
+  {
+    X = 1;
+  }
+
+  return X != 1;
+}

diff  --git a/openmp/docs/design/Runtimes.rst b/openmp/docs/design/Runtimes.rst
index 4e3137abd6fb7..14e300a0f531f 100644
--- a/openmp/docs/design/Runtimes.rst
+++ b/openmp/docs/design/Runtimes.rst
@@ -741,6 +741,7 @@ variables is defined below.
     * ``LIBOMPTARGET_JIT_REPLACEMENT_MODULE=<in:Filename> (LLVM-IR file)``
     * ``LIBOMPTARGET_JIT_PRE_OPT_IR_MODULE=<out:Filename> (LLVM-IR file)``
     * ``LIBOMPTARGET_JIT_POST_OPT_IR_MODULE=<out:Filename> (LLVM-IR file)``
+    * ``LIBOMPTARGET_JIT_SAVE_IMAGE_FILENAME=<out:Filename> (device image file)``
     * ``LIBOMPTARGET_MIN_THREADS_FOR_LOW_TRIP_COUNT=<Num> (default: 32)``
     * ``LIBOMPTARGET_REUSE_BLOCKS_FOR_HIGH_TRIP_COUNT=[TRUE/FALSE] (default TRUE)``
     * ``OFFLOAD_TRACK_ALLOCATION_TRACES=[TRUE/FALSE] (default FALSE)``
@@ -1162,6 +1163,14 @@ which the LLVM-IR module is written. The module can be the analyzed, and
 transformed and loaded back into the JIT pipeline via
 :ref:`LIBOMPTARGET_JIT_REPLACEMENT_MODULE`.
 
+.. _libomptarget_jit_save_image_filename:
+
+LIBOMPTARGET_JIT_SAVE_IMAGE_FILENAME
+""""""""""""""""""""""""""""""""""""
+
+This environment variable can be used to save the device image produced by the
+device JIT after target-specific post-processing. The value is expected to be a
+filename into which the binary device image is written.
 
 LIBOMPTARGET_MIN_THREADS_FOR_LOW_TRIP_COUNT
 """""""""""""""""""""""""""""""""""""""""""


        

From openmp-commits at lists.llvm.org  Mon Aug  3 21:34:05 2026
From: openmp-commits at lists.llvm.org (Kevin Sala Penades via Openmp-commits)
Date: Mon, 03 Aug 2026 21:34:05 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [Offload] Save jit image (PR
	#212384)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/212384@github.com>
Message-ID: <6a716bbd.160324b0.f2e18.cbfc@mx.google.com>


https://github.com/kevinsala closed https://github.com/llvm/llvm-project/pull/212384

From openmp-commits at lists.llvm.org  Mon Aug  3 21:43:06 2026
From: openmp-commits at lists.llvm.org (LLVM Continuous Integration via Openmp-commits)
Date: Mon, 03 Aug 2026 21:43:06 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [Offload] Save jit image (PR
	#212384)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/212384@github.com>
Message-ID: <6a716dda.824e11a6.18590a.2a2f@mx.google.com>


llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `intel-sycl-gpu` running on `intel-sycl-gpu-01` while building `offload,openmp` at step 6 "test-build-unified-tree-check-all".

Full details are available at: https://lab.llvm.org/buildbot/#/builders/225/builds/14449

<details>
<summary>Here is the relevant piece of the build log for the reference</summary>

```
Step 6 (test-build-unified-tree-check-all) failure: test (failure)
...
UNSUPPORTED: libomptarget :: spirv64-intel :: offloading/fortran/target-map-dtype-multi-explicit-array-member.f90 (98680 of 103029)
PASS: libomp :: worksharing/for/omp_for_bigbounds.c (98681 of 103029)
UNSUPPORTED: libomptarget :: spirv64-intel :: mapping/lambda_mapping.cpp (98682 of 103029)
UNSUPPORTED: libomptarget :: spirv64-intel :: offloading/fortran/target-map-dtype-multi-explicit-member.f90 (98683 of 103029)
UNSUPPORTED: libomptarget :: spirv64-intel :: offloading/fortran/target-map-dynamic.f90 (98684 of 103029)
UNSUPPORTED: libomptarget :: spirv64-intel :: offloading/fortran/target-map-enter-exit-array.f90 (98685 of 103029)
PASS: libomptarget :: spirv64-intel :: mapping/chained_containing_structs_3.cc (98686 of 103029)
UNSUPPORTED: libomptarget :: spirv64-intel :: offloading/fortran/double-target-call-with-declare-target.f90 (98687 of 103029)
UNSUPPORTED: libomptarget :: spirv64-intel :: offloading/fortran/target-map-first-common-block-member.f90 (98688 of 103029)
XPASS: libomptarget :: spirv64-intel :: jit/save_image.c (98689 of 103029)
******************** TEST 'libomptarget :: spirv64-intel :: jit/save_image.c' FAILED ********************
Exit Code: 0

Command Output (stdout):
--
# RUN: at line 2
/home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/./bin/clang -fopenmp    -I /home/test-user/llvm-buildbot-worker/intel-sycl-gpu/llvm-project/offload/test -I /home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/runtimes/runtimes-bins/openmp/runtime/src -L /home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/./lib/x86_64-unknown-linux-gnu -L /home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/./lib/x86_64-unknown-linux-gnu -L /home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/./lib -L /home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/runtimes/runtimes-bins/openmp/runtime/src  -Wl,-rpath,/home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/./lib/x86_64-unknown-linux-gnu -Wl,-rpath,/home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/runtimes/runtimes-bins/openmp/runtime/src -Wl,-rpath,/home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/./lib/x86_64-unknown-linux-gnu -Wl,-rpath,/home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/./lib  -fopenmp-targets=spirv64-intel -O3 /home/test-user/llvm-buildbot-worker/intel-sycl-gpu/llvm-project/offload/test/jit/save_image.c -o /home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/runtimes/runtimes-bins/offload/test/spirv64-intel/jit/Output/save_image.c.tmp -fopenmp-target-jit
# executed command: /home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/./bin/clang -fopenmp -I /home/test-user/llvm-buildbot-worker/intel-sycl-gpu/llvm-project/offload/test -I /home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/runtimes/runtimes-bins/openmp/runtime/src -L /home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/./lib/x86_64-unknown-linux-gnu -L /home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/./lib/x86_64-unknown-linux-gnu -L /home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/./lib -L /home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/runtimes/runtimes-bins/openmp/runtime/src -Wl,-rpath,/home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/./lib/x86_64-unknown-linux-gnu -Wl,-rpath,/home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/runtimes/runtimes-bins/openmp/runtime/src -Wl,-rpath,/home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/./lib/x86_64-unknown-linux-gnu -Wl,-rpath,/home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/./lib -fopenmp-targets=spirv64-intel -O3 /home/test-user/llvm-buildbot-worker/intel-sycl-gpu/llvm-project/offload/test/jit/save_image.c -o /home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/runtimes/runtimes-bins/offload/test/spirv64-intel/jit/Output/save_image.c.tmp -fopenmp-target-jit
# RUN: at line 3
rm -f /home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/runtimes/runtimes-bins/offload/test/spirv64-intel/jit/Output/save_image.c.tmp.image
# executed command: rm -f /home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/runtimes/runtimes-bins/offload/test/spirv64-intel/jit/Output/save_image.c.tmp.image
# RUN: at line 4
env LIBOMPTARGET_JIT_SAVE_IMAGE_FILENAME=/home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/runtimes/runtimes-bins/offload/test/spirv64-intel/jit/Output/save_image.c.tmp.image /home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/runtimes/runtimes-bins/offload/test/spirv64-intel/jit/Output/save_image.c.tmp
# executed command: env LIBOMPTARGET_JIT_SAVE_IMAGE_FILENAME=/home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/runtimes/runtimes-bins/offload/test/spirv64-intel/jit/Output/save_image.c.tmp.image /home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/runtimes/runtimes-bins/offload/test/spirv64-intel/jit/Output/save_image.c.tmp
# .---command stderr------------
# | 'unknown' is not a recognized processor for this target (ignoring processor)
# | 'unknown' is not a recognized processor for this target (ignoring processor)
# `-----------------------------
# RUN: at line 5
test -s /home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/runtimes/runtimes-bins/offload/test/spirv64-intel/jit/Output/save_image.c.tmp.image
# executed command: test -s /home/test-user/llvm-buildbot-worker/intel-sycl-gpu/build/runtimes/runtimes-bins/offload/test/spirv64-intel/jit/Output/save_image.c.tmp.image

--

********************
UNSUPPORTED: libomptarget :: spirv64-intel :: offloading/fortran/target-map-individual-dtype-member-map.f90 (98690 of 103029)
PASS: libomptarget :: spirv64-intel :: mapping/declare_mapper_target_checks.cpp (98691 of 103029)
UNSUPPORTED: libomptarget :: spirv64-intel :: offloading/fortran/dtype-member-map-syntax-1.f90 (98692 of 103029)
UNSUPPORTED: libomptarget :: spirv64-intel :: offloading/fortran/dtype-member-overlap-map.f90 (98693 of 103029)
UNSUPPORTED: libomptarget :: spirv64-intel :: offloading/fortran/target-map-multi-alloca-dtypes-with-multi-alloca-members.f90 (98694 of 103029)
UNSUPPORTED: libomptarget :: spirv64-intel :: offloading/fortran/target-map-multi-alloca-dtypes-with-multi-mixed-members.f90 (98695 of 103029)
UNSUPPORTED: libomptarget :: spirv64-intel :: offloading/fortran/formatted-io.f90 (98696 of 103029)
UNSUPPORTED: libomptarget :: spirv64-intel :: offloading/fortran/implicit-derived-enter-exit.f90 (98697 of 103029)
UNSUPPORTED: libomptarget :: spirv64-intel :: offloading/fortran/implicit-record-field-mapping.f90 (98698 of 103029)
PASS: libomptarget :: spirv64-intel :: mapping/mapper_map_ptee_only_2_ptr_indirections.c (98699 of 103029)
PASS: libomptarget :: spirv64-intel :: offloading/disable_default_device.c (98700 of 103029)
UNSUPPORTED: libomptarget :: spirv64-intel :: offloading/fortran/target-map-nested-dtype-alloca-array.f90 (98701 of 103029)
PASS: libomptarget :: spirv64-intel :: api/omp_indirect_func_struct.c (98702 of 103029)
PASS: libomptarget :: spirv64-intel :: mapping/pr38704.c (98703 of 103029)

```

</details>

https://github.com/llvm/llvm-project/pull/212384

From openmp-commits at lists.llvm.org  Tue Aug  4 02:18:55 2026
From: openmp-commits at lists.llvm.org (Jan Patrick Lehr via Openmp-commits)
Date: Tue, 04 Aug 2026 02:18:55 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP][OMPT] Add ompt_set_frame_enter
 support to libomptarget (PR #201127)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/201127@github.com>
Message-ID: <6a71ae7f.d681cab9.19d854.506a@mx.google.com>


https://github.com/jplehr closed https://github.com/llvm/llvm-project/pull/201127

From openmp-commits at lists.llvm.org  Tue Aug  4 02:33:42 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Tue, 04 Aug 2026 02:33:42 -0700 (PDT)
Subject: [Openmp-commits] [flang] [llvm] [openmp] [OpenMP][Flang] Add
 integer-kind wrappers to omp_lib (PR #213505)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213505@github.com>
Message-ID: <6a71b1f6.492e23e4.45e2c.c85c@mx.google.com>


shivaramaarao wrote:

@Meinersbur  can you please review this?

https://github.com/llvm/llvm-project/pull/213505

From openmp-commits at lists.llvm.org  Tue Aug  4 05:09:19 2026
From: openmp-commits at lists.llvm.org (Robert Imschweiler via Openmp-commits)
Date: Tue, 04 Aug 2026 05:09:19 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [offload] Add global addrspace qualifier
 to KLE and member pointers (PR #213918)
Message-ID: <llvm.org/llvm/llvm-project/pull/213918@github.com>


https://github.com/ro-i created https://github.com/llvm/llvm-project/pull/213918

Claude assisted with this patch.

>From f661a4b1c9acb60243f50cc1bcb5ab614f43a870 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Tue, 4 Aug 2026 07:06:10 -0500
Subject: [PATCH] [offload] Add global addrspace qualifier to KLE and member
 pointers

Claude assisted with this patch.
---
 openmp/device/include/State.h   |  2 +-
 openmp/device/src/Reduction.cpp |  3 ++-
 openmp/device/src/State.cpp     | 10 +++++-----
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/openmp/device/include/State.h b/openmp/device/include/State.h
index d3cd3d981e29d..f6e3250a9dfe1 100644
--- a/openmp/device/include/State.h
+++ b/openmp/device/include/State.h
@@ -123,7 +123,7 @@ void init(bool IsSPMD, KernelEnvironmentTy &KernelEnvironment,
 /// holds for all instances of the kernel. The latter is dynamic and provides
 /// per-launch information.
 KernelEnvironmentTy &getKernelEnvironment();
-KernelLaunchEnvironmentTy &getKernelLaunchEnvironment();
+Global<KernelLaunchEnvironmentTy> &getKernelLaunchEnvironment();
 
 /// TODO
 enum ValueKind {
diff --git a/openmp/device/src/Reduction.cpp b/openmp/device/src/Reduction.cpp
index df444fb9d90e4..9d0a5121995af 100644
--- a/openmp/device/src/Reduction.cpp
+++ b/openmp/device/src/Reduction.cpp
@@ -256,7 +256,8 @@ int32_t __kmpc_gpu_xteam_reduce_nowait(IdentTy *Loc, void *reduce_data,
   if (NumTeams <= 1)
     return ThreadId == 0;
 
-  uint32_t &TeamsDone = state::getKernelLaunchEnvironment().ReductionTeamsDone;
+  Global<uint32_t> &TeamsDone =
+      state::getKernelLaunchEnvironment().ReductionTeamsDone;
   void *GlobalBuffer = state::getKernelLaunchEnvironment().ReductionBuffer;
   [[clang::loader_uninitialized]] static Local<uint32_t> TeamsDoneResult;
 
diff --git a/openmp/device/src/State.cpp b/openmp/device/src/State.cpp
index 243af1f2cb5e2..779f4d5159311 100644
--- a/openmp/device/src/State.cpp
+++ b/openmp/device/src/State.cpp
@@ -36,9 +36,9 @@ using namespace ompx;
     KernelEnvironmentPtr;
 
 /// The kernel launch environment passed as argument to the kernel by the
-/// runtime.
-[[clang::loader_uninitialized]] static Local<KernelLaunchEnvironmentTy *>
-    KernelLaunchEnvironmentPtr;
+/// runtime. (The runtime allocates it in device global memory.)
+[[clang::loader_uninitialized]] static Local<
+    Global<KernelLaunchEnvironmentTy> *> KernelLaunchEnvironmentPtr;
 
 /// The pointer type for dynamic shared memory. This is important to keep
 /// the alignment and address space information.
@@ -295,7 +295,7 @@ void state::init(bool IsSPMD, KernelEnvironmentTy &KernelEnvironment,
     TeamState.init(IsSPMD);
     ThreadStates = nullptr;
     KernelEnvironmentPtr = &KernelEnvironment;
-    KernelLaunchEnvironmentPtr = KLE;
+    KernelLaunchEnvironmentPtr = (Global<KernelLaunchEnvironmentTy> *)KLE;
   }
 }
 
@@ -303,7 +303,7 @@ KernelEnvironmentTy &state::getKernelEnvironment() {
   return *KernelEnvironmentPtr;
 }
 
-KernelLaunchEnvironmentTy &state::getKernelLaunchEnvironment() {
+Global<KernelLaunchEnvironmentTy> &state::getKernelLaunchEnvironment() {
   return *KernelLaunchEnvironmentPtr;
 }
 


From openmp-commits at lists.llvm.org  Tue Aug  4 07:10:00 2026
From: openmp-commits at lists.llvm.org (Jan Patrick Lehr via Openmp-commits)
Date: Tue, 04 Aug 2026 07:10:00 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMP][OMPT] Error when
 registering EMI and non-EMI callbacks (PR #213697)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213697@github.com>
Message-ID: <6a71f2b8.21a19891.112c1c.2852@mx.google.com>


jplehr wrote:

Out of curiosity: Is that restriction something that the runtime has to enforce?

https://github.com/llvm/llvm-project/pull/213697

From openmp-commits at lists.llvm.org  Tue Aug  4 07:19:45 2026
From: openmp-commits at lists.llvm.org (Jan Patrick Lehr via Openmp-commits)
Date: Tue, 04 Aug 2026 07:19:45 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP][OMPT] Emit `dispatch` callback
 for serial and zero-trip loops (PR #207773)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/207773@github.com>
Message-ID: <6a71f501.27e38385.2ac6b.66e5@mx.google.com>


https://github.com/jplehr edited https://github.com/llvm/llvm-project/pull/207773

From openmp-commits at lists.llvm.org  Tue Aug  4 07:19:46 2026
From: openmp-commits at lists.llvm.org (Jan Patrick Lehr via Openmp-commits)
Date: Tue, 04 Aug 2026 07:19:46 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP][OMPT] Emit `dispatch` callback
 for serial and zero-trip loops (PR #207773)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/207773@github.com>
Message-ID: <6a71f502.c5504292.20ce5f.cf85@mx.google.com>


https://github.com/jplehr commented:

Thank you for putting this up

https://github.com/llvm/llvm-project/pull/207773

From openmp-commits at lists.llvm.org  Tue Aug  4 07:19:46 2026
From: openmp-commits at lists.llvm.org (Jan Patrick Lehr via Openmp-commits)
Date: Tue, 04 Aug 2026 07:19:46 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP][OMPT] Emit `dispatch` callback
 for serial and zero-trip loops (PR #207773)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/207773@github.com>
Message-ID: <6a71f502.7270d2fd.15a00f.9dbf@mx.google.com>


================
@@ -241,6 +259,24 @@ static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid,
           ompt_work_type, ompt_scope_begin, &(team_info->parallel_data),
           &(task_info->task_data), *pstride, codeptr);
     }
+    if (ompt_enabled.ompt_callback_dispatch) {
----------------
jplehr wrote:

Is it required that we have the exact same code twice here or am I missing something?

https://github.com/llvm/llvm-project/pull/207773

From openmp-commits at lists.llvm.org  Tue Aug  4 07:25:43 2026
From: openmp-commits at lists.llvm.org (=?UTF-8?Q?Jan_Andr=C3=A9_Reuter?= via Openmp-commits)
Date: Tue, 04 Aug 2026 07:25:43 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMP][OMPT] Error when
 registering EMI and non-EMI callbacks (PR #213697)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213697@github.com>
Message-ID: <6a71f667.1728eaf3.1f780a.613a@mx.google.com>


Thyre wrote:

> Out of curiosity: Is that restriction something that the runtime has to enforce?

That's a good question. With the wording in OpenMP v5.1 / v5.2

>  These callbacks must not be registered at the same time.

and v6.0 / TR15:

> The deprecated target callback must not be registered if a target_emi callback is registered.

I'd understand this as a user not being allowed to register both callbacks at the same time. From my perspective, "must not" is pretty clear in this case. A runtime returning `ompt_set_always` in a case where a user violates this restriction seems incorrect.

A runtime could decide to ignore this restriction, but this opens a wholer can of worms for implementation-defined behavior. For example, does a tool get both callbacks? If so, in which order? Does `target` still set `target_id`, and will the `target_emi` callback receive that `target_id` via `target_data->value`, or its own set `target_data`?
I think this is one reason why this restriction exists in the first place.

I wasn't involved in any of that discussion though. Maybe @jprotze has a better answer.



https://github.com/llvm/llvm-project/pull/213697

From openmp-commits at lists.llvm.org  Tue Aug  4 07:44:21 2026
From: openmp-commits at lists.llvm.org (=?UTF-8?Q?Jan_Andr=C3=A9_Reuter?= via Openmp-commits)
Date: Tue, 04 Aug 2026 07:44:21 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP][OMPT] Emit `dispatch` callback
 for serial and zero-trip loops (PR #207773)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/207773@github.com>
Message-ID: <6a71fac5.27e38385.2ac6b.f56c@mx.google.com>


================
@@ -241,6 +259,24 @@ static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid,
           ompt_work_type, ompt_scope_begin, &(team_info->parallel_data),
           &(task_info->task_data), *pstride, codeptr);
     }
+    if (ompt_enabled.ompt_callback_dispatch) {
----------------
Thyre wrote:

I'm not that fond of the OMPT callback handling in that method.
Basically, three separate cases are handled in `__kmp_for_static_init`:

1. Special handling for zero-trip loops. Previously, only a _work-begin_ event was dispatched here.
2. Handling for a serialized team. Similarly, only a _work-begin_ event was dispatched.
3. Your "normal" for loop. Here, _work-begin_ and the dispatch callback were already implemented.

This PR adds the dispatch callback for the first two cases. Hence the exact same code twice.
In general, handling for both callbacks is pretty much the same for all three cases. One could question if an implementation ever provides a zero-trip loop for sections. We might be able to strip that part, but I think it doesn't hurt to keep it.

Maybe it's worth trying to refactor the dispatch of both callbacks to reduce duplication a bit.

https://github.com/llvm/llvm-project/pull/207773

From openmp-commits at lists.llvm.org  Tue Aug  4 08:25:24 2026
From: openmp-commits at lists.llvm.org (Kevin Sala Penades via Openmp-commits)
Date: Tue, 04 Aug 2026 08:25:24 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [Offload] Save jit image (PR
	#212384)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/212384@github.com>
Message-ID: <6a720464.d76d81af.2931b1.055d@mx.google.com>


kevinsala wrote:

CC @jandrovins

https://github.com/llvm/llvm-project/pull/212384

From openmp-commits at lists.llvm.org  Tue Aug  4 08:26:54 2026
From: openmp-commits at lists.llvm.org (Nick Sarnie via Openmp-commits)
Date: Tue, 04 Aug 2026 08:26:54 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [Offload] Save jit image (PR
	#212384)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/212384@github.com>
Message-ID: <6a7204be.9edb5c9d.168f3.5130@mx.google.com>


sarnex wrote:

It's an XPASS, i'll fix it.

https://github.com/llvm/llvm-project/pull/212384

From openmp-commits at lists.llvm.org  Tue Aug  4 08:30:36 2026
From: openmp-commits at lists.llvm.org (Nick Sarnie via Openmp-commits)
Date: Tue, 04 Aug 2026 08:30:36 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [Offload] Save jit image (PR
	#212384)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/212384@github.com>
Message-ID: <6a72059c.6a5690f2.27895d.7690@mx.google.com>


sarnex wrote:

https://github.com/llvm/llvm-project/pull/213984

https://github.com/llvm/llvm-project/pull/212384

From openmp-commits at lists.llvm.org  Tue Aug  4 09:46:01 2026
From: openmp-commits at lists.llvm.org (Jan Patrick Lehr via Openmp-commits)
Date: Tue, 04 Aug 2026 09:46:01 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMP][OMPT] Error when
 registering EMI and non-EMI callbacks (PR #213697)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213697@github.com>
Message-ID: <6a721749.c4d3d80a.254a08.7b4d@mx.google.com>


https://github.com/jplehr approved this pull request.

The change seems reasonable to me.

Let's try to give @mhalk and @jprotze a chance to chime in before landing.

https://github.com/llvm/llvm-project/pull/213697

From openmp-commits at lists.llvm.org  Tue Aug  4 09:56:28 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Tue, 04 Aug 2026 09:56:28 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP] Simplify generic microtask
	dispatch (PR #214004)
Message-ID: <llvm.org/llvm/llvm-project/pull/214004@github.com>


https://github.com/arsnyder16 created https://github.com/llvm/llvm-project/pull/214004

Replace the generic microtask dispatcher's handwritten function-pointer types and calls with a variadic-template helper and a macro for each supported argument count.

The dispatcher still casts each microtask to the exact fixed-arity signature required by platforms such as WebAssembly. This is intended as a refactor only: the existing limit of 15 microtask arguments and the diagnostic for larger argument counts remain unchanged.

This reduces the repetitive code and makes future changes to the supported argument range easier to review.

#211071 

>From e4b2bc24ec37e31d230ed13f9e60ed06b032b672 Mon Sep 17 00:00:00 2001
From: Andrew Snyder <asnyder at minitab.com>
Date: Tue, 4 Aug 2026 12:53:56 -0400
Subject: [PATCH] [OpenMP] Simplify generic microtask dispatch

---
 openmp/runtime/src/kmp_invoke_microtask.cpp | 149 +++++---------------
 1 file changed, 39 insertions(+), 110 deletions(-)

diff --git a/openmp/runtime/src/kmp_invoke_microtask.cpp b/openmp/runtime/src/kmp_invoke_microtask.cpp
index f14f27cc06ee3..e3cae9938bcd3 100644
--- a/openmp/runtime/src/kmp_invoke_microtask.cpp
+++ b/openmp/runtime/src/kmp_invoke_microtask.cpp
@@ -1,49 +1,27 @@
 #include "kmp.h"
 
+#include <utility>
+
 #if !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_MIC || KMP_ARCH_AARCH64 ||        \
       KMP_ARCH_PPC64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 ||            \
       KMP_ARCH_ARM || KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_PPC_XCOFF ||   \
       KMP_ARCH_AARCH64_32)
 
-// Because WebAssembly will use `call_indirect` to invoke the microtask and
-// WebAssembly indirect calls check that the called signature is a precise
-// match, we need to cast each microtask function pointer back from `void *` to
-// its original type.
-typedef void (*microtask_t0)(int *, int *);
-typedef void (*microtask_t1)(int *, int *, void *);
-typedef void (*microtask_t2)(int *, int *, void *, void *);
-typedef void (*microtask_t3)(int *, int *, void *, void *, void *);
-typedef void (*microtask_t4)(int *, int *, void *, void *, void *, void *);
-typedef void (*microtask_t5)(int *, int *, void *, void *, void *, void *,
-                             void *);
-typedef void (*microtask_t6)(int *, int *, void *, void *, void *, void *,
-                             void *, void *);
-typedef void (*microtask_t7)(int *, int *, void *, void *, void *, void *,
-                             void *, void *, void *);
-typedef void (*microtask_t8)(int *, int *, void *, void *, void *, void *,
-                             void *, void *, void *, void *);
-typedef void (*microtask_t9)(int *, int *, void *, void *, void *, void *,
-                             void *, void *, void *, void *, void *);
-typedef void (*microtask_t10)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *);
-typedef void (*microtask_t11)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *);
-typedef void (*microtask_t12)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *);
-typedef void (*microtask_t13)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *, void *);
-typedef void (*microtask_t14)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *, void *, void *);
-typedef void (*microtask_t15)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *);
+template <size_t> using microtask_argument_t = void *;
+
+template <size_t... Indices>
+static void invokeMicrotask(microtask_t pkfn, int *gtid, int *tid,
+                            void *p_argv[], std::index_sequence<Indices...>) {
+  // WebAssembly's `call_indirect` requires the callee type to exactly match the
+  // call site. Cast the variadic microtask_t to the fixed-arity signature that
+  // matches argc before invoking it.
+  using typed_microtask_t =
+      void (*)(int *, int *, microtask_argument_t<Indices>...);
+  (*(typed_microtask_t)pkfn)(gtid, tid, p_argv[Indices]...);
+}
 
-// we really only need the case with 1 argument, because CLANG always build
-// a struct of pointers to shared variables referenced in the outlined function
+// Keep a bounded set of exact signatures for targets that cannot dynamically
+// construct a variable-argument microtask call.
 int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,
                            void *p_argv[]
 #if OMPT_SUPPORT
@@ -55,85 +33,36 @@ int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,
   *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
 #endif
 
+#define KMP_INVOKE_MICROTASK_CASE(N)                                           \
+  case N:                                                                      \
+    invokeMicrotask(pkfn, &gtid, &tid, p_argv, std::make_index_sequence<N>{}); \
+    break
+
   switch (argc) {
+    KMP_INVOKE_MICROTASK_CASE(0);
+    KMP_INVOKE_MICROTASK_CASE(1);
+    KMP_INVOKE_MICROTASK_CASE(2);
+    KMP_INVOKE_MICROTASK_CASE(3);
+    KMP_INVOKE_MICROTASK_CASE(4);
+    KMP_INVOKE_MICROTASK_CASE(5);
+    KMP_INVOKE_MICROTASK_CASE(6);
+    KMP_INVOKE_MICROTASK_CASE(7);
+    KMP_INVOKE_MICROTASK_CASE(8);
+    KMP_INVOKE_MICROTASK_CASE(9);
+    KMP_INVOKE_MICROTASK_CASE(10);
+    KMP_INVOKE_MICROTASK_CASE(11);
+    KMP_INVOKE_MICROTASK_CASE(12);
+    KMP_INVOKE_MICROTASK_CASE(13);
+    KMP_INVOKE_MICROTASK_CASE(14);
+    KMP_INVOKE_MICROTASK_CASE(15);
   default:
     fprintf(stderr, "Too many args to microtask: %d!\n", argc);
     fflush(stderr);
     exit(-1);
-  case 0:
-    (*(microtask_t0)pkfn)(&gtid, &tid);
-    break;
-  case 1:
-    (*(microtask_t1)pkfn)(&gtid, &tid, p_argv[0]);
-    break;
-  case 2:
-    (*(microtask_t2)pkfn)(&gtid, &tid, p_argv[0], p_argv[1]);
-    break;
-  case 3:
-    (*(microtask_t3)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2]);
-    break;
-  case 4:
-    (*(microtask_t4)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3]);
-    break;
-  case 5:
-    (*(microtask_t5)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3], p_argv[4]);
-    break;
-  case 6:
-    (*(microtask_t6)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3], p_argv[4], p_argv[5]);
-    break;
-  case 7:
-    (*(microtask_t7)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3], p_argv[4], p_argv[5], p_argv[6]);
-    break;
-  case 8:
-    (*(microtask_t8)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                          p_argv[7]);
-    break;
-  case 9:
-    (*(microtask_t9)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3], p_argv[4], p_argv[5], p_argv[6], p_argv[7],
-                          p_argv[8]);
-    break;
-  case 10:
-    (*(microtask_t10)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9]);
-    break;
-  case 11:
-    (*(microtask_t11)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9], p_argv[10]);
-    break;
-  case 12:
-    (*(microtask_t12)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9], p_argv[10],
-                           p_argv[11]);
-    break;
-  case 13:
-    (*(microtask_t13)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9], p_argv[10],
-                           p_argv[11], p_argv[12]);
-    break;
-  case 14:
-    (*(microtask_t14)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9], p_argv[10],
-                           p_argv[11], p_argv[12], p_argv[13]);
-    break;
-  case 15:
-    (*(microtask_t15)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9], p_argv[10],
-                           p_argv[11], p_argv[12], p_argv[13], p_argv[14]);
-    break;
   }
 
+#undef KMP_INVOKE_MICROTASK_CASE
+
   return 1;
 }
 


From openmp-commits at lists.llvm.org  Tue Aug  4 09:56:37 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Tue, 04 Aug 2026 09:56:37 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP] Simplify generic microtask
	dispatch (PR #214004)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/214004@github.com>
Message-ID: <6a7219c5.3cce8983.33cfba.d5a0@mx.google.com>


arsnyder16 wrote:

@sbc100 

https://github.com/llvm/llvm-project/pull/214004

From openmp-commits at lists.llvm.org  Tue Aug  4 10:47:41 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Tue, 04 Aug 2026 10:47:41 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP] Simplify generic microtask
	dispatch (PR #214004)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/214004@github.com>
Message-ID: <6a7225bd.af4834b4.181aa.db2d@mx.google.com>


================
@@ -1,49 +1,27 @@
 #include "kmp.h"
 
+#include <utility>
+
 #if !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_MIC || KMP_ARCH_AARCH64 ||        \
       KMP_ARCH_PPC64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 ||            \
       KMP_ARCH_ARM || KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_PPC_XCOFF ||   \
       KMP_ARCH_AARCH64_32)
 
-// Because WebAssembly will use `call_indirect` to invoke the microtask and
-// WebAssembly indirect calls check that the called signature is a precise
-// match, we need to cast each microtask function pointer back from `void *` to
-// its original type.
-typedef void (*microtask_t0)(int *, int *);
-typedef void (*microtask_t1)(int *, int *, void *);
-typedef void (*microtask_t2)(int *, int *, void *, void *);
-typedef void (*microtask_t3)(int *, int *, void *, void *, void *);
-typedef void (*microtask_t4)(int *, int *, void *, void *, void *, void *);
-typedef void (*microtask_t5)(int *, int *, void *, void *, void *, void *,
-                             void *);
-typedef void (*microtask_t6)(int *, int *, void *, void *, void *, void *,
-                             void *, void *);
-typedef void (*microtask_t7)(int *, int *, void *, void *, void *, void *,
-                             void *, void *, void *);
-typedef void (*microtask_t8)(int *, int *, void *, void *, void *, void *,
-                             void *, void *, void *, void *);
-typedef void (*microtask_t9)(int *, int *, void *, void *, void *, void *,
-                             void *, void *, void *, void *, void *);
-typedef void (*microtask_t10)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *);
-typedef void (*microtask_t11)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *);
-typedef void (*microtask_t12)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *);
-typedef void (*microtask_t13)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *, void *);
-typedef void (*microtask_t14)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *, void *, void *);
-typedef void (*microtask_t15)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *);
+template <size_t> using microtask_argument_t = void *;
+
+template <size_t... Indices>
+static void invokeMicrotask(microtask_t pkfn, int *gtid, int *tid,
+                            void *p_argv[], std::index_sequence<Indices...>) {
+  // WebAssembly's `call_indirect` requires the callee type to exactly match the
+  // call site. Cast the variadic microtask_t to the fixed-arity signature that
+  // matches argc before invoking it.
+  using typed_microtask_t =
+      void (*)(int *, int *, microtask_argument_t<Indices>...);
+  (*(typed_microtask_t)pkfn)(gtid, tid, p_argv[Indices]...);
+}
 
-// we really only need the case with 1 argument, because CLANG always build
-// a struct of pointers to shared variables referenced in the outlined function
----------------
arsnyder16 wrote:

was added 12 years ago to  [z_Linux_util.c](https://github.com/llvm/llvm-project/commit/3051f9761ef43877ec3017d955999e07cb0b09ac#r194945222)

https://github.com/llvm/llvm-project/pull/214004

From openmp-commits at lists.llvm.org  Tue Aug  4 10:49:04 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Tue, 04 Aug 2026 10:49:04 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP] Simplify generic microtask
	dispatch (PR #214004)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/214004@github.com>
Message-ID: <6a722610.0f490953.111d61.49eb@mx.google.com>


================
@@ -1,49 +1,27 @@
 #include "kmp.h"
 
+#include <utility>
+
 #if !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_MIC || KMP_ARCH_AARCH64 ||        \
       KMP_ARCH_PPC64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 ||            \
       KMP_ARCH_ARM || KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_PPC_XCOFF ||   \
       KMP_ARCH_AARCH64_32)
 
-// Because WebAssembly will use `call_indirect` to invoke the microtask and
-// WebAssembly indirect calls check that the called signature is a precise
-// match, we need to cast each microtask function pointer back from `void *` to
-// its original type.
-typedef void (*microtask_t0)(int *, int *);
-typedef void (*microtask_t1)(int *, int *, void *);
-typedef void (*microtask_t2)(int *, int *, void *, void *);
-typedef void (*microtask_t3)(int *, int *, void *, void *, void *);
-typedef void (*microtask_t4)(int *, int *, void *, void *, void *, void *);
-typedef void (*microtask_t5)(int *, int *, void *, void *, void *, void *,
-                             void *);
-typedef void (*microtask_t6)(int *, int *, void *, void *, void *, void *,
-                             void *, void *);
-typedef void (*microtask_t7)(int *, int *, void *, void *, void *, void *,
-                             void *, void *, void *);
-typedef void (*microtask_t8)(int *, int *, void *, void *, void *, void *,
-                             void *, void *, void *, void *);
-typedef void (*microtask_t9)(int *, int *, void *, void *, void *, void *,
-                             void *, void *, void *, void *, void *);
-typedef void (*microtask_t10)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *);
-typedef void (*microtask_t11)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *);
-typedef void (*microtask_t12)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *);
-typedef void (*microtask_t13)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *, void *);
-typedef void (*microtask_t14)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *, void *, void *);
-typedef void (*microtask_t15)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *);
+template <size_t> using microtask_argument_t = void *;
+
+template <size_t... Indices>
+static void invokeMicrotask(microtask_t pkfn, int *gtid, int *tid,
+                            void *p_argv[], std::index_sequence<Indices...>) {
+  // WebAssembly's `call_indirect` requires the callee type to exactly match the
+  // call site. Cast the variadic microtask_t to the fixed-arity signature that
+  // matches argc before invoking it.
+  using typed_microtask_t =
+      void (*)(int *, int *, microtask_argument_t<Indices>...);
+  (*(typed_microtask_t)pkfn)(gtid, tid, p_argv[Indices]...);
+}
 
-// we really only need the case with 1 argument, because CLANG always build
-// a struct of pointers to shared variables referenced in the outlined function
----------------
arsnyder16 wrote:

https://discourse.llvm.org/t/ppc64-patch-from-intels-fourth-cmake-patch/33203

https://github.com/llvm/llvm-project/pull/214004

From openmp-commits at lists.llvm.org  Tue Aug  4 10:52:37 2026
From: openmp-commits at lists.llvm.org (Sam Clegg via Openmp-commits)
Date: Tue, 04 Aug 2026 10:52:37 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP] Simplify generic microtask
	dispatch (PR #214004)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/214004@github.com>
Message-ID: <6a7226e5.dbb26164.10167b.352a@mx.google.com>


https://github.com/sbc100 edited https://github.com/llvm/llvm-project/pull/214004

From openmp-commits at lists.llvm.org  Tue Aug  4 10:52:48 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Tue, 04 Aug 2026 10:52:48 -0700 (PDT)
Subject: [Openmp-commits] [openmp] af14456 - [OpenMP] Simplify generic
	microtask dispatch (#214004)
Message-ID: <6a7226f0.666420a4.2af57e.5317@mx.google.com>


Author: arsnyder16
Date: 2026-08-04T10:52:44-07:00
New Revision: af14456a2bac55026b2d57cbf14b3905c452b8a4

URL: https://github.com/llvm/llvm-project/commit/af14456a2bac55026b2d57cbf14b3905c452b8a4
DIFF: https://github.com/llvm/llvm-project/commit/af14456a2bac55026b2d57cbf14b3905c452b8a4.diff

LOG: [OpenMP] Simplify generic microtask dispatch (#214004)

Replace the generic microtask dispatcher's handwritten function-pointer
types and calls with a variadic-template helper and a macro for each
supported argument count.

The dispatcher still casts each microtask to the exact fixed-arity
signature required by platforms such as WebAssembly. This is intended as
a refactor only: the existing limit of 15 microtask arguments and the
diagnostic for larger argument counts remain unchanged.

This reduces the repetitive code and makes future changes to the
supported argument range easier to review.

Split out from #211071

Added: 
    

Modified: 
    openmp/runtime/src/kmp_invoke_microtask.cpp

Removed: 
    


################################################################################
diff  --git a/openmp/runtime/src/kmp_invoke_microtask.cpp b/openmp/runtime/src/kmp_invoke_microtask.cpp
index f14f27cc06ee3..e3cae9938bcd3 100644
--- a/openmp/runtime/src/kmp_invoke_microtask.cpp
+++ b/openmp/runtime/src/kmp_invoke_microtask.cpp
@@ -1,49 +1,27 @@
 #include "kmp.h"
 
+#include <utility>
+
 #if !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_MIC || KMP_ARCH_AARCH64 ||        \
       KMP_ARCH_PPC64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 ||            \
       KMP_ARCH_ARM || KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_PPC_XCOFF ||   \
       KMP_ARCH_AARCH64_32)
 
-// Because WebAssembly will use `call_indirect` to invoke the microtask and
-// WebAssembly indirect calls check that the called signature is a precise
-// match, we need to cast each microtask function pointer back from `void *` to
-// its original type.
-typedef void (*microtask_t0)(int *, int *);
-typedef void (*microtask_t1)(int *, int *, void *);
-typedef void (*microtask_t2)(int *, int *, void *, void *);
-typedef void (*microtask_t3)(int *, int *, void *, void *, void *);
-typedef void (*microtask_t4)(int *, int *, void *, void *, void *, void *);
-typedef void (*microtask_t5)(int *, int *, void *, void *, void *, void *,
-                             void *);
-typedef void (*microtask_t6)(int *, int *, void *, void *, void *, void *,
-                             void *, void *);
-typedef void (*microtask_t7)(int *, int *, void *, void *, void *, void *,
-                             void *, void *, void *);
-typedef void (*microtask_t8)(int *, int *, void *, void *, void *, void *,
-                             void *, void *, void *, void *);
-typedef void (*microtask_t9)(int *, int *, void *, void *, void *, void *,
-                             void *, void *, void *, void *, void *);
-typedef void (*microtask_t10)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *);
-typedef void (*microtask_t11)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *);
-typedef void (*microtask_t12)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *);
-typedef void (*microtask_t13)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *, void *);
-typedef void (*microtask_t14)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *, void *, void *);
-typedef void (*microtask_t15)(int *, int *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *, void *,
-                              void *, void *, void *, void *, void *);
+template <size_t> using microtask_argument_t = void *;
+
+template <size_t... Indices>
+static void invokeMicrotask(microtask_t pkfn, int *gtid, int *tid,
+                            void *p_argv[], std::index_sequence<Indices...>) {
+  // WebAssembly's `call_indirect` requires the callee type to exactly match the
+  // call site. Cast the variadic microtask_t to the fixed-arity signature that
+  // matches argc before invoking it.
+  using typed_microtask_t =
+      void (*)(int *, int *, microtask_argument_t<Indices>...);
+  (*(typed_microtask_t)pkfn)(gtid, tid, p_argv[Indices]...);
+}
 
-// we really only need the case with 1 argument, because CLANG always build
-// a struct of pointers to shared variables referenced in the outlined function
+// Keep a bounded set of exact signatures for targets that cannot dynamically
+// construct a variable-argument microtask call.
 int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,
                            void *p_argv[]
 #if OMPT_SUPPORT
@@ -55,85 +33,36 @@ int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,
   *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
 #endif
 
+#define KMP_INVOKE_MICROTASK_CASE(N)                                           \
+  case N:                                                                      \
+    invokeMicrotask(pkfn, &gtid, &tid, p_argv, std::make_index_sequence<N>{}); \
+    break
+
   switch (argc) {
+    KMP_INVOKE_MICROTASK_CASE(0);
+    KMP_INVOKE_MICROTASK_CASE(1);
+    KMP_INVOKE_MICROTASK_CASE(2);
+    KMP_INVOKE_MICROTASK_CASE(3);
+    KMP_INVOKE_MICROTASK_CASE(4);
+    KMP_INVOKE_MICROTASK_CASE(5);
+    KMP_INVOKE_MICROTASK_CASE(6);
+    KMP_INVOKE_MICROTASK_CASE(7);
+    KMP_INVOKE_MICROTASK_CASE(8);
+    KMP_INVOKE_MICROTASK_CASE(9);
+    KMP_INVOKE_MICROTASK_CASE(10);
+    KMP_INVOKE_MICROTASK_CASE(11);
+    KMP_INVOKE_MICROTASK_CASE(12);
+    KMP_INVOKE_MICROTASK_CASE(13);
+    KMP_INVOKE_MICROTASK_CASE(14);
+    KMP_INVOKE_MICROTASK_CASE(15);
   default:
     fprintf(stderr, "Too many args to microtask: %d!\n", argc);
     fflush(stderr);
     exit(-1);
-  case 0:
-    (*(microtask_t0)pkfn)(&gtid, &tid);
-    break;
-  case 1:
-    (*(microtask_t1)pkfn)(&gtid, &tid, p_argv[0]);
-    break;
-  case 2:
-    (*(microtask_t2)pkfn)(&gtid, &tid, p_argv[0], p_argv[1]);
-    break;
-  case 3:
-    (*(microtask_t3)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2]);
-    break;
-  case 4:
-    (*(microtask_t4)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3]);
-    break;
-  case 5:
-    (*(microtask_t5)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3], p_argv[4]);
-    break;
-  case 6:
-    (*(microtask_t6)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3], p_argv[4], p_argv[5]);
-    break;
-  case 7:
-    (*(microtask_t7)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3], p_argv[4], p_argv[5], p_argv[6]);
-    break;
-  case 8:
-    (*(microtask_t8)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                          p_argv[7]);
-    break;
-  case 9:
-    (*(microtask_t9)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                          p_argv[3], p_argv[4], p_argv[5], p_argv[6], p_argv[7],
-                          p_argv[8]);
-    break;
-  case 10:
-    (*(microtask_t10)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9]);
-    break;
-  case 11:
-    (*(microtask_t11)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9], p_argv[10]);
-    break;
-  case 12:
-    (*(microtask_t12)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9], p_argv[10],
-                           p_argv[11]);
-    break;
-  case 13:
-    (*(microtask_t13)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9], p_argv[10],
-                           p_argv[11], p_argv[12]);
-    break;
-  case 14:
-    (*(microtask_t14)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9], p_argv[10],
-                           p_argv[11], p_argv[12], p_argv[13]);
-    break;
-  case 15:
-    (*(microtask_t15)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
-                           p_argv[3], p_argv[4], p_argv[5], p_argv[6],
-                           p_argv[7], p_argv[8], p_argv[9], p_argv[10],
-                           p_argv[11], p_argv[12], p_argv[13], p_argv[14]);
-    break;
   }
 
+#undef KMP_INVOKE_MICROTASK_CASE
+
   return 1;
 }
 


        

From openmp-commits at lists.llvm.org  Tue Aug  4 10:52:51 2026
From: openmp-commits at lists.llvm.org (Sam Clegg via Openmp-commits)
Date: Tue, 04 Aug 2026 10:52:51 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP] Simplify generic microtask
	dispatch (PR #214004)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/214004@github.com>
Message-ID: <6a7226f3.436e7ae1.196892.e182@mx.google.com>


https://github.com/sbc100 closed https://github.com/llvm/llvm-project/pull/214004

From openmp-commits at lists.llvm.org  Tue Aug  4 10:54:14 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Tue, 04 Aug 2026 10:54:14 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP] Simplify generic microtask
	dispatch (PR #214004)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/214004@github.com>
Message-ID: <6a722746.6c8668f2.39fcc1.8cc2@mx.google.com>


https://github.com/arsnyder16 edited https://github.com/llvm/llvm-project/pull/214004

From openmp-commits at lists.llvm.org  Tue Aug  4 12:50:38 2026
From: openmp-commits at lists.llvm.org (Robert Imschweiler via Openmp-commits)
Date: Tue, 04 Aug 2026 12:50:38 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [offload] Add global addrspace qualifier
 to KLE and member pointers (PR #213918)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213918@github.com>
Message-ID: <6a72428e.83ce629a.4777c.b7ff@mx.google.com>


================
@@ -123,7 +123,7 @@ void init(bool IsSPMD, KernelEnvironmentTy &KernelEnvironment,
 /// holds for all instances of the kernel. The latter is dynamic and provides
 /// per-launch information.
 KernelEnvironmentTy &getKernelEnvironment();
-KernelLaunchEnvironmentTy &getKernelLaunchEnvironment();
+Global<KernelLaunchEnvironmentTy> &getKernelLaunchEnvironment();
----------------
ro-i wrote:

they are unrelated. And for the other PR: pinned memory buys not a lot for upstream, but for ROCm.
For every KLE, we copy host -> device, but that has way more latency than pinned host -> device, at least for ROCm (maybe also because the host -> device path is more cluttered with OMPT and other stuff)

https://github.com/llvm/llvm-project/pull/213918

From openmp-commits at lists.llvm.org  Tue Aug  4 12:50:47 2026
From: openmp-commits at lists.llvm.org (Jan Patrick Lehr via Openmp-commits)
Date: Tue, 04 Aug 2026 12:50:47 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OMPT] Upstream device tracing
	implementation (PR #200165)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/200165@github.com>
Message-ID: <6a724297.27e38385.2ac6b.263d@mx.google.com>


https://github.com/jplehr updated https://github.com/llvm/llvm-project/pull/200165

>From 07fd4fbed4b671ce18e918b0a89c3b39e22cc2e0 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Tue, 4 Aug 2026 13:39:45 -0500
Subject: [PATCH 01/11] [OpenMP][OMPT] Test libomptarget connector lookup

Verify that the private target connector only exposes supported callbacks and does not retain the unwired frame-enter extension.

Co-Authored-By: Claude <noreply at anthropic.com>
---
 openmp/runtime/test/ompt/misc/target_lookup.c | 36 +++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 openmp/runtime/test/ompt/misc/target_lookup.c

diff --git a/openmp/runtime/test/ompt/misc/target_lookup.c b/openmp/runtime/test/ompt/misc/target_lookup.c
new file mode 100644
index 0000000000000..88c0f5f46b959
--- /dev/null
+++ b/openmp/runtime/test/ompt/misc/target_lookup.c
@@ -0,0 +1,36 @@
+// RUN: %libomp-compile-and-run | FileCheck %s
+// REQUIRES: ompt
+
+#include <omp-tools.h>
+#include <stdio.h>
+
+void ompt_libomp_connect(ompt_start_tool_result_t *result);
+
+static int tool_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                           ompt_data_t *tool_data) {
+  return 1;
+}
+
+static int target_initialize(ompt_function_lookup_t lookup,
+                             int initial_device_num, ompt_data_t *tool_data) {
+  printf("ompt_set_frame_enter: %s\n",
+         lookup("ompt_set_frame_enter") ? "present" : "absent");
+  return 1;
+}
+
+static void ompt_finalize(ompt_data_t *tool_data) {}
+
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t result = {&tool_initialize, &ompt_finalize, 0};
+  return &result;
+}
+
+int main() {
+  static ompt_start_tool_result_t result = {&target_initialize, &ompt_finalize, 0};
+  // This entry point initializes OMPT before invoking the target connector.
+  ompt_libomp_connect(&result);
+  return 0;
+}
+
+// CHECK: ompt_set_frame_enter: absent

>From b6694bec124d528570a62704cae59de2d88a7b71 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 2 Apr 2026 07:27:58 -0500
Subject: [PATCH 02/11] [Offload] Add GenericProfilerTy abstraction and
 APITypes extensions

Introduce a GenericProfilerTy base class that decouples the plugin
layer from OMPT. This allows profiling/tracing backends to be
implemented independently of the plugin code.

Key changes:
- Add GenericProfiler.h/cpp with virtual hooks for device lifecycle
  events (init, deinit, loadBinary), kernel launch, data alloc/delete,
  kernel completion, and data transfer timing
- Add ProfTimerTy RAII timer for scoped alloc/delete measurements
- Add weak getProfilerToAttach() factory (overridable by OMPT)
- Add ExecAsync and ProfilerData fields to __tgt_async_info
- Replace direct OMPT callback invocations in PluginInterface with
  profiler hook calls (handleInit, handleDeinit, handleLoadBinary,
  handlePreKernelLaunch)
- Add getDeviceTimeStamp() virtual to GenericDeviceTy
- Add Profiler member and getProfiler() to GenericPluginTy
- Suppress ProfilerData during KLE upload to avoid spurious traces
- Add sync path in AsyncInfoWrapperTy::finalize for !ExecAsync

Made-with: Cursor
Co-Authored-By: Claude <noreply at anthropic.com>
---
 offload/include/Shared/APITypes.h             |   5 +
 offload/plugins-nextgen/common/CMakeLists.txt |   1 +
 .../common/include/GenericProfiler.h          | 182 ++++++++++++++++++
 .../common/include/PluginInterface.h          |  35 ++--
 .../common/src/GenericProfiler.cpp            |  35 ++++
 .../common/src/PluginInterface.cpp            |  72 +++----
 6 files changed, 268 insertions(+), 62 deletions(-)
 create mode 100644 offload/plugins-nextgen/common/include/GenericProfiler.h
 create mode 100644 offload/plugins-nextgen/common/src/GenericProfiler.cpp

diff --git a/offload/include/Shared/APITypes.h b/offload/include/Shared/APITypes.h
index 948c12a27107e..592f6e2be1096 100644
--- a/offload/include/Shared/APITypes.h
+++ b/offload/include/Shared/APITypes.h
@@ -85,6 +85,11 @@ struct __tgt_async_info {
   /// ensure it is a valid location while the transfer to the device is
   /// happening.
   KernelLaunchEnvironmentTy KernelLaunchEnvironment;
+
+  /// Opaque handle for profiler-specific data (e.g., OMPT trace record info).
+  /// Owned by the profiler; the runtime threads this pointer through the plugin
+  /// layer to associate async operations with trace records.
+  void *ProfilerData = nullptr;
 };
 
 /// This struct contains all of the arguments to a target kernel region launch.
diff --git a/offload/plugins-nextgen/common/CMakeLists.txt b/offload/plugins-nextgen/common/CMakeLists.txt
index aad8d209e931a..ad018da333400 100644
--- a/offload/plugins-nextgen/common/CMakeLists.txt
+++ b/offload/plugins-nextgen/common/CMakeLists.txt
@@ -11,6 +11,7 @@ add_public_tablegen_target(PluginErrcodes)
 # don't want to export `PluginInterface` while `add_llvm_library` requires that.
 add_library(PluginCommon OBJECT
   src/PluginInterface.cpp
+  src/GenericProfiler.cpp
   src/GlobalHandler.cpp
   src/JIT.cpp
   src/RecordReplay.cpp
diff --git a/offload/plugins-nextgen/common/include/GenericProfiler.h b/offload/plugins-nextgen/common/include/GenericProfiler.h
new file mode 100644
index 0000000000000..ea3c3766554c8
--- /dev/null
+++ b/offload/plugins-nextgen/common/include/GenericProfiler.h
@@ -0,0 +1,182 @@
+//===- GenericProfiler.h - GenericProfiler interface for use in Plugins ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The GenericProfiler interface allows to implement profiler logic for various
+// backends, such as OMPT or other tracing mechanisms.
+// This enables the plugins to be agnostic of the actual high-level language
+// that is implemented.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OFFLOAD_PLUGINS_NEXTGEN_COMMON_INCLUDE_GENERICPROFILER_H
+#define OFFLOAD_PLUGINS_NEXTGEN_COMMON_INCLUDE_GENERICPROFILER_H
+
+#include "Shared/APITypes.h"
+
+#include <cstdint>
+#include <functional>
+#include <tuple>
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace plugin {
+
+struct GenericDeviceTy;
+struct GenericPluginTy;
+class GenericProfilerTy;
+
+template <typename FunT, typename... ArgsT, size_t... IdxSequence>
+void callViaIndexSeq(FunT F, GenericProfilerTy *P, uint64_t StartNanos,
+                     uint64_t EndNanos, std::tuple<ArgsT...> Args,
+                     std::index_sequence<IdxSequence...>) {
+  F(P, StartNanos, EndNanos, std::get<IdxSequence>(Args)...);
+}
+
+template <typename FunT, typename... ArgsT>
+void callViaUnpack(FunT F, GenericProfilerTy *P, uint64_t StartNanos,
+                   uint64_t EndNanos, std::tuple<ArgsT...> Tup) {
+  callViaIndexSeq(F, P, StartNanos, EndNanos, Tup,
+                  std::index_sequence_for<ArgsT...>{});
+}
+
+/// Abstraction layer to implement different profiler backends.
+///
+/// The plugins call into the GenericProfilerTy to handle the specific events
+/// with whatever specific backend was instantiated. For now, the supported
+/// backends are limited to an OMPT implementation.
+class GenericProfilerTy {
+public:
+  GenericProfilerTy() = default;
+  virtual ~GenericProfilerTy() = default;
+
+  /// Obtain a pointer to profiler-specific data, if any.
+  virtual void *getProfilerSpecificData() { return nullptr; }
+
+  virtual bool isProfilingEnabled() { return false; }
+
+  /// Set the factors which are used to interpolate the device clock compared to
+  /// the host clock. This follows a simple linear interpolation: Slope * <time>
+  /// + Offset.
+  void setTimeConversionFactors(double Slope, double Offset) {
+    HostToDeviceSlope = Slope;
+    HostToDeviceOffset = Offset;
+    setTimeConversionFactorsImpl(HostToDeviceSlope, HostToDeviceOffset);
+  }
+
+  /// Hook that is called when the plugin is initialized.
+  virtual void handleInit(GenericDeviceTy *Device, GenericPluginTy *Plugin) {}
+
+  /// Hook that is called when the plugin is de-initialized.
+  virtual void handleDeinit(GenericDeviceTy *Device, GenericPluginTy *Plugin) {}
+
+  /// Hook that is called when the device image is loaded.
+  virtual void handleLoadBinary(GenericDeviceTy *Device,
+                                GenericPluginTy *Plugin,
+                                const StringRef InputTgtImage) {}
+
+  /// Hook that is called when memory is allocated on the device.
+  virtual void handleDataAlloc(uint64_t StartNanos, uint64_t EndNanos,
+                               void *HostPtr, uint64_t Size, void *Data) {}
+
+  /// Hook that is called when memory is freed on the device.
+  virtual void handleDataDelete(uint64_t StartNanos, uint64_t EndNanos,
+                                void *TgtPtr, void *Data) {}
+
+  /// Hook that is called before launching a kernel.
+  virtual void handlePreKernelLaunch(GenericDeviceTy *Device,
+                                     uint32_t NumBlocks[3],
+                                     __tgt_async_info *AI) {}
+
+  /// Hook that is called when the kernel is finished to extract the specific
+  /// timing info for that kernel execution.
+  virtual void handleKernelCompletion(uint64_t StartNanos, uint64_t EndNanos,
+                                      void *Data) {}
+
+  /// Hook that is called when a data transfer happens to extract timing info
+  /// for that transfer.
+  virtual void handleDataTransfer(uint64_t StartNanos, uint64_t EndNanos,
+                                  void *Data) {}
+
+  /// Allow factors for time conversion between host and device.
+  virtual void setTimeConversionFactorsImpl(double Slope, double Offset) {}
+
+  /// RAII style timer that measures the elapsed time between construction and
+  /// destruction, then invokes a callback with the profiler, start/end times,
+  /// and any captured arguments.
+  template <typename FnT, typename... ArgsT> class ProfTimerTy {
+  public:
+    ProfTimerTy(FnT &&F, GenericProfilerTy *P, GenericDeviceTy *D, ArgsT... As)
+        : Fun(F), Prof(P), Dev(D), Args(As...) {
+      assert(Prof && "GenericProfilerTy is null");
+      assert(Dev && "GenericDeviceTy is null");
+      if (Prof)
+        StartTime = Prof->getDeviceTimeStamp(Dev);
+    }
+
+    ~ProfTimerTy() {
+      assert(Prof && "GenericProfilerTy is null");
+      assert(Dev && "GenericDeviceTy is null");
+      if (Prof) {
+        uint64_t EndTime = Prof->getDeviceTimeStamp(Dev);
+        callViaUnpack(Fun, Prof, StartTime, EndTime, Args);
+      }
+    }
+
+  private:
+    FnT Fun;
+    GenericProfilerTy *Prof;
+    GenericDeviceTy *Dev;
+    uint64_t StartTime = 0;
+    std::tuple<ArgsT...> Args;
+  };
+
+  template <typename FnT, typename... ArgsT>
+  ProfTimerTy(FnT &&, GenericProfilerTy *, ArgsT...)
+      -> ProfTimerTy<FnT, ArgsT...>;
+
+  template <typename FnT, typename... ArgsT> friend class ProfTimerTy;
+
+  /// Returns an RAII style timer, which will handle data allocation timing.
+  [[nodiscard]] auto getScopedDataAllocTimer(GenericDeviceTy *Dev,
+                                             void *HostPtr, uint64_t Size,
+                                             void *ProfData = nullptr) {
+    return ProfTimerTy(
+        [](GenericProfilerTy *P, auto... args) {
+          assert(P && "P was null");
+          P->handleDataAlloc(args...);
+        },
+        this, Dev, HostPtr, Size, ProfData);
+  }
+
+  /// Returns an RAII style timer, which will handle data deletion timing.
+  [[nodiscard]] auto getScopedDataDeleteTimer(GenericDeviceTy *Dev,
+                                              void *TgtPtr,
+                                              void *ProfData = nullptr) {
+    return ProfTimerTy(
+        [](GenericProfilerTy *P, auto... args) {
+          assert(P && "P was null");
+          P->handleDataDelete(args...);
+        },
+        this, Dev, TgtPtr, ProfData);
+  }
+
+protected:
+  double HostToDeviceSlope = 1.0;
+  double HostToDeviceOffset = .0;
+
+private:
+  /// Vendor-specific implementation to obtain device time.
+  uint64_t getDeviceTimeStamp(GenericDeviceTy *D);
+};
+} // namespace plugin
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+#endif // OFFLOAD_PLUGINS_NEXTGEN_COMMON_INCLUDE_GENERICPROFILER_H
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index f99a0e817fd58..5e29ab576a999 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -29,6 +29,7 @@
 #include "Shared/Requirements.h"
 #include "Shared/Utils.h"
 
+#include "GenericProfiler.h"
 #include "GlobalHandler.h"
 #include "JIT.h"
 #include "MemoryManager.h"
@@ -57,6 +58,12 @@
 
 using namespace llvm::offload::debug;
 
+/// Factory function for creating a profiler. The default (weak) implementation
+/// returns a no-op GenericProfilerTy. When OMPT is enabled, a strong override
+/// returns an OmptProfilerTy instance.
+std::unique_ptr<llvm::omp::target::plugin::GenericProfilerTy>
+getProfilerToAttach();
+
 namespace llvm {
 namespace omp {
 namespace target {
@@ -1120,6 +1127,11 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
   uint32_t getDebugKind() const { return OMPX_DebugKind; }
   virtual uint64_t getClockFrequency() const { return CLOCKS_PER_SEC; }
 
+  /// Get a device-specific timestamp in nanoseconds, used by the profiler
+  /// for timing device operations. Subclasses should override this to provide
+  /// hardware-accurate timestamps (e.g., via HSA system info).
+  virtual uint64_t getDeviceTimeStamp() { return 0; }
+
   /// Get target compute unit kind (e.g., sm_80, or gfx908).
   virtual std::string getComputeUnitKind() const { return "unknown"; }
 
@@ -1376,16 +1388,6 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
   /// This is used to run the RPC server during task synchronization.
   RPCServerTy *RPCServer;
 
-#ifdef OMPT_SUPPORT
-  /// OMPT callback functions
-#define defineOmptCallback(Name, Type, Code) Name##_t Name##_fn = nullptr;
-  FOREACH_OMPT_DEVICE_EVENT(defineOmptCallback)
-#undef defineOmptCallback
-
-  /// Internal representation for OMPT device (initialize & finalize)
-  std::atomic<bool> OmptInitialized;
-#endif
-
   /// The total per-block native shared memory that a kernel may use.
   size_t MaxBlockSharedMemSize = 0;
 };
@@ -1397,7 +1399,8 @@ struct GenericPluginTy {
 
   /// Construct a plugin instance.
   GenericPluginTy(Triple::ArchType TA)
-      : GlobalHandler(nullptr), JIT(TA), RPCServer(nullptr) {}
+      : GlobalHandler(nullptr), JIT(TA), RPCServer(nullptr),
+        Profiler(getProfilerToAttach()) {}
 
   virtual ~GenericPluginTy() {}
 
@@ -1488,7 +1491,12 @@ struct GenericPluginTy {
   /// Tear down any target-specific doorbell resources.
   virtual Error deinitRPCDoorbell() { return Plugin::success(); }
 
-  /// Get a reference to the record and replay interface for the plugin.
+  /// Get a pointer to the profiler attached to this plugin.
+  GenericProfilerTy *getProfiler() {
+    assert(Profiler && "Profiler not initialized");
+    return Profiler.get();
+  }
+
   /// Initialize a device within the plugin.
   Error initDevice(int32_t DeviceId);
 
@@ -1760,6 +1768,9 @@ struct GenericPluginTy {
 
   /// The interface between the plugin and the GPU for host services.
   RPCServerTy *RPCServer;
+
+  /// The profiler backend attached to this plugin (e.g., OMPT).
+  std::unique_ptr<GenericProfilerTy> Profiler;
 };
 
 /// Auxiliary interface class for GenericDeviceResourceManagerTy. This class
diff --git a/offload/plugins-nextgen/common/src/GenericProfiler.cpp b/offload/plugins-nextgen/common/src/GenericProfiler.cpp
new file mode 100644
index 0000000000000..7269a8aaf8991
--- /dev/null
+++ b/offload/plugins-nextgen/common/src/GenericProfiler.cpp
@@ -0,0 +1,35 @@
+//===- GenericProfiler.cpp - GenericProfiler implementation ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#include "GenericProfiler.h"
+#include "PluginInterface.h"
+
+#include <cstdint>
+#include <memory>
+
+__attribute__((weak)) std::unique_ptr<llvm::omp::target::plugin::GenericProfilerTy>
+getProfilerToAttach() {
+  return std::make_unique<llvm::omp::target::plugin::GenericProfilerTy>();
+}
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace plugin {
+
+uint64_t GenericProfilerTy::getDeviceTimeStamp(GenericDeviceTy *D) {
+  if (D)
+    return D->getDeviceTimeStamp();
+  return 0;
+}
+} // namespace plugin
+} // namespace target
+} // namespace omp
+} // namespace llvm
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index 1e05c6ae66fdf..db0c9819e69bb 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -21,10 +21,7 @@
 #include "Utils/ELF.h"
 #include "omptarget.h"
 
-#ifdef OMPT_SUPPORT
-#include "OpenMP/OMPT/Callback.h"
-#include "omp-tools.h"
-#endif
+#include "GenericProfiler.h"
 
 #include "llvm/Bitcode/BitcodeReader.h"
 #include "llvm/Frontend/OpenMP/OMPConstants.h"
@@ -47,7 +44,9 @@ using namespace llvm::offload::debug;
 AsyncInfoWrapperTy::AsyncInfoWrapperTy(GenericDeviceTy &Device,
                                        __tgt_async_info *AsyncInfoPtr)
     : Device(Device),
-      AsyncInfoPtr(AsyncInfoPtr ? AsyncInfoPtr : &LocalAsyncInfo) {}
+      AsyncInfoPtr(AsyncInfoPtr ? AsyncInfoPtr : &LocalAsyncInfo) {
+  this->AsyncInfoPtr->ProfilerData = nullptr;
+}
 
 Error AsyncInfoWrapperTy::synchronize() {
   assert(AsyncInfoPtr && "AsyncInfoWrapperTy already finalized");
@@ -162,9 +161,15 @@ GenericKernelTy::getKernelLaunchEnvironment(
        DPxPTR(&LocalKLE), DPxPTR(*AllocOrErr),
        sizeof(KernelLaunchEnvironmentTy));
 
+  // Temporarily suppress ProfilerData so the KLE upload is not traced as
+  // a user data operation.
+  __tgt_async_info *AI = AsyncInfoWrapper;
+  void *SavedProfilerData = AI->ProfilerData;
+  AI->ProfilerData = nullptr;
   auto Err = GenericDevice.dataSubmit(*AllocOrErr, &LocalKLE,
                                       sizeof(KernelLaunchEnvironmentTy),
                                       AsyncInfoWrapper);
+  AI->ProfilerData = SavedProfilerData;
   if (Err)
     return Err;
   return static_cast<KernelLaunchEnvironmentTy *>(*AllocOrErr);
@@ -323,6 +328,9 @@ Error GenericKernelTy::launch(GenericDeviceTy &GenericDevice, void **ArgPtrs,
     RRHandle = *RRHandleOrErr;
   }
 
+  GenericDevice.Plugin.getProfiler()->handlePreKernelLaunch(
+      &GenericDevice, EffectiveNumBlocks, AsyncInfoWrapper);
+
   if (auto Err = launchImpl(GenericDevice, EffectiveNumThreads,
                             EffectiveNumBlocks, DynBlockMemConf.NativeSize,
                             KernelArgs, LaunchParams, AsyncInfoWrapper))
@@ -498,22 +506,6 @@ GenericDeviceTy::GenericDeviceTy(GenericPluginTy &Plugin, int32_t DeviceId,
   // vendor (u)uid will become available later.
   setDeviceUidFromVendorUid(std::to_string(static_cast<uint64_t>(DeviceId)));
 
-#ifdef OMPT_SUPPORT
-  OmptInitialized.store(false);
-  // Bind the callbacks to this device's member functions
-#define bindOmptCallback(Name, Type, Code)                                     \
-  if (ompt::Initialized && ompt::lookupCallbackByCode) {                       \
-    ompt::lookupCallbackByCode((ompt_callbacks_t)(Code),                       \
-                               ((ompt_callback_t *)&(Name##_fn)));             \
-    ODBG(OLDT_Tool) << "OMPT: class bound " << #Name << "="                    \
-                    << ((void *)(uint64_t)Name##_fn);                          \
-  }
-
-  FOREACH_OMPT_DEVICE_EVENT(bindOmptCallback);
-#undef bindOmptCallback
-
-#endif
-
   // Envar that indicates whether mapped host buffers should be locked
   // automatically. The possible values are boolean (on/off) and a special:
   //   off:       Mapped host buffers are not locked.
@@ -545,17 +537,7 @@ Error GenericDeviceTy::init(GenericPluginTy &Plugin) {
   if (auto Err = initImpl(Plugin))
     return Err;
 
-#ifdef OMPT_SUPPORT
-  if (ompt::Initialized) {
-    bool ExpectedStatus = false;
-    if (OmptInitialized.compare_exchange_strong(ExpectedStatus, true))
-      performOmptCallback(device_initialize, Plugin.getUserId(DeviceId),
-                          /*type=*/getComputeUnitKind().c_str(),
-                          /*device=*/reinterpret_cast<ompt_device_t *>(this),
-                          /*lookup=*/ompt::lookupCallbackByName,
-                          /*documentation=*/nullptr);
-  }
-#endif
+  Plugin.getProfiler()->handleInit(this, &Plugin);
 
   // Read and reinitialize the envars that depend on the device initialization.
   // Notice these two envars may change the stack size and heap size of the
@@ -645,13 +627,7 @@ Error GenericDeviceTy::deinit(GenericPluginTy &Plugin) {
     if (auto Err = RPCServer->deinitDevice(*this))
       return Err;
 
-#ifdef OMPT_SUPPORT
-  if (ompt::Initialized) {
-    bool ExpectedStatus = true;
-    if (OmptInitialized.compare_exchange_strong(ExpectedStatus, false))
-      performOmptCallback(device_finalize, Plugin.getUserId(DeviceId));
-  }
-#endif
+  Plugin.getProfiler()->handleDeinit(this, &Plugin);
 
   return deinitImpl();
 }
@@ -686,17 +662,7 @@ Expected<DeviceImageTy *> GenericDeviceTy::loadBinary(GenericPluginTy &Plugin,
   if (auto Err = setupRPCServer(Plugin, *Image))
     return std::move(Err);
 
-#ifdef OMPT_SUPPORT
-  if (ompt::Initialized) {
-    size_t Bytes = InputTgtImage.size();
-    performOmptCallback(
-        device_load, Plugin.getUserId(DeviceId),
-        /*FileName=*/nullptr, /*FileOffset=*/0, /*VmaInFile=*/nullptr,
-        /*ImgSize=*/Bytes,
-        /*HostAddr=*/const_cast<unsigned char *>(InputTgtImage.bytes_begin()),
-        /*DeviceAddr=*/nullptr, /* FIXME: ModuleId */ 0);
-  }
-#endif
+  Plugin.getProfiler()->handleLoadBinary(this, &Plugin, InputTgtImage);
 
   // Call any global constructors present on the device.
   if (auto Err = callGlobalConstructors(Plugin, *Image))
@@ -978,6 +944,9 @@ Error GenericDeviceTy::getDeviceMemorySize(uint64_t &DSize) {
 
 Expected<void *> GenericDeviceTy::dataAlloc(int64_t Size, void *HostPtr,
                                             TargetAllocTy Kind) {
+  auto ProfTimer =
+      Plugin.getProfiler()->getScopedDataAllocTimer(this, HostPtr, Size);
+
   void *Alloc = nullptr;
 
   if (RecordReplay && RecordReplay->isRecordingOrReplaying())
@@ -1044,6 +1013,9 @@ Expected<void *> GenericDeviceTy::dataAlloc(int64_t Size, void *HostPtr,
 }
 
 Error GenericDeviceTy::dataDelete(void *TgtPtr, TargetAllocTy Kind) {
+  auto ProfTimer =
+      Plugin.getProfiler()->getScopedDataDeleteTimer(this, TgtPtr);
+
   // Free is a noop when recording or replaying.
   if (RecordReplay && RecordReplay->isRecordingOrReplaying())
     return RecordReplay->deallocate(TgtPtr);

>From 325aaa5346bd10cda2a719aa47ce01dd4d9563a3 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 2 Apr 2026 07:36:31 -0500
Subject: [PATCH 03/11] [Offload][AMDGPU] Wire HSA profiling into
 GenericProfiler abstraction

Add device profiling infrastructure to the AMDGPU plugin so that the
GenericProfiler can receive nanosecond-accurate kernel execution and
data transfer timestamps from the HSA runtime.

Key changes:
- Add ProfilingInfoTy struct to transport HSA profiling data
- Add timeKernelInNsAsync/timeDataTransferInNsAsync callbacks that
  extract dispatch/copy times from HSA signals and call
  handleKernelCompletion/handleDataTransfer on the profiler
- Add getOrNullProfilerSpecificData helper to extract ProfilerData
  from AsyncInfoWrapperTy
- Add getDeviceTimeStamp() override using hsa_system_get_info
- Add getSystemTimestampInNs() for HSA system timestamp queries
- Add schedProfilerKernelTiming/schedProfilerDataTransferTiming to
  StreamSlotTy for scheduling profiler callbacks on stream slots
- Thread ProfilerSpecificData through pushKernelLaunch,
  pushMemoryCopyH2DAsync, pushMemoryCopyD2HAsync, pushMemoryCopyD2DAsync
- Extract ProfilerSpecificData in dataSubmitImpl, dataRetrieveImpl,
  dataExchangeImpl, and launchImpl
- Add stub getDeviceTimeStamp() to CUDA plugin

Made-with: Cursor
Co-Authored-By: Claude <noreply at anthropic.com>
---
 .../amdgpu/dynamic_hsa/hsa.cpp                |   1 +
 .../amdgpu/dynamic_hsa/hsa_ext_amd.h          |   9 +
 offload/plugins-nextgen/amdgpu/src/rtl.cpp    | 204 +++++++++++++++++-
 offload/plugins-nextgen/cuda/src/rtl.cpp      |   3 +
 4 files changed, 209 insertions(+), 8 deletions(-)

diff --git a/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa.cpp b/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa.cpp
index 5c7ec186b0ceb..99202574bfecc 100644
--- a/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa.cpp
+++ b/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa.cpp
@@ -71,6 +71,7 @@ DLWRAP(hsa_amd_signal_create, 5)
 DLWRAP(hsa_amd_signal_async_handler, 5)
 DLWRAP(hsa_amd_pointer_info, 5)
 DLWRAP(hsa_amd_profiling_get_dispatch_time, 3)
+DLWRAP(hsa_amd_profiling_get_async_copy_time, 2)
 DLWRAP(hsa_amd_profiling_set_profiler_enabled, 2)
 DLWRAP(hsa_code_object_reader_create_from_memory, 3)
 DLWRAP(hsa_code_object_reader_destroy, 1)
diff --git a/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa_ext_amd.h b/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa_ext_amd.h
index d26f9248e27ef..35b3fc1a594cc 100644
--- a/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa_ext_amd.h
+++ b/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa_ext_amd.h
@@ -206,6 +206,15 @@ hsa_amd_profiling_get_dispatch_time(hsa_agent_t agent, hsa_signal_t signal,
 hsa_status_t hsa_amd_profiling_set_profiler_enabled(hsa_queue_t *queue,
                                                     int enable);
 
+typedef struct hsa_amd_profiling_async_copy_time_s {
+  uint64_t start;
+  uint64_t end;
+} hsa_amd_profiling_async_copy_time_t;
+
+hsa_status_t
+hsa_amd_profiling_get_async_copy_time(hsa_signal_t signal,
+                                      hsa_amd_profiling_async_copy_time_t *time);
+
 hsa_status_t hsa_amd_vmem_address_reserve(void **va, size_t size,
                                           uint64_t address, uint64_t flags);
 
diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index d1c6d0de11280..331b16cef852f 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -30,6 +30,7 @@
 #include "Shared/Utils.h"
 #include "Utils/ELF.h"
 
+#include "GenericProfiler.h"
 #include "GlobalHandler.h"
 #include "OffloadAPI.h"
 #include "OpenMP/OMPT/Callback.h"
@@ -95,6 +96,88 @@ struct AMDGPUEventManagerTy;
 struct AMDGPUDeviceImageTy;
 struct AMDGPUMemoryManagerTy;
 struct AMDGPUMemoryPoolTy;
+struct AMDGPUSignalTy;
+
+/// Use to transport information to profiler timing functions.
+struct ProfilingInfoTy {
+  GenericPluginTy *Plugin;
+  hsa_agent_t Agent;
+  AMDGPUSignalTy *Signal;
+  double TicksToTime;
+  void *ProfilerSpecificData;
+};
+
+static ProfilingInfoTy *getProfilingInfo(void *Data);
+
+static std::pair<uint64_t, uint64_t>
+getKernelStartAndEndTime(const ProfilingInfoTy *Args);
+
+static std::pair<uint64_t, uint64_t>
+getCopyStartAndEndTime(const ProfilingInfoTy *Args);
+
+static Error timeKernelInNsAsync(void *Data);
+
+static Error timeDataTransferInNsAsync(void *Data) {
+  auto Args = getProfilingInfo(Data);
+  auto [Start, End] = getCopyStartAndEndTime(Args);
+  Args->Plugin->getProfiler()->handleDataTransfer(Start, End,
+                                                  Args->ProfilerSpecificData);
+  return Plugin::success();
+}
+
+static void *
+getOrNullProfilerSpecificData(AsyncInfoWrapperTy &AsyncInfoWrapper) {
+  __tgt_async_info *AI = AsyncInfoWrapper;
+  return AI ? AI->ProfilerData : nullptr;
+}
+
+} // namespace plugin
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+static double setTicksToTime() {
+  uint64_t TicksFrequency = 1;
+  double TicksToTime = 1.0;
+
+  hsa_status_t Status =
+      hsa_system_get_info(HSA_SYSTEM_INFO_TIMESTAMP_FREQUENCY, &TicksFrequency);
+  if (Status == HSA_STATUS_SUCCESS)
+    TicksToTime = (double)1e9 / (double)TicksFrequency;
+
+  return TicksToTime;
+}
+
+static double TicksToTime = 1.0;
+
+static void setHSATicksToTimeConstant() { TicksToTime = setTicksToTime(); }
+
+/// Get the current HSA-based system timestamp in nanoseconds.
+/// Called by OmptTracing.cpp (from PluginOmpt) for device time queries.
+uint64_t getSystemTimestampInNs() {
+  uint64_t TimeStamp = 0;
+  hsa_status_t Status =
+      hsa_system_get_info(HSA_SYSTEM_INFO_TIMESTAMP, &TimeStamp);
+  if (Status != HSA_STATUS_SUCCESS)
+    return 0;
+  return TimeStamp;
+}
+
+/// Enable or disable HSA async copy profiling for OMPT device tracing.
+/// Called by OmptTracing.cpp when a tool activates/deactivates device tracing.
+/// Full HSA profiling integration (enabling per-copy timing signals) will be
+/// wired up in a follow-on commit.
+void setOmptAsyncCopyProfile(bool Enable) {}
+
+/// Enable or disable HSA queue kernel profiling for OMPT device tracing.
+/// Called by OmptTracing.cpp when a tool activates/deactivates device tracing.
+/// Full HSA queue profiling integration will be wired up in a follow-on commit.
+void setGlobalOmptKernelProfile(void *Device, int Enable) {}
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace plugin {
 
 namespace hsa_utils {
 
@@ -1024,6 +1107,7 @@ struct AMDGPUStreamTy {
       MemcpyArgsTy MemcpyArgs;
       ReleaseBufferArgsTy ReleaseBufferArgs;
       ReleaseSignalArgsTy ReleaseSignalArgs;
+      ProfilingInfoTy ProfilerArgs;
       void *CallbackArgs;
     };
 
@@ -1061,7 +1145,32 @@ struct AMDGPUStreamTy {
       return Plugin::success();
     }
 
-    /// Register a callback to be called on compleition
+    /// Schedule kernel timing measurement via the profiler on the slot.
+    Error schedProfilerKernelTiming(GenericDeviceTy *Device, hsa_agent_t Agent,
+                                    AMDGPUSignalTy *OutputSignal,
+                                    double TicksToTime,
+                                    void *ProfilerSpecificData) {
+      Callbacks.emplace_back(timeKernelInNsAsync);
+      ActionArgs.emplace_back().ProfilerArgs =
+          ProfilingInfoTy{&(Device->Plugin), Agent, OutputSignal, TicksToTime,
+                          ProfilerSpecificData};
+      return Plugin::success();
+    }
+
+    /// Schedule data transfer timing via the profiler on the slot.
+    Error schedProfilerDataTransferTiming(GenericDeviceTy *Device,
+                                          hsa_agent_t Agent,
+                                          AMDGPUSignalTy *OutputSignal,
+                                          double TicksToTime,
+                                          void *ProfilerSpecificData) {
+      Callbacks.emplace_back(timeDataTransferInNsAsync);
+      ActionArgs.emplace_back().ProfilerArgs =
+          ProfilingInfoTy{&(Device->Plugin), Agent, OutputSignal, TicksToTime,
+                          ProfilerSpecificData};
+      return Plugin::success();
+    }
+
+    /// Register a callback to be called on completion.
     Error schedCallback(AMDGPUStreamCallbackTy *Func, void *Data) {
       Callbacks.emplace_back(Func);
       ActionArgs.emplace_back().CallbackArgs = Data;
@@ -1086,6 +1195,12 @@ struct AMDGPUStreamTy {
         } else if (Callback == releaseSignalAction) {
           if (auto Err = releaseSignalAction(&ActionArg))
             return Err;
+        } else if (Callback == timeKernelInNsAsync) {
+          if (auto Err = timeKernelInNsAsync(&ActionArg))
+            return Err;
+        } else if (Callback == timeDataTransferInNsAsync) {
+          if (auto Err = timeDataTransferInNsAsync(&ActionArg))
+            return Err;
         } else if (Callback) {
           if (auto Err = Callback(ActionArg.CallbackArgs))
             return Err;
@@ -1357,7 +1472,8 @@ struct AMDGPUStreamTy {
   Error pushKernelLaunch(const AMDGPUKernelTy &Kernel, void *KernelArgs,
                          uint32_t NumThreads[3], uint32_t NumBlocks[3],
                          uint32_t GroupSize, uint64_t StackSize,
-                         AMDGPUMemoryManagerTy &MemoryManager) {
+                         AMDGPUMemoryManagerTy &MemoryManager,
+                         void *ProfilerSpecificData = nullptr) {
     if (Queue == nullptr)
       return Plugin::error(ErrorCode::INVALID_NULL_POINTER,
                            "target queue was nullptr");
@@ -1378,6 +1494,14 @@ struct AMDGPUStreamTy {
     if (auto Err = Slots[Curr].schedReleaseBuffer(KernelArgs, MemoryManager))
       return Err;
 
+#ifdef OMPT_SUPPORT
+    if (ProfilerSpecificData) {
+      if (auto Err = Slots[Curr].schedProfilerKernelTiming(
+              &Device, Agent, OutputSignal, TicksToTime, ProfilerSpecificData))
+        return Err;
+    }
+#endif
+
     // If we are running an RPC server we want to wake up the server thread
     // whenever there is a kernel running and let it sleep otherwise.
     if (Device.getRPCServer())
@@ -1440,7 +1564,8 @@ struct AMDGPUStreamTy {
   /// manager once the operation completes.
   Error pushMemoryCopyD2HAsync(void *Dst, const void *Src, void *Inter,
                                uint64_t CopySize,
-                               AMDGPUMemoryManagerTy &MemoryManager) {
+                               AMDGPUMemoryManagerTy &MemoryManager,
+                               void *ProfilerSpecificData = nullptr) {
     // Retrieve available signals for the operation's outputs.
     AMDGPUSignalTy *OutputSignals[2] = {};
     if (auto Err = SignalManager.getResources(/*Num=*/2, OutputSignals))
@@ -1503,6 +1628,7 @@ struct AMDGPUStreamTy {
   Error pushMemoryCopyH2DAsync(void *Dst, const void *Src, void *Inter,
                                uint64_t CopySize,
                                AMDGPUMemoryManagerTy &MemoryManager,
+                               void *ProfilerSpecificData = nullptr,
                                size_t NumTimes = 1) {
     // Retrieve available signals for the operation's outputs.
     AMDGPUSignalTy *OutputSignals[2] = {};
@@ -1578,7 +1704,8 @@ struct AMDGPUStreamTy {
 
   // AMDGPUDeviceTy is incomplete here, passing the underlying agent instead
   Error pushMemoryCopyD2DAsync(void *Dst, hsa_agent_t DstAgent, const void *Src,
-                               hsa_agent_t SrcAgent, uint64_t CopySize) {
+                               hsa_agent_t SrcAgent, uint64_t CopySize,
+                               void *ProfilerSpecificData = nullptr) {
     AMDGPUSignalTy *OutputSignal;
     if (auto Err = SignalManager.getResources(/*Num=*/1, &OutputSignal))
       return Err;
@@ -2244,6 +2371,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     if (auto Err = initMemoryPools())
       return Err;
 
+    setHSATicksToTimeConstant();
+
     char GPUName[64];
     if (auto Err = getDeviceAttr(HSA_AGENT_INFO_NAME, GPUName))
       return Err;
@@ -2576,6 +2705,9 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
   /// Returns the clock frequency for the given AMDGPU device.
   uint64_t getClockFrequency() const override { return ClockFrequency; }
 
+  /// Returns the current HSA system timestamp for profiling.
+  uint64_t getDeviceTimeStamp() override { return getSystemTimestampInNs(); }
+
   /// Returns the HSA system timestamp frequency. Zero means unavailable.
   uint64_t getSystemTimestampFrequency() const {
     return SystemTimestampFrequency;
@@ -2784,6 +2916,9 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     AMDGPUStreamTy *Stream = nullptr;
     void *PinnedPtr = nullptr;
 
+    auto ProfilerSpecificData =
+        getOrNullProfilerSpecificData(AsyncInfoWrapper);
+
     // Use one-step asynchronous operation when host memory is already pinned.
     if (void *PinnedPtr =
             PinnedAllocs.getDeviceAccessiblePtrFromPinnedBuffer(HstPtr)) {
@@ -2834,7 +2969,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
       return Err;
 
     return Stream->pushMemoryCopyH2DAsync(TgtPtr, HstPtr, PinnedPtr, Size,
-                                          PinnedMemoryManager);
+                                          PinnedMemoryManager,
+                                          ProfilerSpecificData);
   }
 
   /// Retrieve data from the device (device to host transfer).
@@ -2843,6 +2979,9 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     AMDGPUStreamTy *Stream = nullptr;
     void *PinnedPtr = nullptr;
 
+    auto ProfilerSpecificData =
+        getOrNullProfilerSpecificData(AsyncInfoWrapper);
+
     // Use one-step asynchronous operation when host memory is already pinned.
     if (void *PinnedPtr =
             PinnedAllocs.getDeviceAccessiblePtrFromPinnedBuffer(HstPtr)) {
@@ -2894,7 +3033,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
       return Err;
 
     return Stream->pushMemoryCopyD2HAsync(HstPtr, TgtPtr, PinnedPtr, Size,
-                                          PinnedMemoryManager);
+                                          PinnedMemoryManager,
+                                          ProfilerSpecificData);
   }
 
   /// Exchange data between two devices within the plugin.
@@ -2903,6 +3043,9 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
                          AsyncInfoWrapperTy &AsyncInfoWrapper) override {
     AMDGPUDeviceTy &DstDevice = static_cast<AMDGPUDeviceTy &>(DstGenericDevice);
 
+    auto ProfilerSpecificData =
+        getOrNullProfilerSpecificData(AsyncInfoWrapper);
+
     // For large transfers use synchronous behavior.
     if (Size >= OMPX_MaxAsyncCopyBytes) {
       if (AsyncInfoWrapper.hasQueue())
@@ -2931,7 +3074,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
       return Plugin::success();
 
     return Stream->pushMemoryCopyD2DAsync(DstPtr, DstDevice.getAgent(), SrcPtr,
-                                          getAgent(), (uint64_t)Size);
+                                          getAgent(), (uint64_t)Size,
+                                          ProfilerSpecificData);
   }
 
   /// Insert a data fence between previous data operations and the following
@@ -3009,6 +3153,7 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
 
     return Stream->pushMemoryCopyH2DAsync(TgtPtr, PatternPtr, PinnedPtr,
                                           PatternSize, PinnedMemoryManager,
+                                          /*ProfilerSpecificData=*/nullptr,
                                           Size / PatternSize);
   }
 
@@ -4271,10 +4416,12 @@ Error AMDGPUKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
   // HSA requires the group segment size to include both static and dynamic.
   uint32_t TotalBlockMemSize = getStaticBlockMemSize() + DynBlockMemSize;
 
+  auto ProfilerSpecificData = getOrNullProfilerSpecificData(AsyncInfoWrapper);
+
   // Push the kernel launch into the stream.
   return Stream->pushKernelLaunch(*this, AllArgs, NumThreads, NumBlocks,
                                   TotalBlockMemSize, StackSize,
-                                  ArgsMemoryManager);
+                                  ArgsMemoryManager, ProfilerSpecificData);
 }
 
 Error AMDGPUKernelTy::printLaunchInfoDetails(GenericDeviceTy &GenericDevice,
@@ -4441,6 +4588,47 @@ void AMDGPUQueueTy::callbackError(hsa_status_t Status, hsa_queue_t *Source,
   FATAL_MESSAGE(1, "%s", toString(std::move(Err)).data());
 }
 
+/// Implementation of profiling helper functions.
+static ProfilingInfoTy *getProfilingInfo(void *Data) {
+  return reinterpret_cast<ProfilingInfoTy *>(Data);
+}
+
+static std::pair<uint64_t, uint64_t>
+getKernelStartAndEndTime(const ProfilingInfoTy *Args) {
+  hsa_amd_profiling_dispatch_time_t Time = {};
+  hsa_status_t Status =
+      hsa_amd_profiling_get_dispatch_time(Args->Agent, Args->Signal->get(),
+                                          &Time);
+  if (Status != HSA_STATUS_SUCCESS)
+    return {0, 0};
+  return {static_cast<uint64_t>(Time.start * Args->TicksToTime),
+          static_cast<uint64_t>(Time.end * Args->TicksToTime)};
+}
+
+static std::pair<uint64_t, uint64_t>
+getCopyStartAndEndTime(const ProfilingInfoTy *Args) {
+  hsa_amd_profiling_async_copy_time_t Time = {};
+  hsa_status_t Status =
+      hsa_amd_profiling_get_async_copy_time(Args->Signal->get(), &Time);
+  if (Status != HSA_STATUS_SUCCESS)
+    return {0, 0};
+  return {static_cast<uint64_t>(Time.start * Args->TicksToTime),
+          static_cast<uint64_t>(Time.end * Args->TicksToTime)};
+}
+
+static Error timeKernelInNsAsync(void *Data) {
+  assert(Data && "Invalid data pointer");
+  auto ProfilerInfo = getProfilingInfo(Data);
+  assert(ProfilerInfo && "Invalid profiling info");
+  assert(ProfilerInfo->ProfilerSpecificData &&
+         "Invalid ProfilerSpecificData");
+
+  auto [StartTime, EndTime] = getKernelStartAndEndTime(ProfilerInfo);
+  ProfilerInfo->Plugin->getProfiler()->handleKernelCompletion(
+      StartTime, EndTime, ProfilerInfo->ProfilerSpecificData);
+  return Plugin::success();
+}
+
 } // namespace plugin
 } // namespace target
 } // namespace omp
diff --git a/offload/plugins-nextgen/cuda/src/rtl.cpp b/offload/plugins-nextgen/cuda/src/rtl.cpp
index 51e2bdb0c01dc..add99e95b07e9 100644
--- a/offload/plugins-nextgen/cuda/src/rtl.cpp
+++ b/offload/plugins-nextgen/cuda/src/rtl.cpp
@@ -1331,6 +1331,9 @@ struct CUDADeviceTy : public GenericDeviceTy {
   /// Returns the clock frequency for the given NVPTX device.
   uint64_t getClockFrequency() const override { return 1000000000; }
 
+  /// Device timestamp stub for CUDA - full profiling is a future extension.
+  uint64_t getDeviceTimeStamp() override { return 0; }
+
 private:
   using CUDAStreamManagerTy = GenericDeviceResourceManagerTy<CUDAStreamRef>;
   using CUDAEventManagerTy = GenericDeviceResourceManagerTy<CUDAEventRef>;

>From 704ac544b24761ad062fc3b41312c7c675fd2931 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 2 Apr 2026 07:40:01 -0500
Subject: [PATCH 04/11] [Offload][OMPT] Add OmptProfilerTy, common types, and
 plugin tracing shim

Introduce the OMPT-specific profiler backend and shared type
definitions for the device tracing feature.

New files:
- OmptCommonDefs.h: Shared macros (FOREACH_OMPT_DEVICE_TRACING_FN,
  OMPT_IF_TRACING_ENABLED, performOmptCallback), function pointer
  typedefs for libomptarget_ompt_* entry points
- OmptEventInfoTy.h: Small struct carrying NumTeams and TraceRecord
  pointer for async profiler data handoff
- OmptDeviceTracing.h: Plugin-side API surface for device tracing
  state management, device ID mapping, clock correlation, and
  dynamic symbol loading via ensureFuncPtrLoaded template
- OmptProfiler.h/cpp: OmptProfilerTy inheriting GenericProfilerTy
  with OMPT callback bindings and profiler-specific data allocation.
  Provides strong override of getProfilerToAttach() factory
- OmptTracing.cpp (plugin side): Shim that dynamically loads
  libomptarget_ompt_* symbols, implements ompt_set_trace_ompt,
  ompt_start_trace, ompt_stop_trace, ompt_flush_trace,
  ompt_advance_buffer_cursor, ompt_get_record_ompt,
  ompt_get_device_time, ompt_translate_time

Modified:
- common/CMakeLists.txt: Add PluginOmpt static library target

Made-with: Cursor
Co-Authored-By: Claude <noreply at anthropic.com>
---
 offload/include/OpenMP/OMPT/OmptCommonDefs.h  | 127 +++++++
 offload/include/OpenMP/OMPT/OmptEventInfoTy.h |  39 +++
 offload/libomptarget/CMakeLists.txt           |  16 +
 offload/plugins-nextgen/common/CMakeLists.txt |  28 ++
 .../common/OMPT/OmptDeviceTracing.h           | 133 ++++++++
 .../common/OMPT/OmptProfiler.cpp              | 163 +++++++++
 .../common/OMPT/OmptProfiler.h                | 166 ++++++++++
 .../common/OMPT/OmptTracing.cpp               | 313 ++++++++++++++++++
 8 files changed, 985 insertions(+)
 create mode 100644 offload/include/OpenMP/OMPT/OmptCommonDefs.h
 create mode 100644 offload/include/OpenMP/OMPT/OmptEventInfoTy.h
 create mode 100644 offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h
 create mode 100644 offload/plugins-nextgen/common/OMPT/OmptProfiler.cpp
 create mode 100644 offload/plugins-nextgen/common/OMPT/OmptProfiler.h
 create mode 100644 offload/plugins-nextgen/common/OMPT/OmptTracing.cpp

diff --git a/offload/include/OpenMP/OMPT/OmptCommonDefs.h b/offload/include/OpenMP/OMPT/OmptCommonDefs.h
new file mode 100644
index 0000000000000..5391658e80262
--- /dev/null
+++ b/offload/include/OpenMP/OMPT/OmptCommonDefs.h
@@ -0,0 +1,127 @@
+//===------ OmptCommonDefs.h - Common definitions for OMPT --*- C++ -*-----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Common defines and typedefs for OMPT callback and tracing functionality.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OFFLOAD_INCLUDE_OMPTCOMMONDEFS_H
+#define OFFLOAD_INCLUDE_OMPTCOMMONDEFS_H
+
+#ifdef OMPT_SUPPORT
+
+#include "omp-tools.h"
+
+#pragma push_macro("DEBUG_PREFIX")
+#undef DEBUG_PREFIX
+#define DEBUG_PREFIX "OMPT"
+
+#define FUNCPTR_TO_PTR(x) ((void *)(uint64_t)x)
+
+#define FOREACH_OMPT_TARGET_CALLBACK(macro)                                    \
+  FOREACH_OMPT_DEVICE_EVENT(macro)                                             \
+  FOREACH_OMPT_NOEMI_EVENT(macro)                                              \
+  FOREACH_OMPT_EMI_EVENT(macro)
+
+// Common device tracing functions
+#define FOREACH_OMPT_DEVICE_TRACING_FN_COMMON(macro)                           \
+  macro(ompt_set_trace_ompt) macro(ompt_start_trace) macro(ompt_flush_trace)   \
+      macro(ompt_stop_trace) macro(ompt_advance_buffer_cursor)                 \
+          macro(ompt_get_record_type)
+
+// Supported device tracing entry points
+#define FOREACH_OMPT_DEVICE_TRACING_FN(macro)                                  \
+  FOREACH_OMPT_DEVICE_TRACING_FN_COMMON(macro)                                 \
+  macro(ompt_get_record_ompt) macro(ompt_get_device_time)                      \
+      macro(ompt_translate_time)
+
+// Device tracing functionalities, which are also e.g. coupled to mutexes
+#define FOREACH_OMPT_DEVICE_TRACING_FN_IMPLEMENTAIONS(macro)                   \
+  FOREACH_OMPT_DEVICE_TRACING_FN_COMMON(macro)                                 \
+  macro(ompt_set_timestamp) macro(ompt_set_granted_teams)
+
+#define OMPT_API_ROUTINE static
+
+#define OMPT_CALLBACK_AVAILABLE(fn) (llvm::omp::target::ompt::Initialized && fn)
+
+#define OMPT_IF_BUILT(stmt) stmt
+
+#define OMPT_IF_ENABLED(stmts)                                                 \
+  do {                                                                         \
+    if (llvm::omp::target::ompt::Initialized) {                                \
+      stmts                                                                    \
+    }                                                                          \
+  } while (0)
+
+#define OMPT_IF_TRACING_ENABLED(stmts)                                         \
+  do {                                                                         \
+    if (llvm::omp::target::ompt::TracingActive) {                              \
+      stmts                                                                    \
+    }                                                                          \
+  } while (0)
+
+#define OMPT_FRAME_FLAGS (ompt_frame_runtime | OMPT_FRAME_POSITION_DEFAULT)
+
+#if (__PPC64__ | __arm__)
+#define OMPT_GET_FRAME_ADDRESS(level) __builtin_frame_address(level)
+#define OMPT_FRAME_POSITION_DEFAULT ompt_frame_cfa
+#else
+#define OMPT_GET_FRAME_ADDRESS(level) __builtin_frame_address(level)
+#define OMPT_FRAME_POSITION_DEFAULT ompt_frame_framepointer
+#endif
+
+#define OMPT_PTR_UNKNOWN ((void *)0)
+
+#define performIfOmptInitialized(stmt)                                         \
+  do {                                                                         \
+    if (llvm::omp::target::ompt::Initialized) {                                \
+      stmt;                                                                    \
+    }                                                                          \
+  } while (0)
+
+#define performOmptCallback(CallbackName, ...)                                 \
+  do {                                                                         \
+    if (ompt_callback_##CallbackName##_fn)                                     \
+      ompt_callback_##CallbackName##_fn(__VA_ARGS__);                          \
+  } while (0)
+
+typedef ompt_set_result_t (*libomptarget_ompt_set_trace_ompt_t)(
+    int Device, unsigned int Enable, unsigned int EventTy);
+typedef int (*libomptarget_ompt_start_trace_t)(int,
+                                               ompt_callback_buffer_request_t,
+                                               ompt_callback_buffer_complete_t);
+typedef int (*libomptarget_ompt_flush_trace_t)(int);
+typedef int (*libomptarget_ompt_stop_trace_t)(int);
+typedef int (*libomptarget_ompt_advance_buffer_cursor_t)(
+    ompt_device_t *, ompt_buffer_t *, size_t, ompt_buffer_cursor_t,
+    ompt_buffer_cursor_t *);
+typedef ompt_get_record_ompt_t libomptarget_ompt_get_record_ompt_t;
+typedef ompt_device_time_t (*libomptarget_ompt_get_device_time_t)(
+    ompt_device_t *);
+typedef ompt_translate_time_t libomptarget_ompt_translate_time_t;
+typedef ompt_device_time_t (*libomptarget_ompt_get_device_time_t)(
+    ompt_device_t *);
+typedef ompt_record_t (*libomptarget_ompt_get_record_type_t)(
+    ompt_buffer_t *, ompt_buffer_cursor_t);
+typedef void (*libomptarget_ompt_set_timestamp_t)(uint64_t start, uint64_t end);
+typedef void (*libomptarget_ompt_set_granted_teams_t)(uint32_t);
+
+/// Function type def used for maintaining unique target region, target
+/// operations ids
+typedef uint64_t (*IdInterfaceTy)();
+
+#pragma pop_macro("DEBUG_PREFIX")
+
+#else
+#define performIfOmptInitialized(stmt)
+#define OMPT_IF_BUILT(stmt)
+#define OMPT_IF_ENABLED(stmts)
+#define OMPT_IF_TRACING_ENABLED(stmts)
+#endif // OMPT_SUPPORT
+
+#endif // OFFLOAD_INCLUDE_OMPTCOMMONDEFS_H
diff --git a/offload/include/OpenMP/OMPT/OmptEventInfoTy.h b/offload/include/OpenMP/OMPT/OmptEventInfoTy.h
new file mode 100644
index 0000000000000..7124b3a3ff501
--- /dev/null
+++ b/offload/include/OpenMP/OMPT/OmptEventInfoTy.h
@@ -0,0 +1,39 @@
+//===- OmptEventInfoTy.h - OMPT specific trace record data ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Data structure used to communicate OMPT specific profiler data from the
+// high-level libomptarget into the vendor-specific plugins
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OFFLOAD_INCLUDE_OPENMP_OMPT_OMPTEVENTINFOTY_H
+#define OFFLOAD_INCLUDE_OPENMP_OMPT_OMPTEVENTINFOTY_H
+
+#include "Shared/Debug.h"
+
+struct ompt_record_ompt_t;
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace ompt {
+
+/// Holds info needed to fill asynchronous trace records
+struct OmptEventInfoTy {
+  /// The granted number of teams at runtime
+  uint64_t NumTeams;
+  /// Pointer to the actual buffer storage location
+  ompt_record_ompt_t *TraceRecord;
+};
+
+} // namespace ompt
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+#endif // OFFLOAD_INCLUDE_OPENMP_OMPT_OMPTEVENTINFOTY_H
diff --git a/offload/libomptarget/CMakeLists.txt b/offload/libomptarget/CMakeLists.txt
index 8e6314b2a6eae..7909d55eff5c3 100644
--- a/offload/libomptarget/CMakeLists.txt
+++ b/offload/libomptarget/CMakeLists.txt
@@ -47,6 +47,22 @@ endforeach()
 
 target_compile_options(omptarget PRIVATE ${offload_compile_flags})
 target_link_options(omptarget PRIVATE ${offload_link_flags})
+if (OMPT_TARGET_DEFAULT AND LIBOMPTARGET_OMPT_SUPPORT)
+  # OmptProfiler.cpp provides the strong getProfilerToAttach() that routes
+  # plugin device events through OmptProfilerTy. It must be compiled directly
+  # into omptarget so its strong definition unconditionally overrides the weak
+  # fallback in GenericProfiler.cpp (which is embedded into each plugin archive
+  # via the PluginCommon OBJECT library and would otherwise win the link).
+  target_sources(omptarget PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}/../plugins-nextgen/common/OMPT/OmptProfiler.cpp
+  )
+  target_include_directories(omptarget PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}/../plugins-nextgen/common/OMPT
+    ${CMAKE_CURRENT_SOURCE_DIR}/../plugins-nextgen/common/include
+  )
+  # PluginOmpt provides OmptTracing.cpp (plugin-side GPU tracing).
+  target_link_libraries(omptarget PRIVATE PluginOmpt)
+endif()
 
 # libomptarget.so needs to be aware of where the plugins live as they
 # are now separated in the build directory.
diff --git a/offload/plugins-nextgen/common/CMakeLists.txt b/offload/plugins-nextgen/common/CMakeLists.txt
index ad018da333400..a717deb23e5e5 100644
--- a/offload/plugins-nextgen/common/CMakeLists.txt
+++ b/offload/plugins-nextgen/common/CMakeLists.txt
@@ -37,6 +37,34 @@ endif()
 include(FindLibcCommonUtils)
 target_link_libraries(PluginCommon PRIVATE llvm-libc-common-utilities)
 
+if (OMPT_TARGET_DEFAULT AND LIBOMPTARGET_OMPT_SUPPORT)
+  # OmptTracing.cpp calls plugin-specific symbols (HSA profiling etc.) and
+  # is linked per-plugin. OmptProfiler.cpp is added directly to omptarget
+  # in offload/libomptarget/CMakeLists.txt to ensure its strong
+  # getProfilerToAttach() override is unconditionally included.
+  add_library(PluginOmpt STATIC OMPT/OmptTracing.cpp)
+  target_include_directories(PluginOmpt PUBLIC
+    OMPT
+    ${CMAKE_CURRENT_SOURCE_DIR}/include
+    ${CMAKE_CURRENT_BINARY_DIR}/include
+    ${LIBOMPTARGET_LLVM_INCLUDE_DIRS}
+    ${LIBOMPTARGET_BINARY_INCLUDE_DIR}
+    ${LIBOMPTARGET_INCLUDE_DIR}
+  )
+  target_compile_options(PluginOmpt PUBLIC ${offload_compile_flags} -fPIC)
+  target_link_options(PluginOmpt PUBLIC ${offload_link_flags})
+  target_compile_definitions(PluginOmpt PRIVATE
+    TARGET_NAME="Profiler"
+    DEBUG_PREFIX="OMPT"
+  )
+  add_dependencies(PluginOmpt PluginErrcodes)
+endif()
+
+if (TARGET flang_rt.runtime.static)
+  target_link_libraries(PluginCommon PRIVATE flang_rt.runtime.static)
+  target_compile_definitions(PluginCommon PRIVATE OFFLOAD_HAS_FLANG_RT)
+endif()
+
 # Define the TARGET_NAME and DEBUG_PREFIX.
 target_compile_definitions(PluginCommon PRIVATE
   TARGET_NAME=PluginInterface
diff --git a/offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h b/offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h
new file mode 100644
index 0000000000000..5744a784825da
--- /dev/null
+++ b/offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h
@@ -0,0 +1,133 @@
+//===- OmptDeviceTracing.h - Target independent OMPT callbacks --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Interface used by target-independent runtimes to coordinate registration and
+// invocation of OMPT tracing functionality.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_COMMON_OMPTDEVICETRACING_H
+#define OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_COMMON_OMPTDEVICETRACING_H
+
+#ifdef OMPT_SUPPORT
+
+#include "OpenMP/OMPT/OmptCommonDefs.h"
+
+#include "llvm/Support/DynamicLibrary.h"
+
+#include <map>
+#include <memory>
+
+#pragma push_macro("DEBUG_PREFIX")
+#undef DEBUG_PREFIX
+#define DEBUG_PREFIX "OMPT"
+
+extern void setOmptAsyncCopyProfile(bool Enable);
+extern void setGlobalOmptKernelProfile(void *Device, int Enable);
+extern uint64_t getSystemTimestampInNs();
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace ompt {
+
+// Declare OMPT device tracing function entry points
+#define declareOmptTracingFn(Name) extern libomptarget_##Name##_t Name##_fn;
+FOREACH_OMPT_DEVICE_TRACING_FN_IMPLEMENTAIONS(declareOmptTracingFn)
+#undef declareOmptTracingFn
+
+// Declare OMPT device tracing function mutexes
+#define declareOmptTracingFnMutex(Name) extern std::mutex Name##_mutex;
+FOREACH_OMPT_DEVICE_TRACING_FN_IMPLEMENTAIONS(declareOmptTracingFnMutex)
+#undef declareOmptTracingFnMutex
+
+extern std::mutex DeviceIdWritingMutex;
+
+/// Activate tracing on the given device
+void enableDeviceTracing(int DeviceId);
+
+/// Deactivate tracing on the given device
+void disableDeviceTracing(int DeviceId);
+
+/// Set 'start' and 'stop' in trace records
+void setOmptTimestamp(uint64_t StartTime, uint64_t EndTime);
+
+/// Set the linear function correlation between host and device clocks
+void setOmptHostToDeviceRate(double Slope, double Offset);
+
+/// Set / store the number of granted teams in trace records
+void setOmptGrantedNumTeams(uint64_t NumTeams);
+
+/// Lookup the given device pointer and return its RTL device ID
+int getDeviceId(ompt_device_t *Device);
+
+/// Map the given device pointer to the given DeviceId
+void setDeviceId(ompt_device_t *Device, int32_t DeviceId);
+
+/// Rempve the given device pointer from the current mapping
+void removeDeviceId(ompt_device_t *Device);
+
+/// Check whether the provided device is currently traced.
+bool isTracedDevice(int32_t DeviceId);
+
+/// Provide name based lookup for the device tracing functions
+extern ompt_interface_fn_t
+lookupDeviceTracingFn(const char *InterfaceFunctionName);
+
+/// Host to device linear clock correlation
+extern double HostToDeviceSlope;
+
+/// Host to device constant clock offset
+extern double HostToDeviceOffset;
+
+/// Mapping of device pointers to their corresponding RTL device ID
+extern std::map<ompt_device_t *, int32_t> Devices;
+
+/// Mapping of RTL device IDs to their currently enabled tracing event types.
+/// Note: Event type '0' (bit position) indicates if this device is traced.
+extern std::map<int32_t, uint64_t> TracedDevices;
+
+/// OMPT global tracing status. Indicates if at least one device is traced.
+extern bool TracingActive;
+
+/// Parent library pointer
+extern std::shared_ptr<llvm::sys::DynamicLibrary> ParentLibrary;
+
+/// Get the parent library by pointer. If it is not already set, it will set the
+/// parent library pointer.
+std::shared_ptr<llvm::sys::DynamicLibrary> getParentLibrary();
+
+/// Set the parent library by filename
+void setParentLibrary(const char *Filename);
+
+/// Search for FuncName inside the parent library and assign to FuncPtr.
+/// IMPORTANT: This function assumes that the *caller* holds the respective lock
+/// for FuncPtr.
+template <typename FT>
+void ensureFuncPtrLoaded(const std::string &FuncName, FT *FuncPtr) {
+  if (*FuncPtr == nullptr) {
+    if ((ParentLibrary == nullptr && getParentLibrary() == nullptr) ||
+        !ParentLibrary->isValid())
+      return;
+    void *SymbolPtr = ParentLibrary->getAddressOfSymbol(FuncName.c_str());
+    if (SymbolPtr == nullptr)
+      return;
+    *FuncPtr = reinterpret_cast<FT>(SymbolPtr);
+  }
+}
+
+} // namespace ompt
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+#pragma pop_macro("DEBUG_PREFIX")
+
+#endif // OMPT_SUPPORT
+
+#endif // OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_COMMON_OMPTDEVICETRACING_H
diff --git a/offload/plugins-nextgen/common/OMPT/OmptProfiler.cpp b/offload/plugins-nextgen/common/OMPT/OmptProfiler.cpp
new file mode 100644
index 0000000000000..4b74492352c44
--- /dev/null
+++ b/offload/plugins-nextgen/common/OMPT/OmptProfiler.cpp
@@ -0,0 +1,163 @@
+//===- OmptProfiler.cpp - OMPT impl of GenericProfilerTy --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementation of OmptProfilerTy
+//
+//===----------------------------------------------------------------------===//
+
+#include "OmptProfiler.h"
+#include "OpenMP/OMPT/Interface.h"
+#include "PluginInterface.h"
+#include "Shared/Debug.h"
+
+#include <memory>
+
+using namespace llvm::omp::target;
+
+/// Strong override of getProfilerToAttach() that returns an OmptProfilerTy
+/// when OMPT is enabled. This overrides the weak default in GenericProfiler.cpp.
+std::unique_ptr<llvm::omp::target::plugin::GenericProfilerTy>
+getProfilerToAttach() {
+  return std::make_unique<llvm::omp::target::ompt::OmptProfilerTy>();
+}
+
+void ompt::OmptProfilerTy::handleInit(plugin::GenericDeviceTy *Device,
+                                      plugin::GenericPluginTy *Plugin) {
+  auto DeviceId = Device->getDeviceId();
+  auto DevicePtr = reinterpret_cast<ompt_device_t *>(Device);
+  ompt::setDeviceId(DevicePtr, Plugin->getUserId(DeviceId));
+
+  if (ompt::Initialized) {
+    bool ExpectedStatus = false;
+    if (OmptInitialized.compare_exchange_strong(ExpectedStatus, true))
+      performOmptCallback(device_initialize, Plugin->getUserId(DeviceId),
+                          /*type=*/Device->getComputeUnitKind().c_str(),
+                          /*device=*/DevicePtr,
+                          /*lookup=*/ompt::lookupDeviceTracingFn,
+                          /*documentation=*/nullptr);
+  }
+}
+
+void ompt::OmptProfilerTy::handleDeinit(
+    plugin::GenericDeviceTy *Device, target::plugin::GenericPluginTy *Plugin) {
+  auto DeviceId = Device->getDeviceId();
+
+  if (ompt::Initialized) {
+    bool ExpectedStatus = true;
+    if (OmptInitialized.compare_exchange_strong(ExpectedStatus, false))
+      performOmptCallback(device_finalize, Plugin->getUserId(DeviceId));
+  }
+  ompt::removeDeviceId(reinterpret_cast<ompt_device_t *>(Device));
+}
+
+void ompt::OmptProfilerTy::handleLoadBinary(plugin::GenericDeviceTy *Device,
+                                            plugin::GenericPluginTy *Plugin,
+                                            const StringRef InputTgtImage) {
+
+  if (!ompt::Initialized)
+    return;
+
+  auto DeviceId = Device->getDeviceId();
+  size_t Bytes = InputTgtImage.size();
+  performOmptCallback(
+      device_load, Plugin->getUserId(DeviceId),
+      /*FileName=*/nullptr, /*FileOffset=*/0, /*VmaInFile=*/nullptr,
+      /*ImgSize=*/Bytes,
+      /*HostAddr=*/const_cast<unsigned char *>(InputTgtImage.bytes_begin()),
+      /*DeviceAddr=*/nullptr, /* FIXME: ModuleId */ 0);
+}
+
+void ompt::OmptProfilerTy::handleDataAlloc(uint64_t StartNanos,
+                                           uint64_t EndNanos, void *HostPtr,
+                                           uint64_t Size, void *Data) {
+  ompt::setOmptTimestamp(StartNanos, EndNanos);
+}
+
+void ompt::OmptProfilerTy::handleDataDelete(uint64_t StartNanos,
+                                            uint64_t EndNanos, void *TgtPtr,
+                                            void *Data) {
+  ompt::setOmptTimestamp(StartNanos, EndNanos);
+}
+
+void ompt::OmptProfilerTy::handlePreKernelLaunch(
+    plugin::GenericDeviceTy *Device, uint32_t NumBlocks[3],
+    __tgt_async_info *AI) {
+  if (!ompt::isTracedDevice(
+          ompt::getDeviceId(reinterpret_cast<ompt_device_t *>(Device))))
+    return;
+
+  if (AI->ProfilerData == nullptr)
+    return;
+
+  auto ProfilerSpecificData =
+      reinterpret_cast<ompt::OmptEventInfoTy *>(AI->ProfilerData);
+  assert(ProfilerSpecificData && "Invalid ProfilerSpecificData");
+  // Set number of granted teams for OMPT
+  setOmptGrantedNumTeams(NumBlocks[0]);
+  ProfilerSpecificData->NumTeams = NumBlocks[0];
+}
+
+void ompt::OmptProfilerTy::handleKernelCompletion(uint64_t StartNanos,
+                                                  uint64_t EndNanos,
+                                                  void *Data) {
+
+  if (!isProfilingEnabled())
+    return;
+
+  /// Empty data means no tracing in OMPT
+  /// offload/include/OpenMP/OMPT/Interface.h line 492
+  if (!Data)
+    return;
+
+  ODBG(ODT_Tool) << "OMPT-Async: Time kernel for asynchronous execution: Start "
+                 << StartNanos << " End " << EndNanos;
+
+  auto OmptEventInfo = reinterpret_cast<ompt::OmptEventInfoTy *>(Data);
+  assert(OmptEventInfo && "Invalid OmptEventInfo");
+  assert(OmptEventInfo->TraceRecord && "Invalid TraceRecord");
+
+  ompt::RegionInterface.stopTargetSubmitTraceAsync(OmptEventInfo->TraceRecord,
+                                                   OmptEventInfo->NumTeams,
+                                                   StartNanos, EndNanos);
+
+  // Done processing, our responsibility to free the memory
+  freeProfilerDataEntry(OmptEventInfo);
+}
+
+void ompt::OmptProfilerTy::handleDataTransfer(uint64_t StartNanos,
+                                              uint64_t EndNanos, void *Data) {
+
+  if (!isProfilingEnabled())
+    return;
+
+  /// Empty data means no tracing in OMPT
+  /// offload/include/OpenMP/OMPT/Interface.h line 492
+  if (!Data)
+    return;
+
+  ODBG(ODT_Tool) << "OMPT-Async: Time data for asynchronous execution: Start "
+                 << StartNanos << " End " << EndNanos;
+
+  auto OmptEventInfo = reinterpret_cast<ompt::OmptEventInfoTy *>(Data);
+  assert(OmptEventInfo && "Invalid OmptEventInfo");
+  assert(OmptEventInfo->TraceRecord && "Invalid TraceRecord");
+
+  ompt::RegionInterface.stopTargetDataMovementTraceAsync(
+      OmptEventInfo->TraceRecord, StartNanos, EndNanos);
+
+  // Done processing, our responsibility to free the memory
+  freeProfilerDataEntry(OmptEventInfo);
+}
+
+bool ompt::OmptProfilerTy::isProfilingEnabled() { return ompt::TracingActive; }
+
+void ompt::OmptProfilerTy::setTimeConversionFactorsImpl(double Slope,
+                                                        double Offset) {
+  ODBG(ODT_Tool) << "Using Time Slope: " << Slope << " and Offset: " << Offset;
+  setOmptHostToDeviceRate(Slope, Offset);
+}
diff --git a/offload/plugins-nextgen/common/OMPT/OmptProfiler.h b/offload/plugins-nextgen/common/OMPT/OmptProfiler.h
new file mode 100644
index 0000000000000..9a12d0eda4a53
--- /dev/null
+++ b/offload/plugins-nextgen/common/OMPT/OmptProfiler.h
@@ -0,0 +1,166 @@
+//===- OmptProfiler.h - OMPT specific impl of GenericProfilerTy -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// OMPT specific implementation of the GenericProfilerTy class.
+// This class uses the already existing implementation of OMPT to invoke
+// callbacks and perform tracing.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OFFLOAD_PLUGINS_NEXTGEN_COMMON_OMPT_OMPTPROFILERTY_H
+#define OFFLOAD_PLUGINS_NEXTGEN_COMMON_OMPT_OMPTPROFILERTY_H
+
+#include "GenericProfiler.h"
+
+#include "OmptDeviceTracing.h"
+#include "OpenMP/OMPT/Callback.h"
+#include "OpenMP/OMPT/OmptEventInfoTy.h"
+#include "Shared/Debug.h"
+#include "omp-tools.h"
+
+#include <functional>
+#include <tuple>
+
+#pragma push_macro("DEBUG_PREFIX")
+#undef DEBUG_PREFIX
+#define DEBUG_PREFIX "OMPT"
+
+extern uint64_t getSystemTimestampInNs();
+
+using namespace llvm::omp::target::debug;
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace plugin {
+struct GenericDeviceTy;
+struct GenericPluginTy;
+class GenericProfilerTy;
+
+} // namespace plugin
+
+namespace ompt {
+
+// From Callback.h / Callback.cpp
+extern bool Initialized;
+
+/**
+ * Implements an OMPT backend for the Profiler interface used in the plugins.
+ *
+ * Forwards / Implements the different generic hooks with OMPT semantics.
+ */
+class OmptProfilerTy : public plugin::GenericProfilerTy {
+public:
+  /** Public members **/
+  OmptProfilerTy() {
+
+    OmptInitialized.store(false);
+    // Bind the callbacks to this device's member functions
+#define bindOmptCallback(Name, Type, Code)                                     \
+  if (ompt::Initialized && ompt::lookupCallbackByCode) {                       \
+    ompt::lookupCallbackByCode((ompt_callbacks_t)(Code),                       \
+                               ((ompt_callback_t *)&(Name##_fn)));             \
+    ODBG(ODT_Tool) << "class bound " << #Name                                  \
+                   << "=" << ((void *)(uint64_t)Name##_fn);                    \
+  }
+
+    FOREACH_OMPT_DEVICE_EVENT(bindOmptCallback);
+#undef bindOmptCallback
+
+#define bindOmptTracingFunction(FunctionName)                                  \
+  if (ompt::Initialized && ompt::lookupDeviceTracingFn) {                      \
+    FunctionName##_fn = ompt::lookupDeviceTracingFn(#FunctionName);            \
+    ODBG(ODT_Tool) << "device tracing fn bound " << #FunctionName              \
+                   << "=" << ((void *)(uint64_t)FunctionName##_fn);            \
+  }
+
+    FOREACH_OMPT_DEVICE_TRACING_FN_COMMON(bindOmptTracingFunction);
+#undef bindOmptTracingFunction
+  }
+
+  bool isProfilingEnabled() override;
+
+  void handleInit(plugin::GenericDeviceTy *Device,
+                  plugin::GenericPluginTy *Plugin) override;
+
+  void handleDeinit(plugin::GenericDeviceTy *Device,
+                    plugin::GenericPluginTy *Plugin) override;
+
+  void handleLoadBinary(plugin::GenericDeviceTy *Device,
+                        plugin::GenericPluginTy *Plugin,
+                        const StringRef InputTgtImage) override;
+
+  void handleDataAlloc(uint64_t StartNanos, uint64_t EndNanos, void *HostPtr,
+                       uint64_t Size, void *Data) override;
+  void handleDataDelete(uint64_t StartNanos, uint64_t EndNanos, void *TgtPtr,
+                        void *Data) override;
+
+  void handlePreKernelLaunch(plugin::GenericDeviceTy *Device,
+                             uint32_t NumBlocks[3],
+                             __tgt_async_info *AI) override;
+
+  void handleKernelCompletion(uint64_t StartNanos, uint64_t EndNanos,
+                              void *Data) override;
+
+  void handleDataTransfer(uint64_t StartNanos, uint64_t EndNanos,
+                          void *Data) override;
+
+  void setTimeConversionFactorsImpl(double Slope, double Offset) override;
+
+  void *getProfilerSpecificData() override {
+    // TODO: This is ID is not used currently
+    uint64_t Id = OmptProfDataId.fetch_add(1);
+    {
+      std::scoped_lock Lock(ProfilerDataMutex);
+      ProfilerData[Id] = std::make_unique<OmptEventInfoTy>();
+      return ProfilerData[Id].get();
+    }
+  }
+
+  void freeProfilerDataEntry(OmptEventInfoTy *DataPtr) {
+    std::scoped_lock Lock(ProfilerDataMutex);
+
+    for (auto &Entry : ProfilerData)
+      if (Entry.second.get() == DataPtr) {
+        ProfilerData.erase(Entry.first);
+        break;
+      }
+  }
+
+private:
+  /// Holds a unique ID for each allocation of OmptEventInfoTy
+  std::atomic<uint64_t> OmptProfDataId{0};
+
+  /// Holds memory used to store OMPT specific data and pass it down from
+  /// libomptarget into the plugins.
+  std::map<uint64_t, std::unique_ptr<OmptEventInfoTy>> ProfilerData;
+
+  /// Lock to guard STL ProfilerData map
+  std::mutex ProfilerDataMutex;
+
+  /// OMPT callback functions
+#define defineOmptCallback(Name, Type, Code) Name##_t Name##_fn = nullptr;
+  FOREACH_OMPT_DEVICE_EVENT(defineOmptCallback)
+#undef defineOmptCallback
+
+  /// OMPT device tracing functions
+#define defineOmptTracingFunction(Name) ompt_interface_fn_t Name##_fn = nullptr;
+  FOREACH_OMPT_DEVICE_TRACING_FN_COMMON(defineOmptTracingFunction);
+#undef defineOmptTracingFunction
+
+  /// Internal representation for OMPT device (initialize & finalize)
+  std::atomic<bool> OmptInitialized;
+};
+} // namespace ompt
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+#pragma pop_macro("DEBUG_PREFIX")
+
+#endif
diff --git a/offload/plugins-nextgen/common/OMPT/OmptTracing.cpp b/offload/plugins-nextgen/common/OMPT/OmptTracing.cpp
new file mode 100644
index 0000000000000..d5be9b8d9cd8a
--- /dev/null
+++ b/offload/plugins-nextgen/common/OMPT/OmptTracing.cpp
@@ -0,0 +1,313 @@
+//===-- OmptTracing.cpp - Target independent OpenMP target RTL --- C++ -*--===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementation of OMPT tracing interfaces for PluginInterface
+//
+//===----------------------------------------------------------------------===//
+
+#ifdef OMPT_SUPPORT
+
+#include "Shared/Debug.h"
+#include "OmptDeviceTracing.h"
+#include "omp-tools.h"
+
+#include "llvm/Support/DynamicLibrary.h"
+
+#include <atomic>
+#include <cassert>
+#include <cstdlib>
+#include <cstring>
+#include <memory>
+#include <mutex>
+
+#pragma push_macro("DEBUG_PREFIX")
+#undef DEBUG_PREFIX
+#define DEBUG_PREFIX "OMPT"
+
+// Define OMPT device tracing function entry points
+#define defineOmptTracingFn(Name)                                              \
+  libomptarget_##Name##_t llvm::omp::target::ompt::Name##_fn = nullptr;
+FOREACH_OMPT_DEVICE_TRACING_FN_IMPLEMENTAIONS(defineOmptTracingFn)
+#undef defineOmptTracingFn
+
+// Define OMPT device tracing function mutexes
+#define defineOmptTracingFnMutex(Name)                                         \
+  std::mutex llvm::omp::target::ompt::Name##_mutex;
+FOREACH_OMPT_DEVICE_TRACING_FN_IMPLEMENTAIONS(defineOmptTracingFnMutex)
+#undef defineOmptTracingFnMutex
+
+std::mutex llvm::omp::target::ompt::DeviceIdWritingMutex;
+
+using namespace llvm::omp::target::ompt;
+using namespace llvm::omp::target::debug;
+
+std::shared_ptr<llvm::sys::DynamicLibrary>
+    llvm::omp::target::ompt::ParentLibrary(nullptr);
+
+double llvm::omp::target::ompt::HostToDeviceSlope = .0;
+double llvm::omp::target::ompt::HostToDeviceOffset = .0;
+
+std::map<ompt_device_t *, int32_t> llvm::omp::target::ompt::Devices;
+
+std::shared_ptr<llvm::sys::DynamicLibrary>
+llvm::omp::target::ompt::getParentLibrary() {
+  static bool ParentLibraryAssigned = false;
+  if (!ParentLibraryAssigned) {
+    setParentLibrary("libomptarget.so");
+    ParentLibraryAssigned = true;
+  }
+  return ParentLibrary;
+}
+
+void llvm::omp::target::ompt::setParentLibrary(const char *Filename) {
+  if (ParentLibrary)
+    return;
+  std::string ErrorMsg;
+  ParentLibrary = std::make_shared<llvm::sys::DynamicLibrary>(
+      llvm::sys::DynamicLibrary::getPermanentLibrary(Filename, &ErrorMsg));
+  if ((ParentLibrary == nullptr) || (!ParentLibrary->isValid()))
+    REPORT() << "Failed to set parent library: " << ErrorMsg.c_str();
+}
+
+int llvm::omp::target::ompt::getDeviceId(ompt_device_t *Device) {
+  // Block other threads, which might trigger an erase (for the same device)
+  std::unique_lock<std::mutex> Lock(DeviceIdWritingMutex);
+  auto DeviceIterator = Devices.find(Device);
+  if (Device == nullptr || DeviceIterator == Devices.end()) {
+    REPORT() << "Failed to get ID for Device=" << Device;
+    return -1;
+  }
+  return DeviceIterator->second;
+}
+
+void llvm::omp::target::ompt::setDeviceId(ompt_device_t *Device,
+                                          int32_t DeviceId) {
+  assert(Device && "Mapping device ID to nullptr is not allowed");
+  if (Device == nullptr || DeviceId < 0) {
+    REPORT() << "Failed to set ID=%d for Device=" << DeviceId << Device;
+    return;
+  }
+  std::unique_lock<std::mutex> Lock(DeviceIdWritingMutex);
+  auto DeviceIterator = Devices.find(Device);
+  if (DeviceIterator != Devices.end()) {
+    auto CurrentDeviceId = DeviceIterator->second;
+    if (DeviceId == CurrentDeviceId) {
+      REPORT() << "Tried to duplicate OMPT Device= " << Device <<  " ID=" << DeviceId;
+    } else {
+      REPORT() << "Tried to overwrite OMPT Device=" << Device << " (ID=" << CurrentDeviceId << " with new ID=" << DeviceId;
+    }
+    return;
+  }
+  Devices.emplace(Device, DeviceId);
+}
+
+void llvm::omp::target::ompt::removeDeviceId(ompt_device_t *Device) {
+  int DeviceId = getDeviceId(Device);
+  if (DeviceId < 0) {
+    REPORT() << "Tried to remove Device= " << Device <<  " ID=" << DeviceId;
+    return;
+  }
+  std::unique_lock<std::mutex> Lock(DeviceIdWritingMutex);
+  Devices.erase(Device);
+  TracedDevices.erase(DeviceId);
+}
+
+OMPT_API_ROUTINE ompt_set_result_t ompt_set_trace_ompt(ompt_device_t *Device,
+                                                       unsigned int Enable,
+                                                       unsigned int EventTy) {
+  ODBG(ODT_Tool) << "Executing ompt_set_trace_ompt";
+
+  int DeviceId = getDeviceId(Device);
+  if (DeviceId < 0) {
+    REPORT() << "Failed to set trace events for Device=" << Device <<
+                 " (Unknown device) [Enable=" << Enable << " EventTy=" << EventTy;
+    return ompt_set_never;
+  }
+
+  std::unique_lock<std::mutex> Lock(ompt_set_trace_ompt_mutex);
+  ensureFuncPtrLoaded<libomptarget_ompt_set_trace_ompt_t>(
+      "libomptarget_ompt_set_trace_ompt", &ompt_set_trace_ompt_fn);
+  assert(ompt_set_trace_ompt_fn && "libomptarget_ompt_set_trace_ompt loaded");
+  return ompt_set_trace_ompt_fn(DeviceId, Enable, EventTy);
+}
+
+OMPT_API_ROUTINE int
+ompt_start_trace(ompt_device_t *Device, ompt_callback_buffer_request_t Request,
+                 ompt_callback_buffer_complete_t Complete) {
+  ODBG(ODT_Tool) << "Executing ompt_start_trace";
+
+  int DeviceId = getDeviceId(Device);
+  if (DeviceId < 0) {
+    REPORT() << "Failed to start trace for Device=" << Device << " (Unknown device";
+    // Indicate failure
+    return 0;
+  }
+
+  {
+    // Protect the function pointer
+    std::unique_lock<std::mutex> Lock(ompt_start_trace_mutex);
+
+    if (Request && Complete) {
+      llvm::omp::target::ompt::enableDeviceTracing(DeviceId);
+      // Enable asynchronous memory copy profiling
+      setOmptAsyncCopyProfile(/*Enable=*/true);
+      // Enable queue dispatch profiling
+      if (DeviceId >= 0)
+        setGlobalOmptKernelProfile(Device, /*Enable=*/1);
+      else
+        REPORT() << "May not enable kernel profiling for invalid device id=" <<
+               DeviceId;
+    }
+
+    // Call libomptarget specific function
+    ensureFuncPtrLoaded<libomptarget_ompt_start_trace_t>(
+        "libomptarget_ompt_start_trace", &ompt_start_trace_fn);
+    assert(ompt_start_trace_fn && "libomptarget_ompt_start_trace loaded");
+  }
+  return ompt_start_trace_fn(DeviceId, Request, Complete);
+}
+
+OMPT_API_ROUTINE int ompt_flush_trace(ompt_device_t *Device) {
+  ODBG(ODT_Tool) << "Executing ompt_flush_trace";
+
+  std::unique_lock<std::mutex> Lock(ompt_flush_trace_mutex);
+  ensureFuncPtrLoaded<libomptarget_ompt_flush_trace_t>(
+      "libomptarget_ompt_flush_trace", &ompt_flush_trace_fn);
+  assert(ompt_flush_trace_fn && "libomptarget_ompt_flush_trace loaded");
+  return ompt_flush_trace_fn(getDeviceId(Device));
+}
+
+OMPT_API_ROUTINE int ompt_stop_trace(ompt_device_t *Device) {
+  ODBG(ODT_Tool) << "Executing ompt_stop_trace";
+
+  int DeviceId = getDeviceId(Device);
+  if (DeviceId < 0) {
+    REPORT() << "Failed to stop trace for Device=" << Device << " (Unknown device)";
+    // Indicate failure
+    return 0;
+  }
+
+  {
+    // Protect the function pointer
+    std::unique_lock<std::mutex> Lock(ompt_stop_trace_mutex);
+    llvm::omp::target::ompt::disableDeviceTracing(DeviceId);
+    // Disable asynchronous memory copy profiling
+    setOmptAsyncCopyProfile(/*Enable=*/false);
+    // Disable queue dispatch profiling
+    if (DeviceId >= 0)
+      setGlobalOmptKernelProfile(Device, /*Enable=*/0);
+    else
+      REPORT() << "May not disable kernel profiling for invalid device id=" <<
+             DeviceId;
+    ensureFuncPtrLoaded<libomptarget_ompt_stop_trace_t>(
+        "libomptarget_ompt_stop_trace", &ompt_stop_trace_fn);
+    assert(ompt_stop_trace_fn && "libomptarget_ompt_stop_trace loaded");
+  }
+  return ompt_stop_trace_fn(DeviceId);
+}
+
+OMPT_API_ROUTINE ompt_record_ompt_t *
+ompt_get_record_ompt(ompt_buffer_t *Buffer, ompt_buffer_cursor_t CurrentPos) {
+  // TODO In debug mode, get the metadata associated with this buffer
+  // and assert that there are enough bytes for the current record
+
+  // Currently, no synchronization required since a disjoint set of
+  // trace records is handed over to a thread.
+
+  // Note that CurrentPos can be nullptr. In that case, we return
+  // nullptr. The tool has to handle that properly.
+  return (ompt_record_ompt_t *)CurrentPos;
+}
+
+OMPT_API_ROUTINE int ompt_advance_buffer_cursor(ompt_device_t *Device,
+                                                ompt_buffer_t *Buffer,
+                                                size_t Size,
+                                                ompt_buffer_cursor_t CurrentPos,
+                                                ompt_buffer_cursor_t *NextPos) {
+  // Note: The input parameter size is unused here. It refers to the
+  // bytes returned in the corresponding callback.
+  // Advance can be called concurrently, so synchronize setting the
+  // function pointer. The actual libomptarget function does not need
+  // to be synchronized since it must be working on logically disjoint
+  // buffers.
+  std::unique_lock<std::mutex> Lock(ompt_advance_buffer_cursor_mutex);
+  ensureFuncPtrLoaded<libomptarget_ompt_advance_buffer_cursor_t>(
+      "libomptarget_ompt_advance_buffer_cursor",
+      &ompt_advance_buffer_cursor_fn);
+  assert(ompt_advance_buffer_cursor_fn &&
+         "libomptarget_ompt_advance_buffer_cursor loaded");
+  return ompt_advance_buffer_cursor_fn(Device, Buffer, Size, CurrentPos,
+                                       NextPos);
+}
+
+OMPT_API_ROUTINE ompt_record_t
+ompt_get_record_type(ompt_buffer_t *Buffer, ompt_buffer_cursor_t CurrentPos) {
+  std::unique_lock<std::mutex> Lock(ompt_get_record_type_mutex);
+  ensureFuncPtrLoaded<libomptarget_ompt_get_record_type_t>(
+      "libomptarget_ompt_get_record_type", &ompt_get_record_type_fn);
+  assert(ompt_get_record_type_fn && "libomptarget_ompt_get_record_type loaded");
+  return ompt_get_record_type_fn(Buffer, CurrentPos);
+}
+
+OMPT_API_ROUTINE ompt_device_time_t
+ompt_get_device_time(ompt_device_t *Device) {
+  ODBG(ODT_Tool) << "Executing ompt_get_device_time";
+  return getSystemTimestampInNs();
+}
+
+OMPT_API_ROUTINE double ompt_translate_time(ompt_device_t *Device,
+                                            ompt_device_time_t DeviceTime) {
+  // Translate a device time to a meaningful timepoint in host time
+  // We do not need to account for clock-skew / drift. So simple linear
+  // translation using the host to device rate we obtained.
+  double TranslatedTime = DeviceTime * HostToDeviceSlope + HostToDeviceOffset;
+  ODBG(ODT_Tool) << "D2H translated time: " << TranslatedTime;
+
+  return TranslatedTime;
+}
+
+void llvm::omp::target::ompt::setOmptTimestamp(uint64_t StartTime,
+                                               uint64_t EndTime) {
+  std::unique_lock<std::mutex> Lock(ompt_set_timestamp_mutex);
+  ensureFuncPtrLoaded<libomptarget_ompt_set_timestamp_t>(
+      "libomptarget_ompt_set_timestamp", &ompt_set_timestamp_fn);
+  // No need to hold a lock
+  ompt_set_timestamp_fn(StartTime, EndTime);
+}
+
+void llvm::omp::target::ompt::setOmptHostToDeviceRate(double Slope,
+                                                      double Offset) {
+  HostToDeviceSlope = Slope;
+  HostToDeviceOffset = Offset;
+}
+
+void llvm::omp::target::ompt::setOmptGrantedNumTeams(uint64_t NumTeams) {
+  std::unique_lock<std::mutex> Lock(ompt_set_granted_teams_mutex);
+  ensureFuncPtrLoaded<libomptarget_ompt_set_granted_teams_t>(
+      "libomptarget_ompt_set_granted_teams", &ompt_set_granted_teams_fn);
+  // No need to hold a lock
+  ompt_set_granted_teams_fn(NumTeams);
+}
+
+ompt_interface_fn_t llvm::omp::target::ompt::lookupDeviceTracingFn(
+    const char *InterfaceFunctionName) {
+#define compareAgainst(AvailableFunction)                                      \
+  if (strcmp(InterfaceFunctionName, #AvailableFunction) == 0)                  \
+    return (ompt_interface_fn_t)AvailableFunction;
+
+  FOREACH_OMPT_DEVICE_TRACING_FN(compareAgainst);
+#undef compareAgainst
+
+  ODBG(ODT_Tool) << "Warning: Could not find requested function "
+                 << InterfaceFunctionName;
+  return (ompt_interface_fn_t) nullptr;
+}
+
+#pragma pop_macro("DEBUG_PREFIX")
+
+#endif // OMPT_SUPPORT

>From 97552f7a62979139a4073f0eaa5e38f0aa7a3556 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 2 Apr 2026 07:40:55 -0500
Subject: [PATCH 05/11] [Offload][OMPT] Add tracing extensions to Interface.h

Extend the OMPT Interface class with trace record creation methods and
the async-aware TracerInterfaceRAII for device tracing support.

Key changes to Interface.h:
- Add includes for OmptEventInfoTy.h, APITypes.h, GenericProfiler.h
- Add extern TracingActive flag and isTracingEnabled() declaration
- Add trace start/stop methods for all target operations:
  startTargetDataAllocTrace, stopTargetDataAllocTrace,
  startTargetDataSubmitTrace, startTargetDataDeleteTrace,
  startTargetDataRetrieveTrace, stopTargetDataMovementTraceAsync,
  startTargetSubmitTrace, stopTargetSubmitTraceAsync, and
  corresponding methods for enter/exit/update/target regions
- Add getTraceGenerators<>() template overloads parallel to
  existing getCallbacks<>()
- Add private helpers: setTraceRecordCommon, setTraceRecordTargetDataOp,
  setTraceRecordTargetKernel, setTraceRecordTarget, announceTargetRegion
- Add TracerInterfaceRAII class: async-aware RAII that checks
  isTracingEnabled, calls begin trace generator, allocates
  OmptEventInfoTy via profiler, sets AsyncInfo->ProfilerData
- Rename CallbackPairTy to FunctionPairTy in InterfaceRAII
- Make InvokeInterfaceFunction return a value
- Replace OMPT_IF_BUILT with OMPT_IF_TRACING_ENABLED

Key changes to Callback.h:
- Replace inline macro/typedef definitions with include of
  OmptCommonDefs.h (shared definitions moved there in PR3)

Made-with: Cursor
Co-Authored-By: Claude <noreply at anthropic.com>
---
 offload/include/OpenMP/OMPT/Callback.h  |  26 +--
 offload/include/OpenMP/OMPT/Interface.h | 255 ++++++++++++++++++++++--
 2 files changed, 243 insertions(+), 38 deletions(-)

diff --git a/offload/include/OpenMP/OMPT/Callback.h b/offload/include/OpenMP/OMPT/Callback.h
index 9d545c643223f..7aff68ced0d79 100644
--- a/offload/include/OpenMP/OMPT/Callback.h
+++ b/offload/include/OpenMP/OMPT/Callback.h
@@ -16,34 +16,12 @@
 
 #ifdef OMPT_SUPPORT
 
-#include "omp-tools.h"
+#include "OmptCommonDefs.h"
 
 #pragma push_macro("DEBUG_PREFIX")
 #undef DEBUG_PREFIX
 #define DEBUG_PREFIX "OMPT"
 
-#define FOREACH_OMPT_TARGET_CALLBACK(macro)                                    \
-  FOREACH_OMPT_DEVICE_EVENT(macro)                                             \
-  FOREACH_OMPT_NOEMI_EVENT(macro)                                              \
-  FOREACH_OMPT_EMI_EVENT(macro)
-
-#define performIfOmptInitialized(stmt)                                         \
-  do {                                                                         \
-    if (llvm::omp::target::ompt::Initialized) {                                \
-      stmt;                                                                    \
-    }                                                                          \
-  } while (0)
-
-#define performOmptCallback(CallbackName, ...)                                 \
-  do {                                                                         \
-    if (ompt_callback_##CallbackName##_fn)                                     \
-      ompt_callback_##CallbackName##_fn(__VA_ARGS__);                          \
-  } while (0)
-
-/// Function type def used for maintaining unique target region, target
-/// operations ids
-typedef uint64_t (*IdInterfaceTy)();
-
 namespace llvm {
 namespace omp {
 namespace target {
@@ -98,8 +76,6 @@ extern bool Initialized;
 
 #pragma pop_macro("DEBUG_PREFIX")
 
-#else
-#define performIfOmptInitialized(stmt)
 #endif // OMPT_SUPPORT
 
 #endif // OFFLOAD_INCLUDE_OPENMP_OMPT_CALLBACK_H
diff --git a/offload/include/OpenMP/OMPT/Interface.h b/offload/include/OpenMP/OMPT/Interface.h
index 6961641769b76..54e7564c21ad8 100644
--- a/offload/include/OpenMP/OMPT/Interface.h
+++ b/offload/include/OpenMP/OMPT/Interface.h
@@ -16,14 +16,21 @@
 // Only provide functionality if target OMPT support is enabled
 #ifdef OMPT_SUPPORT
 #include "Callback.h"
+#include "OmptEventInfoTy.h"
+#include "Shared/APITypes.h"
+#include "Shared/Debug.h"
 #include "omp-tools.h"
 
+#include "GenericProfiler.h"
+
 #include "llvm/Support/ErrorHandling.h"
 
 #include <functional>
 #include <tuple>
 
-#define OMPT_IF_BUILT(stmt) stmt
+#pragma push_macro("DEBUG_PREFIX")
+#undef DEBUG_PREFIX
+#define DEBUG_PREFIX "OMPT"
 
 /// Callbacks for target regions require task_data representing the
 /// encountering task.
@@ -42,9 +49,17 @@ namespace ompt {
 static ompt_get_task_data_t ompt_get_task_data_fn;
 static ompt_get_target_task_data_t ompt_get_target_task_data_fn;
 
+/// OMPT global tracing status. Indicates if at least one device is traced.
+extern bool TracingActive;
+
+/// Check if this device traces the given event type
+extern bool isTracingEnabled(int DeviceId, unsigned int EventTy);
+
 /// Used to maintain execution state for this thread
 class Interface {
 public:
+  // Target data callbacks
+
   /// Top-level function for invoking callback before device data allocation
   void beginTargetDataAlloc(int64_t DeviceId, void *HstPtrBegin,
                             void **TgtPtrBegin, size_t Size, void *Code);
@@ -142,6 +157,85 @@ class Interface {
   /// Top-level function for invoking callback after target construct
   void endTarget(int64_t DeviceId, void *Code);
 
+  // Target data tracing
+
+  /// Top-level function for starting trace before device data allocation
+  void startTargetDataAllocTrace(int64_t DeviceId, void *HstPtrBegin,
+                                 void **TgtPtrBegin, size_t Size, void *Code);
+
+  /// Top-level function for stopping trace after device data allocation
+  ompt_record_ompt_t *stopTargetDataAllocTrace(int64_t DeviceId,
+                                               void *HstPtrBegin,
+                                               void **TgtPtrBegin, size_t Size,
+                                               void *Code);
+
+  /// Top-level function for starting trace before data submit
+  ompt_record_ompt_t *startTargetDataSubmitTrace(int64_t SrcDeviceId,
+                                                 void *SrcPtrBegin,
+                                                 int64_t DstDeviceId,
+                                                 void *DstPtrBegin, size_t Size,
+                                                 void *Code);
+
+  /// Top-level function for starting trace before device data deallocation
+  void startTargetDataDeleteTrace(int64_t DeviceId, void *TgtPtrBegin,
+                                  void *Code);
+
+  /// Top-level function for stopping trace after device data deallocation
+  ompt_record_ompt_t *stopTargetDataDeleteTrace(int64_t DeviceId,
+                                                void *TgtPtrBegin, void *Code);
+
+  /// Top-level function for starting trace before data retrieve
+  ompt_record_ompt_t *startTargetDataRetrieveTrace(int64_t SrcDeviceId,
+                                                   void *SrcPtrBegin,
+                                                   int64_t DstDeviceId,
+                                                   void *DstPtrBegin,
+                                                   size_t Size, void *Code);
+
+  ompt_record_ompt_t *
+  stopTargetDataMovementTraceAsync(ompt_record_ompt_t *DataPtr,
+                                   uint64_t NanosStart, uint64_t NanosEnd);
+
+  /// Top-level function for starting trace before kernel dispatch
+  ompt_record_ompt_t *startTargetSubmitTrace(int64_t DeviceId,
+                                             unsigned int NumTeams = 1);
+
+  ompt_record_ompt_t *stopTargetSubmitTraceAsync(ompt_record_ompt_t *DataPtr,
+                                                 unsigned int NumTeams,
+                                                 uint64_t NanosStart,
+                                                 uint64_t NanosStop);
+
+  // Target region tracing
+
+  /// Top-level function for starting trace before target enter data
+  /// construct
+  ompt_record_ompt_t *startTargetDataEnterTrace(int64_t DeviceId, void *Code);
+
+  /// Top-level function for stopping trace after target enter data
+  /// construct
+  ompt_record_ompt_t *stopTargetDataEnterTrace(int64_t DeviceId, void *Code);
+
+  /// Top-level function for starting trace before target exit data
+  /// construct
+  ompt_record_ompt_t *startTargetDataExitTrace(int64_t DeviceId, void *Code);
+
+  /// Top-level function for stopping trace after target exit data
+  /// construct
+  ompt_record_ompt_t *stopTargetDataExitTrace(int64_t DeviceId, void *Code);
+
+  /// Top-level function for starting trace before target update construct
+  ompt_record_ompt_t *startTargetUpdateTrace(int64_t DeviceId, void *Code);
+
+  /// Top-level function for stopping trace after target update construct
+  ompt_record_ompt_t *stopTargetUpdateTrace(int64_t DeviceId, void *Code);
+
+  // Target kernel tracing
+
+  /// Top-level function for starting trace before target construct
+  ompt_record_ompt_t *startTargetTrace(int64_t DeviceId, void *Code);
+
+  /// Top-level function for stopping trace after target construct
+  ompt_record_ompt_t *stopTargetTrace(int64_t DeviceId, void *Code);
+
   // Callback getter: Target data operations
   template <ompt_target_data_op_t OpType> auto getCallbacks() {
     if constexpr (OpType == ompt_target_data_alloc ||
@@ -217,6 +311,69 @@ class Interface {
     llvm_unreachable("Unhandled target operation!");
   }
 
+  // Callback getter: Target data operations
+  template <ompt_target_data_op_t OpType> auto getTraceGenerators() {
+    if constexpr (OpType == ompt_target_data_alloc ||
+                  OpType == ompt_target_data_alloc_async)
+      return std::make_pair(std::mem_fn(&Interface::startTargetDataAllocTrace),
+                            std::mem_fn(&Interface::stopTargetDataAllocTrace));
+
+    if constexpr (OpType == ompt_target_data_delete ||
+                  OpType == ompt_target_data_delete_async)
+      return std::make_pair(std::mem_fn(&Interface::startTargetDataDeleteTrace),
+                            std::mem_fn(&Interface::stopTargetDataDeleteTrace));
+
+    if constexpr (OpType == ompt_target_data_transfer_to_device ||
+                  OpType == ompt_target_data_transfer_to_device_async)
+      return std::make_pair(
+          std::mem_fn(&Interface::startTargetDataSubmitTrace),
+          std::mem_fn(&Interface::stopTargetDataMovementTraceAsync));
+
+    if constexpr (OpType == ompt_target_data_transfer_from_device ||
+                  OpType == ompt_target_data_transfer_from_device_async)
+      return std::make_pair(
+          std::mem_fn(&Interface::startTargetDataRetrieveTrace),
+          std::mem_fn(&Interface::stopTargetDataMovementTraceAsync));
+
+    llvm_unreachable("Unhandled target data operation type!");
+  }
+
+  // Callback getter: Target region operations
+  template <ompt_target_t OpType> auto getTraceGenerators() {
+    if constexpr (OpType == ompt_target_enter_data ||
+                  OpType == ompt_target_enter_data_nowait)
+      return std::make_pair(std::mem_fn(&Interface::startTargetDataEnterTrace),
+                            std::mem_fn(&Interface::stopTargetDataEnterTrace));
+
+    if constexpr (OpType == ompt_target_exit_data ||
+                  OpType == ompt_target_exit_data_nowait)
+      return std::make_pair(std::mem_fn(&Interface::startTargetDataExitTrace),
+                            std::mem_fn(&Interface::stopTargetDataExitTrace));
+
+    if constexpr (OpType == ompt_target_update ||
+                  OpType == ompt_target_update_nowait)
+      return std::make_pair(std::mem_fn(&Interface::startTargetUpdateTrace),
+                            std::mem_fn(&Interface::stopTargetUpdateTrace));
+
+    if constexpr (OpType == ompt_target || OpType == ompt_target_nowait)
+      return std::make_pair(std::mem_fn(&Interface::startTargetTrace),
+                            std::mem_fn(&Interface::stopTargetTrace));
+
+    llvm_unreachable("Unknown target region operation type!");
+  }
+
+  // Callback getter: Kernel launch operation
+  template <ompt_callbacks_t OpType> auto getTraceGenerators() {
+    // We use 'ompt_callbacks_t', because no other enum is currently available
+    // to model a kernel launch / target submit operation.
+    if constexpr (OpType == ompt_callback_target_submit)
+      return std::make_pair(
+          std::mem_fn(&Interface::startTargetSubmitTrace),
+          std::mem_fn(&Interface::stopTargetSubmitTraceAsync));
+
+    llvm_unreachable("Unhandled target operation!");
+  }
+
   /// Setters for target region and target operation correlation ids
   void setTargetDataValue(uint64_t DataValue) { TargetData.value = DataValue; }
   void setTargetDataPtr(void *DataPtr) { TargetData.ptr = DataPtr; }
@@ -240,6 +397,9 @@ class Interface {
   /// Target task data representing the target task region
   ompt_data_t *TargetTaskData = nullptr;
 
+  /// Used for marking begin of a data operation
+  void announceTargetRegion(const char *RegionName);
+
   /// Used for marking begin of a data operation
   void beginTargetDataOperation();
 
@@ -251,6 +411,23 @@ class Interface {
 
   /// Used for marking end of a target region
   void endTargetRegion();
+
+  // Called by all trace generation routines
+  void setTraceRecordCommon(ompt_record_ompt_t *DataPtr,
+                            ompt_callbacks_t CallbackType);
+  // Type specific helpers
+  void setTraceRecordTargetDataOp(ompt_record_target_data_op_t *Record,
+                                  ompt_target_data_op_t DataOpType,
+                                  void *SrcAddr, int64_t SrcDeviceNum,
+                                  void *DstAddr, int64_t DstDeviceNum,
+                                  size_t Bytes, void *CodePtr);
+
+  void setTraceRecordTargetKernel(ompt_record_target_kernel_t *Record,
+                                  unsigned int NumTeams);
+
+  void setTraceRecordTarget(ompt_record_target_t *Record, int64_t DeviceId,
+                            ompt_target_t TargetKind,
+                            ompt_scope_endpoint_t Endpoint, void *CodePtr);
 };
 
 /// Thread local state for target region and associated metadata
@@ -262,14 +439,14 @@ extern thread_local Interface RegionInterface;
 extern thread_local void *ReturnAddress;
 
 template <typename FuncTy, typename ArgsTy, size_t... IndexSeq>
-void InvokeInterfaceFunction(FuncTy Func, ArgsTy Args,
+auto InvokeInterfaceFunction(FuncTy Func, ArgsTy Args,
                              std::index_sequence<IndexSeq...>) {
-  std::invoke(Func, RegionInterface, std::get<IndexSeq>(Args)...);
+  return std::invoke(Func, RegionInterface, std::get<IndexSeq>(Args)...);
 }
 
-template <typename CallbackPairTy, typename... ArgsTy> class InterfaceRAII {
+template <typename FunctionPairTy, typename... ArgsTy> class InterfaceRAII {
 public:
-  InterfaceRAII(CallbackPairTy Callbacks, ArgsTy... Args)
+  InterfaceRAII(FunctionPairTy Callbacks, ArgsTy... Args)
       : Arguments(Args...), beginFunction(std::get<0>(Callbacks)),
         endFunction(std::get<1>(Callbacks)) {
     performIfOmptInitialized(begin());
@@ -290,14 +467,66 @@ template <typename CallbackPairTy, typename... ArgsTy> class InterfaceRAII {
   }
 
   std::tuple<ArgsTy...> Arguments;
-  typename CallbackPairTy::first_type beginFunction;
-  typename CallbackPairTy::second_type endFunction;
+  typename FunctionPairTy::first_type beginFunction;
+  typename FunctionPairTy::second_type endFunction;
 };
 
 // InterfaceRAII's class template argument deduction guide
-template <typename CallbackPairTy, typename... ArgsTy>
-InterfaceRAII(CallbackPairTy Callbacks, ArgsTy... Args)
-    -> InterfaceRAII<CallbackPairTy, ArgsTy...>;
+template <typename FunctionPairTy, typename... ArgsTy>
+InterfaceRAII(FunctionPairTy Callbacks, ArgsTy... Args)
+    -> InterfaceRAII<FunctionPairTy, ArgsTy...>;
+
+/// Similar to the original InterfaceRAII this class is used for tracing and
+/// extends the original with async capabilities. That is: It takes an
+/// additional AsyncInfo reference as argument to populate the relevant fields.
+/// The AsyncInfoTy propagates the info into the RTL / plugins.
+/// TracedDeviceId represents the trace record's device affinity. EventType is
+/// the callback type that needs to be enabled via ompt_set_trace_ompt.
+template <typename FunctionPairTy, typename AsyncInfoTy, typename... ArgsTy>
+class TracerInterfaceRAII {
+public:
+  TracerInterfaceRAII(FunctionPairTy Callbacks, AsyncInfoTy &AsyncInfo,
+                      plugin::GenericProfilerTy *Prof, int TracedDeviceId,
+                      ompt_callbacks_t EventType, ArgsTy... Args)
+      : Arguments(Args...), beginFunction(std::get<0>(Callbacks)) {
+    __tgt_async_info *AI = AsyncInfo;
+    if (isTracingEnabled(TracedDeviceId, EventType)) {
+      auto Record = begin();
+
+      // The Profiler can allocate specific data to be used to pass information
+      // from here to lower parts of the runtime system.
+      // NOTE: It is the responsibility of the programmer to ensure type
+      // compatibility and correct usage of the data. The profiler, however,
+      // OWNS the pointer and frees it at an appropriate time.
+      OmptEventInfoTy *ProfilerData =
+          reinterpret_cast<OmptEventInfoTy *>(Prof->getProfilerSpecificData());
+      ProfilerData->TraceRecord = Record;
+      ProfilerData->NumTeams = 0;
+
+      // Allows to pass down into the plugins via AsyncInfoTy
+      AI->ProfilerData = ProfilerData;
+    } else {
+      // Actively prevent further tracing of this event
+      AI->ProfilerData = nullptr;
+    }
+  }
+
+private:
+  auto begin() {
+    auto IndexSequence =
+        std::make_index_sequence<std::tuple_size_v<decltype(Arguments)>>{};
+    return InvokeInterfaceFunction(beginFunction, Arguments, IndexSequence);
+  }
+
+  std::tuple<ArgsTy...> Arguments;
+  typename FunctionPairTy::first_type beginFunction;
+  /// No end-function here, since the end is called asynchronously from the
+  /// plugins, once the operation has completed.
+};
+
+template <typename FunctionPairTy, typename... ArgsTy>
+TracerInterfaceRAII(FunctionPairTy Callbacks, ArgsTy... Args)
+    -> TracerInterfaceRAII<FunctionPairTy, ArgsTy...>;
 
 /// Used to set and reset the thread-local return address. The RAII is expected
 /// to be created at a runtime entry point when the return address should be
@@ -335,8 +564,8 @@ class ReturnAddressSetterRAII {
 // The getter returns the address stored in the thread local variable.
 #define OMPT_GET_RETURN_ADDRESS llvm::omp::target::ompt::ReturnAddress
 
-#else
-#define OMPT_IF_BUILT(stmt)
-#endif
+#pragma pop_macro("DEBUG_PREFIX")
+
+#endif // OMPT_SUPPORT
 
 #endif // OFFLOAD_INCLUDE_OPENMP_OMPT_INTERFACE_H

>From 75c70af2fb7c636fc15b8782ec38e476e97b54ff Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 2 Apr 2026 07:44:36 -0500
Subject: [PATCH 06/11] [Offload][OMPT] Add OmptTracingBufferMgr trace buffer
 infrastructure

Add the trace buffer management subsystem for OMPT device tracing. This
manages allocation, population, and flushing of trace record buffers
between OpenMP worker threads and helper threads.

New files:
- OmptTracingBuffer.h: OmptTracingBufferMgr class with Buffer metadata,
  FlushInfo, TraceRecord types, thread-local per-device buffer pointers,
  helper thread management, and cursor-based record allocation
- OmptTracingBuffer.cpp: Full implementation of buffer lifecycle:
  assignCursor (lock-free fast path for existing buffers, locked path
  for new allocations), triggerFlushOnBufferFull, driveCompletion
  (helper thread main loop), invokeCallbacks, flushBuffer (dispatches
  buffer-completion callbacks for ranges of ready records),
  flushAllBuffers, helper thread start/shutdown/flush-and-shutdown
- OmptTracing.h: Libomptarget-side declarations for tracing state,
  buffer management callback registration, device tracing control,
  timestamp/clock correlation, and thread-local trace record fields.
  Implementation deferred to subsequent PR.

Changes to existing files:
- PluginManager.h: Add OmptTracingBufferMgr* TraceRecordManager member
  with getTraceRecordManager() accessor, include OmptTracingBuffer.h
- libomptarget/CMakeLists.txt: Add OmptTracingBuffer.cpp to omptarget

Key design points:
- Per-thread buffer assignment avoids contention on the common path
- Helper threads process buffers in flush-id order via condition variables
- LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL / FLUSH_ON_SHUTDOWN envars
  control automatic flushing behavior
- Buffer states: init -> ready -> released track record lifecycle
- Flush states: waiting -> processing prevent concurrent helper access

Made-with: Cursor
---
 offload/include/OpenMP/OMPT/OmptTracing.h     | 154 ++++
 .../include/OpenMP/OMPT/OmptTracingBuffer.h   | 412 ++++++++++
 offload/include/PluginManager.h               |  10 +-
 offload/libomptarget/CMakeLists.txt           |   1 +
 .../OpenMP/OMPT/OmptTracingBuffer.cpp         | 763 ++++++++++++++++++
 5 files changed, 1339 insertions(+), 1 deletion(-)
 create mode 100644 offload/include/OpenMP/OMPT/OmptTracing.h
 create mode 100644 offload/include/OpenMP/OMPT/OmptTracingBuffer.h
 create mode 100644 offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp

diff --git a/offload/include/OpenMP/OMPT/OmptTracing.h b/offload/include/OpenMP/OMPT/OmptTracing.h
new file mode 100644
index 0000000000000..2a892582923d5
--- /dev/null
+++ b/offload/include/OpenMP/OMPT/OmptTracing.h
@@ -0,0 +1,154 @@
+//===---- OmptTracing.h - Target independent OMPT callbacks --*- C++ -*----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Interface used by target-independent runtimes to coordinate registration and
+// invocation of OMPT tracing functionality.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OPENMP_LIBOMPTARGET_INCLUDE_OMPTTRACING_H
+#define OPENMP_LIBOMPTARGET_INCLUDE_OMPTTRACING_H
+
+#ifdef OMPT_SUPPORT
+
+#include <unordered_map>
+
+#include "OmptCommonDefs.h"
+#include "OmptTracingBuffer.h"
+
+#pragma push_macro("DEBUG_PREFIX")
+#undef DEBUG_PREFIX
+#define DEBUG_PREFIX "OMPT"
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace ompt {
+
+/// After a timestamp has been read, reset it.
+void resetTimestamp(uint64_t *T);
+
+/// A tool may register unique buffer-request and buffer-completion
+/// callback functions for a device. The following are utility functions to
+/// manage those functions.
+
+/// Given a device-id, return the corresponding buffer-request callback
+/// function.
+ompt_callback_buffer_request_t getBufferRequestFn(int DeviceId);
+
+/// Give a device-id, return the corresponding buffer-completion callback
+/// function.
+ompt_callback_buffer_complete_t getBufferCompleteFn(int DeviceId);
+
+/// Given a device-id, set the corresponding buffer-request and
+/// buffer-completion callback functions.
+void setBufferManagementFns(int DeviceId, ompt_callback_buffer_request_t ReqFn,
+                            ompt_callback_buffer_complete_t CmpltFn);
+
+/// Given a device-id, remove the corresponding buffer-request and
+/// buffer-completion callback functions.
+void removeBufferManagementFns(int DeviceId);
+
+/// Is device tracing stopped for all devices?
+bool isAllDeviceTracingStopped();
+
+/// Invoke callback function for buffer request events
+void ompt_callback_buffer_request(int DeviceId, ompt_buffer_t **BufferPtr,
+                                  size_t *Bytes);
+
+/// Invoke callback function for buffer complete events
+void ompt_callback_buffer_complete(int DeviceId, ompt_buffer_t *Buffer,
+                                   size_t Bytes,
+                                   ompt_buffer_cursor_t BeginCursor,
+                                   int BufferOwned);
+
+/// Set 'start' and 'stop' for the current trace record
+void setOmptTimestamp(uint64_t StartTime, uint64_t EndTime);
+
+/// Set the linear function correlation between host and device clocks
+void setOmptHostToDeviceRate(double Slope, double Offset);
+
+/// Set / store the number of granted teams
+void setOmptGrantedNumTeams(uint64_t NumTeams);
+
+/// Check if (1) tracing is globally active (2) the given device is actively
+/// traced and (3) the given event type is traced on the device
+bool isTracingEnabled(int DeviceId, unsigned int EventTy);
+
+/// Check if the given device is actively traced
+bool isTracedDevice(int DeviceId);
+
+/// Check if the given device is monitoring the provided tracing type
+bool isTracingTypeEnabled(int DeviceId, unsigned int EventTy);
+
+/// Check if the given device is monitoring the provided tracing type 'group'
+/// Where group means we will check for both: EMI and non-EMI event types
+bool isTracingTypeGroupEnabled(int DeviceId, unsigned int EventTy);
+
+/// Set whether the given tracing type should be monitored (or not) on the
+/// device
+void setTracingTypeEnabled(uint64_t &TracedEventTy, bool Enable,
+                           unsigned int EventTy);
+
+/// Set / reset the given tracing types (EventTy = 0 corresponds to 'all')
+ompt_set_result_t setTraceEventTy(int DeviceId, unsigned int Enable,
+                                  unsigned int EventTy);
+
+/// Return thread id
+uint64_t getThreadId();
+
+/// See TracedDevices in OmptDeviceTracing.h
+extern std::map<int32_t, uint64_t> TracedDevices;
+/// Activate tracing on the given device
+void enableDeviceTracing(int DeviceId);
+/// Deactivate tracing on the given device
+void disableDeviceTracing(int DeviceId);
+
+/// Mutexes to serialize invocation of device registration and checks
+extern std::mutex DeviceAccessMutex;
+
+/// Mutexes to serialize invocation of device-independent entry points
+extern std::mutex TraceAccessMutex;
+extern std::mutex TraceControlMutex;
+
+/// Ensure serialization of calls to std::hash
+extern std::mutex TraceHashThreadMutex;
+
+/// Protect map from device-id to the corresponding buffer-request and
+/// buffer-completion callback functions.
+extern std::mutex BufferManagementFnMutex;
+
+/// Map from device-id to the corresponding buffer-request and buffer-completion
+/// callback functions.
+extern std::unordered_map<int, std::pair<ompt_callback_buffer_request_t,
+                                         ompt_callback_buffer_complete_t>>
+    BufferManagementFns;
+
+/// Thread local variables used by the plugin to communicate OMPT information
+/// that are then used to populate trace records. This method assumes a
+/// synchronous implementation, otherwise it won't work.
+extern thread_local uint32_t TraceRecordNumGrantedTeams;
+extern thread_local uint64_t TraceRecordStartTime;
+extern thread_local uint64_t TraceRecordStopTime;
+
+/// Thread local thread-id.
+extern thread_local uint64_t ThreadId;
+
+/// OMPT global tracing status. Indicates if at least one device is traced.
+extern bool TracingActive;
+
+} // namespace ompt
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+#pragma pop_macro("DEBUG_PREFIX")
+
+#endif // OMPT_SUPPORT
+
+#endif // OPENMP_LIBOMPTARGET_INCLUDE_OMPTTRACING_H
diff --git a/offload/include/OpenMP/OMPT/OmptTracingBuffer.h b/offload/include/OpenMP/OMPT/OmptTracingBuffer.h
new file mode 100644
index 0000000000000..5c9f4bf33dae8
--- /dev/null
+++ b/offload/include/OpenMP/OMPT/OmptTracingBuffer.h
@@ -0,0 +1,412 @@
+//===- OmptTracingBuffer.h - Target independent OpenMP target RTL -- C++ *-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Interface to be used for generating and flushing OMPT device trace records.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OPENMP_LIBOMPTARGET_OMPTTRACINGBUFFER_H
+#define OPENMP_LIBOMPTARGET_OMPTTRACINGBUFFER_H
+
+#ifdef OMPT_SUPPORT
+
+#include <atomic>
+#include <cassert>
+#include <condition_variable>
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <set>
+#include <thread>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#include <omp-tools.h>
+
+#include "Shared/EnvironmentVar.h"
+
+// Maximum number of devices supported in device tracing. No device tracing
+// will be performed for any device-id larger than 1023.
+#define MAX_NUM_DEVICES 1024
+
+// TODO Start with 1 helper thread and add dynamically if required
+// Number of helper threads must not execeed 32 since the
+// thread-wait-tracker is 32 bits in length.
+#define OMPT_NUM_HELPER_THREADS 1
+
+/*
+ * Buffer manager for trace records generated by OpenMP master and
+ * worker threads. During device init, a tool may register a
+ * buffer-request and a buffer-completion callback. The buffer-request
+ * callback should be used to allocate new buffers as required. The
+ * buffer-complete callback should be used to return trace records to
+ * the tool.
+ *
+ * In addition to trace records, this class manages the helper threads
+ * for dispatching a range of trace records to the tool.
+ */
+class OmptTracingBufferMgr {
+public:
+  /*
+   * A trace record (TR) holds the trace data. Its type
+   * can be ompt or native. Currently, only ompt type is implemented.
+   */
+
+  /*
+   * A TR can be in the following states:
+   * TR_init: initial state
+   * TR_ready: An OpenMP thread marks a TR ready when it is done
+   * populating the TR
+   * TR_released: A helper thread marks a TR released after it has
+   * completed returning the TR to the tool
+   */
+  enum TRStatus { TR_init, TR_ready, TR_released };
+
+  /*
+   * Metadata capturing the state of a buffer of trace records. Once a
+   * buffer is allocated by an OpenMP worker thread, trace records are
+   * carved out from that buffer by that same OpenMP thread alone. Thus
+   * the allocated buffer is thread-specific from the allocation/population
+   * standpoint. But it may be manipulated by helper threads.
+   *
+   * Id, DeviceId, Start, and TotalBytes are not changed once set.
+   * RemainingBytes could be written multiple times but only by the same
+   * thread. But Cursor and IsFull may be read/written by an OpenMP worker
+   * thread and read by helper threads. Hence, accesses of
+   * this 2nd set of locations need to be atomic or synchronized.
+   */
+  struct Buffer {
+    uint64_t Id;                // Unique identifier of the buffer
+    int64_t DeviceId;           // Device for which this buffer is allocated
+    void *Start;                // Start of allocated space for trace records
+    size_t TotalBytes;          // Total number of bytes in the allocated space
+    size_t RemainingBytes;      // Total number of unused bytes
+                                // corresponding to Cursor
+    std::atomic<void *> Cursor; // Address of the last trace record carved out
+    std::atomic<bool> IsFull;   // true if no more trace records can be
+                                // accomodated, otherwise false
+    Buffer(uint64_t BufId, int64_t DevId, void *S, size_t Bytes, size_t Rem,
+           void *C, bool F)
+        : Id(BufId), DeviceId(DevId), Start(S), TotalBytes(Bytes),
+          RemainingBytes(Rem), Cursor(C), IsFull(F) {}
+    Buffer() = delete;
+    Buffer(const Buffer &) = delete;
+    Buffer &operator=(const Buffer &) = delete;
+  };
+  using BufPtr = std::shared_ptr<Buffer>;
+
+private:
+  /// Envar to control whether a buffer should be flushed when it gets full.
+  BoolEnvar OMPX_FlushOnBufferFull;
+
+  /// Envar to control whether all buffers should be flushed during shutdown.
+  BoolEnvar OMPX_FlushOnShutdown;
+
+  // Internal variable for tracking threads to wait for flush
+  uint32_t ThreadFlushTracker;
+
+  // Internal variable for tracking threads shutting down
+  uint32_t ThreadShutdownTracker;
+
+  using MapId2Buf = std::map<uint64_t, BufPtr>;
+
+  // Map from id to corresponding buffer. The ids are assigned in
+  // increasing order of creation.
+  MapId2Buf Id2BufferMap;
+
+  // Trace record. We currently support OMPT data type only. The state
+  // (TRStatus type) is maintained inline in the trace record. The
+  // tool is expected to access only the OMPT record.
+  struct TraceRecord {
+    ompt_record_ompt_t TR;
+    std::atomic<TRStatus> TRState;
+  };
+
+  // Thread-specific array of pointers to a buffer. The buffer pointed to
+  // is the last one allocated by this thread for a given device. The ith
+  // element points to the buffer for the ith device. At most MAX_NUM_DEVICES
+  // devices are supported.
+  static thread_local BufPtr ArrayOfBufPtr[MAX_NUM_DEVICES];
+
+  /*
+   * A buffer is flushed when it fills up or when the tool invokes
+   * flush_trace. So it's possible that the same buffer may be flushed
+   * more than once. When a buffer is flushed the first time, a unique
+   * id (flush-id) is generated and assigned to that buffer. Even if
+   * it is flushed again, the previously assigned id is maintained for
+   * that buffer. This id is loosely used to determine the order in
+   * which the buffers are processed and the corresponding trace
+   * records released to the tool.
+   */
+
+  struct FlushInfo {
+    uint64_t FlushId;
+    void *FlushCursor;
+    BufPtr FlushBuf;
+    FlushInfo() = default;
+    FlushInfo(uint64_t Id, void *CR, BufPtr Buf)
+        : FlushId{Id}, FlushCursor{CR}, FlushBuf{Buf} {}
+  };
+
+  /*
+   * A buffer may be in the following states:
+   * Flush_waiting: when a buffer is flushed, either because it is
+   * full or because the tool invokes ompt_flush_trace
+   * Flush_processing: when a helper thread claims the waiting buffer
+   * and is in the process of dispatching buffer-completion callbacks
+   * on an associated range of trace records. If all trace records are
+   * not released, the state may be reset to Flush_waiting after the
+   * buffer-completion callbacks return
+   */
+  enum BufferFlushStatus { Flush_waiting, Flush_processing };
+  struct FlushMd {
+    void *FlushCursor;
+    BufPtr FlushBuf;
+    BufferFlushStatus FlushStatus;
+    FlushMd(void *CR, BufPtr Buf, BufferFlushStatus Status)
+        : FlushCursor{CR}, FlushBuf{Buf}, FlushStatus{Status} {}
+    FlushMd() = delete;
+  };
+
+  using MapId2Md = std::map<uint64_t, FlushMd>;
+
+  /*
+   * A map from a flush-id to metadata containing the current
+   * cursor. the corresponding buffer, and its flushed status. If a
+   * buffer is flushed multiple times, the cursor is updated to the
+   * furthest one
+   */
+  MapId2Md Id2FlushMdMap;
+
+  using UMapBufPtr2Id = std::unordered_map<BufPtr, uint64_t>;
+
+  // A hash map from a buffer address to the corresponding flush-id
+  UMapBufPtr2Id FlushBufPtr2IdMap;
+
+  using USetCursor = std::unordered_set<void *>;
+
+  USetCursor LastCursors;
+
+  using UMapThd2Id = std::unordered_map<std::thread::id, uint32_t>;
+
+  // A hash map from a helper thread id to an integer
+  UMapThd2Id HelperThreadIdMap;
+
+  // Mutex to protect Id2BufferMap and Cursor2BufMdMap
+  std::mutex BufferMgrMutex;
+
+  // Mutex to protect FlushBufPtr2IdMap and Id2FlushMdMap
+  std::mutex FlushMutex;
+
+  // Mutex to protect metadata tracking last cursors of buffer-completion
+  // callbacks
+  std::mutex LastCursorMutex;
+
+  // Condition variable used by helper thread to signal that flush is requested
+  std::condition_variable FlushCv;
+
+  // Condition variable used while waiting for flushing to complete
+  std::condition_variable ThreadFlushCv;
+
+  // Condition variable used while waiting for threads to shutdown
+  std::condition_variable ThreadShutdownCv;
+
+  // TODO Separate out the helper thread into its own class
+  std::vector<std::thread> CompletionThreads;
+
+  /// Called when a buffer \p Buf may be flushed with \p Cursor as the
+  /// last allocated trace record in the buffer.
+  /// triggerFlushOnBufferFull should be called without holding any lock.
+  void triggerFlushOnBufferFull(void *Cursor, BufPtr Buf);
+
+  // Called to dispatch buffer-completion callbacks for the trace records in
+  // this buffer
+  void flushBuffer(FlushInfo);
+
+  // Dispatch a buffer-completion callback with a range of trace records
+  void dispatchCallback(int64_t DeviceId, void *Buffer, void *FirstCursor,
+                        void *LastCursor);
+
+  // Add a last cursor
+  void addLastCursor(void *Cursor) {
+    std::unique_lock<std::mutex> Lock(LastCursorMutex);
+    LastCursors.emplace(Cursor);
+  }
+
+  // Remove a last cursor
+  void removeLastCursor(void *Cursor) {
+    std::unique_lock<std::mutex> Lock(LastCursorMutex);
+    assert(LastCursors.find(Cursor) != LastCursors.end());
+    LastCursors.erase(Cursor);
+  }
+
+  // Given a trace record pointer, initialize its metadata
+  void initTraceRecordMetaData(void *Rec);
+
+  // Given a device-id, get/set a pointer to the last allocated buffer metadata.
+  BufPtr getDeviceSpecificBuffer(int64_t DevId);
+  void setDeviceSpecificBuffer(int64_t DevId, BufPtr Buf);
+
+  // Reserve a candidate buffer for flushing, preventing other helper threads
+  // from accessing it
+  FlushInfo findAndReserveFlushedBuf(uint64_t FlushId);
+
+  // Unreserve a buffer so that other helper threads can process it
+  void unreserveFlushedBuf(const FlushInfo &);
+
+  // All done with this buffer, so the buffer and its metadata can be removed
+  void destroyFlushedBuf(const FlushInfo &);
+
+  // Add a new buffer by an OpenMP thread so that a helper thread can process it
+  uint64_t addNewFlushEntry(BufPtr Buf, void *Cursor);
+
+  // Get the next trace record
+  void *getNextTR(void *TR);
+
+  // Given a buffer, return the latest cursor
+  void *getBufferCursor(BufPtr);
+
+  // Is no more space remaining for trace records in this buffer?
+  bool isBufferFull(const FlushInfo &);
+
+  // Have all trace records in this buffer been returned to the tool?
+  bool isBufferOwned(const FlushInfo &);
+
+  // Dispatch a buffer-completion callback and indicate that the buffer can be
+  // deallocated
+  void dispatchBufferOwnedCallback(const FlushInfo &);
+
+  // Main entry point for a helper thread
+  void driveCompletion();
+
+  // Examine the flushed buffers and dispatch buffer-completion callbacks
+  void invokeCallbacks();
+
+  // The caller does not hold a lock while calling this method
+  void waitForFlushCompletion();
+
+  // Given a thread number, set the corresponding bit in the flush
+  // tracker. The caller must hold the flush lock.
+  void setThreadFlush(uint32_t ThreadNum) {
+    ThreadFlushTracker |= (1 << ThreadNum);
+  }
+
+  // Reset this thread's flush bit. The caller must hold the flush lock
+  void resetThisThreadFlush() {
+    std::thread::id ID = std::this_thread::get_id();
+    assert(HelperThreadIdMap.find(ID) != HelperThreadIdMap.end());
+    ThreadFlushTracker &= ~(1 << HelperThreadIdMap[ID]);
+  }
+
+  // Given a thread number, set the corresponding bit in the shutdown
+  // tracker. The caller must hold the flush lock.
+  void setThreadShutdown(uint32_t ThreadNum) {
+    ThreadShutdownTracker |= (1 << ThreadNum);
+  }
+
+  // Reset this thread's shutdown bit. The caller must hold the flush
+  // lock
+  void resetThisThreadShutdown() {
+    std::thread::id ID = std::this_thread::get_id();
+    assert(HelperThreadIdMap.find(ID) != HelperThreadIdMap.end());
+    ThreadShutdownTracker &= ~(1 << HelperThreadIdMap[ID]);
+  }
+
+  // Return true if this thread's flush bit is set. The caller must
+  // hold the flush lock
+  bool isThisThreadFlushWaitedUpon() {
+    std::thread::id ID = std::this_thread::get_id();
+    assert(HelperThreadIdMap.find(ID) != HelperThreadIdMap.end());
+    return (ThreadFlushTracker & (1 << HelperThreadIdMap[ID])) != 0;
+  }
+
+  // Return true if this thread's shutdown bit is set. The caller must
+  // hold the flush lock
+  bool isThisThreadShutdownWaitedUpon() {
+    std::thread::id ID = std::this_thread::get_id();
+    assert(HelperThreadIdMap.find(ID) != HelperThreadIdMap.end());
+    return (ThreadShutdownTracker & (1 << HelperThreadIdMap[ID])) != 0;
+  }
+
+  // The caller must not hold the flush lock
+  bool amIHelperThread() {
+    std::unique_lock<std::mutex> flush_lock(FlushMutex);
+    if (HelperThreadIdMap.find(std::this_thread::get_id()) !=
+        HelperThreadIdMap.end())
+      return true;
+    return false;
+  }
+
+  // The caller must not hold the flush lock
+  bool areHelperThreadsAvailable();
+
+  // The caller must hold the appropriate lock
+  void init();
+
+  // The caller must hold the flush lock
+  void createHelperThreads();
+
+  // The caller must hold the flush lock
+  void destroyHelperThreads();
+
+public:
+  OmptTracingBufferMgr()
+      : OMPX_FlushOnBufferFull("LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL", true),
+        OMPX_FlushOnShutdown("LIBOMPTARGET_OMPT_FLUSH_ON_SHUTDOWN", true) {
+    // no need to hold locks for init() since object is getting constructed
+    // here.
+    init();
+  }
+
+  OmptTracingBufferMgr(const OmptTracingBufferMgr &) = delete;
+  OmptTracingBufferMgr &operator=(const OmptTracingBufferMgr &) = delete;
+
+  // The caller must not hold the flush lock
+  void startHelperThreads();
+
+  // The caller must not hold the flush lock. The helper threads are shut down
+  // without flushing any outstanding trace records.
+  void shutdownHelperThreads();
+
+  // The caller must not hold the flush lock. The helper threads are shut down
+  // after flushing all outstanding trace records for all devices.
+  void flushAndShutdownHelperThreads();
+
+  // Assign a cursor for a new trace record. This will assign a trace record
+  // for the provided device-id, allocating a new buffer if required.
+  void *assignCursor(ompt_callbacks_t Type, int64_t DeviceId);
+
+  // Get the size of a trace record
+  size_t getTRSize() { return sizeof(TraceRecord); }
+
+  // Get the status of a trace record. This function does not acquire
+  // a lock. If locking is required, the caller must hold a lock.
+  TRStatus getTRStatus(void *Rec);
+
+  // Set the status of a trace record. This function does not acquire
+  // a lock. If locking is required, the caller must hold a lock.
+  void setTRStatus(void *Rec, TRStatus);
+
+  // Is this a last cursor of a buffer completion callback?
+  bool isLastCursor(void *Cursor) {
+    std::unique_lock<std::mutex> Lock(LastCursorMutex);
+    return LastCursors.find(Cursor) != LastCursors.end();
+  }
+
+  // Called for flushing outstanding buffers for the provided device-id.
+  int flushAllBuffers(int DeviceId);
+};
+
+#else
+class OmptTracingBufferMgr {};
+#endif // OMPT_SUPPORT
+
+#endif // OPENMP_LIBOMPTARGET_OMPTTRACINGBUFFER_H
diff --git a/offload/include/PluginManager.h b/offload/include/PluginManager.h
index 6c6fdebe76dff..972afd962cbc1 100644
--- a/offload/include/PluginManager.h
+++ b/offload/include/PluginManager.h
@@ -13,6 +13,7 @@
 #ifndef OMPTARGET_PLUGIN_MANAGER_H
 #define OMPTARGET_PLUGIN_MANAGER_H
 
+#include "OpenMP/OMPT/OmptTracingBuffer.h"
 #include "PluginInterface.h"
 
 #include "DeviceImage.h"
@@ -48,7 +49,7 @@ struct PluginManager {
   /// Exclusive accessor type for the device container.
   using ExclusiveDevicesAccessorTy = Accessor<DeviceContainerTy>;
 
-  PluginManager() {}
+  PluginManager() : TraceRecordManager(nullptr) {}
 
   void init();
 
@@ -150,6 +151,11 @@ struct PluginManager {
     return count;
   }
 
+  auto getTraceRecordManager() const {
+    assert(TraceRecordManager && "Trace record manager not initialized");
+    return TraceRecordManager;
+  }
+
 private:
   bool RTLsLoaded = false;
   llvm::SmallVector<__tgt_bin_desc *> DelayedBinDesc;
@@ -176,6 +182,8 @@ struct PluginManager {
   /// Devices associated with plugins, accesses to the container are exclusive.
   ProtectedObj<DeviceContainerTy> Devices;
 
+  OmptTracingBufferMgr *TraceRecordManager;
+
   /// References to upgraded legacy offloading entries.
   std::list<llvm::SmallVector<llvm::offloading::EntryTy, 0>> LegacyEntries;
   std::list<llvm::SmallVector<__tgt_device_image, 0>> LegacyImages;
diff --git a/offload/libomptarget/CMakeLists.txt b/offload/libomptarget/CMakeLists.txt
index 7909d55eff5c3..3301c01ac90a3 100644
--- a/offload/libomptarget/CMakeLists.txt
+++ b/offload/libomptarget/CMakeLists.txt
@@ -17,6 +17,7 @@ add_library(omptarget SHARED
   OpenMP/Mapping.cpp
   OpenMP/InteropAPI.cpp
   OpenMP/OMPT/Callback.cpp
+  OpenMP/OMPT/OmptTracingBuffer.cpp
 
   KernelLanguage/API.cpp
 )
diff --git a/offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp b/offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp
new file mode 100644
index 0000000000000..2d4b32e3554ac
--- /dev/null
+++ b/offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp
@@ -0,0 +1,763 @@
+//=== OmptTracingBuffer.cpp - Target independent OpenMP target RTL -- C++ -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementation of OMPT device trace record generation and flushing.
+//
+//===----------------------------------------------------------------------===//
+
+#ifdef OMPT_SUPPORT
+
+#include "OpenMP/OMPT/OmptTracingBuffer.h"
+#include "OpenMP/OMPT/OmptTracing.h"
+#include "Shared/Debug.h"
+
+#include <atomic>
+#include <cassert>
+#include <cstdlib>
+#include <limits>
+
+using namespace llvm::omp::target::debug;
+
+// When set to true, helper threads terminate their work
+static bool DoneTracing{false};
+
+// Unique buffer id in creation order
+static std::atomic<uint64_t> BufId{0};
+
+// Unique id in buffer flush order
+static std::atomic<uint64_t> FlushId{0};
+
+thread_local OmptTracingBufferMgr::BufPtr
+    OmptTracingBufferMgr::ArrayOfBufPtr[MAX_NUM_DEVICES];
+
+static uint64_t get_and_inc_buf_id() { return BufId++; }
+
+static uint64_t get_and_inc_flush_id() { return FlushId++; }
+static uint64_t get_flush_id() { return FlushId; }
+
+/*
+ * Used by OpenMP threads for assigning space for a trace record. If
+ * there is no space in the last buffer allocated by this thread, the
+ * last buffer is marked full and scheduled for flushing. Otherwise,
+ * space is assigned for a trace record and the new cursor returned.
+ * Since the memory allocated by a thread is used by that thread alone
+ * for creating trace records, a lock need not be held. In the less
+ * common branch when memory is allocated, a lock needs to be acquired
+ * for updating shared metadata. The common path of allocating a trace
+ * record from an existing buffer proceeds without locking.
+ */
+void *OmptTracingBufferMgr::assignCursor(ompt_callbacks_t Type,
+                                         int64_t DeviceId) {
+  // The caller should handle nullptr by not tracing for this event.
+  if (DeviceId < 0 || DeviceId > MAX_NUM_DEVICES - 1)
+    return nullptr;
+
+  size_t RecSize = getTRSize();
+
+  // If the buffer fills up, it will be scheduled for flushing with the
+  // following cursor.
+  void *ToBeFlushedCursor = nullptr;
+  BufPtr ToBeFlushedBuf = nullptr;
+
+  // Thread local buffer pointer should be non-null once an allocation
+  // has been done by this thread.
+  BufPtr DeviceBuf = getDeviceSpecificBuffer(DeviceId);
+  if (DeviceBuf != nullptr) {
+    assert(DeviceBuf->DeviceId == DeviceId && "Unexpected device id in buffer");
+    void *OldCursor = DeviceBuf->Cursor.load(std::memory_order_acquire);
+    // Try to assign a trace record from the last allocated buffer
+    if (RecSize <= DeviceBuf->RemainingBytes) {
+      assert((char *)DeviceBuf->Start + DeviceBuf->TotalBytes -
+                 DeviceBuf->RemainingBytes ==
+             (char *)OldCursor + RecSize);
+      DeviceBuf->RemainingBytes -= RecSize;
+
+      // Note the trace record status must be initialized before setting
+      // the cursor, ensuring that a helper thread always sees an initialized
+      // trace record status.
+      void *NewCursor = (char *)OldCursor + RecSize;
+      initTraceRecordMetaData(NewCursor);
+      DeviceBuf->Cursor.store(NewCursor, std::memory_order_release);
+
+      ODBG(ODT_Tool) << "Thread " << llvm::omp::target::ompt::getThreadId()
+                     << ": Assigned " << RecSize << " bytes at " << NewCursor
+                     << " in existing buffer " << DeviceBuf->Start
+                     << " for device " << DeviceId;
+      return NewCursor;
+    } else {
+      ToBeFlushedCursor = OldCursor;
+      ToBeFlushedBuf = DeviceBuf;
+
+      // Mark that no space is present for any more trace records.
+      // The following is atomic but there is no logical order between when
+      // it is set here and when it is checked by a helper thread. That works
+      // because the helper thread uses this info to decide whether a buffer
+      // can be scheduled for removal. In the worst case, the buffer will be
+      // removed late.
+      DeviceBuf->IsFull.store(true, std::memory_order_release);
+    }
+  }
+  void *NewBuffer = nullptr;
+  size_t TotalBytes = 0;
+  // TODO Move the buffer allocation to a helper thread
+  llvm::omp::target::ompt::ompt_callback_buffer_request(DeviceId, &NewBuffer,
+                                                        &TotalBytes);
+
+  // The caller should handle nullptr by not tracing for this event.
+  if (NewBuffer == nullptr || TotalBytes < RecSize)
+    return nullptr;
+
+  uint64_t NewBufId = get_and_inc_buf_id();
+  auto new_buf = std::make_shared<Buffer>(
+      NewBufId, DeviceId, /*Start=*/NewBuffer, TotalBytes,
+      /*RemainingBytes=*/TotalBytes - RecSize,
+      /*Cursor=*/NewBuffer,
+      /*IsFull=*/false);
+
+  // Initialize trace record status before publishing it to helper threads.
+  initTraceRecordMetaData(new_buf->Cursor.load(std::memory_order_acquire));
+  setDeviceSpecificBuffer(DeviceId, new_buf);
+
+  // Make this trace record visible to helper threads by adding to shared
+  // metadata.
+  std::unique_lock<std::mutex> lck(BufferMgrMutex);
+  assert(Id2BufferMap.find(NewBufId) == Id2BufferMap.end());
+  Id2BufferMap[NewBufId] = new_buf;
+  lck.unlock();
+
+  // Schedule the full buffer for flushing till the corresponding cursor.
+  if (OMPX_FlushOnBufferFull && ToBeFlushedCursor)
+    triggerFlushOnBufferFull(ToBeFlushedCursor, ToBeFlushedBuf);
+
+  ODBG(ODT_Tool) << "Thread " << llvm::omp::target::ompt::getThreadId()
+                 << ": Assigned " << RecSize << " bytes at " << NewBuffer
+                 << " in new buffer with id " << NewBufId << " for device "
+                 << DeviceId;
+  return NewBuffer;
+}
+
+/*
+ * Called by an OpenMP thread when a buffer fills up and should be
+ * flushed. This function assigns a new FlushId to the buffer, adds
+ * to the flush-related metadata and wakes up a helper thread to
+ * dispatch a buffer-completion callback. This function should be
+ * called without holding any lock.
+ * Note lock order: buf_lock -> flush_lock
+ */
+void OmptTracingBufferMgr::triggerFlushOnBufferFull(void *cursor, BufPtr Buf) {
+  std::unique_lock<std::mutex> buf_lock(BufferMgrMutex);
+
+  // Between calling this function and this check, a flush-all may have
+  // delivered this buffer to the tool and deleted it. So the buffer
+  // may not exist.
+  if (Id2BufferMap.find(Buf->Id) == Id2BufferMap.end())
+    return;
+
+  // Cannot assert that the state of the cursor is ready since a
+  // different thread may be in the process of populating it. If it
+  // remains in init state when the range of trace records is
+  // determined for dispatching the buffer-completion callback, it
+  // will not be included.
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  uint64_t flush_id;
+  auto flush_itr = FlushBufPtr2IdMap.find(Buf);
+  if (flush_itr == FlushBufPtr2IdMap.end()) {
+    // This buffer has not been flushed yet
+    addNewFlushEntry(Buf, cursor);
+  } else {
+    // This buffer has been flushed before
+    flush_id = flush_itr->second;
+    auto flush_md_itr = Id2FlushMdMap.find(flush_id);
+    assert(flush_md_itr != Id2FlushMdMap.end());
+    flush_md_itr->second.FlushCursor = cursor; // update the cursor
+    // Do not update the flush status since it may be under processing
+    // by another thread
+    ODBG(ODT_Tool) << "Updated id " << flush_id << " cursor " << cursor
+                   << " buf " << flush_md_itr->second.FlushBuf->Start;
+  }
+  flush_lock.unlock();
+  buf_lock.unlock();
+
+  // Wake up a helper thread to invoke the buffer-completion callback
+  FlushCv.notify_one();
+}
+
+// This is the driver routine for the completion thread
+void OmptTracingBufferMgr::driveCompletion() {
+  while (true) {
+    bool should_signal_workers = false;
+    std::unique_lock<std::mutex> flush_lock(FlushMutex);
+    if (DoneTracing) {
+      // An upper layer serializes flush_trace and stop_trace. In
+      // addition, before DoneTracing is set, a flush is performed as
+      // part of stop_trace. So assert that no flush is in progress.
+      assert(ThreadFlushTracker == 0);
+      break;
+    }
+    FlushCv.wait(flush_lock, [this] {
+      return DoneTracing ||
+             (!Id2FlushMdMap.empty() &&
+              llvm::omp::target::ompt::TracingActive) ||
+             isThisThreadFlushWaitedUpon();
+    });
+    if (isThisThreadFlushWaitedUpon()) {
+      resetThisThreadFlush();
+      if (ThreadFlushTracker == 0)
+        should_signal_workers = true;
+    }
+    flush_lock.unlock();
+
+    invokeCallbacks();
+
+    if (should_signal_workers)
+      ThreadFlushCv.notify_all();
+
+    // There is a scenario where a buffer was processed but not full
+    // or owned, so it was put back in waiting state. So this thread
+    // would not wait but keep on looping without having any actual
+    // work until new trace records are added and this thread
+    // signaled. Hence, this thread yields.
+    std::this_thread::yield();
+  }
+  bool is_last_helper = false;
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  assert(DoneTracing && "Helper thread exiting but not yet done");
+  assert(isThisThreadShutdownWaitedUpon() &&
+         "Helper thread exiting but not waited upon");
+  resetThisThreadShutdown();
+  if (ThreadShutdownTracker == 0)
+    is_last_helper = true;
+  flush_lock.unlock();
+  if (is_last_helper)
+    ThreadShutdownCv.notify_all();
+
+  // Note that some trace records may have been written but not
+  // delivered to the tool. If flush/stop APIs are not called by the
+  // tool, those trace records may never be delivered to the tool and
+  // the corresponding buffers not reclaimed. TODO Explore whether
+  // this cleanup must be done.
+}
+
+/*
+ * Called by a buffer-completion helper thread. This function examines
+ * the flushed buffers in flush order and dispatches
+ * callbacks. Lock holding is minimized by reserving a buffer,
+ * processing it, and then unreserving it if there are more trace
+ * records to flush later. If all trace records are flushed, a
+ * callback is dispatched informing the tool that the buffer can be
+ * deallocated. If the buffer can be deallocated, all metadata is
+ * destroyed.
+ * Note that this function must be called without holding any locks.
+ */
+void OmptTracingBufferMgr::invokeCallbacks() {
+  ODBG(ODT_Tool) << "Looking for callbacks to invoke";
+  auto max_id = std::numeric_limits<uint64_t>::max();
+  auto curr_id = max_id;
+  auto end_id = get_flush_id();
+  ODBG(ODT_Tool) << "End id is " << end_id;
+  while (true) {
+    // Set the status of the flushed buffer to in-processing so that
+    // another helper thread does not process it concurrently. An
+    // OpenMP worker thread may, however, populate a trace record in a
+    // reserved buffer concurrently.
+    FlushInfo flush_info = findAndReserveFlushedBuf(curr_id);
+
+    // no entry found, nothing to process
+    if (curr_id == max_id && flush_info.FlushCursor == nullptr)
+      return;
+
+    if (flush_info.FlushCursor != nullptr) {
+      // increment curr_id to get the candidate for the next iteration
+      curr_id = flush_info.FlushId + 1;
+    } else {
+      assert(curr_id != max_id && "Cannot increment max id");
+      ++curr_id;
+    }
+
+    ODBG(ODT_Tool) << "Next id will be " << curr_id;
+
+    if (flush_info.FlushCursor == nullptr) {
+      // This buffer must have been processed already
+      if (curr_id < end_id)
+        continue;
+      else
+        return; // nothing else to process
+    }
+
+    ODBG(ODT_Tool) << "Buf " << flush_info.FlushBuf->Start
+             << " Cursor " << flush_info.FlushCursor
+             << " Id " << flush_info.FlushId
+             << " will be flushed";
+
+    // Examine the status of the trace records and dispatch
+    // buffer-completion callbacks as appropriate.
+    flushBuffer(flush_info);
+
+    // TODO optimize to set buffer-owned in the same pass above.
+    // Currently, this is the only way a buffer is deallocated
+    if (isBufferFull(flush_info)) {
+      // All trace records have been delivered to the tool
+      if (isBufferOwned(flush_info)) {
+        // erase element from buffer and flush maps
+        destroyFlushedBuf(flush_info);
+
+        // dispatch callback with a null range and have the tool
+        // deallocate the buffer
+        dispatchBufferOwnedCallback(flush_info);
+      } else {
+        unreserveFlushedBuf(flush_info);
+      }
+    } else {
+      unreserveFlushedBuf(flush_info);
+    }
+    if (curr_id >= end_id)
+      return;
+  }
+}
+
+/*
+ * This function is called on a buffer that is already reserved by
+ * this thread. Buffer-completion callbacks are dispatched for every
+ * range of trace records that are ready.
+ * This routine must be called without holding locks
+ */
+void OmptTracingBufferMgr::flushBuffer(FlushInfo flush_info) {
+  assert(flush_info.FlushBuf && "Cannot flush an empty buffer");
+  assert(flush_info.FlushCursor && "Cannot flush upto a null cursor");
+
+  void *curr_tr = flush_info.FlushBuf->Start;
+  void *last_tr = flush_info.FlushCursor;
+  // Compute a range [first_cursor,last_cursor] to flush
+  void *first_cursor = nullptr;
+  void *last_cursor = nullptr;
+  while (curr_tr <= last_tr) {
+    TRStatus tr_status = getTRStatus(curr_tr);
+    if (tr_status == TR_init || tr_status == TR_released) {
+      if (first_cursor == nullptr) {
+        // This TR won't be part of a range
+        assert(last_cursor == nullptr &&
+               "Begin/last cursors mutually inconsistent");
+      } else {
+        // End the current interval
+        dispatchCallback(flush_info.FlushBuf->DeviceId,
+                         flush_info.FlushBuf->Start, first_cursor, last_cursor);
+        first_cursor = last_cursor = nullptr;
+      }
+    } else {
+      assert(tr_status == TR_ready && "Unknown trace record status");
+      setTRStatus(curr_tr, TR_released);
+      if (first_cursor == nullptr)
+        first_cursor = curr_tr;
+      last_cursor = curr_tr;
+    }
+    curr_tr = getNextTR(curr_tr);
+  }
+  if (first_cursor != nullptr) {
+    assert(last_cursor != nullptr);
+    dispatchCallback(flush_info.FlushBuf->DeviceId, flush_info.FlushBuf->Start,
+                     first_cursor, last_cursor);
+  }
+}
+
+// Given a range of trace records, dispatch a buffer-completion callback
+void OmptTracingBufferMgr::dispatchCallback(int64_t DeviceId, void *Buffer,
+                                            void *FirstCursor,
+                                            void *LastCursor) {
+  assert(FirstCursor != nullptr && LastCursor != nullptr &&
+         "Callback with nullptr");
+  addLastCursor(LastCursor);
+
+  // This is best effort.
+  // There is a small window when the buffer-completion callback may
+  // be invoked even after tracing has been disabled.
+  // Note that we don't want to hold a lock when dispatching the callback.
+  if (llvm::omp::target::ompt::isTracedDevice(DeviceId)) {
+    ODBG(ODT_Tool) << "Dispatch callback w/ range (inclusive) to be flushed: "
+                   << FirstCursor << " -> " << LastCursor;
+    llvm::omp::target::ompt::ompt_callback_buffer_complete(
+        DeviceId, Buffer,
+        /* bytes returned in this callback */
+        (char *)getNextTR(LastCursor) - (char *)FirstCursor,
+        (ompt_buffer_cursor_t)FirstCursor, false /* buffer_owned */);
+  }
+
+  removeLastCursor(LastCursor);
+}
+
+// Dispatch a buffer-completion callback with buffer_owned set so that
+// the tool can deallocate the buffer
+void OmptTracingBufferMgr::dispatchBufferOwnedCallback(
+    const FlushInfo &flush_info) {
+  // This is best effort.
+  // There is a small window when the buffer-completion callback may
+  // be invoked even after tracing has been disabled.
+  // Note that we don't want to hold a lock when dispatching the callback.
+  if (llvm::omp::target::ompt::isTracedDevice(flush_info.FlushBuf->DeviceId)) {
+    ODBG(ODT_Tool) << "Dispatch callback with buffer "
+             << flush_info.FlushBuf->Start << " owned";
+    llvm::omp::target::ompt::ompt_callback_buffer_complete(
+        flush_info.FlushBuf->DeviceId, flush_info.FlushBuf->Start, 0,
+        (ompt_buffer_cursor_t)0, true /* buffer owned */);
+  }
+}
+
+void OmptTracingBufferMgr::initTraceRecordMetaData(void *Rec) {
+  setTRStatus(Rec, TR_init);
+}
+
+OmptTracingBufferMgr::BufPtr
+OmptTracingBufferMgr::getDeviceSpecificBuffer(int64_t DeviceId) {
+  if (DeviceId < 0 || DeviceId > MAX_NUM_DEVICES - 1) {
+    REPORT() << "getDeviceSpecificBuffer: Device id " << DeviceId
+             << " invalid or exceeds supported max: "
+             << MAX_NUM_DEVICES - 1;
+    return nullptr;
+  }
+  return ArrayOfBufPtr[DeviceId];
+}
+
+void OmptTracingBufferMgr::setDeviceSpecificBuffer(int64_t DeviceId,
+                                                   BufPtr Buf) {
+  if (DeviceId < 0 || DeviceId > MAX_NUM_DEVICES - 1) {
+    REPORT() << "setDeviceSpecificBuffer: Device id " << DeviceId
+             << " invalid or exceeds supported max: "
+             << MAX_NUM_DEVICES - 1;
+    return;
+  }
+  ArrayOfBufPtr[DeviceId] = Buf;
+}
+
+void OmptTracingBufferMgr::setTRStatus(void *Rec, TRStatus Status) {
+  TraceRecord *TR = static_cast<TraceRecord *>(Rec);
+  TR->TRState.store(Status, std::memory_order_release);
+}
+
+OmptTracingBufferMgr::TRStatus OmptTracingBufferMgr::getTRStatus(void *Rec) {
+  return static_cast<TraceRecord *>(Rec)->TRState.load(
+      std::memory_order_acquire);
+}
+
+void *OmptTracingBufferMgr::getNextTR(void *TR) {
+  size_t RecSize = getTRSize();
+  // warning: no overflow check done
+  return (char *)TR + RecSize;
+}
+
+bool OmptTracingBufferMgr::isBufferFull(const FlushInfo &flush_info) {
+  std::unique_lock<std::mutex> buf_lock(BufferMgrMutex);
+  return flush_info.FlushBuf->IsFull;
+}
+
+void *OmptTracingBufferMgr::getBufferCursor(BufPtr buf) {
+  return buf->Cursor.load(std::memory_order_acquire);
+}
+
+/*
+ * Traverse all the trace records of a buffer and return true if all
+ * of them have been released to the tool, otherwise return false
+ */
+bool OmptTracingBufferMgr::isBufferOwned(const FlushInfo &flush_info) {
+  assert(isBufferFull(flush_info) && "Compute buffer-owned when it is full");
+  void *curr_tr = flush_info.FlushBuf->Start;
+  // Since the buffer is full, the cursor must be the last valid
+  // TR. Note that this may be more up-to-date than the cursor in the
+  // flush_info. Use the last valid TR to avoid dropping trace records
+  void *last_tr = getBufferCursor(flush_info.FlushBuf);
+  while (curr_tr <= last_tr) {
+    if (getTRStatus(curr_tr) != TR_released)
+      return false;
+    curr_tr = getNextTR(curr_tr);
+  }
+  return true;
+}
+
+/*
+ * A buffer must be reserved by a thread before it can be processed
+ * and callbacks dispatched for that buffer. Reservation is done by
+ * setting the status to in-processing.
+ *
+ * If a buffer is found in the flush metadata for the given id and it
+ * is not in in-processing mode, reserve it by setting its mode to
+ * in-processing and return the corresponding flush metadata. If the
+ * given id is set to max, return the first waiting buffer in the
+ * list of buffers to be flushed.
+ */
+OmptTracingBufferMgr::FlushInfo
+OmptTracingBufferMgr::findAndReserveFlushedBuf(uint64_t FlushId) {
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  MapId2Md::iterator flush_itr;
+  if (FlushId == std::numeric_limits<uint64_t>::max()) {
+    // Reserve the first waiting buffer and return it
+    if (Id2FlushMdMap.empty())
+      return FlushInfo();
+    for (flush_itr = Id2FlushMdMap.begin(); flush_itr != Id2FlushMdMap.end();
+         ++flush_itr) {
+      // Reserve only if waiting
+      if (flush_itr->second.FlushStatus == Flush_waiting)
+        break;
+    }
+    if (flush_itr == Id2FlushMdMap.end())
+      return FlushInfo();
+  } else {
+    flush_itr = Id2FlushMdMap.find(FlushId);
+    if (flush_itr == Id2FlushMdMap.end() ||
+        flush_itr->second.FlushStatus == Flush_processing)
+      return FlushInfo();
+  }
+  assert(flush_itr->second.FlushStatus == Flush_waiting);
+  flush_itr->second.FlushStatus = Flush_processing;
+  // Update the metadata cursor since more trace records may have been
+  // generated.
+  flush_itr->second.FlushCursor =
+      flush_itr->second.FlushBuf->Cursor.load(std::memory_order_acquire);
+
+  FlushInfo flush_info(flush_itr->first, flush_itr->second.FlushCursor,
+                       flush_itr->second.FlushBuf);
+  ODBG(ODT_Tool) << "Reserved buffer: flush_id:" << flush_itr->first
+                 << ", cursor:" << flush_itr->second.FlushCursor
+                 << ", buf:" << flush_itr->second.FlushBuf->Start;
+  return flush_info;
+}
+
+/*
+ * Given a buffer, verify that it is in processing state and set its
+ * status to waiting, removing the reservation. The same thread that
+ * reserved it should be unreserving it but currently there is no such
+ * check.
+ */
+void OmptTracingBufferMgr::unreserveFlushedBuf(const FlushInfo &flush_info) {
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  auto itr = Id2FlushMdMap.find(flush_info.FlushId);
+  assert(itr != Id2FlushMdMap.end() &&
+         itr->second.FlushStatus == Flush_processing);
+  itr->second.FlushStatus = Flush_waiting;
+  ODBG(ODT_Tool) << "Unreserved buffer: flush_id:" << flush_info.FlushId
+                 << ", cursor:" << flush_info.FlushCursor
+                 << ", buf:" << flush_info.FlushBuf->Start;
+}
+
+/*
+ * This function must be called after all of the trace records in the
+ * buffer have been released to the tool. The buffer is removed from
+ * all metadata maps.
+ * Note lock order: buf_lock -> flush_lock
+ */
+void OmptTracingBufferMgr::destroyFlushedBuf(const FlushInfo &flush_info) {
+  ODBG(ODT_Tool) << "Destroying buffer: flush_id:" << flush_info.FlushId
+                 << ", cursor:" << flush_info.FlushCursor
+                 << ", buf:" << flush_info.FlushBuf->Start;
+
+  BufPtr buf = flush_info.FlushBuf;
+
+  std::unique_lock<std::mutex> buf_lock(BufferMgrMutex);
+  Id2BufferMap.erase(buf->Id);
+
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  auto flush_itr = Id2FlushMdMap.find(flush_info.FlushId);
+  assert(flush_itr != Id2FlushMdMap.end());
+  assert(flush_itr->second.FlushBuf == buf);
+  Id2FlushMdMap.erase(flush_itr);
+  FlushBufPtr2IdMap.erase(buf);
+}
+
+/*
+ * Generate a new flush id and add the buffer to the flush metadata
+ * maps. This function must be called while holding the flush lock.
+ */
+uint64_t OmptTracingBufferMgr::addNewFlushEntry(BufPtr Buf, void *Cursor) {
+  assert(FlushBufPtr2IdMap.find(Buf) == FlushBufPtr2IdMap.end());
+  uint64_t FlushId = get_and_inc_flush_id();
+  FlushBufPtr2IdMap.emplace(Buf, FlushId);
+  assert(Id2FlushMdMap.find(FlushId) == Id2FlushMdMap.end());
+  Id2FlushMdMap.emplace(FlushId, FlushMd(Cursor, Buf, Flush_waiting));
+
+  ODBG(ODT_Tool) << "Added new flush id "
+           << FlushId << " cursor "
+           << Cursor << " buf " << Buf->Start;
+
+  return FlushId;
+}
+
+/*
+ * Called by ompt_flush_trace and ompt_stop_trace. Traverse the
+ * existing buffers in creation order and flush all the ready TRs
+ */
+int OmptTracingBufferMgr::flushAllBuffers(int DeviceId) {
+  ODBG(ODT_Tool) << "Flushing buffers for device " << DeviceId << " :: START";
+  // Overloading MAX_NUM_DEVICES to mean all devices.
+  if (DeviceId < 0 || DeviceId > MAX_NUM_DEVICES)
+    return 0; // failed to flush
+
+  if (!areHelperThreadsAvailable())
+    return 0; // failed to flush
+
+  // If flush is called from a helper thread, just bail out
+  if (amIHelperThread())
+    return 0; // failed to flush
+
+  // To avoid holding the mutex for too long, get the ids of the first
+  // and the last TRs under lock, and then go through that range,
+  // holding the mutex for an individual TR
+  std::unique_lock<std::mutex> buf_lock(BufferMgrMutex);
+  if (Id2BufferMap.empty())
+    return 1; // no trace records to flush
+  uint64_t curr_buf_id = Id2BufferMap.begin()->first;
+  uint64_t last_buf_id = Id2BufferMap.rbegin()->first;
+  buf_lock.unlock();
+
+  while (curr_buf_id <= last_buf_id) {
+    std::unique_lock<std::mutex> buf_lock(BufferMgrMutex);
+    // Another thread may have deleted this buffer by now
+    auto buf_itr = Id2BufferMap.find(curr_buf_id);
+    if (buf_itr == Id2BufferMap.end()) {
+      ++curr_buf_id;
+      continue;
+    }
+    BufPtr curr_buf = buf_itr->second;
+
+    // If the device-id does not match, skip it. A device-id of MAX_NUM_DEVICES
+    // indicates flushing for all devices.
+    if (DeviceId != MAX_NUM_DEVICES && curr_buf->DeviceId != DeviceId) {
+      ++curr_buf_id;
+      continue;
+    }
+
+    // If this buffer is in the flush-map, skip it. It is either in
+    // process by another thread or will be processed
+    std::unique_lock<std::mutex> flush_lock(FlushMutex);
+    auto flush_itr = FlushBufPtr2IdMap.find(curr_buf);
+    if (flush_itr != FlushBufPtr2IdMap.end()) {
+      ++curr_buf_id;
+      continue;
+    }
+    // This buffer has not been flushed yet
+    void *CurrBufCursor = getBufferCursor(curr_buf);
+    uint64_t flush_id = addNewFlushEntry(curr_buf, CurrBufCursor);
+    (void)flush_id; // Silence warning.
+    ODBG(ODT_Tool) << "flushAllBuffers: Added new id "
+             << flush_id << " cursor " << CurrBufCursor
+             << " buf " << curr_buf->Start;
+
+    flush_lock.unlock();
+    buf_lock.unlock();
+
+    ++curr_buf_id;
+  }
+
+  ODBG(ODT_Tool) << "Flushing buffers for device " << DeviceId << " :: WAIT";
+
+  // This is best effort. It is possible that some trace records are
+  // not flushed when the wait is done.
+  waitForFlushCompletion();
+
+  ODBG(ODT_Tool) << "Flushing buffers for device " << DeviceId << " :: STOP";
+
+  return 1; // success
+}
+
+void OmptTracingBufferMgr::waitForFlushCompletion() {
+  {
+    std::unique_lock<std::mutex> flush_lock(FlushMutex);
+    // Setting the flush bit for a given helper thread indicates that the worker
+    // thread is ready for the helper thread to do some work.
+    for (uint32_t i = 0; i < OMPT_NUM_HELPER_THREADS; ++i)
+      setThreadFlush(i);
+  }
+
+  // Wake up all helper threads to invoke buffer-completion callbacks.
+  FlushCv.notify_all();
+
+  // Now wait for all helper threads  to complete flushing.
+  {
+    std::unique_lock<std::mutex> flush_lock(FlushMutex);
+    ThreadFlushCv.wait(flush_lock, [this] { return ThreadFlushTracker == 0; });
+  }
+}
+
+void OmptTracingBufferMgr::init() {
+  for (int i = 0; i < MAX_NUM_DEVICES; ++i)
+    ArrayOfBufPtr[i] = nullptr;
+  ThreadFlushTracker = 0;
+  ThreadShutdownTracker = 0;
+  DoneTracing = false; // TODO make it a class member
+}
+
+void OmptTracingBufferMgr::startHelperThreads() {
+  // All helper threads are stopped while holding FlushMutex. So if
+  // any helper thread is present, just return. This takes care of
+  // repeated calls to start-trace.
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  if (!HelperThreadIdMap.empty()) {
+    assert(!DoneTracing && "Helper threads exist but tracing is done");
+    return;
+  }
+  init();
+  createHelperThreads();
+}
+
+bool OmptTracingBufferMgr::areHelperThreadsAvailable() {
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  if (DoneTracing // If another thread called stop, assume there are no threads
+      || HelperThreadIdMap.empty() // Threads were never started
+  ) {
+    // Don't assert on HelperThreadIdMap since shutdown by another
+    // thread may be in progress
+    return false;
+  }
+  return true;
+}
+
+void OmptTracingBufferMgr::shutdownHelperThreads() {
+  if (!areHelperThreadsAvailable())
+    return;
+
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  // If I am destroying the threads, then at least one thread must be present
+  assert(!CompletionThreads.empty());
+  assert(!HelperThreadIdMap.empty());
+  assert(ThreadShutdownTracker == 0);
+
+  // Set the done flag which helper threads will look at
+  DoneTracing = true;
+  // Wait to make sure all helper threads exit
+  for (uint32_t i = 0; i < OMPT_NUM_HELPER_THREADS; ++i)
+    setThreadShutdown(i);
+  // Signal indicating that DoneTracing is set
+  FlushCv.notify_all();
+  ThreadShutdownCv.wait(flush_lock,
+                        [this] { return ThreadShutdownTracker == 0; });
+
+  // Now destroy all the helper threads
+  destroyHelperThreads();
+}
+
+void OmptTracingBufferMgr::flushAndShutdownHelperThreads() {
+  std::unique_lock<std::mutex> Lock(llvm::omp::target::ompt::TraceControlMutex);
+  // Flush buffers for all devices.
+  if (OMPX_FlushOnShutdown)
+    flushAllBuffers(MAX_NUM_DEVICES);
+  else
+    waitForFlushCompletion(); // Dont initiate but wait for outstanding flushes.
+  shutdownHelperThreads();
+}
+
+void OmptTracingBufferMgr::createHelperThreads() {
+  for (uint32_t i = 0; i < OMPT_NUM_HELPER_THREADS; ++i) {
+    CompletionThreads.emplace_back(
+        std::thread(&OmptTracingBufferMgr::driveCompletion, this));
+    HelperThreadIdMap[CompletionThreads.back().get_id()] = i;
+  }
+}
+
+void OmptTracingBufferMgr::destroyHelperThreads() {
+  for (auto &thd : CompletionThreads)
+    thd.join();
+  CompletionThreads.clear();
+  HelperThreadIdMap.clear();
+}
+#endif

>From ca9365b67045c1046225a42a92b9e7d32c32f240 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 2 Apr 2026 07:52:40 -0500
Subject: [PATCH 07/11] [Offload][OMPT] Add tracing orchestration and
 libomptarget integration

Wire the OMPT device tracing subsystem into libomptarget, completing the tracing pipeline from record production through buffer management to tool delivery. Include the synchronous fallback and initialization fixes in the integration that requires them.

Co-Authored-By: Claude <noreply at anthropic.com>
---
 offload/libomptarget/CMakeLists.txt           |   1 +
 offload/libomptarget/OpenMP/OMPT/Callback.cpp |  12 +
 .../libomptarget/OpenMP/OMPT/OmptTracing.cpp  | 885 ++++++++++++++++++
 offload/libomptarget/PluginManager.cpp        |  22 +
 offload/libomptarget/device.cpp               |  58 +-
 offload/libomptarget/exports                  |   8 +
 offload/libomptarget/interface.cpp            |  44 +-
 offload/libomptarget/omptarget.cpp            |   6 +-
 8 files changed, 1008 insertions(+), 28 deletions(-)
 create mode 100644 offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp

diff --git a/offload/libomptarget/CMakeLists.txt b/offload/libomptarget/CMakeLists.txt
index 3301c01ac90a3..7f2ff7329a536 100644
--- a/offload/libomptarget/CMakeLists.txt
+++ b/offload/libomptarget/CMakeLists.txt
@@ -17,6 +17,7 @@ add_library(omptarget SHARED
   OpenMP/Mapping.cpp
   OpenMP/InteropAPI.cpp
   OpenMP/OMPT/Callback.cpp
+  OpenMP/OMPT/OmptTracing.cpp
   OpenMP/OMPT/OmptTracingBuffer.cpp
 
   KernelLanguage/API.cpp
diff --git a/offload/libomptarget/OpenMP/OMPT/Callback.cpp b/offload/libomptarget/OpenMP/OMPT/Callback.cpp
index 1e03f1455d1b2..071276b3903ca 100644
--- a/offload/libomptarget/OpenMP/OMPT/Callback.cpp
+++ b/offload/libomptarget/OpenMP/OMPT/Callback.cpp
@@ -12,9 +12,13 @@
 
 #ifdef OMPT_SUPPORT
 
+#include <atomic>
+#include <cassert>
 #include <cstdlib>
 #include <cstring>
 #include <memory>
+#include <mutex>
+#include <thread>
 
 #include "Shared/Debug.h"
 
@@ -23,7 +27,9 @@
 #include "OpenMP/OMPT/Interface.h"
 
 #include "llvm/Support/DynamicLibrary.h"
+#include "llvm/Support/ErrorHandling.h"
 
+#pragma push_macro("DEBUG_PREFIX")
 #undef DEBUG_PREFIX
 #define DEBUG_PREFIX "OMPT"
 
@@ -441,6 +447,11 @@ void Interface::endTarget(int64_t DeviceId, void *Code) {
   endTargetRegion();
 }
 
+void Interface::announceTargetRegion(const char *RegionName) {
+  ODBG(ODT_Tool) << "in Interface::target_region_" << RegionName
+                 << " target_id=" << TargetData.value;
+}
+
 void Interface::beginTargetDataOperation() {
   ODBG(ODT_Tool) << "in ompt_target_region_begin (TargetRegionId = "
                  << TargetData.value << ")";
@@ -562,4 +573,5 @@ void llvm::omp::target::ompt::connectLibrary() {
   ODBG(ODT_Tool) << "Exiting connectLibrary";
 }
 
+#pragma pop_macro("DEBUG_PREFIX")
 #endif // OMPT_SUPPORT
diff --git a/offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp b/offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp
new file mode 100644
index 0000000000000..a74b10137f2c3
--- /dev/null
+++ b/offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp
@@ -0,0 +1,885 @@
+//===-- OmptTracing.cpp - Target independent OpenMP target RTL --- C++ -*--===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementation of OMPT tracing interfaces for target independent layer
+//
+//===----------------------------------------------------------------------===//
+
+#ifdef OMPT_SUPPORT
+
+#include "OpenMP/OMPT/OmptTracing.h"
+#include "OpenMP/OMPT/Callback.h"
+#include "OpenMP/OMPT/Interface.h"
+#include "OpenMP/OMPT/OmptTracingBuffer.h"
+#include "PluginManager.h"
+#include "Shared/Debug.h"
+#include "omp-tools.h"
+
+#include "llvm/Support/DynamicLibrary.h"
+
+#include <atomic>
+#include <cassert>
+#include <cstdlib>
+#include <cstring>
+#include <memory>
+#include <mutex>
+#include <thread>
+
+#pragma push_macro("DEBUG_PREFIX")
+#undef DEBUG_PREFIX
+#define DEBUG_PREFIX "OMPT"
+
+using namespace llvm::omp::target::ompt;
+
+std::mutex llvm::omp::target::ompt::DeviceAccessMutex;
+std::mutex llvm::omp::target::ompt::TraceAccessMutex;
+std::mutex llvm::omp::target::ompt::TraceControlMutex;
+std::mutex llvm::omp::target::ompt::TraceHashThreadMutex;
+std::mutex llvm::omp::target::ompt::BufferManagementFnMutex;
+
+std::unordered_map<int /*DeviceId*/, std::pair<ompt_callback_buffer_request_t,
+                                               ompt_callback_buffer_complete_t>>
+    llvm::omp::target::ompt::BufferManagementFns;
+
+thread_local uint32_t llvm::omp::target::ompt::TraceRecordNumGrantedTeams = 0;
+thread_local uint64_t llvm::omp::target::ompt::TraceRecordStartTime = 0;
+thread_local uint64_t llvm::omp::target::ompt::TraceRecordStopTime = 0;
+thread_local uint64_t llvm::omp::target::ompt::ThreadId =
+    std::numeric_limits<uint64_t>::max();
+
+std::map<int32_t, uint64_t> llvm::omp::target::ompt::TracedDevices;
+
+bool llvm::omp::target::ompt::TracingActive = false;
+
+void llvm::omp::target::ompt::resetTimestamp(uint64_t *T) { *T = 0; }
+
+ompt_callback_buffer_request_t
+llvm::omp::target::ompt::getBufferRequestFn(int DeviceId) {
+  std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
+  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
+  if (BufferMgrItr == BufferManagementFns.end()) {
+    return nullptr;
+  }
+  return BufferMgrItr->second.first;
+}
+
+ompt_callback_buffer_complete_t
+llvm::omp::target::ompt::getBufferCompleteFn(int DeviceId) {
+  std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
+  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
+  if (BufferMgrItr == BufferManagementFns.end()) {
+    return nullptr;
+  }
+  return BufferMgrItr->second.second;
+}
+
+void llvm::omp::target::ompt::setBufferManagementFns(
+    int DeviceId, ompt_callback_buffer_request_t ReqFn,
+    ompt_callback_buffer_complete_t CmpltFn) {
+  std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
+  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
+  if (BufferMgrItr != BufferManagementFns.end()) {
+    REPORT() << "Buffer request and complete functions already exist for device  "
+             << DeviceId  << "ignoring ...";
+    return;
+  }
+  BufferManagementFns[DeviceId] = std::make_pair(ReqFn, CmpltFn);
+}
+
+void llvm::omp::target::ompt::removeBufferManagementFns(int DeviceId) {
+  std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
+  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
+  if (BufferMgrItr == BufferManagementFns.end()) {
+    REPORT() << "Buffer request and complete functions don't exist for device  "
+             << DeviceId  << "ignoring ...";
+    return;
+  }
+  BufferManagementFns.erase(BufferMgrItr);
+}
+
+bool llvm::omp::target::ompt::isAllDeviceTracingStopped() {
+  std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
+  return BufferManagementFns.empty();
+}
+
+void llvm::omp::target::ompt::ompt_callback_buffer_request(
+    int DeviceId, ompt_buffer_t **BufferPtr, size_t *Bytes) {
+  if (auto Fn = getBufferRequestFn(DeviceId))
+    Fn(DeviceId, BufferPtr, Bytes);
+}
+
+void llvm::omp::target::ompt::ompt_callback_buffer_complete(
+    int DeviceId, ompt_buffer_t *Buffer, size_t Bytes,
+    ompt_buffer_cursor_t BeginCursor, int BufferOwned) {
+  if (auto Fn = getBufferCompleteFn(DeviceId))
+    Fn(DeviceId, Buffer, Bytes, BeginCursor, BufferOwned);
+}
+
+inline void setDeviceTracing(uint64_t &TracingTypes) {
+  // Set bit 0 to indicate generally enabled device tracing.
+  TracingTypes |= 1UL;
+}
+
+inline void resetDeviceTracing(uint64_t &TracingTypes) {
+  // Reset bit 0 to indicate generally disabled device tracing.
+  TracingTypes &= ~(1UL);
+}
+
+inline bool checkDeviceTracingState(const uint64_t &TracingTypes) {
+  // Return state of bit 0 to indicate if device is actively traced.
+  return TracingTypes & 1UL;
+}
+
+void llvm::omp::target::ompt::enableDeviceTracing(int DeviceId) {
+  std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
+  auto Device = TracedDevices.find(DeviceId);
+  if (Device == TracedDevices.end()) {
+    uint64_t TracingTypes{0};
+    setDeviceTracing(TracingTypes);
+    TracedDevices.emplace(DeviceId, TracingTypes);
+  } else
+    setDeviceTracing(Device->second);
+  // In any case: at least one device is traced
+  TracingActive = true;
+}
+
+void llvm::omp::target::ompt::disableDeviceTracing(int DeviceId) {
+  std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
+  auto Device = TracedDevices.find(DeviceId);
+  if (Device == TracedDevices.end()) {
+    uint64_t TracingTypes{0};
+    resetDeviceTracing(TracingTypes);
+    TracedDevices.emplace(DeviceId, TracingTypes);
+  } else
+    resetDeviceTracing(Device->second);
+
+  // Check for actively traced devices
+  for (auto &Dev : TracedDevices)
+    if (checkDeviceTracingState(Dev.second))
+      return;
+
+  // If no device is currently traced: set global tracing flag to false
+  TracingActive = false;
+}
+
+bool llvm::omp::target::ompt::isTracingEnabled(int DeviceId,
+                                               unsigned int EventTy) {
+  return TracingActive && isTracedDevice(DeviceId) &&
+         isTracingTypeGroupEnabled(DeviceId, EventTy);
+}
+
+bool llvm::omp::target::ompt::isTracedDevice(int DeviceId) {
+  std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
+  auto Device = TracedDevices.find(DeviceId);
+  if (Device != TracedDevices.end())
+    return checkDeviceTracingState(Device->second);
+
+  return false;
+}
+
+bool llvm::omp::target::ompt::isTracingTypeEnabled(int DeviceId,
+                                                   unsigned int EventTy) {
+  std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
+  // Make sure we do not shift more than std::numeric_limits<uint64_t>::digits
+  assert(EventTy < 64 && "Shift limit exceeded: EventTy must be less than 64");
+  auto Device = TracedDevices.find(DeviceId);
+  if (Device != TracedDevices.end() && EventTy < 64)
+    return (Device->second & (1UL << EventTy));
+  return false;
+}
+
+bool llvm::omp::target::ompt::isTracingTypeGroupEnabled(int DeviceId,
+                                                        unsigned int EventTy) {
+  std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
+  // Make sure we do not shift more than std::numeric_limits<uint64_t>::digits
+  assert(EventTy < 64 && "Shift limit exceeded: EventTy must be less than 64");
+  auto Device = TracedDevices.find(DeviceId);
+  if (Device != TracedDevices.end() && EventTy < 64) {
+    auto TracedEvents = Device->second;
+    switch (EventTy) {
+    case ompt_callbacks_t::ompt_callback_target:
+    case ompt_callbacks_t::ompt_callback_target_emi:
+      return ((TracedEvents & (1UL << ompt_callback_target))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_emi)));
+    case ompt_callbacks_t::ompt_callback_target_data_op:
+    case ompt_callbacks_t::ompt_callback_target_data_op_emi:
+      return ((TracedEvents & (1UL << ompt_callback_target_data_op))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_data_op_emi)));
+    case ompt_callbacks_t::ompt_callback_target_submit:
+    case ompt_callbacks_t::ompt_callback_target_submit_emi:
+      return ((TracedEvents & (1UL << ompt_callback_target_submit))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_submit_emi)));
+    // Special case: EventTy == 0 -> Check all EventTy
+    case 0:
+      return ((TracedEvents & (1UL << ompt_callback_target))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_emi))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_data_op))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_data_op_emi))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_submit))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_submit_emi)));
+    }
+  }
+  return false;
+}
+
+void llvm::omp::target::ompt::setTracingTypeEnabled(uint64_t &TracedEventTy,
+                                                    bool Enable,
+                                                    unsigned int EventTy) {
+  // Make sure we do not shift more than std::numeric_limits<uint64_t>::digits
+  assert(EventTy < 64 && "Shift limit exceeded: EventTy must be less than 64");
+  if (EventTy < 64) {
+    if (Enable)
+      TracedEventTy |= (1UL << EventTy);
+    else
+      TracedEventTy &= ~(1UL << EventTy);
+  }
+}
+
+ompt_set_result_t
+llvm::omp::target::ompt::setTraceEventTy(int DeviceId, unsigned int Enable,
+                                         unsigned int EventTy) {
+  if (DeviceId < 0) {
+    REPORT() << "Failed to set trace event type for DeviceId=" << DeviceId;
+    return ompt_set_never;
+  }
+
+  ODBG(debug::ODT_Tool) << "Executing setTraceEventTy: DeviceId=" << DeviceId
+                 << " Enable=" << Enable << " EventTy=" << EventTy;
+
+  std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
+  if (TracedDevices.find(DeviceId) == TracedDevices.end())
+    TracedDevices.emplace(DeviceId, 0UL);
+
+  auto &TracedEventTy = TracedDevices[DeviceId];
+  bool Enabled = Enable > 0;
+  if (EventTy == 0) {
+    // Set / reset all supported types
+    setTracingTypeEnabled(TracedEventTy, Enabled,
+                          ompt_callbacks_t::ompt_callback_target);
+    setTracingTypeEnabled(TracedEventTy, Enabled,
+                          ompt_callbacks_t::ompt_callback_target_data_op);
+    setTracingTypeEnabled(TracedEventTy, Enabled,
+                          ompt_callbacks_t::ompt_callback_target_submit);
+    setTracingTypeEnabled(TracedEventTy, Enabled,
+                          ompt_callbacks_t::ompt_callback_target_emi);
+    setTracingTypeEnabled(TracedEventTy, Enabled,
+                          ompt_callbacks_t::ompt_callback_target_data_op_emi);
+    setTracingTypeEnabled(TracedEventTy, Enabled,
+                          ompt_callbacks_t::ompt_callback_target_submit_emi);
+
+    if (Enabled) {
+      // Event subset is enabled
+      return ompt_set_sometimes;
+    } else {
+      // All events are disabled
+      return ompt_set_always;
+    }
+  }
+
+  switch (EventTy) {
+  case ompt_callbacks_t::ompt_callback_target:
+  case ompt_callbacks_t::ompt_callback_target_data_op:
+  case ompt_callbacks_t::ompt_callback_target_submit:
+  case ompt_callbacks_t::ompt_callback_target_emi:
+  case ompt_callbacks_t::ompt_callback_target_data_op_emi:
+  case ompt_callbacks_t::ompt_callback_target_submit_emi: {
+    setTracingTypeEnabled(TracedEventTy, Enabled, EventTy);
+    return ompt_set_always;
+  }
+  default: {
+    if (Enabled) {
+      // Unimplemented
+      return ompt_set_never;
+    } else {
+      // Always disabled anyways
+      return ompt_set_always;
+    }
+  }
+  }
+}
+
+uint64_t llvm::omp::target::ompt::getThreadId() {
+  // Grab the value from thread local storage, if valid.
+  if (ThreadId != std::numeric_limits<uint64_t>::max())
+    return ThreadId;
+  // Otherwise set it, protecting the hash with a lock.
+  std::unique_lock<std::mutex> Lock(TraceHashThreadMutex);
+  ThreadId = std::hash<std::thread::id>()(std::this_thread::get_id());
+  return ThreadId;
+}
+
+void Interface::setTraceRecordCommon(ompt_record_ompt_t *DataPtr,
+                                     ompt_callbacks_t CallbackType) {
+  DataPtr->type = CallbackType;
+
+  if (CallbackType == ompt_callback_target)
+    DataPtr->time = 0; // Currently, no consumer, so no need to set it
+  else {
+    DataPtr->time = TraceRecordStartTime;
+    resetTimestamp(&TraceRecordStartTime);
+  }
+
+  DataPtr->thread_id = getThreadId();
+  DataPtr->target_id = TargetData.value;
+}
+
+void Interface::setTraceRecordTargetDataOp(ompt_record_target_data_op_t *Record,
+                                           ompt_target_data_op_t DataOpType,
+                                           void *SrcAddr, int64_t SrcDeviceNum,
+                                           void *DstAddr, int64_t DstDeviceNum,
+                                           size_t Bytes, void *CodePtr) {
+  Record->host_op_id = HostOpId;
+  Record->optype = DataOpType;
+  Record->src_addr = SrcAddr;
+  Record->src_device_num = SrcDeviceNum;
+  Record->dest_addr = DstAddr;
+  Record->dest_device_num = DstDeviceNum;
+  Record->bytes = Bytes;
+
+  Record->end_time = TraceRecordStopTime;
+  resetTimestamp(&TraceRecordStopTime);
+
+  Record->codeptr_ra = CodePtr;
+}
+
+void Interface::setTraceRecordTargetKernel(ompt_record_target_kernel_t *Record,
+                                           unsigned int NumTeams) {
+  Record->host_op_id = HostOpId;
+  Record->requested_num_teams = NumTeams;
+  Record->granted_num_teams = TraceRecordNumGrantedTeams;
+
+  Record->end_time = TraceRecordStopTime;
+  resetTimestamp(&TraceRecordStopTime);
+}
+
+void Interface::setTraceRecordTarget(ompt_record_target_t *Record,
+                                     int64_t DeviceId, ompt_target_t TargetKind,
+                                     ompt_scope_endpoint_t Endpoint,
+                                     void *CodePtr) {
+  Record->kind = TargetKind;
+  Record->endpoint = Endpoint;
+  Record->device_num = DeviceId;
+  assert(TaskData);
+  Record->task_id = TaskData->value;
+  Record->target_id = TargetData.value;
+  Record->codeptr_ra = CodePtr;
+}
+
+void Interface::startTargetDataAllocTrace(int64_t DeviceId, void *HstPtrBegin,
+                                          void **TgtPtrBegin, size_t Size,
+                                          void *Code) {}
+
+ompt_record_ompt_t *Interface::stopTargetDataAllocTrace(int64_t DeviceId,
+                                                        void *HstPtrBegin,
+                                                        void **TgtPtrBegin,
+                                                        size_t Size,
+                                                        void *Code) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target_data_op))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr = (ompt_record_ompt_t *)TRM->assignCursor(
+      ompt_callback_target_data_op, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target_data_op);
+  setTraceRecordTargetDataOp(&DataPtr->record.target_data_op,
+                             ompt_target_data_alloc, HstPtrBegin,
+                             /*SrcDeviceNum=*/omp_get_initial_device(),
+                             *TgtPtrBegin, DeviceId, Size, Code);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Generated trace record: " << DataPtr
+                 << " (ompt_target_data_alloc)";
+  return DataPtr;
+}
+
+void Interface::startTargetDataDeleteTrace(int64_t DeviceId, void *TgtPtrBegin,
+                                           void *Code) {}
+
+ompt_record_ompt_t *Interface::stopTargetDataDeleteTrace(int64_t DeviceId,
+                                                         void *TgtPtrBegin,
+                                                         void *Code) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target_data_op))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr = (ompt_record_ompt_t *)TRM->assignCursor(
+      ompt_callback_target_data_op, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target_data_op);
+  setTraceRecordTargetDataOp(&DataPtr->record.target_data_op,
+                             ompt_target_data_delete, TgtPtrBegin, DeviceId,
+                             /*DstAddr=*/nullptr,
+                             /*DstDeviceNum=*/-1, /*Bytes=*/0, Code);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Generated trace record: " << DataPtr
+                 << " (ompt_target_data_delete)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *
+Interface::startTargetDataSubmitTrace(int64_t SrcDeviceId, void *SrcPtrBegin,
+                                      int64_t DstDeviceId, void *DstPtrBegin,
+                                      size_t Size, void *Code) {
+  if (!isTracingEnabled(DstDeviceId, ompt_callback_target_data_op))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr = (ompt_record_ompt_t *)TRM->assignCursor(
+      ompt_callback_target_data_op, DstDeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target_data_op);
+  DataPtr->time = 0; // Set to sanity value and let "stop" function fix it
+
+  // Set some of the data-op specific fields here
+  setTraceRecordTargetDataOp(&DataPtr->record.target_data_op,
+                             ompt_target_data_transfer_to_device, SrcPtrBegin,
+                             SrcDeviceId, DstPtrBegin, DstDeviceId, Size, Code);
+
+  ODBG(debug::ODT_Tool) << "OMPT-Async: Returning data trace record buf ptr " << DataPtr;
+  return DataPtr;
+}
+
+ompt_record_ompt_t *
+Interface::startTargetDataRetrieveTrace(int64_t SrcDeviceId, void *SrcPtrBegin,
+                                        int64_t DstDeviceId, void *DstPtrBegin,
+                                        size_t Size, void *Code) {
+  if (!isTracingEnabled(SrcDeviceId, ompt_callback_target_data_op))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr = (ompt_record_ompt_t *)TRM->assignCursor(
+      ompt_callback_target_data_op, SrcDeviceId);
+
+  if (!DataPtr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target_data_op);
+  DataPtr->time = 0; // Set to sanity value and let "stop" function fix it
+
+  // Set some of the data-op specific fields here
+  setTraceRecordTargetDataOp(&DataPtr->record.target_data_op,
+                             ompt_target_data_transfer_from_device, SrcPtrBegin,
+                             SrcDeviceId, DstPtrBegin, DstDeviceId, Size, Code);
+
+  ODBG(debug::ODT_Tool) << "OMPT-Async: Returning data trace record buf ptr " << DataPtr;
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::stopTargetDataMovementTraceAsync(
+    ompt_record_ompt_t *DataPtr, uint64_t NanosStart, uint64_t NanosEnd) {
+  // Finalize the data that comes from the plugin.
+  DataPtr->time = NanosStart;
+  auto Record = static_cast<ompt_record_target_data_op_t *>(
+      &DataPtr->record.target_data_op);
+  Record->end_time = NanosEnd;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "OMPT-Async: Completed target_data trace record " << DataPtr;
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::startTargetSubmitTrace(int64_t DeviceId,
+                                                      unsigned int NumTeams) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target_submit))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr = (ompt_record_ompt_t *)TRM->assignCursor(
+      ompt_callback_target_submit, DeviceId);
+
+  // Set all known entries and leave remaining to the stop function
+  setTraceRecordCommon(DataPtr, ompt_callback_target_submit);
+  DataPtr->time = 0; // Set to sanity value and let "stop" function fix it
+  // Kernel specific things
+  DataPtr->record.target_kernel.requested_num_teams = NumTeams;
+  DataPtr->record.target_kernel.host_op_id = getHostOpId();
+
+  // May be null if event is not traced
+  ODBG(debug::ODT_Tool) << "OMPT-Async: Returning kernel trace record buf ptr " << DataPtr;
+  return DataPtr;
+}
+
+ompt_record_ompt_t *
+Interface::stopTargetSubmitTraceAsync(ompt_record_ompt_t *DataPtr,
+                                      unsigned int NumTeams,
+                                      uint64_t NanosStart, uint64_t NanosStop) {
+  // Common fields
+  DataPtr->time = NanosStart;
+  // Submit specific
+  DataPtr->record.target_kernel.end_time = NanosStop;
+  DataPtr->record.target_kernel.granted_num_teams = NumTeams;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  // Ready Record
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "OMPT-Async: Completed trace record buf ptr " << DataPtr;
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::startTargetDataEnterTrace(int64_t DeviceId,
+                                                         void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId,
+                       ompt_target_enter_data, ompt_scope_begin, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Returning trace record buf ptr " << DataPtr
+                 << " (ompt_target_enter_data)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::stopTargetDataEnterTrace(int64_t DeviceId,
+                                                        void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId,
+                       ompt_target_enter_data, ompt_scope_end, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Generated trace record " << DataPtr
+                 << " (ompt_target_enter_data)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::startTargetDataExitTrace(int64_t DeviceId,
+                                                        void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId, ompt_target_exit_data,
+                       ompt_scope_begin, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Returning trace record buf ptr " << DataPtr
+                 << " (ompt_target_exit_data)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::stopTargetDataExitTrace(int64_t DeviceId,
+                                                       void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId, ompt_target_exit_data,
+                       ompt_scope_end, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Generated trace record " << DataPtr
+                 << " (ompt_target_exit_data)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::startTargetUpdateTrace(int64_t DeviceId,
+                                                      void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId, ompt_target_update,
+                       ompt_scope_begin, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Returning trace record buf ptr " << DataPtr
+                 << " (ompt_target_update)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::stopTargetUpdateTrace(int64_t DeviceId,
+                                                     void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId, ompt_target_update,
+                       ompt_scope_end, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Generated trace record " << DataPtr
+                 << " (ompt_target_update)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::startTargetTrace(int64_t DeviceId,
+                                                void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId, ompt_target,
+                       ompt_scope_begin, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Returning trace record buf ptr " << DataPtr
+                 << " (ompt_target)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::stopTargetTrace(int64_t DeviceId,
+                                               void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId, ompt_target,
+                       ompt_scope_end, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+
+  ODBG(debug::ODT_Tool) << "Generated trace record " << DataPtr
+                 << " (ompt_target)";
+  return DataPtr;
+}
+
+extern "C" {
+// Device-independent entry point for ompt_set_trace_ompt
+ompt_set_result_t libomptarget_ompt_set_trace_ompt(int DeviceId,
+                                                   unsigned int Enable,
+                                                   unsigned int EventTy) {
+  std::unique_lock<std::mutex> Lock(TraceAccessMutex);
+  return llvm::omp::target::ompt::setTraceEventTy(DeviceId, Enable, EventTy);
+}
+
+// Device-independent entry point for ompt_start_trace
+int libomptarget_ompt_start_trace(int DeviceId,
+                                  ompt_callback_buffer_request_t Request,
+                                  ompt_callback_buffer_complete_t Complete) {
+  if (!PM) {
+    REPORT() << "Failed to start trace for DeviceId="
+             << DeviceId << " (invalid plugin manager)";
+    // Indicate failure
+    return 0;
+  }
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  std::unique_lock<std::mutex> Lock(TraceControlMutex);
+  if (Request && Complete) {
+    // Set buffer related functions
+    llvm::omp::target::ompt::setBufferManagementFns(DeviceId, Request,
+                                                    Complete);
+    llvm::omp::target::ompt::enableDeviceTracing(DeviceId);
+    TRM->startHelperThreads();
+    // Success
+    return 1;
+  }
+  // Failure
+  return 0;
+}
+
+// Device-independent entry point for ompt_flush_trace
+int libomptarget_ompt_flush_trace(int DeviceId) {
+  if (!PM) {
+    REPORT() << "Failed to flush trace for DeviceId="
+             << DeviceId << " (invalid plugin manager)";
+    // Indicate failure
+    return 0;
+  }
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  std::unique_lock<std::mutex> Lock(TraceControlMutex);
+  return TRM->flushAllBuffers(DeviceId);
+}
+
+// Device independent entry point for ompt_stop_trace
+int libomptarget_ompt_stop_trace(int DeviceId) {
+  if (!PM) {
+    REPORT() << "Failed to stop trace for DeviceId="
+             << DeviceId << " (invalid plugin manager)";
+    // Indicate failure
+    return 0;
+  }
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  std::unique_lock<std::mutex> Lock(TraceControlMutex);
+
+  // Schedule flushing of trace records for this device
+  int Status = TRM->flushAllBuffers(DeviceId);
+
+  // De-register this device so that no more traces are collected
+  // or delivered for this device until an ompt_start_trace is
+  // invoked for this device.
+  removeBufferManagementFns(DeviceId);
+
+  // If no device is being traced, shut down the helper threads. A
+  // subsequent ompt_start_trace will start up the helper threads.
+  if (isAllDeviceTracingStopped()) {
+    // TODO shutdown should perhaps return a status
+    TRM->shutdownHelperThreads();
+    llvm::omp::target::ompt::disableDeviceTracing(DeviceId);
+  }
+  return Status;
+}
+
+// Device independent entry point for ompt_advance_buffer_cursor
+// Note: The input parameter size is unused here. It refers to the
+// bytes returned in the corresponding callback.
+int libomptarget_ompt_advance_buffer_cursor(ompt_device_t *Device,
+                                            ompt_buffer_t *Buffer, size_t Size,
+                                            ompt_buffer_cursor_t CurrentPos,
+                                            ompt_buffer_cursor_t *NextPos) {
+  if (!PM) {
+    REPORT() << "Failed to advance buffer cursor for Device="
+             << Device << " (invalid plugin manager)";
+    // Indicate failure
+    return false;
+  }
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  char *TraceRecord = (char *)CurrentPos;
+  // Don't assert if CurrentPos is null, just indicate end of buffer
+  if (TraceRecord == nullptr || TRM->isLastCursor(TraceRecord)) {
+    *NextPos = 0;
+    return false;
+  }
+  // TODO In debug mode, assert that the metadata points to the
+  // input parameter buffer
+
+  size_t TRSize = TRM->getTRSize();
+  *NextPos = (ompt_buffer_cursor_t)(TraceRecord + TRSize);
+  ODBG(llvm::omp::target::debug::ODT_Tool) << "Advanced buffer pointer by "
+                 << TRSize << " bytes to "
+                 << TraceRecord + TRSize;
+  return true;
+}
+
+// This function is invoked before the kernel launch. So, when the trace record
+// is populated after kernel completion, TraceRecordNumGrantedTeams is already
+// updated.
+void libomptarget_ompt_set_granted_teams(uint32_t NumTeams) {
+  TraceRecordNumGrantedTeams = NumTeams;
+}
+
+// Assume a synchronous implementation and set thread local variables to track
+// timestamps. The thread local variables can then be used to populate trace
+// records.
+void libomptarget_ompt_set_timestamp(uint64_t Start, uint64_t Stop) {
+  TraceRecordStartTime = Start;
+  TraceRecordStopTime = Stop;
+}
+
+// Device-independent entry point to query for the trace format used.
+// Currently, only OMPT format is supported.
+ompt_record_t
+libomptarget_ompt_get_record_type(ompt_buffer_t *Buffer,
+                                  ompt_buffer_cursor_t CurrentPos) {
+  // TODO: When different OMPT trace buffer formats supported, this needs to be
+  // fixed.
+  return ompt_record_t::ompt_record_ompt;
+}
+} // extern "C"
+
+#pragma pop_macro("DEBUG_PREFIX")
+
+#endif // OMPT_SUPPORT
diff --git a/offload/libomptarget/PluginManager.cpp b/offload/libomptarget/PluginManager.cpp
index 41b653a60adfd..33c35b4f8db17 100644
--- a/offload/libomptarget/PluginManager.cpp
+++ b/offload/libomptarget/PluginManager.cpp
@@ -11,6 +11,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "PluginManager.h"
+#include "OpenMP/OMPT/Callback.h"
+#include "OpenMP/OMPT/OmptCommonDefs.h"
+#include "OpenMP/OMPT/OmptTracing.h"
 #include "OffloadPolicy.h"
 #include "Shared/Debug.h"
 #include "Shared/Profile.h"
@@ -47,6 +50,12 @@ void PluginManager::init() {
   } while (false);
 #include "Shared/Targets.def"
 
+#ifdef OMPT_SUPPORT
+  assert(TraceRecordManager == nullptr &&
+         "Expected trace record manager to be null");
+  TraceRecordManager = new OmptTracingBufferMgr();
+#endif
+
   ODBG(ODT_Init) << "RTLs loaded!";
 }
 
@@ -54,6 +63,16 @@ void PluginManager::deinit() {
   TIMESCOPE();
   ODBG(ODT_Deinit) << "Unloading RTLs...";
 
+#ifdef OMPT_SUPPORT
+  // When offloading is disabled, the TRM is not initialized.
+  if (!OffloadPolicy::isOffloadDisabled()) {
+    assert(TraceRecordManager != nullptr &&
+           "Trace record manager should have been non-null");
+    delete TraceRecordManager;
+    TraceRecordManager = nullptr;
+  }
+#endif
+
   for (auto &Plugin : Plugins) {
     if (!Plugin->is_initialized())
       continue;
@@ -323,6 +342,9 @@ int target(ident_t *Loc, DeviceTy &Device, void *HostPtr,
 void PluginManager::unregisterLib(__tgt_bin_desc *Desc) {
   ODBG(ODT_Deinit) << "Unloading target library!";
 
+  OMPT_IF_TRACING_ENABLED(
+      PM->getTraceRecordManager()->flushAndShutdownHelperThreads(););
+
   Desc = upgradeLegacyEntries(Desc);
 
   PM->RTLsMtx.lock();
diff --git a/offload/libomptarget/device.cpp b/offload/libomptarget/device.cpp
index 546f679353544..db5968b90ee78 100644
--- a/offload/libomptarget/device.cpp
+++ b/offload/libomptarget/device.cpp
@@ -15,6 +15,8 @@
 #include "OpenMP/Mapping.h"
 #include "OpenMP/OMPT/Callback.h"
 #include "OpenMP/OMPT/Interface.h"
+#include "OpenMP/OMPT/OmptCommonDefs.h"
+#include "OpenMP/OMPT/OmptTracing.h"
 #include "PluginManager.h"
 #include "Shared/APITypes.h"
 #include "Shared/Debug.h"
@@ -34,7 +36,8 @@
 #include <thread>
 
 #ifdef OMPT_SUPPORT
-using namespace llvm::omp::target::ompt;
+using namespace llvm::omp::target;
+using namespace ompt;
 #endif
 
 using namespace llvm::omp::target::plugin;
@@ -242,10 +245,15 @@ DeviceTy::loadBinary(__tgt_device_image *Img) {
 void *DeviceTy::allocData(int64_t Size, void *HstPtr, int32_t Kind) {
   /// RAII to establish tool anchors before and after data allocation
   void *TargetPtr = nullptr;
-  OMPT_IF_BUILT(InterfaceRAII TargetDataAllocRAII(
-                    RegionInterface.getCallbacks<ompt_target_data_alloc>(),
-                    DeviceID, HstPtr, &TargetPtr, Size,
-                    /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
+  OMPT_IF_BUILT(
+      InterfaceRAII TargetDataAllocRAII(
+          RegionInterface.getCallbacks<ompt_target_data_alloc>(), DeviceID,
+          HstPtr, &TargetPtr, Size,
+          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+      InterfaceRAII TargetDataAllocTraceRAII(
+          RegionInterface.getTraceGenerators<ompt_target_data_alloc>(),
+          RTLDeviceID, HstPtr, &TargetPtr, Size,
+          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
 
   TargetPtr = RTL->data_alloc(RTLDeviceID, Size, HstPtr, Kind);
   return TargetPtr;
@@ -253,11 +261,15 @@ void *DeviceTy::allocData(int64_t Size, void *HstPtr, int32_t Kind) {
 
 int32_t DeviceTy::deleteData(void *TgtAllocBegin, int32_t Kind) {
   /// RAII to establish tool anchors before and after data deletion
-  OMPT_IF_BUILT(InterfaceRAII TargetDataDeleteRAII(
-                    RegionInterface.getCallbacks<ompt_target_data_delete>(),
-                    DeviceID, TgtAllocBegin,
-                    /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
-
+  OMPT_IF_BUILT(
+      InterfaceRAII TargetDataDeleteRAII(
+          RegionInterface.getCallbacks<ompt_target_data_delete>(), DeviceID,
+          TgtAllocBegin,
+          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+      InterfaceRAII TargetDataDeleteTraceRAII(
+          RegionInterface.getTraceGenerators<ompt_target_data_delete>(),
+          DeviceID, TgtAllocBegin,
+          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
   return RTL->data_delete(RTLDeviceID, TgtAllocBegin, Kind);
 }
 
@@ -274,6 +286,13 @@ int32_t DeviceTy::submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size,
       InterfaceRAII TargetDataSubmitRAII(
           RegionInterface.getCallbacks<ompt_target_data_transfer_to_device>(),
           omp_initial_device, HstPtrBegin, DeviceID, TgtPtrBegin, Size,
+          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+      TracerInterfaceRAII TargetDataSubmitTraceRAII(
+          RegionInterface
+              .getTraceGenerators<ompt_target_data_transfer_to_device>(),
+          AsyncInfo, RTL->getProfiler(), /*TracedDeviceId=*/DeviceID,
+          /*EventType=*/ompt_callback_target_data_op, omp_initial_device,
+          HstPtrBegin, DeviceID, TgtPtrBegin, Size,
           /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
 
   return RTL->data_submit_async(RTLDeviceID, TgtPtrBegin, HstPtrBegin, Size,
@@ -294,6 +313,13 @@ int32_t DeviceTy::retrieveData(void *HstPtrBegin, void *TgtPtrBegin,
       InterfaceRAII TargetDataRetrieveRAII(
           RegionInterface.getCallbacks<ompt_target_data_transfer_from_device>(),
           DeviceID, TgtPtrBegin, omp_initial_device, HstPtrBegin, Size,
+          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+      TracerInterfaceRAII TargetDataRetrieveTraceRAII(
+          RegionInterface
+              .getTraceGenerators<ompt_target_data_transfer_from_device>(),
+          AsyncInfo, RTL->getProfiler(), /*TracedDeviceId=*/DeviceID,
+          /*EventType=*/ompt_callback_target_data_op, DeviceID, TgtPtrBegin,
+          omp_initial_device, HstPtrBegin, Size,
           /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
 
   return RTL->data_retrieve_async(RTLDeviceID, HstPtrBegin, TgtPtrBegin, Size,
@@ -313,11 +339,15 @@ int32_t DeviceTy::dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr,
       InterfaceRAII TargetDataExchangeRAII(
           RegionInterface.getCallbacks<ompt_target_data_transfer_from_device>(),
           RTLDeviceID, SrcPtr, DstDev.RTLDeviceID, DstPtr, Size,
+          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+      TracerInterfaceRAII TargetDataExchangeTraceRAII(
+          RegionInterface
+              .getTraceGenerators<ompt_target_data_transfer_from_device>(),
+          AsyncInfo, RTL->getProfiler(), /*TracedDeviceId=*/RTLDeviceID,
+          /*EventType=*/ompt_callback_target_data_op, RTLDeviceID, SrcPtr,
+          DstDev.RTLDeviceID, DstPtr, Size,
           /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
-  if (!AsyncInfo) {
-    return RTL->data_exchange(RTLDeviceID, SrcPtr, DstDev.RTLDeviceID, DstPtr,
-                              Size);
-  }
+
   return RTL->data_exchange_async(RTLDeviceID, SrcPtr, DstDev.RTLDeviceID,
                                   DstPtr, Size, AsyncInfo);
 }
diff --git a/offload/libomptarget/exports b/offload/libomptarget/exports
index 1831c43cc5f29..6e404ed059096 100644
--- a/offload/libomptarget/exports
+++ b/offload/libomptarget/exports
@@ -67,6 +67,14 @@ VERS1.0 {
     llvm_omp_target_unlock_mem;
     __tgt_set_info_flag;
     __tgt_print_device_info;
+    libomptarget_ompt_set_trace_ompt;
+    libomptarget_ompt_start_trace;
+    libomptarget_ompt_flush_trace;
+    libomptarget_ompt_stop_trace;
+    libomptarget_ompt_set_granted_teams;
+    libomptarget_ompt_set_timestamp;
+    libomptarget_ompt_advance_buffer_cursor;
+    libomptarget_ompt_get_record_type;
     omp_get_interop_ptr;
     omp_get_interop_str;
     omp_get_interop_int;
diff --git a/offload/libomptarget/interface.cpp b/offload/libomptarget/interface.cpp
index a436708814c90..1bbc3f63ce292 100644
--- a/offload/libomptarget/interface.cpp
+++ b/offload/libomptarget/interface.cpp
@@ -14,6 +14,7 @@
 #include "OpenMP/OMPT/Interface.h"
 #include "OffloadPolicy.h"
 #include "OpenMP/OMPT/Callback.h"
+#include "OpenMP/OMPT/OmptCommonDefs.h"
 #include "OpenMP/omp.h"
 #include "PluginManager.h"
 #include "omptarget.h"
@@ -153,19 +154,30 @@ targetData(ident_t *Loc, int64_t DeviceId, int32_t ArgNum, void **ArgsBase,
   AsyncInfoTy &AsyncInfo = TargetAsyncInfo;
 
   /// RAII to establish tool anchors before and after data begin / end / update
-  OMPT_IF_BUILT(assert((TargetDataFunction == targetDataBegin ||
-                        TargetDataFunction == targetDataEnd ||
-                        TargetDataFunction == targetDataUpdate) &&
-                       "Encountered unexpected TargetDataFunction during "
-                       "execution of targetData");
-                auto CallbackFunctions =
-                    (TargetDataFunction == targetDataBegin)
-                        ? RegionInterface.getCallbacks<ompt_target_enter_data>()
-                    : (TargetDataFunction == targetDataEnd)
-                        ? RegionInterface.getCallbacks<ompt_target_exit_data>()
-                        : RegionInterface.getCallbacks<ompt_target_update>();
-                InterfaceRAII TargetDataRAII(CallbackFunctions, DeviceId,
-                                             OMPT_GET_RETURN_ADDRESS);)
+  OMPT_IF_BUILT(
+      assert((TargetDataFunction == targetDataBegin ||
+              TargetDataFunction == targetDataEnd ||
+              TargetDataFunction == targetDataUpdate) &&
+             "Encountered unexpected TargetDataFunction during "
+             "execution of targetData");
+      auto CallbackFunctions =
+          (TargetDataFunction == targetDataBegin)
+              ? RegionInterface.getCallbacks<ompt_target_enter_data>()
+          : (TargetDataFunction == targetDataEnd)
+              ? RegionInterface.getCallbacks<ompt_target_exit_data>()
+              : RegionInterface.getCallbacks<ompt_target_update>();
+
+      auto TraceGenerators =
+          (TargetDataFunction == targetDataBegin)
+              ? RegionInterface.getTraceGenerators<ompt_target_enter_data>()
+          : (TargetDataFunction == targetDataEnd)
+              ? RegionInterface.getTraceGenerators<ompt_target_exit_data>()
+              : RegionInterface.getTraceGenerators<ompt_target_update>();
+
+      InterfaceRAII TargetDataRAII(CallbackFunctions, DeviceId,
+                                   /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+      InterfaceRAII TargetDataTraceRAII(TraceGenerators, DeviceId,
+                                        /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
 
   int Rc = OFFLOAD_SUCCESS;
 
@@ -433,6 +445,9 @@ static inline int targetKernel(ident_t *Loc, int64_t DeviceId, int32_t NumTeams,
   /// RAII to establish tool anchors before and after target region
   OMPT_IF_BUILT(InterfaceRAII TargetRAII(
                     RegionInterface.getCallbacks<ompt_target>(), DeviceId,
+                    /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+                InterfaceRAII TargetTraceRAII(
+                    RegionInterface.getTraceGenerators<ompt_target>(), DeviceId,
                     /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
 
   int Rc = OFFLOAD_SUCCESS;
@@ -542,6 +557,9 @@ EXTERN int __tgt_target_kernel_replay(
   /// RAII to establish tool anchors before and after target region
   OMPT_IF_BUILT(InterfaceRAII TargetRAII(
                     RegionInterface.getCallbacks<ompt_target>(), DeviceId,
+                    /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+                InterfaceRAII TargetTraceRAII(
+                    RegionInterface.getTraceGenerators<ompt_target>(), DeviceId,
                     /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
 
   AsyncInfoTy AsyncInfo(*DeviceOrErr);
diff --git a/offload/libomptarget/omptarget.cpp b/offload/libomptarget/omptarget.cpp
index 17b215732d51b..2d55e2bbf00f2 100644
--- a/offload/libomptarget/omptarget.cpp
+++ b/offload/libomptarget/omptarget.cpp
@@ -2347,8 +2347,12 @@ int target(ident_t *Loc, DeviceTy &Device, void *HostPtr,
     // No need to guard this with OMPT_IF_BUILT
     InterfaceRAII TargetSubmitRAII(
         RegionInterface.getCallbacks<ompt_callback_target_submit>(), NumTeams);
-#endif
 
+    TracerInterfaceRAII TargetTraceRAII(
+        RegionInterface.getTraceGenerators<ompt_callback_target_submit>(),
+        AsyncInfo, Device.RTL->getProfiler(), /*TracedDeviceId=*/DeviceId,
+        /*EventType=*/ompt_callback_target_submit, DeviceId, NumTeams);
+#endif
     Ret = Device.launchKernel(TgtEntryPtr, TgtArgs.data(), TgtOffsets.data(),
                               KernelArgs, nullptr, AsyncInfo);
   }

>From 1c81d11b2cdfdf3757975ada83c04b7f779f585f Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Mon, 13 Apr 2026 06:26:35 -0500
Subject: [PATCH 08/11] [Offload][OMPT] Fix global destructor warnings in OMPT
 tracing globals

Replace five globals with non-trivial destructors with function-local
statics, following the LLVM canonical pattern (see Signals.cpp). This
resolves -Werror=global-constructors failures that were triggered by
the destructor of std::map, std::unordered_map, std::shared_ptr, and
a thread_local array of shared_ptr.

Affected globals:
- ParentLibrary (shared_ptr) in plugin OmptTracing.cpp -> file-local
  getParentLibraryStorage() helper
- Devices (map) in plugin OmptTracing.cpp -> getDevices()
- BufferManagementFns (unordered_map) in libomptarget OmptTracing.cpp
  -> file-local getBufferMgmt() helper
- TracedDevices (map) in libomptarget OmptTracing.cpp -> getTracedDevices()
- ArrayOfBufPtr (thread_local shared_ptr array) in OmptTracingBuffer.cpp
  -> getArrayOfBufPtr()

extern declarations are replaced with getter function declarations in
the corresponding headers (OmptDeviceTracing.h, OmptTracing.h,
OmptTracingBuffer.h). All use-sites are updated accordingly.

Made-with: Cursor
---
 offload/include/OpenMP/OMPT/OmptTracing.h     | 11 +--
 .../include/OpenMP/OMPT/OmptTracingBuffer.h   |  5 +-
 .../libomptarget/OpenMP/OMPT/OmptTracing.cpp  | 72 ++++++++++---------
 .../OpenMP/OMPT/OmptTracingBuffer.cpp         | 12 ++--
 .../common/OMPT/OmptDeviceTracing.h           | 20 +++---
 .../common/OMPT/OmptTracing.cpp               | 41 ++++++-----
 6 files changed, 85 insertions(+), 76 deletions(-)

diff --git a/offload/include/OpenMP/OMPT/OmptTracing.h b/offload/include/OpenMP/OMPT/OmptTracing.h
index 2a892582923d5..4c8c049a9ae48 100644
--- a/offload/include/OpenMP/OMPT/OmptTracing.h
+++ b/offload/include/OpenMP/OMPT/OmptTracing.h
@@ -102,8 +102,9 @@ ompt_set_result_t setTraceEventTy(int DeviceId, unsigned int Enable,
 /// Return thread id
 uint64_t getThreadId();
 
-/// See TracedDevices in OmptDeviceTracing.h
-extern std::map<int32_t, uint64_t> TracedDevices;
+/// Access the map of RTL device IDs to their currently enabled tracing event
+/// types. See also OmptDeviceTracing.h.
+std::map<int32_t, uint64_t> &getTracedDevices();
 /// Activate tracing on the given device
 void enableDeviceTracing(int DeviceId);
 /// Deactivate tracing on the given device
@@ -123,12 +124,6 @@ extern std::mutex TraceHashThreadMutex;
 /// buffer-completion callback functions.
 extern std::mutex BufferManagementFnMutex;
 
-/// Map from device-id to the corresponding buffer-request and buffer-completion
-/// callback functions.
-extern std::unordered_map<int, std::pair<ompt_callback_buffer_request_t,
-                                         ompt_callback_buffer_complete_t>>
-    BufferManagementFns;
-
 /// Thread local variables used by the plugin to communicate OMPT information
 /// that are then used to populate trace records. This method assumes a
 /// synchronous implementation, otherwise it won't work.
diff --git a/offload/include/OpenMP/OMPT/OmptTracingBuffer.h b/offload/include/OpenMP/OMPT/OmptTracingBuffer.h
index 5c9f4bf33dae8..a9b7f8d569fcb 100644
--- a/offload/include/OpenMP/OMPT/OmptTracingBuffer.h
+++ b/offload/include/OpenMP/OMPT/OmptTracingBuffer.h
@@ -133,8 +133,9 @@ class OmptTracingBufferMgr {
   // Thread-specific array of pointers to a buffer. The buffer pointed to
   // is the last one allocated by this thread for a given device. The ith
   // element points to the buffer for the ith device. At most MAX_NUM_DEVICES
-  // devices are supported.
-  static thread_local BufPtr ArrayOfBufPtr[MAX_NUM_DEVICES];
+  // devices are supported. Stored as a function-local thread_local to avoid
+  // a global destructor.
+  static BufPtr *getArrayOfBufPtr();
 
   /*
    * A buffer is flushed when it fills up or when the tool invokes
diff --git a/offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp b/offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp
index a74b10137f2c3..1045801842e90 100644
--- a/offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp
+++ b/offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp
@@ -42,17 +42,25 @@ std::mutex llvm::omp::target::ompt::TraceControlMutex;
 std::mutex llvm::omp::target::ompt::TraceHashThreadMutex;
 std::mutex llvm::omp::target::ompt::BufferManagementFnMutex;
 
-std::unordered_map<int /*DeviceId*/, std::pair<ompt_callback_buffer_request_t,
-                                               ompt_callback_buffer_complete_t>>
-    llvm::omp::target::ompt::BufferManagementFns;
-
 thread_local uint32_t llvm::omp::target::ompt::TraceRecordNumGrantedTeams = 0;
 thread_local uint64_t llvm::omp::target::ompt::TraceRecordStartTime = 0;
 thread_local uint64_t llvm::omp::target::ompt::TraceRecordStopTime = 0;
 thread_local uint64_t llvm::omp::target::ompt::ThreadId =
     std::numeric_limits<uint64_t>::max();
 
-std::map<int32_t, uint64_t> llvm::omp::target::ompt::TracedDevices;
+// File-local helper returning a reference to the buffer-management map stored
+// as a function-local static, avoiding a global destructor.
+static auto &getBufferMgmt() {
+  static std::unordered_map<int, std::pair<ompt_callback_buffer_request_t,
+                                           ompt_callback_buffer_complete_t>>
+      M;
+  return M;
+}
+
+std::map<int32_t, uint64_t> &llvm::omp::target::ompt::getTracedDevices() {
+  static std::map<int32_t, uint64_t> TracedDevices;
+  return TracedDevices;
+}
 
 bool llvm::omp::target::ompt::TracingActive = false;
 
@@ -61,8 +69,8 @@ void llvm::omp::target::ompt::resetTimestamp(uint64_t *T) { *T = 0; }
 ompt_callback_buffer_request_t
 llvm::omp::target::ompt::getBufferRequestFn(int DeviceId) {
   std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
-  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
-  if (BufferMgrItr == BufferManagementFns.end()) {
+  auto BufferMgrItr = getBufferMgmt().find(DeviceId);
+  if (BufferMgrItr == getBufferMgmt().end()) {
     return nullptr;
   }
   return BufferMgrItr->second.first;
@@ -71,8 +79,8 @@ llvm::omp::target::ompt::getBufferRequestFn(int DeviceId) {
 ompt_callback_buffer_complete_t
 llvm::omp::target::ompt::getBufferCompleteFn(int DeviceId) {
   std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
-  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
-  if (BufferMgrItr == BufferManagementFns.end()) {
+  auto BufferMgrItr = getBufferMgmt().find(DeviceId);
+  if (BufferMgrItr == getBufferMgmt().end()) {
     return nullptr;
   }
   return BufferMgrItr->second.second;
@@ -82,29 +90,29 @@ void llvm::omp::target::ompt::setBufferManagementFns(
     int DeviceId, ompt_callback_buffer_request_t ReqFn,
     ompt_callback_buffer_complete_t CmpltFn) {
   std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
-  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
-  if (BufferMgrItr != BufferManagementFns.end()) {
+  auto BufferMgrItr = getBufferMgmt().find(DeviceId);
+  if (BufferMgrItr != getBufferMgmt().end()) {
     REPORT() << "Buffer request and complete functions already exist for device  "
              << DeviceId  << "ignoring ...";
     return;
   }
-  BufferManagementFns[DeviceId] = std::make_pair(ReqFn, CmpltFn);
+  getBufferMgmt()[DeviceId] = std::make_pair(ReqFn, CmpltFn);
 }
 
 void llvm::omp::target::ompt::removeBufferManagementFns(int DeviceId) {
   std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
-  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
-  if (BufferMgrItr == BufferManagementFns.end()) {
+  auto BufferMgrItr = getBufferMgmt().find(DeviceId);
+  if (BufferMgrItr == getBufferMgmt().end()) {
     REPORT() << "Buffer request and complete functions don't exist for device  "
              << DeviceId  << "ignoring ...";
     return;
   }
-  BufferManagementFns.erase(BufferMgrItr);
+  getBufferMgmt().erase(BufferMgrItr);
 }
 
 bool llvm::omp::target::ompt::isAllDeviceTracingStopped() {
   std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
-  return BufferManagementFns.empty();
+  return getBufferMgmt().empty();
 }
 
 void llvm::omp::target::ompt::ompt_callback_buffer_request(
@@ -137,11 +145,11 @@ inline bool checkDeviceTracingState(const uint64_t &TracingTypes) {
 
 void llvm::omp::target::ompt::enableDeviceTracing(int DeviceId) {
   std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
-  auto Device = TracedDevices.find(DeviceId);
-  if (Device == TracedDevices.end()) {
+  auto Device = getTracedDevices().find(DeviceId);
+  if (Device == getTracedDevices().end()) {
     uint64_t TracingTypes{0};
     setDeviceTracing(TracingTypes);
-    TracedDevices.emplace(DeviceId, TracingTypes);
+    getTracedDevices().emplace(DeviceId, TracingTypes);
   } else
     setDeviceTracing(Device->second);
   // In any case: at least one device is traced
@@ -150,16 +158,16 @@ void llvm::omp::target::ompt::enableDeviceTracing(int DeviceId) {
 
 void llvm::omp::target::ompt::disableDeviceTracing(int DeviceId) {
   std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
-  auto Device = TracedDevices.find(DeviceId);
-  if (Device == TracedDevices.end()) {
+  auto Device = getTracedDevices().find(DeviceId);
+  if (Device == getTracedDevices().end()) {
     uint64_t TracingTypes{0};
     resetDeviceTracing(TracingTypes);
-    TracedDevices.emplace(DeviceId, TracingTypes);
+    getTracedDevices().emplace(DeviceId, TracingTypes);
   } else
     resetDeviceTracing(Device->second);
 
   // Check for actively traced devices
-  for (auto &Dev : TracedDevices)
+  for (auto &Dev : getTracedDevices())
     if (checkDeviceTracingState(Dev.second))
       return;
 
@@ -175,8 +183,8 @@ bool llvm::omp::target::ompt::isTracingEnabled(int DeviceId,
 
 bool llvm::omp::target::ompt::isTracedDevice(int DeviceId) {
   std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
-  auto Device = TracedDevices.find(DeviceId);
-  if (Device != TracedDevices.end())
+  auto Device = getTracedDevices().find(DeviceId);
+  if (Device != getTracedDevices().end())
     return checkDeviceTracingState(Device->second);
 
   return false;
@@ -187,8 +195,8 @@ bool llvm::omp::target::ompt::isTracingTypeEnabled(int DeviceId,
   std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
   // Make sure we do not shift more than std::numeric_limits<uint64_t>::digits
   assert(EventTy < 64 && "Shift limit exceeded: EventTy must be less than 64");
-  auto Device = TracedDevices.find(DeviceId);
-  if (Device != TracedDevices.end() && EventTy < 64)
+  auto Device = getTracedDevices().find(DeviceId);
+  if (Device != getTracedDevices().end() && EventTy < 64)
     return (Device->second & (1UL << EventTy));
   return false;
 }
@@ -198,8 +206,8 @@ bool llvm::omp::target::ompt::isTracingTypeGroupEnabled(int DeviceId,
   std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
   // Make sure we do not shift more than std::numeric_limits<uint64_t>::digits
   assert(EventTy < 64 && "Shift limit exceeded: EventTy must be less than 64");
-  auto Device = TracedDevices.find(DeviceId);
-  if (Device != TracedDevices.end() && EventTy < 64) {
+  auto Device = getTracedDevices().find(DeviceId);
+  if (Device != getTracedDevices().end() && EventTy < 64) {
     auto TracedEvents = Device->second;
     switch (EventTy) {
     case ompt_callbacks_t::ompt_callback_target:
@@ -252,10 +260,10 @@ llvm::omp::target::ompt::setTraceEventTy(int DeviceId, unsigned int Enable,
                  << " Enable=" << Enable << " EventTy=" << EventTy;
 
   std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
-  if (TracedDevices.find(DeviceId) == TracedDevices.end())
-    TracedDevices.emplace(DeviceId, 0UL);
+  if (getTracedDevices().find(DeviceId) == getTracedDevices().end())
+    getTracedDevices().emplace(DeviceId, 0UL);
 
-  auto &TracedEventTy = TracedDevices[DeviceId];
+  auto &TracedEventTy = getTracedDevices()[DeviceId];
   bool Enabled = Enable > 0;
   if (EventTy == 0) {
     // Set / reset all supported types
diff --git a/offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp b/offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp
index 2d4b32e3554ac..cf93b5f968ac7 100644
--- a/offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp
+++ b/offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp
@@ -32,8 +32,10 @@ static std::atomic<uint64_t> BufId{0};
 // Unique id in buffer flush order
 static std::atomic<uint64_t> FlushId{0};
 
-thread_local OmptTracingBufferMgr::BufPtr
-    OmptTracingBufferMgr::ArrayOfBufPtr[MAX_NUM_DEVICES];
+OmptTracingBufferMgr::BufPtr *OmptTracingBufferMgr::getArrayOfBufPtr() {
+  thread_local BufPtr Array[MAX_NUM_DEVICES]{};
+  return Array;
+}
 
 static uint64_t get_and_inc_buf_id() { return BufId++; }
 
@@ -418,7 +420,7 @@ OmptTracingBufferMgr::getDeviceSpecificBuffer(int64_t DeviceId) {
              << MAX_NUM_DEVICES - 1;
     return nullptr;
   }
-  return ArrayOfBufPtr[DeviceId];
+  return getArrayOfBufPtr()[DeviceId];
 }
 
 void OmptTracingBufferMgr::setDeviceSpecificBuffer(int64_t DeviceId,
@@ -429,7 +431,7 @@ void OmptTracingBufferMgr::setDeviceSpecificBuffer(int64_t DeviceId,
              << MAX_NUM_DEVICES - 1;
     return;
   }
-  ArrayOfBufPtr[DeviceId] = Buf;
+  getArrayOfBufPtr()[DeviceId] = Buf;
 }
 
 void OmptTracingBufferMgr::setTRStatus(void *Rec, TRStatus Status) {
@@ -681,7 +683,7 @@ void OmptTracingBufferMgr::waitForFlushCompletion() {
 
 void OmptTracingBufferMgr::init() {
   for (int i = 0; i < MAX_NUM_DEVICES; ++i)
-    ArrayOfBufPtr[i] = nullptr;
+    getArrayOfBufPtr()[i] = nullptr;
   ThreadFlushTracker = 0;
   ThreadShutdownTracker = 0;
   DoneTracing = false; // TODO make it a class member
diff --git a/offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h b/offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h
index 5744a784825da..ba9e1dcd6ccc3 100644
--- a/offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h
+++ b/offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h
@@ -85,19 +85,17 @@ extern double HostToDeviceSlope;
 /// Host to device constant clock offset
 extern double HostToDeviceOffset;
 
-/// Mapping of device pointers to their corresponding RTL device ID
-extern std::map<ompt_device_t *, int32_t> Devices;
+/// Access the map of device pointers to their corresponding RTL device ID
+std::map<ompt_device_t *, int32_t> &getDevices();
 
-/// Mapping of RTL device IDs to their currently enabled tracing event types.
-/// Note: Event type '0' (bit position) indicates if this device is traced.
-extern std::map<int32_t, uint64_t> TracedDevices;
+/// Access the map of RTL device IDs to their currently enabled tracing event
+/// types. Note: Event type '0' (bit position) indicates if this device is
+/// traced.
+std::map<int32_t, uint64_t> &getTracedDevices();
 
 /// OMPT global tracing status. Indicates if at least one device is traced.
 extern bool TracingActive;
 
-/// Parent library pointer
-extern std::shared_ptr<llvm::sys::DynamicLibrary> ParentLibrary;
-
 /// Get the parent library by pointer. If it is not already set, it will set the
 /// parent library pointer.
 std::shared_ptr<llvm::sys::DynamicLibrary> getParentLibrary();
@@ -111,10 +109,10 @@ void setParentLibrary(const char *Filename);
 template <typename FT>
 void ensureFuncPtrLoaded(const std::string &FuncName, FT *FuncPtr) {
   if (*FuncPtr == nullptr) {
-    if ((ParentLibrary == nullptr && getParentLibrary() == nullptr) ||
-        !ParentLibrary->isValid())
+    auto Lib = getParentLibrary();
+    if (!Lib || !Lib->isValid())
       return;
-    void *SymbolPtr = ParentLibrary->getAddressOfSymbol(FuncName.c_str());
+    void *SymbolPtr = Lib->getAddressOfSymbol(FuncName.c_str());
     if (SymbolPtr == nullptr)
       return;
     *FuncPtr = reinterpret_cast<FT>(SymbolPtr);
diff --git a/offload/plugins-nextgen/common/OMPT/OmptTracing.cpp b/offload/plugins-nextgen/common/OMPT/OmptTracing.cpp
index d5be9b8d9cd8a..6e900143c4012 100644
--- a/offload/plugins-nextgen/common/OMPT/OmptTracing.cpp
+++ b/offload/plugins-nextgen/common/OMPT/OmptTracing.cpp
@@ -46,39 +46,44 @@ std::mutex llvm::omp::target::ompt::DeviceIdWritingMutex;
 using namespace llvm::omp::target::ompt;
 using namespace llvm::omp::target::debug;
 
-std::shared_ptr<llvm::sys::DynamicLibrary>
-    llvm::omp::target::ompt::ParentLibrary(nullptr);
-
 double llvm::omp::target::ompt::HostToDeviceSlope = .0;
 double llvm::omp::target::ompt::HostToDeviceOffset = .0;
 
-std::map<ompt_device_t *, int32_t> llvm::omp::target::ompt::Devices;
+// File-local helper to hold the parent library shared_ptr as a function-local
+// static, avoiding a global destructor.
+static std::shared_ptr<llvm::sys::DynamicLibrary> &getParentLibraryStorage() {
+  static std::shared_ptr<llvm::sys::DynamicLibrary> Lib(nullptr);
+  return Lib;
+}
 
 std::shared_ptr<llvm::sys::DynamicLibrary>
 llvm::omp::target::ompt::getParentLibrary() {
-  static bool ParentLibraryAssigned = false;
-  if (!ParentLibraryAssigned) {
+  if (!getParentLibraryStorage()) {
     setParentLibrary("libomptarget.so");
-    ParentLibraryAssigned = true;
   }
-  return ParentLibrary;
+  return getParentLibraryStorage();
 }
 
 void llvm::omp::target::ompt::setParentLibrary(const char *Filename) {
-  if (ParentLibrary)
+  if (getParentLibraryStorage())
     return;
   std::string ErrorMsg;
-  ParentLibrary = std::make_shared<llvm::sys::DynamicLibrary>(
+  getParentLibraryStorage() = std::make_shared<llvm::sys::DynamicLibrary>(
       llvm::sys::DynamicLibrary::getPermanentLibrary(Filename, &ErrorMsg));
-  if ((ParentLibrary == nullptr) || (!ParentLibrary->isValid()))
+  if (!getParentLibraryStorage() || !getParentLibraryStorage()->isValid())
     REPORT() << "Failed to set parent library: " << ErrorMsg.c_str();
 }
 
+std::map<ompt_device_t *, int32_t> &llvm::omp::target::ompt::getDevices() {
+  static std::map<ompt_device_t *, int32_t> Devices;
+  return Devices;
+}
+
 int llvm::omp::target::ompt::getDeviceId(ompt_device_t *Device) {
   // Block other threads, which might trigger an erase (for the same device)
   std::unique_lock<std::mutex> Lock(DeviceIdWritingMutex);
-  auto DeviceIterator = Devices.find(Device);
-  if (Device == nullptr || DeviceIterator == Devices.end()) {
+  auto DeviceIterator = getDevices().find(Device);
+  if (Device == nullptr || DeviceIterator == getDevices().end()) {
     REPORT() << "Failed to get ID for Device=" << Device;
     return -1;
   }
@@ -93,8 +98,8 @@ void llvm::omp::target::ompt::setDeviceId(ompt_device_t *Device,
     return;
   }
   std::unique_lock<std::mutex> Lock(DeviceIdWritingMutex);
-  auto DeviceIterator = Devices.find(Device);
-  if (DeviceIterator != Devices.end()) {
+  auto DeviceIterator = getDevices().find(Device);
+  if (DeviceIterator != getDevices().end()) {
     auto CurrentDeviceId = DeviceIterator->second;
     if (DeviceId == CurrentDeviceId) {
       REPORT() << "Tried to duplicate OMPT Device= " << Device <<  " ID=" << DeviceId;
@@ -103,7 +108,7 @@ void llvm::omp::target::ompt::setDeviceId(ompt_device_t *Device,
     }
     return;
   }
-  Devices.emplace(Device, DeviceId);
+  getDevices().emplace(Device, DeviceId);
 }
 
 void llvm::omp::target::ompt::removeDeviceId(ompt_device_t *Device) {
@@ -113,8 +118,8 @@ void llvm::omp::target::ompt::removeDeviceId(ompt_device_t *Device) {
     return;
   }
   std::unique_lock<std::mutex> Lock(DeviceIdWritingMutex);
-  Devices.erase(Device);
-  TracedDevices.erase(DeviceId);
+  getDevices().erase(Device);
+  getTracedDevices().erase(DeviceId);
 }
 
 OMPT_API_ROUTINE ompt_set_result_t ompt_set_trace_ompt(ompt_device_t *Device,

>From 7cc2d00113ee78eaa09ac78bfd9b7e01b092c0c8 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 21 May 2026 08:43:39 -0500
Subject: [PATCH 09/11] [OpenMP] Add OMPT device tracing tests

---
 .../callbacks.h                               | 340 ++++++++++++++++++
 .../veccopy_ompt_selective_tracing_dataop.cpp |  66 ++++
 .../callbacks.h                               | 340 ++++++++++++++++++
 .../veccopy_ompt_selective_tracing_kernel.cpp |  70 ++++
 .../callbacks.h                               | 332 +++++++++++++++++
 .../veccopy-ompt-target-data-tracing-emi.cpp  | 146 ++++++++
 .../callbacks.h                               | 333 +++++++++++++++++
 .../veccopy-ompt-target-emi-tracing-dag.cpp   |  81 +++++
 .../callbacks.h                               | 332 +++++++++++++++++
 .../veccopy-ompt-target-emi-tracing.cpp       |  78 ++++
 .../callbacks.h                               | 332 +++++++++++++++++
 .../veccopy-ompt-target-emi-tracing.cpp       |  67 ++++
 .../callbacks.h                               | 332 +++++++++++++++++
 .../veccopy-ompt-target-emi-tracing.cpp       |  66 ++++
 .../veccopy-ompt-target-tracing/callbacks.h   | 292 +++++++++++++++
 .../veccopy-ompt-target-tracing.cpp           |  78 ++++
 16 files changed, 3285 insertions(+)
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/veccopy_ompt_selective_tracing_dataop.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/veccopy_ompt_selective_tracing_kernel.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/veccopy-ompt-target-data-tracing-emi.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/veccopy-ompt-target-emi-tracing-dag.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/veccopy-ompt-target-emi-tracing.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/veccopy-ompt-target-emi-tracing.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/veccopy-ompt-target-emi-tracing.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing/veccopy-ompt-target-tracing.cpp

diff --git a/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/callbacks.h
new file mode 100644
index 0000000000000..5cd611990d32b
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/callbacks.h
@@ -0,0 +1,340 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+  // Only enable device tracing for dataop events explicitly
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d filename:%s host_adddr:%p device_addr:%p"
+         " bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+  assert(DeviceMapPtr != nullptr && "DeviceMapPtr must be valid");
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d optype=%d target_task_data=%p "
+         "(0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) src=%p "
+         "src_device_num=%d dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, optype, target_task_data, target_task_data_value,
+         target_data, target_data->value, host_op_id, *host_op_id, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d req_num_teams=%d target_data=%p "
+         "(0x%lx) host_op_id=%p (0x%lx)\n",
+         endpoint, requested_num_teams, target_data, target_data->value,
+         host_op_id, *host_op_id);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+#if EMI
+  register_ompt_callback(ompt_callback_target_data_op_emi);
+  register_ompt_callback(ompt_callback_target_emi);
+  register_ompt_callback(ompt_callback_target_submit_emi);
+#else
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+#endif
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/veccopy_ompt_selective_tracing_dataop.cpp b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/veccopy_ompt_selective_tracing_dataop.cpp
new file mode 100644
index 0000000000000..c07bff4230688
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/veccopy_ompt_selective_tracing_dataop.cpp
@@ -0,0 +1,66 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
+/// CHECK: rec={{0x[0-9a-fA-F]+}} type=9
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=10
+/// CHECK: rec={{0x[0-9a-fA-F]+}} type=9
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
+
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
+/// CHECK: rec={{0x[0-9a-fA-F]+}} type=9
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=10
+/// CHECK: rec={{0x[0-9a-fA-F]+}} type=9
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
diff --git a/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/callbacks.h
new file mode 100644
index 0000000000000..9655d718c7c48
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/callbacks.h
@@ -0,0 +1,340 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+  // Only enable device tracing for kernel submit events
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d filename:%s host_adddr:%p device_addr:%p"
+         " bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+  assert(DeviceMapPtr != nullptr && "DeviceMapPtr must be valid");
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d optype=%d target_task_data=%p "
+         "(0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) src=%p "
+         "src_device_num=%d dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, optype, target_task_data, target_task_data_value,
+         target_data, target_data->value, host_op_id, *host_op_id, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d req_num_teams=%d target_data=%p "
+         "(0x%lx) host_op_id=%p (0x%lx)\n",
+         endpoint, requested_num_teams, target_data, target_data->value,
+         host_op_id, *host_op_id);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+#if EMI
+  register_ompt_callback(ompt_callback_target_data_op_emi);
+  register_ompt_callback(ompt_callback_target_emi);
+  register_ompt_callback(ompt_callback_target_submit_emi);
+#else
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+#endif
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/veccopy_ompt_selective_tracing_kernel.cpp b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/veccopy_ompt_selective_tracing_kernel.cpp
new file mode 100644
index 0000000000000..285f1f1011268
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/veccopy_ompt_selective_tracing_kernel.cpp
@@ -0,0 +1,70 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// XFAIL: *
+// clang-format on
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  for (auto Device : *DeviceMapPtr)
+    flush_trace(Device);
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=9
+/// CHECK: rec={{0x[0-9a-fA-F]+}} type=10
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=9
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
+
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=9
+/// CHECK: rec={{0x[0-9a-fA-F]+}} type=10
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=9
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/callbacks.h
new file mode 100644
index 0000000000000..d05f05a4a4501
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/callbacks.h
@@ -0,0 +1,332 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+#define EMI 1
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d filename:%s host_adddr:%p device_addr:%p"
+         " bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d optype=%d target_task_data=%p "
+         "(0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) src=%p "
+         "src_device_num=%d dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, optype, target_task_data, target_task_data_value,
+         target_data, target_data->value, host_op_id, *host_op_id, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d req_num_teams=%d target_data=%p "
+         "(0x%lx) host_op_id=%p (0x%lx)\n",
+         endpoint, requested_num_teams, target_data, target_data->value,
+         host_op_id, *host_op_id);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+#if EMI
+  register_ompt_callback(ompt_callback_target_data_op_emi);
+  register_ompt_callback(ompt_callback_target_emi);
+  register_ompt_callback(ompt_callback_target_submit_emi);
+#else
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+#endif
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/veccopy-ompt-target-data-tracing-emi.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/veccopy-ompt-target-data-tracing-emi.cpp
new file mode 100644
index 0000000000000..35a38f4eba739
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/veccopy-ompt-target-data-tracing-emi.cpp
@@ -0,0 +1,146 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+#define N 100000
+
+#pragma omp declare target
+int c[N];
+#pragma omp end declare target
+
+int main() {
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+  for (i = 0; i < N; i++)
+    c[i] = 0;
+
+#pragma omp target enter data map(to : a)
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+#pragma omp target exit data map(from : a)
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+#pragma omp target parallel for map(alloc : c)
+  {
+    for (int j = 0; j < N; j++)
+      c[j] = 2 * j + 1;
+  }
+#pragma omp target update from(c) nowait
+#pragma omp barrier
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+  int rc = 0;
+  for (i = 0; i < N; i++) {
+    if (a[i] != i) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+  }
+
+  for (i = 0; i < N; i++) {
+    if (c[i] != 2 * i + 1) {
+      rc++;
+      printf("Wrong value: c[%d]=%d\n", i, c[i]);
+    }
+  }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+// > OMPT device callback related checks below. <
+
+/// CHECK-NOT: Callback Target EMI:
+/// CHECK-NOT: device_num=-1
+
+/// CHECK: Callback Init:
+/// CHECK: Callback Load:
+
+/// CHECK-DAG: Callback Target EMI: kind=2 endpoint=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=2
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=2
+/// CHECK-DAG: Callback Target EMI: kind=2 endpoint=2
+
+/// CHECK-DAG: Callback Target EMI: kind=1 endpoint=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=2
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=2
+/// CHECK-DAG: Callback Submit EMI: endpoint=1 req_num_teams=1
+/// CHECK-DAG: Callback Submit EMI: endpoint=2 req_num_teams=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=3
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=3
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=4
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=4
+/// CHECK-DAG: Callback Target EMI: kind=1 endpoint=2
+/// CHECK-DAG: Callback Target EMI: kind=3 endpoint=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=3
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=3
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=4
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=4
+/// CHECK-DAG: Callback Target EMI: kind=3 endpoint=2
+
+/// CHECK-DAG: Callback Target EMI: kind=1 endpoint=1
+/// CHECK-DAG: Callback Submit EMI: endpoint=1 req_num_teams=1
+/// CHECK-DAG: Callback Submit EMI: endpoint=2 req_num_teams=1
+/// CHECK-DAG: Callback Target EMI: kind=1 endpoint=2
+/// CHECK-DAG: Callback Target EMI: kind=4 endpoint=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=3
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=3
+/// CHECK-DAG: Callback Target EMI: kind=4 endpoint=2
+
+/// CHECK-DAG: Success
+/// CHECK-DAG: Callback Fini:
+
+// > OMPT device tracing related checks below. <
+
+
+
+// Note: Split checks for record address and content. That way we do not imply
+//       any order. Records 01-06 and 12-17 occur interleaved and belong to the
+//       first target region. 07-11 occur interleaved with 18-22 and belong to
+//       the second target region.
+
+// Note: These addresses will only occur once. They are only captured to
+//       indicate their existence.
+
+/// CHECK-DAG: type=8 (Target task)
+/// CHECK-DAG: type=9 (Target data op)
+
+// Note: ADDRX_11 may not trigger a final callback.
+
+// Note: ADDRX_11 may not be deallocated.
+
+/// CHECK-NOT: host_op_id=0x0
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/callbacks.h
new file mode 100644
index 0000000000000..f5724fa7a6852
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/callbacks.h
@@ -0,0 +1,333 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// Enable / disable OMPT's 'External Monitoring Interface'
+#define EMI 1
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d filename:%s host_adddr:%p device_addr:%p"
+         " bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d optype=%d target_task_data=%p "
+         "(0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) src=%p "
+         "src_device_num=%d dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, optype, target_task_data, target_task_data_value,
+         target_data, target_data->value, host_op_id, *host_op_id, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d req_num_teams=%d target_data=%p "
+         "(0x%lx) host_op_id=%p (0x%lx)\n",
+         endpoint, requested_num_teams, target_data, target_data->value,
+         host_op_id, *host_op_id);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+#if EMI
+  register_ompt_callback(ompt_callback_target_data_op_emi);
+  register_ompt_callback(ompt_callback_target_emi);
+  register_ompt_callback(ompt_callback_target_submit_emi);
+#else
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+#endif
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/veccopy-ompt-target-emi-tracing-dag.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/veccopy-ompt-target-emi-tracing-dag.cpp
new file mode 100644
index 0000000000000..0fdae19017ab2
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/veccopy-ompt-target-emi-tracing-dag.cpp
@@ -0,0 +1,81 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+/// CHECK-NOT: host_op_id=0x0
+
+
+
+// Note: Split checks for record address and content. That way we do not imply
+//       any order. Records 01-06 and 12-17 occur interleaved and belong to the
+//       first target region. 07-11 occur interleaved with 18-22 and belong to
+//       the second target region.
+
+/// CHECK-DAG: type=8 (Target task)
+/// CHECK-DAG: type=9 (Target data op)
+
+// Note: These addresses will only occur once. They are only captured to
+//       indicate their existence.
+
+
+// Note: ADDRX_11 may not trigger a final callback.
+
+// Note: ADDRX_11 may not be deallocated.
+
+/// CHECK-DAG: Success
+
+/// CHECK-NOT: host_op_id=0x0
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/callbacks.h
new file mode 100644
index 0000000000000..d05f05a4a4501
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/callbacks.h
@@ -0,0 +1,332 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+#define EMI 1
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d filename:%s host_adddr:%p device_addr:%p"
+         " bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d optype=%d target_task_data=%p "
+         "(0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) src=%p "
+         "src_device_num=%d dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, optype, target_task_data, target_task_data_value,
+         target_data, target_data->value, host_op_id, *host_op_id, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d req_num_teams=%d target_data=%p "
+         "(0x%lx) host_op_id=%p (0x%lx)\n",
+         endpoint, requested_num_teams, target_data, target_data->value,
+         host_op_id, *host_op_id);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+#if EMI
+  register_ompt_callback(ompt_callback_target_data_op_emi);
+  register_ompt_callback(ompt_callback_target_emi);
+  register_ompt_callback(ompt_callback_target_submit_emi);
+#else
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+#endif
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/veccopy-ompt-target-emi-tracing.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/veccopy-ompt-target-emi-tracing.cpp
new file mode 100644
index 0000000000000..e682e9b20bfd8
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/veccopy-ompt-target-emi-tracing.cpp
@@ -0,0 +1,78 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-generic && env LIBOMPTARGET_OMPT_FLUSH_ON_SHUTDOWN=false LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL=false %libomptarget-run-generic | %fcheck-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+/*
+ * This test is run with LIBOMPTARGET_OMPT_FLUSH_ON_SHUTDOWN=false and
+ * LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL=false while doing explicit flushing
+ * using ompt_flush_trace. The intention is to check whether trace records are
+ * properly flushed when the program/tool uses the ompt_flush_trace API. There
+ * should be 23 trace records returned to the tool.
+ */
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+/// CHECK-NOT: host_op_id=0x0
+
+/// CHECK: rec=
+
+/// CHECK-DAG: Success
+
+/// The user calls flush before printing success, so 
+/// no more records should be returned here.
+
+/// CHECK-NOT: host_op_id=0x0
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/callbacks.h
new file mode 100644
index 0000000000000..d05f05a4a4501
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/callbacks.h
@@ -0,0 +1,332 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+#define EMI 1
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d filename:%s host_adddr:%p device_addr:%p"
+         " bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d optype=%d target_task_data=%p "
+         "(0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) src=%p "
+         "src_device_num=%d dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, optype, target_task_data, target_task_data_value,
+         target_data, target_data->value, host_op_id, *host_op_id, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d req_num_teams=%d target_data=%p "
+         "(0x%lx) host_op_id=%p (0x%lx)\n",
+         endpoint, requested_num_teams, target_data, target_data->value,
+         host_op_id, *host_op_id);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+#if EMI
+  register_ompt_callback(ompt_callback_target_data_op_emi);
+  register_ompt_callback(ompt_callback_target_emi);
+  register_ompt_callback(ompt_callback_target_submit_emi);
+#else
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+#endif
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/veccopy-ompt-target-emi-tracing.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/veccopy-ompt-target-emi-tracing.cpp
new file mode 100644
index 0000000000000..abf2426d2eee9
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/veccopy-ompt-target-emi-tracing.cpp
@@ -0,0 +1,67 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-generic && env LIBOMPTARGET_OMPT_FLUSH_ON_SHUTDOWN=false %libomptarget-run-generic | %fcheck-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+/*
+ * This test is run with LIBOMPTARGET_OMPT_FLUSH_ON_SHUTDOWN=false and
+ * ompt_flush_trace is not invoked by the user/tool. The intention is to check
+ * whether trace records are properly flushed as a buffer fills up. Currently,
+ * the buffer size in callbacks.h implies that every buffer holds 2 trace
+ * records. With this assumption, there should be 22 trace records. The last
+ * trace record is not flushed because the last buffer is not yet full.
+ */
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+/// CHECK-NOT: host_op_id=0x0
+
+/// CHECK: rec=
+/// CHECK-NOT: host_op_id=0x0
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/callbacks.h
new file mode 100644
index 0000000000000..d05f05a4a4501
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/callbacks.h
@@ -0,0 +1,332 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+#define EMI 1
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d filename:%s host_adddr:%p device_addr:%p"
+         " bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d optype=%d target_task_data=%p "
+         "(0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) src=%p "
+         "src_device_num=%d dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, optype, target_task_data, target_task_data_value,
+         target_data, target_data->value, host_op_id, *host_op_id, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d req_num_teams=%d target_data=%p "
+         "(0x%lx) host_op_id=%p (0x%lx)\n",
+         endpoint, requested_num_teams, target_data, target_data->value,
+         host_op_id, *host_op_id);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+#if EMI
+  register_ompt_callback(ompt_callback_target_data_op_emi);
+  register_ompt_callback(ompt_callback_target_emi);
+  register_ompt_callback(ompt_callback_target_submit_emi);
+#else
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+#endif
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/veccopy-ompt-target-emi-tracing.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/veccopy-ompt-target-emi-tracing.cpp
new file mode 100644
index 0000000000000..7f4e99b83e4e3
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/veccopy-ompt-target-emi-tracing.cpp
@@ -0,0 +1,66 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-generic && env LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL=false %libomptarget-run-generic | %fcheck-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+/*
+ * This test is run with LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL=false and
+ * ompt_flush_trace is not invoked by the user/tool. The intention is to check
+ * whether trace records are properly flushed on shutdown. 23 trace records
+ * should be flushed during shutdown.
+ */
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+/// CHECK-NOT: host_op_id=0x0
+
+/// CHECK: rec=
+
+/// CHECK-NOT: host_op_id=0x0
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-tracing/callbacks.h
new file mode 100644
index 0000000000000..cfb5fd4147ebf
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing/callbacks.h
@@ -0,0 +1,292 @@
+// Copyright © Advanced Micro Devices, Inc., or its affiliates.
+//
+// SPDX-License-Identifier:  MIT
+
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// Enable / disable OMPT's 'External Monitoring Interface'
+#define EMI 0
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE (4 * 1024)
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+static ompt_get_record_type_t ompt_get_record_type_fn = 0;
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+// Note: This callback must handle a null begin cursor. Currently,
+// ompt_get_record_ompt, print_record_ompt, and
+// ompt_advance_buffer_cursor handle a null cursor.
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+
+    if (ompt_get_record_type_fn(buffer, current) != ompt_record_ompt) {
+      printf("WARNING: received non-ompt type buffer object\n");
+    }
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n", device_num,
+         type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  ompt_get_record_type_fn =
+      (ompt_get_record_type_t)lookup("ompt_get_record_type");
+  if (!ompt_get_record_type_fn) {
+    printf("WARNING: No function ompt_get_record_type found in device "
+           "callbacks\n");
+  }
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Load: device_num:%d filename:%s host_adddr:%p device_addr:%p "
+         "bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: target_id=%lu host_op_id=%lu optype=%d src=%p "
+         "src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         target_id, host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: target_id=%lu kind=%d endpoint=%d device_num=%d "
+         "code=%p\n",
+         target_id, kind, endpoint, device_num, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing/veccopy-ompt-target-tracing.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-tracing/veccopy-ompt-target-tracing.cpp
new file mode 100644
index 0000000000000..d393bc63b2e14
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing/veccopy-ompt-target-tracing.cpp
@@ -0,0 +1,78 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+// Copyright © Advanced Micro Devices, Inc., or its affiliates.
+//
+// SPDX-License-Identifier:  MIT
+
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+// > OMPT device tracing related checks below. <
+
+// Note: This test will allocate one buffer, big enough to hold all trace
+//       records, hence there will be only one allocation.
+
+
+// Note: Split checks for record address and content. That way we do not imply
+//       any order. Records may / will occur interleaved.
+
+// Note: These addresses will only occur once. They are only captured to
+//       indicate their existence.
+
+/// CHECK-DAG: type=8 (Target task)
+/// CHECK-DAG: type=9 (Target data op)
+
+// Note: ADDRX_01 may not trigger a final callback.
+// Note: ADDRX_01 may not be deallocated.
+
+/// CHECK-NOT: host_op_id=0x0

>From 28c11105340076a071689571c2b07839e48d9d95 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 21 May 2026 08:44:24 -0500
Subject: [PATCH 10/11] [Openmp] Add OMPT device tracing support tests

These tests concern functionality used by parts of the OMPT device
tracing implementation.
---
 .../veccopy-ompt-ctor-1/Inputs/callbacks.h    | 373 ++++++++++++++++++
 .../Inputs/veccopy-ctor-1.cpp                 |  37 ++
 .../tracing/veccopy-ompt-ctor-1/veccopy.cpp   |  52 +++
 .../callbacks.h                               | 286 ++++++++++++++
 .../veccopy-ompt-target-default-device.cpp    |  86 ++++
 .../veccopy-ompt-target-devices/callbacks.h   | 324 +++++++++++++++
 .../veccopy-ompt-target-devices.cpp           |  86 ++++
 .../callbacks.h                               | 166 ++++++++
 .../veccopy-ompt-target.c                     |  78 ++++
 .../callbacks.h                               | 160 ++++++++
 .../veccopy-ompt-target.c                     |  78 ++++
 11 files changed, 1726 insertions(+)
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/veccopy-ctor-1.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-ctor-1/veccopy.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-default-device/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-default-device/veccopy-ompt-target-default-device.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-devices/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-devices/veccopy-ompt-target-devices.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-translate-time/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-translate-time/veccopy-ompt-target.c
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/veccopy-ompt-target.c

diff --git a/offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/callbacks.h
new file mode 100644
index 0000000000000..d206c46567a3e
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/callbacks.h
@@ -0,0 +1,373 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+#define EMI 1
+
+// Tool related code below
+#include <omp-tools.h>
+
+ompt_id_t  next_op_id = 0x8000000000000001;
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t*> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL) return;
+  
+  printf("rec=%p type=%2d time=%lu thread_id=%lu target_id=%lu\n",
+	 rec, rec->type, rec->time, rec->thread_id, rec->target_id);
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi:
+    {
+      ompt_record_target_t target_rec = rec->record.target;
+      printf("\tRecord Target task: target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu codeptr=%p\n",
+	     target_rec.target_id, 
+	     target_rec.kind, target_rec.endpoint, target_rec.device_num,
+	     target_rec.task_id, target_rec.codeptr_ra);
+      break;
+    }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi:
+    {
+      ompt_record_target_data_op_t target_data_op_rec = rec->record.target_data_op;
+      printf("\t  Record Target data op: target_id=0x%lx host_op_id=0x%lx optype=%d src_addr=%p src_device=%d "
+	     "dest_addr=%p dest_device=%d bytes=%lu end_time=%lu duration=%lu ns codeptr=%p\n",
+             rec->target_id,
+	     target_data_op_rec.host_op_id, target_data_op_rec.optype,
+	     target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+	     target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+	     target_data_op_rec.bytes, target_data_op_rec.end_time,
+	     target_data_op_rec.end_time - rec->time,
+	     target_data_op_rec.codeptr_ra);
+      break;
+    }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi:
+    {
+      ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+      printf("\t  Record Target kernel:  target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u granted_num_teams=%u "
+	     "end_time=%lu duration=%lu ns\n",
+             rec->target_id,
+	     target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+	     target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+	     target_kernel_rec.end_time - rec->time);
+    break;
+    }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request (
+  int device_num,
+  ompt_buffer_t **buffer,
+  size_t *bytes
+) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes, *buffer);
+}
+
+static void on_ompt_callback_buffer_complete (
+  int device_num,
+  ompt_buffer_t *buffer,
+  size_t bytes, /* bytes returned in this callback */
+  ompt_buffer_cursor_t begin,
+  int buffer_owned
+) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n",
+	 device_num, buffer, bytes, (void*)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+					buffer,
+					bytes,
+					current,
+					&current);
+  }
+  if (buffer_owned) delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt) return ompt_set_error;
+
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+#endif
+  
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace) return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+	 "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+  
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+			  &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace) return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace) return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize
+(
+  int device_num,
+  const char *type,
+  ompt_device_t *device,
+  ompt_function_lookup_t lookup,
+  const char *documentation
+ ) {
+  printf("Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+	 device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t) lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t) lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t) lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t) lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t) lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor = (ompt_advance_buffer_cursor_t) lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_load
+    (
+     int device_num,
+     const char *filename,
+     int64_t offset_in_file,
+     void *vma_in_file,
+     size_t bytes,
+     void *host_addr,
+     void *device_addr,
+     uint64_t module_id
+     ) {
+  printf("Load: device_num:%d filename:%s host_adddr:%p device_addr:%p bytes:%lu\n",
+	 device_num, filename, host_addr, device_addr, bytes);
+}
+
+
+static void on_ompt_callback_target_data_op
+    (
+     ompt_id_t target_id,
+     ompt_id_t host_op_id,
+     ompt_target_data_op_t optype,
+     void *src_addr,
+     int src_device_num,
+     void *dest_addr,
+     int dest_device_num,
+     size_t bytes,
+     const void *codeptr_ra
+     ) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+	 "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+	 host_op_id, optype, src_addr, src_device_num,
+	 dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+
+static void on_ompt_callback_target_data_op_emi
+    (
+     ompt_scope_endpoint_t endpoint,
+     ompt_data_t *target_task_data,
+     ompt_data_t *target_data,
+     ompt_id_t *host_op_id,
+     ompt_target_data_op_t optype,
+     void *src_addr,
+     int src_device_num,
+     void *dest_addr,
+     int dest_device_num,
+     size_t bytes,
+     const void *codeptr_ra
+     ) {
+     if (endpoint == ompt_scope_begin) *host_op_id = next_op_id++;
+     // target_task_data may be null, avoid dereferencing it
+     uint64_t target_task_data_value = (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d target_task_data=%p (0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) optype=%d src=%p src_device_num=%d "
+	 "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+	 endpoint, 
+         target_task_data, target_task_data_value, 
+         target_data, target_data->value, 
+         host_op_id, *host_op_id, 
+         optype, src_addr, src_device_num,
+	 dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+
+static void on_ompt_callback_target
+    (
+     ompt_target_t kind,
+     ompt_scope_endpoint_t endpoint,
+     int device_num,
+     ompt_data_t *task_data,
+     ompt_id_t target_id,
+     const void *codeptr_ra
+     ) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu code=%p\n",
+	 kind, endpoint, device_num, target_id, codeptr_ra);
+}
+
+
+static void on_ompt_callback_target_emi
+    (
+     ompt_target_t kind,
+     ompt_scope_endpoint_t endpoint,
+     int device_num,
+     ompt_data_t *task_data,
+     ompt_data_t *target_task_data,
+     ompt_data_t *target_data,
+     const void *codeptr_ra
+     ) {
+     if (endpoint == ompt_scope_begin) target_data->value = next_op_id++;
+     // target_task_data may be null, avoid dereferencing it
+     uint64_t target_task_data_value = (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p (0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+	 kind, endpoint, device_num, 
+         task_data, task_data->value, 
+         target_task_data, target_task_data_value, 
+         target_data, target_data->value, 
+         codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit
+    (
+     ompt_id_t target_id,
+     ompt_id_t host_op_id,
+     unsigned int requested_num_teams
+     ) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+     target_id, host_op_id, requested_num_teams);
+}
+
+
+static void on_ompt_callback_target_submit_emi
+    (
+     ompt_scope_endpoint_t endpoint,
+     ompt_data_t *target_data,
+     ompt_id_t *host_op_id,
+     unsigned int requested_num_teams
+     ) {
+  printf("  Callback Submit EMI: endpoint=%d target_data=%p (0x%lx) host_op_id=%p (0x%lx) req_num_teams=%d\n",
+     endpoint, 
+     target_data, target_data->value, 
+     host_op_id, *host_op_id, 
+     requested_num_teams);
+}
+
+// Init functions
+int ompt_initialize(
+  ompt_function_lookup_t lookup,
+  int initial_device_num,
+  ompt_data_t *tool_data)
+{
+  ompt_set_callback = (ompt_set_callback_t) lookup("ompt_set_callback");
+
+  ompt_set_callback(ompt_callback_device_initialize,
+		    (ompt_callback_t)&on_ompt_callback_device_initialize);
+  ompt_set_callback(ompt_callback_device_load,
+		    (ompt_callback_t)&on_ompt_callback_device_load);
+#if EMI
+  ompt_set_callback(ompt_callback_target_submit_emi,
+		    (ompt_callback_t)&on_ompt_callback_target_submit_emi);
+  ompt_set_callback(ompt_callback_target_data_op_emi,
+		    (ompt_callback_t)&on_ompt_callback_target_data_op_emi);
+  ompt_set_callback(ompt_callback_target_emi,
+		    (ompt_callback_t)&on_ompt_callback_target_emi);
+#else
+  ompt_set_callback(ompt_callback_target_submit,
+		    (ompt_callback_t)&on_ompt_callback_target_submit);
+  ompt_set_callback(ompt_callback_target_data_op,
+		    (ompt_callback_t)&on_ompt_callback_target_data_op);
+  ompt_set_callback(ompt_callback_target,
+		    (ompt_callback_t)&on_ompt_callback_target);
+#endif
+  return 1; //success
+}
+
+void ompt_finalize(ompt_data_t *tool_data)
+{
+}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(
+  unsigned int omp_version,
+  const char *runtime_version)
+{
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,&ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
+
+
diff --git a/offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/veccopy-ctor-1.cpp b/offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/veccopy-ctor-1.cpp
new file mode 100644
index 0000000000000..c944d0b5831fa
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/veccopy-ctor-1.cpp
@@ -0,0 +1,37 @@
+#include <stdio.h>
+#include <omp.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int status;
+
+__attribute__((constructor(101))) void veccopy_init() {
+  int N = 10;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i=0; i<N; i++)
+    a[i]=0;
+
+  for (i=0; i<N; i++)
+    b[i]=i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j< N; j++)
+      a[j]=b[j];
+  }
+
+  if (a[5] != b[5])
+    status = 1;
+  else
+    status = 0;
+}
+
+
diff --git a/offload/test/ompt/tracing/veccopy-ompt-ctor-1/veccopy.cpp b/offload/test/ompt/tracing/veccopy-ompt-ctor-1/veccopy.cpp
new file mode 100644
index 0000000000000..3d6533ad45f0e
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-ctor-1/veccopy.cpp
@@ -0,0 +1,52 @@
+// clang-format off
+// RUN: %clangxx-generic %s %S/Inputs/veccopy-ctor-1.cpp -o %t -Xoffload-linker -lompdevice && %libomptarget-run-generic | %fcheck-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+/*
+ * Test whether OMPT support works for OpenMP directives 
+ * in a global constructor without using a shared library. 
+ * This currently succeeds.
+ */
+
+#include <stdio.h>
+#include <omp.h>
+
+extern int status;
+
+int main()
+{
+  int N = 10;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i=0; i<N; i++)
+    a[i]=0;
+
+  for (i=0; i<N; i++)
+    b[i]=i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j< N; j++)
+      a[j]=b[j];
+  }
+
+  int rc = 0;
+  for (i=0; i<N; i++)
+    if (a[i] != b[i] ) {
+      rc++;
+      printf ("Wrong varlue: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc && !status)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+/// CHECK: Record Target task
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-default-device/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-default-device/callbacks.h
new file mode 100644
index 0000000000000..5df9985ced0c3
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-default-device/callbacks.h
@@ -0,0 +1,286 @@
+#include <cassert>
+#include <memory>
+#include <unordered_set>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// Enable / disable OMPT's 'External Monitoring Interface'
+#define EMI 0
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+static ompt_get_record_type_t ompt_get_record_type_fn = 0;
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request_default_device(
+    int device_num, ompt_buffer_t **buffer, size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback for default "
+         "device %d\n",
+         *bytes, *buffer, device_num);
+}
+
+// Note: This callback must handle a null begin cursor. Currently,
+// ompt_get_record_ompt, print_record_ompt, and
+// ompt_advance_buffer_cursor handle a null cursor.
+static void on_ompt_callback_buffer_complete_default_device(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback for default device: \
+  device_num=%d, buffer=%p, num_bytes=%lu, begin=%p, buffer_owned=%d\n",
+         device_num, buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+
+    if (ompt_get_record_type_fn(buffer, current) != ompt_record_ompt) {
+      printf("WARNING: received non-ompt type buffer object\n");
+    }
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+  ompt_set_trace_ompt(Device, /*enable=*/1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, /*enable=*/1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, /*enable=*/1, ompt_callback_target_submit);
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+  if (device_num != omp_get_default_device())
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device,
+                          &on_ompt_callback_buffer_request_default_device,
+                          &on_ompt_callback_buffer_complete_default_device);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n", device_num,
+         type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  ompt_get_record_type_fn =
+      (ompt_get_record_type_t)lookup("ompt_get_record_type");
+  if (!ompt_get_record_type_fn) {
+    printf("WARNING: No function ompt_get_record_type found in device "
+           "callbacks\n");
+  }
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Load: device_num:%d filename:%s host_adddr:%p device_addr:%p "
+         "bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: target_id=%lu host_op_id=%lu optype=%d src=%p "
+         "src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         target_id, host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: target_id=%lu kind=%d endpoint=%d device_num=%d "
+         "code=%p\n",
+         target_id, kind, endpoint, device_num, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-default-device/veccopy-ompt-target-default-device.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-default-device/veccopy-ompt-target-default-device.cpp
new file mode 100644
index 0000000000000..ccf11f88b9cb3
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-default-device/veccopy-ompt-target-default-device.cpp
@@ -0,0 +1,86 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+#include <cassert>
+#include <omp.h>
+#include <stdio.h>
+
+// This test starts device tracing on the default device only (see
+// start_trace in callbacks.h). However, if more devices are
+// available, it calls flush and stop on the other devices as
+// well. The intention is to check correct runtime behavior if a tool
+// invokes flush or stop on a device that was not started. The runtime
+// should just return without doing anything.
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+    // Warm up
+#pragma omp target
+  {}
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+  for (int dev = 0; dev < omp_get_num_devices(); ++dev) {
+#pragma omp target teams distribute parallel for device(dev)
+    {
+      for (int j = 0; j < N; j++)
+        a[j] = b[j];
+    }
+  }
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+// > OMPT device tracing related checks below. <
+
+
+
+// Note: Split checks for record address and content. That way we do not imply
+//       any order. Records may / will occur interleaved.
+
+// Note: These addresses will only occur once. They are only captured to
+//       indicate their existence.
+
+/// CHECK-DAG: type=8 (Target task)
+/// CHECK-DAG: type=9 (Target data op)
+
+// Note: ADDRX_07 may not trigger a final callback.
+
+// Note: ADDRX_07 may not be deallocated.
+
+/// CHECK-NOT: host_op_id=0x0
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-devices/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-devices/callbacks.h
new file mode 100644
index 0000000000000..0625b1a7b1522
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-devices/callbacks.h
@@ -0,0 +1,324 @@
+#include <cassert>
+#include <memory>
+#include <unordered_set>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// Enable / disable OMPT's 'External Monitoring Interface'
+#define EMI 0
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+static ompt_get_record_type_t ompt_get_record_type_fn = 0;
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request_default_device(
+    int device_num, ompt_buffer_t **buffer, size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback for default "
+         "device %d\n",
+         *bytes, *buffer, device_num);
+}
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback for device %d\n",
+         *bytes, *buffer, device_num);
+}
+
+// Note: This callback must handle a null begin cursor. Currently,
+// ompt_get_record_ompt, print_record_ompt, and
+// ompt_advance_buffer_cursor handle a null cursor.
+static void on_ompt_callback_buffer_complete_default_device(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback for default device: \
+  device_num=%d, buffer=%p, num_bytes=%lu, begin=%p, buffer_owned=%d\n",
+         device_num, buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+
+    if (ompt_get_record_type_fn(buffer, current) != ompt_record_ompt) {
+      printf("WARNING: received non-ompt type buffer object\n");
+    }
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+// Note: This callback must handle a null begin cursor. Currently,
+// ompt_get_record_ompt, print_record_ompt, and
+// ompt_advance_buffer_cursor handle a null cursor.
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: \
+  device_num=%d, buffer=%p, num_bytes=%lu, begin=%p, buffer_owned=%d\n",
+         device_num, buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+
+    if (ompt_get_record_type_fn(buffer, current) != ompt_record_ompt) {
+      printf("WARNING: received non-ompt type buffer object\n");
+    }
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+  ompt_set_trace_ompt(Device, /*enable=*/1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, /*enable=*/1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, /*enable=*/1, ompt_callback_target_submit);
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  if (device_num == omp_get_default_device())
+    return ompt_start_trace(Device,
+                            &on_ompt_callback_buffer_request_default_device,
+                            &on_ompt_callback_buffer_complete_default_device);
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n", device_num,
+         type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  ompt_get_record_type_fn =
+      (ompt_get_record_type_t)lookup("ompt_get_record_type");
+  if (!ompt_get_record_type_fn) {
+    printf("WARNING: No function ompt_get_record_type found in device "
+           "callbacks\n");
+  }
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Load: device_num:%d filename:%s host_adddr:%p device_addr:%p "
+         "bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: target_id=%lu host_op_id=%lu optype=%d src=%p "
+         "src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         target_id, host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: target_id=%lu kind=%d endpoint=%d device_num=%d "
+         "code=%p\n",
+         target_id, kind, endpoint, device_num, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-devices/veccopy-ompt-target-devices.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-devices/veccopy-ompt-target-devices.cpp
new file mode 100644
index 0000000000000..7805e8c89de40
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-devices/veccopy-ompt-target-devices.cpp
@@ -0,0 +1,86 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+#include <cassert>
+#include <omp.h>
+#include <stdio.h>
+
+// This test starts device tracing on all available devices (see
+// start_trace in callbacks.h). It subsequently calls flush and stop
+// on all devices like a tool would normally do.
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+    // Warm up
+#pragma omp target
+  {}
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+  for (int dev = 0; dev < omp_get_num_devices(); ++dev) {
+#pragma omp target teams distribute parallel for device(dev)
+    {
+      for (int j = 0; j < N; j++)
+        a[j] = b[j];
+    }
+  }
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+// > OMPT device tracing related checks below. <
+
+// Note: Since this test may (theoretically) run on an arbitrary number of
+//       devices, we may only check for the minimal set of records.
+
+
+
+// Note: Split checks for record address and content. That way we do not imply
+//       any order. Records may / will occur interleaved.
+
+// Note: These addresses will only occur once. They are only captured to
+//       indicate their existence.
+
+/// CHECK-DAG: type=8 (Target task)
+/// CHECK-DAG: type=9 (Target data op)
+
+// Note: ADDRX_07 may not trigger a final callback.
+
+// Note: ADDRX_07 may not be deallocated.
+
+/// CHECK-NOT: host_op_id=0x0
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-translate-time/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-translate-time/callbacks.h
new file mode 100644
index 0000000000000..5ce283c1a16b3
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-translate-time/callbacks.h
@@ -0,0 +1,166 @@
+#include <assert.h>
+#include <math.h>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_translate_time_t ompt_translate_time_fn = 0;
+static ompt_get_device_time_t ompt_get_device_time_fn = 0;
+
+// OMPT callbacks
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize
+(
+  int device_num,
+  const char *type,
+  ompt_device_t *device,
+  ompt_function_lookup_t lookup,
+  const char *documentation
+ ) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+	 device_num, type, device, lookup, documentation);
+
+  ompt_device_time_t device_time = 0;
+
+  ompt_translate_time_fn = (ompt_translate_time_t) lookup("ompt_translate_time");
+  ompt_get_device_time_fn = (ompt_get_device_time_t) lookup("ompt_get_device_time");
+
+  if (ompt_get_device_time_fn) {
+    uint64_t time = ompt_get_device_time_fn(NULL);
+    printf("The device time can be queried: %lu\n", time);
+    device_time = time;
+  } else {
+    printf("Could not determine the device time\n");
+  }
+
+  if (ompt_translate_time_fn) {
+    assert(device_time > 0 && "Obtain device time was successfull");
+    printf("Translate time available\n");
+    double omp_wtime = omp_get_wtime();
+    device_time = ompt_get_device_time_fn(NULL);
+    double omp_device_wtime = ompt_translate_time_fn(NULL, device_time);
+    printf("OMPT: Host time: %f, Device time: %f\n", omp_wtime, omp_device_wtime);
+  } else {
+    printf("Could not translate time.\n");
+  }
+}
+
+static void on_ompt_callback_device_finalize
+(
+  int device_num
+ ) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load
+    (
+     int device_num,
+     const char *filename,
+     int64_t offset_in_file,
+     void *vma_in_file,
+     size_t bytes,
+     void *host_addr,
+     void *device_addr,
+     uint64_t module_id
+     ) {
+  printf("Callback Load: device_num:%d module_id:%lu filename:%s host_adddr:%p device_addr:%p bytes:%lu\n",
+	 device_num, module_id, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op
+    (
+     ompt_id_t target_id,
+     ompt_id_t host_op_id,
+     ompt_target_data_op_t optype,
+     void *src_addr,
+     int src_device_num,
+     void *dest_addr,
+     int dest_device_num,
+     size_t bytes,
+     const void *codeptr_ra
+     ) {
+  assert(codeptr_ra != 0);
+  // Both src and dest must not be null
+  assert(src_addr != 0 || dest_addr != 0);
+  printf("  Callback DataOp: target_id=%lu host_op_id=%lu optype=%d src=%p src_device_num=%d "
+	 "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+	 target_id, host_op_id, optype, src_addr, src_device_num,
+	 dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target
+    (
+     ompt_target_t kind,
+     ompt_scope_endpoint_t endpoint,
+     int device_num,
+     ompt_data_t *task_data,
+     ompt_id_t target_id,
+     const void *codeptr_ra
+     ) {
+  assert(codeptr_ra != 0);
+  printf("Callback Target: target_id=%lu kind=%d endpoint=%d device_num=%d code=%p\n",
+	 target_id, kind, endpoint, device_num, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit
+    (
+     ompt_id_t target_id,
+     ompt_id_t host_op_id,
+     unsigned int requested_num_teams
+     ) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+     target_id, host_op_id, requested_num_teams);
+}
+
+// Init functions
+int ompt_initialize(
+  ompt_function_lookup_t lookup,
+  int initial_device_num,
+  ompt_data_t *tool_data)
+{
+  ompt_set_callback = (ompt_set_callback_t) lookup("ompt_set_callback");
+
+  if (!ompt_set_callback) return 0; // failed
+  
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+
+
+
+  return 1; //success
+}
+
+void ompt_finalize(ompt_data_t *tool_data)
+{
+}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(
+  unsigned int omp_version,
+  const char *runtime_version)
+{
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,&ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-translate-time/veccopy-ompt-target.c b/offload/test/ompt/tracing/veccopy-ompt-target-translate-time/veccopy-ompt-target.c
new file mode 100644
index 0000000000000..211da6aaacd33
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-translate-time/veccopy-ompt-target.c
@@ -0,0 +1,78 @@
+// clang-format off
+// RUN: %libomptarget-compile-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+#include <stdio.h>
+#include <omp.h>
+
+#include "callbacks.h"
+
+int main()
+{
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i=0; i<N; i++)
+    a[i]=0;
+
+  for (i=0; i<N; i++)
+    b[i]=i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j< N; j++)
+      a[j]=b[j];
+  }
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j< N; j++)
+      a[j]=b[j];
+  }
+
+  int rc = 0;
+  for (i=0; i<N; i++)
+    if (a[i] != b[i] ) {
+      rc++;
+      printf ("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+  /// CHECK: Callback Init:
+  /// CHECK: The device time can be queried: [[X:[0-9]+]] 
+  /// CHECK: Translate time available
+  /// CHECK: Callback Load:
+  /// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+  /// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+  /// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2
+
+  /// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+  /// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+  /// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2
+  /// CHECK: Callback Fini:
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/callbacks.h
new file mode 100644
index 0000000000000..655e230275314
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/callbacks.h
@@ -0,0 +1,160 @@
+#include <assert.h>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_get_record_type_t ompt_get_record_type_fn = 0;
+static ompt_get_device_time_t ompt_get_device_time_fn = 0;
+
+// OMPT callbacks
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize
+(
+  int device_num,
+  const char *type,
+  ompt_device_t *device,
+  ompt_function_lookup_t lookup,
+  const char *documentation
+ ) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+	 device_num, type, device, lookup, documentation);
+
+  ompt_get_record_type_fn = (ompt_get_record_type_t) lookup("ompt_get_record_type");
+  ompt_get_device_time_fn = (ompt_get_device_time_t) lookup("ompt_get_device_time");
+
+  if (ompt_get_record_type_fn) {
+    ompt_buffer_cursor_t buf;
+    // FIXME: For now, we pass a NULL buffer
+    ompt_record_t rec_type = ompt_get_record_type_fn(NULL, buf);
+    printf("Record Type: %s\n", ((rec_type == ompt_record_ompt) ? "OMPT" : "Unknown"));
+  } else {
+    printf("Could not determine Record Type");
+  }
+
+  if (ompt_get_device_time_fn) {
+    uint64_t time = ompt_get_device_time_fn(NULL);
+    printf("The device time can be queried: %lu\n", time);
+  } else {
+    printf("Could not determine the device time \n");
+  }
+}
+
+static void on_ompt_callback_device_finalize
+(
+  int device_num
+ ) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load
+    (
+     int device_num,
+     const char *filename,
+     int64_t offset_in_file,
+     void *vma_in_file,
+     size_t bytes,
+     void *host_addr,
+     void *device_addr,
+     uint64_t module_id
+     ) {
+  printf("Callback Load: device_num:%d module_id:%lu filename:%s host_adddr:%p device_addr:%p bytes:%lu\n",
+	 device_num, module_id, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op
+    (
+     ompt_id_t target_id,
+     ompt_id_t host_op_id,
+     ompt_target_data_op_t optype,
+     void *src_addr,
+     int src_device_num,
+     void *dest_addr,
+     int dest_device_num,
+     size_t bytes,
+     const void *codeptr_ra
+     ) {
+  assert(codeptr_ra != 0);
+  // Both src and dest must not be null
+  assert(src_addr != 0 || dest_addr != 0);
+  printf("  Callback DataOp: target_id=%lu host_op_id=%lu optype=%d src=%p src_device_num=%d "
+	 "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+	 target_id, host_op_id, optype, src_addr, src_device_num,
+	 dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target
+    (
+     ompt_target_t kind,
+     ompt_scope_endpoint_t endpoint,
+     int device_num,
+     ompt_data_t *task_data,
+     ompt_id_t target_id,
+     const void *codeptr_ra
+     ) {
+  assert(codeptr_ra != 0);
+  printf("Callback Target: target_id=%lu kind=%d endpoint=%d device_num=%d code=%p\n",
+	 target_id, kind, endpoint, device_num, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit
+    (
+     ompt_id_t target_id,
+     ompt_id_t host_op_id,
+     unsigned int requested_num_teams
+     ) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+     target_id, host_op_id, requested_num_teams);
+}
+
+// Init functions
+int ompt_initialize(
+  ompt_function_lookup_t lookup,
+  int initial_device_num,
+  ompt_data_t *tool_data)
+{
+  ompt_set_callback = (ompt_set_callback_t) lookup("ompt_set_callback");
+
+  if (!ompt_set_callback) return 0; // failed
+  
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+
+
+
+  return 1; //success
+}
+
+void ompt_finalize(ompt_data_t *tool_data)
+{
+}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(
+  unsigned int omp_version,
+  const char *runtime_version)
+{
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,&ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/veccopy-ompt-target.c b/offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/veccopy-ompt-target.c
new file mode 100644
index 0000000000000..22374a691b8a3
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/veccopy-ompt-target.c
@@ -0,0 +1,78 @@
+// clang-format off
+// RUN: %libomptarget-compile-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+#include <stdio.h>
+#include <omp.h>
+
+#include "callbacks.h"
+
+int main()
+{
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i=0; i<N; i++)
+    a[i]=0;
+
+  for (i=0; i<N; i++)
+    b[i]=i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j< N; j++)
+      a[j]=b[j];
+  }
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j< N; j++)
+      a[j]=b[j];
+  }
+
+  int rc = 0;
+  for (i=0; i<N; i++)
+    if (a[i] != b[i] ) {
+      rc++;
+      printf ("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+  /// CHECK: Callback Init:
+  /// CHECK: Record Type: OMPT
+  /// CHECK: The device time can be queried: [[X:[0-9]+]] 
+  /// CHECK: Callback Load:
+  /// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+  /// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+  /// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2
+
+  /// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+  /// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+  /// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2
+  /// CHECK: Callback Fini:

>From 7575d3950be6d023b1f2e004dbda819dd044befb Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 21 May 2026 09:11:16 -0500
Subject: [PATCH 11/11] [OMPT] Add test for synchronous path w/ OMPT

---
 .../large_mapping_sync_tracing.cpp            | 81 +++++++++++++++++++
 1 file changed, 81 insertions(+)
 create mode 100644 offload/test/ompt/tracing/large-mapping-sync-tracing/large_mapping_sync_tracing.cpp

diff --git a/offload/test/ompt/tracing/large-mapping-sync-tracing/large_mapping_sync_tracing.cpp b/offload/test/ompt/tracing/large-mapping-sync-tracing/large_mapping_sync_tracing.cpp
new file mode 100644
index 0000000000000..44939536d30f3
--- /dev/null
+++ b/offload/test/ompt/tracing/large-mapping-sync-tracing/large_mapping_sync_tracing.cpp
@@ -0,0 +1,81 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-generic && env LIBOMPTARGET_AMDGPU_MAX_ASYNC_COPY_BYTES=1 LIBOMPTARGET_OMPT_FLUSH_ON_SHUTDOWN=false LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL=false %libomptarget-run-generic | %fcheck-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+
+// Exercises OMPT device tracing for AMDGPU's synchronous large-copy path.
+// The transfer size is intentionally above the plugin's default async-copy
+// threshold, while the RUN line also lowers the threshold to keep the test
+// independent of future default changes.
+
+#include <omp.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "../veccopy-ompt-target-data-tracing-emi/callbacks.h"
+
+#define COPY_BYTES (50 * 1024 * 1024)
+
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  if (omp_get_num_devices() == 0) {
+    printf("Success\n");
+    return 0;
+  }
+
+  const size_t N = COPY_BYTES / sizeof(int);
+  int *Input = (int *)malloc(COPY_BYTES);
+  int *Output = (int *)malloc(COPY_BYTES);
+
+  if (!Input || !Output) {
+    free(Input);
+    free(Output);
+    printf("Failure\n");
+    return 1;
+  }
+
+  for (size_t I = 0; I < N; ++I) {
+    Input[I] = (int)(I & 0x7fff);
+    Output[I] = 0;
+  }
+
+#pragma omp target teams distribute parallel for map(to : Input[0 : N])        \
+    map(from : Output[0 : N])
+  for (size_t I = 0; I < N; ++I)
+    Output[I] = Input[I] + 1;
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+  int Failed = 0;
+  for (size_t I = 0; I < N; ++I) {
+    if (Output[I] != Input[I] + 1) {
+      Failed = 1;
+      break;
+    }
+  }
+
+  free(Input);
+  free(Output);
+
+  printf("%s\n", Failed ? "Failure" : "Success");
+  return Failed;
+}
+
+// clang-format off
+
+/// CHECK: Callback Init:
+/// CHECK: Callback Load:
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=2 {{.*}} bytes=52428800
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=2 {{.*}} bytes=52428800
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=3 {{.*}} bytes=52428800
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=3 {{.*}} bytes=52428800
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=4
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=4
+
+/// CHECK-DAG: type=9 (Target data op) {{.*}} optype=1 {{.*}} bytes=52428800
+/// CHECK: Success


From openmp-commits at lists.llvm.org  Tue Aug  4 12:51:18 2026
From: openmp-commits at lists.llvm.org (Robert Imschweiler via Openmp-commits)
Date: Tue, 04 Aug 2026 12:51:18 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [offload] Add global addrspace qualifier
 to KLE and member pointers (PR #213918)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213918@github.com>
Message-ID: <6a7242b6.d76d81af.2931b1.bcc0@mx.google.com>


https://github.com/ro-i edited https://github.com/llvm/llvm-project/pull/213918

From openmp-commits at lists.llvm.org  Tue Aug  4 14:02:26 2026
From: openmp-commits at lists.llvm.org (Spencer Bryngelson via Openmp-commits)
Date: Tue, 04 Aug 2026 14:02:26 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMP][DeviceRTL] Use the actual
 block size in the SPMD no-loop distribute path (PR #214073)
Message-ID: <llvm.org/llvm/llvm-project/pull/214073@github.com>


https://github.com/sbryngelson created https://github.com/llvm/llvm-project/pull/214073

In the SPMD no-loop path the iteration index is `BId * NumThreads + TId`, so `NumThreads` has to
be the size of the block the threads are actually in. `DistributeFor` uses the caller-supplied
value as-is. If that value is larger than the block size, the index strides past the end of each
block and the iterations in between are never run.

Fixes #198621.

Reproduced on gfx90a, `num_teams(4)`, 128 iterations, varying the value passed as `NumThreads`
against the threads the block actually has:

| NumThreads passed | threads in block | iterations not run |
|---|---|---|
| 256 | 32 | 96 of 128 |
| 256 | 64 | 64 of 128 |
| 128 | 32 | 96 of 128 |
| 64 | 64 | 0 |

Exactly as many iterations run as there are threads in the block, and the rest are dropped. With
this patch all four cases run all 128.

AMD carries the same fix downstream as ROCm/llvm-project#3058.

### Testing

The new test calls `__kmpc_distribute_for_static_loop_4u` directly. That is unusual, and the reason
is that no C or C++ construct reaches it: clang lowers worksharing loops to
`__kmpc_for_static_init_4`, and only flang emits `__kmpc_distribute_for_static_loop_*`. Verified on
device IR from both frontends. Calling the entry directly keeps the test independent of the Fortran
runtime, which upstream does not ship for the device.

Verified the test fails without the patch (`unwritten: 96`) and passes with it (`unwritten: 0`).

Also ran 66 Fortran offload probes covering collapse, reductions, private and firstprivate arrays,
nested parallel, simd, allocatable, atomic and complex arithmetic, in no-loop mode with
`exec_mode` confirmed to be 6 rather than assumed: all pass before and after, so the clamp does not
disturb the cases where the caller's value was already correct.

---

Parts of this change were written or audited with Claude Code. I have reviewed all of it and take
full responsibility for the contribution. See `llvm/docs/AIToolPolicy.md`.


>From f727dc83678fcca51f72fd763a7f7b27bf3598ad Mon Sep 17 00:00:00 2001
From: Spencer Bryngelson <sbryngelson at gmail.com>
Date: Tue, 4 Aug 2026 16:02:09 -0500
Subject: [PATCH] Use the actual block size for NumThreads in the SPMD no-loop
 distribute path.

Assisted-by: Claude
---
 .../distribute_for_no_loop_num_threads.c      | 79 +++++++++++++++++++
 openmp/device/src/Workshare.cpp               |  7 ++
 2 files changed, 86 insertions(+)
 create mode 100644 offload/test/offloading/distribute_for_no_loop_num_threads.c

diff --git a/offload/test/offloading/distribute_for_no_loop_num_threads.c b/offload/test/offloading/distribute_for_no_loop_num_threads.c
new file mode 100644
index 0000000000000..d0e3afa59bfcf
--- /dev/null
+++ b/offload/test/offloading/distribute_for_no_loop_num_threads.c
@@ -0,0 +1,79 @@
+// Regression test for the SPMD no-loop path of __kmpc_distribute_for_static_loop.
+//
+// In that path the index is computed as BId * NumThreads + TId, so NumThreads
+// has to be the size of the block the threads are actually in. If the caller
+// passes a larger value the index strides past the end of each block and the
+// iterations in between are never run.
+//
+// The entry point is called directly here because no C or C++ construct reaches
+// it: clang lowers worksharing loops to __kmpc_for_static_init_4, and only flang
+// emits __kmpc_distribute_for_static_loop_*. Calling it directly keeps the test
+// independent of the Fortran runtime.
+//
+// RUN: %libomptarget-compile-run-and-check-generic
+// REQUIRES: gpu
+
+#include <omp.h>
+#include <stdio.h>
+
+#define N 128
+#define NUM_THREADS 256
+#define THREAD_LIMIT 32
+
+struct Args {
+  double *Out;
+};
+
+#pragma omp begin declare target
+extern void __kmpc_distribute_for_static_loop_4u(
+    void *Loc, void (*Fn)(unsigned, void *), void *Arg, unsigned NumIters,
+    unsigned NumThreads, unsigned BlockChunk, unsigned ThreadChunk,
+    unsigned char OneIterationPerThread);
+
+__attribute__((noinline)) static void body(unsigned I, void *A) {
+  ((struct Args *)A)->Out[I] = (double)(I + 1);
+}
+#pragma omp end declare target
+
+// Only the host fallback copy of the target region needs this; the device uses
+// the definition in the device runtime.
+#ifndef __AMDGCN__
+void __kmpc_distribute_for_static_loop_4u(
+    void *Loc, void (*Fn)(unsigned, void *), void *Arg, unsigned NumIters,
+    unsigned NumThreads, unsigned BlockChunk, unsigned ThreadChunk,
+    unsigned char OneIterationPerThread) {
+  for (unsigned I = 0; I < NumIters; ++I)
+    Fn(I, Arg);
+}
+#endif
+
+int main(void) {
+  static double Out[N];
+  for (int I = 0; I < N; ++I)
+    Out[I] = -1.0;
+
+  // NUM_THREADS deliberately exceeds THREAD_LIMIT, the number of threads the
+  // block actually has.
+#pragma omp target teams map(tofrom : Out[0 : N]) num_teams(4)                 \
+    thread_limit(THREAD_LIMIT)
+  {
+#pragma omp parallel
+    {
+      struct Args A;
+      A.Out = Out;
+      __kmpc_distribute_for_static_loop_4u(0, body, &A, N, NUM_THREADS,
+                                           /*BlockChunk=*/0,
+                                           /*ThreadChunk=*/0,
+                                           /*OneIterationPerThread=*/1);
+    }
+  }
+
+  int Unwritten = 0;
+  for (int I = 0; I < N; ++I)
+    if (Out[I] == -1.0)
+      ++Unwritten;
+
+  // CHECK: unwritten: 0
+  printf("unwritten: %d\n", Unwritten);
+  return 0;
+}
diff --git a/openmp/device/src/Workshare.cpp b/openmp/device/src/Workshare.cpp
index 6e6440b690db0..22971b557103a 100644
--- a/openmp/device/src/Workshare.cpp
+++ b/openmp/device/src/Workshare.cpp
@@ -895,6 +895,13 @@ template <typename Ty> class StaticLoopChunker {
     Ty NumBlocks = mapping::getNumberOfBlocksInKernel();
     Ty BId = mapping::getBlockIdInKernel();
 
+    // In the one-iteration-per-thread case the index is computed as
+    // BId * NumThreads + TId, so NumThreads has to be the size of the block
+    // the threads are actually in. A caller-supplied value that is larger
+    // strides past the end of each block and drops the iterations in between.
+    if (OneIterationPerThread)
+      NumThreads = static_cast<Ty>(mapping::getMaxTeamThreads());
+
     // If the block chunk is not specified we pick a default now.
     if (BlockChunk == 0)
       BlockChunk = NumThreads;


From openmp-commits at lists.llvm.org  Tue Aug  4 14:03:11 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Tue, 04 Aug 2026 14:03:11 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMP][DeviceRTL] Use the actual
 block size in the SPMD no-loop distribute path (PR #214073)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/214073@github.com>
Message-ID: <6a72538f.a1accbb4.2dd632.63bf@mx.google.com>


llvmorg-github-actions[bot] wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-offload

Author: Spencer Bryngelson (sbryngelson)

<details>
<summary>Changes</summary>

In the SPMD no-loop path the iteration index is `BId * NumThreads + TId`, so `NumThreads` has to
be the size of the block the threads are actually in. `DistributeFor` uses the caller-supplied
value as-is. If that value is larger than the block size, the index strides past the end of each
block and the iterations in between are never run.

Fixes #<!-- -->198621.

Reproduced on gfx90a, `num_teams(4)`, 128 iterations, varying the value passed as `NumThreads`
against the threads the block actually has:

| NumThreads passed | threads in block | iterations not run |
|---|---|---|
| 256 | 32 | 96 of 128 |
| 256 | 64 | 64 of 128 |
| 128 | 32 | 96 of 128 |
| 64 | 64 | 0 |

Exactly as many iterations run as there are threads in the block, and the rest are dropped. With
this patch all four cases run all 128.

AMD carries the same fix downstream as ROCm/llvm-project#<!-- -->3058.

### Testing

The new test calls `__kmpc_distribute_for_static_loop_4u` directly. That is unusual, and the reason
is that no C or C++ construct reaches it: clang lowers worksharing loops to
`__kmpc_for_static_init_4`, and only flang emits `__kmpc_distribute_for_static_loop_*`. Verified on
device IR from both frontends. Calling the entry directly keeps the test independent of the Fortran
runtime, which upstream does not ship for the device.

Verified the test fails without the patch (`unwritten: 96`) and passes with it (`unwritten: 0`).

Also ran 66 Fortran offload probes covering collapse, reductions, private and firstprivate arrays,
nested parallel, simd, allocatable, atomic and complex arithmetic, in no-loop mode with
`exec_mode` confirmed to be 6 rather than assumed: all pass before and after, so the clamp does not
disturb the cases where the caller's value was already correct.

---

Parts of this change were written or audited with Claude Code. I have reviewed all of it and take
full responsibility for the contribution. See `llvm/docs/AIToolPolicy.md`.


---
Full diff: https://github.com/llvm/llvm-project/pull/214073.diff


2 Files Affected:

- (added) offload/test/offloading/distribute_for_no_loop_num_threads.c (+79) 
- (modified) openmp/device/src/Workshare.cpp (+7) 


``````````diff
diff --git a/offload/test/offloading/distribute_for_no_loop_num_threads.c b/offload/test/offloading/distribute_for_no_loop_num_threads.c
new file mode 100644
index 0000000000000..d0e3afa59bfcf
--- /dev/null
+++ b/offload/test/offloading/distribute_for_no_loop_num_threads.c
@@ -0,0 +1,79 @@
+// Regression test for the SPMD no-loop path of __kmpc_distribute_for_static_loop.
+//
+// In that path the index is computed as BId * NumThreads + TId, so NumThreads
+// has to be the size of the block the threads are actually in. If the caller
+// passes a larger value the index strides past the end of each block and the
+// iterations in between are never run.
+//
+// The entry point is called directly here because no C or C++ construct reaches
+// it: clang lowers worksharing loops to __kmpc_for_static_init_4, and only flang
+// emits __kmpc_distribute_for_static_loop_*. Calling it directly keeps the test
+// independent of the Fortran runtime.
+//
+// RUN: %libomptarget-compile-run-and-check-generic
+// REQUIRES: gpu
+
+#include <omp.h>
+#include <stdio.h>
+
+#define N 128
+#define NUM_THREADS 256
+#define THREAD_LIMIT 32
+
+struct Args {
+  double *Out;
+};
+
+#pragma omp begin declare target
+extern void __kmpc_distribute_for_static_loop_4u(
+    void *Loc, void (*Fn)(unsigned, void *), void *Arg, unsigned NumIters,
+    unsigned NumThreads, unsigned BlockChunk, unsigned ThreadChunk,
+    unsigned char OneIterationPerThread);
+
+__attribute__((noinline)) static void body(unsigned I, void *A) {
+  ((struct Args *)A)->Out[I] = (double)(I + 1);
+}
+#pragma omp end declare target
+
+// Only the host fallback copy of the target region needs this; the device uses
+// the definition in the device runtime.
+#ifndef __AMDGCN__
+void __kmpc_distribute_for_static_loop_4u(
+    void *Loc, void (*Fn)(unsigned, void *), void *Arg, unsigned NumIters,
+    unsigned NumThreads, unsigned BlockChunk, unsigned ThreadChunk,
+    unsigned char OneIterationPerThread) {
+  for (unsigned I = 0; I < NumIters; ++I)
+    Fn(I, Arg);
+}
+#endif
+
+int main(void) {
+  static double Out[N];
+  for (int I = 0; I < N; ++I)
+    Out[I] = -1.0;
+
+  // NUM_THREADS deliberately exceeds THREAD_LIMIT, the number of threads the
+  // block actually has.
+#pragma omp target teams map(tofrom : Out[0 : N]) num_teams(4)                 \
+    thread_limit(THREAD_LIMIT)
+  {
+#pragma omp parallel
+    {
+      struct Args A;
+      A.Out = Out;
+      __kmpc_distribute_for_static_loop_4u(0, body, &A, N, NUM_THREADS,
+                                           /*BlockChunk=*/0,
+                                           /*ThreadChunk=*/0,
+                                           /*OneIterationPerThread=*/1);
+    }
+  }
+
+  int Unwritten = 0;
+  for (int I = 0; I < N; ++I)
+    if (Out[I] == -1.0)
+      ++Unwritten;
+
+  // CHECK: unwritten: 0
+  printf("unwritten: %d\n", Unwritten);
+  return 0;
+}
diff --git a/openmp/device/src/Workshare.cpp b/openmp/device/src/Workshare.cpp
index 6e6440b690db0..22971b557103a 100644
--- a/openmp/device/src/Workshare.cpp
+++ b/openmp/device/src/Workshare.cpp
@@ -895,6 +895,13 @@ template <typename Ty> class StaticLoopChunker {
     Ty NumBlocks = mapping::getNumberOfBlocksInKernel();
     Ty BId = mapping::getBlockIdInKernel();
 
+    // In the one-iteration-per-thread case the index is computed as
+    // BId * NumThreads + TId, so NumThreads has to be the size of the block
+    // the threads are actually in. A caller-supplied value that is larger
+    // strides past the end of each block and drops the iterations in between.
+    if (OneIterationPerThread)
+      NumThreads = static_cast<Ty>(mapping::getMaxTeamThreads());
+
     // If the block chunk is not specified we pick a default now.
     if (BlockChunk == 0)
       BlockChunk = NumThreads;

``````````

</details>


https://github.com/llvm/llvm-project/pull/214073

From openmp-commits at lists.llvm.org  Tue Aug  4 14:48:04 2026
From: openmp-commits at lists.llvm.org (Spencer Bryngelson via Openmp-commits)
Date: Tue, 04 Aug 2026 14:48:04 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMP][DeviceRTL] Use the actual
 block size in the SPMD no-loop distribute path (PR #214073)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/214073@github.com>
Message-ID: <6a725e14.27e38385.2ac6b.6df2@mx.google.com>


https://github.com/sbryngelson updated https://github.com/llvm/llvm-project/pull/214073

>From d6800680f30d333a59c84be598ecc0f569c1eb75 Mon Sep 17 00:00:00 2001
From: Spencer Bryngelson <sbryngelson at gmail.com>
Date: Tue, 4 Aug 2026 16:02:09 -0500
Subject: [PATCH] Use the actual block size for NumThreads in the SPMD no-loop
 distribute path.

Assisted-by: Claude
---
 .../distribute_for_no_loop_num_threads.c      | 71 +++++++++++++++++++
 openmp/device/src/Workshare.cpp               |  5 ++
 2 files changed, 76 insertions(+)
 create mode 100644 offload/test/offloading/distribute_for_no_loop_num_threads.c

diff --git a/offload/test/offloading/distribute_for_no_loop_num_threads.c b/offload/test/offloading/distribute_for_no_loop_num_threads.c
new file mode 100644
index 0000000000000..4ad4012efb81d
--- /dev/null
+++ b/offload/test/offloading/distribute_for_no_loop_num_threads.c
@@ -0,0 +1,71 @@
+// SPMD no-loop distribute: the index is BId * NumThreads + TId, so a NumThreads
+// larger than the real block size drops iterations.
+//
+// The entry is called directly because no C/C++ construct reaches it: clang
+// emits __kmpc_for_static_init_4, only flang emits this one.
+//
+// RUN: %libomptarget-compile-run-and-check-generic
+// REQUIRES: gpu
+
+#include <omp.h>
+#include <stdio.h>
+
+#define N 128
+#define NUM_THREADS 256
+#define THREAD_LIMIT 32
+
+struct Args {
+  double *Out;
+};
+
+#pragma omp begin declare target
+extern void __kmpc_distribute_for_static_loop_4u(
+    void *Loc, void (*Fn)(unsigned, void *), void *Arg, unsigned NumIters,
+    unsigned NumThreads, unsigned BlockChunk, unsigned ThreadChunk,
+    unsigned char OneIterationPerThread);
+
+__attribute__((noinline)) static void body(unsigned I, void *A) {
+  ((struct Args *)A)->Out[I] = (double)(I + 1);
+}
+#pragma omp end declare target
+
+// For the host fallback copy only; the device uses the runtime's definition.
+#ifndef __AMDGCN__
+void __kmpc_distribute_for_static_loop_4u(
+    void *Loc, void (*Fn)(unsigned, void *), void *Arg, unsigned NumIters,
+    unsigned NumThreads, unsigned BlockChunk, unsigned ThreadChunk,
+    unsigned char OneIterationPerThread) {
+  for (unsigned I = 0; I < NumIters; ++I)
+    Fn(I, Arg);
+}
+#endif
+
+int main(void) {
+  static double Out[N];
+  for (int I = 0; I < N; ++I)
+    Out[I] = -1.0;
+
+  // NUM_THREADS deliberately exceeds the block's actual thread count.
+#pragma omp target teams map(tofrom : Out[0 : N]) num_teams(4)                 \
+    thread_limit(THREAD_LIMIT)
+  {
+#pragma omp parallel
+    {
+      struct Args A;
+      A.Out = Out;
+      __kmpc_distribute_for_static_loop_4u(0, body, &A, N, NUM_THREADS,
+                                           /*BlockChunk=*/0,
+                                           /*ThreadChunk=*/0,
+                                           /*OneIterationPerThread=*/1);
+    }
+  }
+
+  int Unwritten = 0;
+  for (int I = 0; I < N; ++I)
+    if (Out[I] == -1.0)
+      ++Unwritten;
+
+  // CHECK: unwritten: 0
+  printf("unwritten: %d\n", Unwritten);
+  return 0;
+}
diff --git a/openmp/device/src/Workshare.cpp b/openmp/device/src/Workshare.cpp
index 6e6440b690db0..a16ef93707989 100644
--- a/openmp/device/src/Workshare.cpp
+++ b/openmp/device/src/Workshare.cpp
@@ -895,6 +895,11 @@ template <typename Ty> class StaticLoopChunker {
     Ty NumBlocks = mapping::getNumberOfBlocksInKernel();
     Ty BId = mapping::getBlockIdInKernel();
 
+    // The index is BId * NumThreads + TId, so NumThreads must be the real
+    // block size; a larger value strides past each block and drops iterations.
+    if (OneIterationPerThread)
+      NumThreads = static_cast<Ty>(mapping::getMaxTeamThreads());
+
     // If the block chunk is not specified we pick a default now.
     if (BlockChunk == 0)
       BlockChunk = NumThreads;


From openmp-commits at lists.llvm.org  Wed Aug  5 00:06:36 2026
From: openmp-commits at lists.llvm.org (Jan Patrick Lehr via Openmp-commits)
Date: Wed, 05 Aug 2026 00:06:36 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OMPT] Upstream device tracing
	implementation (PR #200165)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/200165@github.com>
Message-ID: <6a72e0fc.6c5720b3.3d12fb.b248@mx.google.com>


https://github.com/jplehr updated https://github.com/llvm/llvm-project/pull/200165

>From 7b69c79a05a704db1100a2813badf6ef35297bce Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Tue, 4 Aug 2026 13:39:45 -0500
Subject: [PATCH 01/11] [OpenMP][OMPT] Test libomptarget connector lookup

Verify that the private target connector only exposes supported callbacks and does not retain the unwired frame-enter extension.

Co-Authored-By: Claude <noreply at anthropic.com>
---
 openmp/runtime/test/ompt/misc/target_lookup.c | 36 +++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 openmp/runtime/test/ompt/misc/target_lookup.c

diff --git a/openmp/runtime/test/ompt/misc/target_lookup.c b/openmp/runtime/test/ompt/misc/target_lookup.c
new file mode 100644
index 0000000000000..88c0f5f46b959
--- /dev/null
+++ b/openmp/runtime/test/ompt/misc/target_lookup.c
@@ -0,0 +1,36 @@
+// RUN: %libomp-compile-and-run | FileCheck %s
+// REQUIRES: ompt
+
+#include <omp-tools.h>
+#include <stdio.h>
+
+void ompt_libomp_connect(ompt_start_tool_result_t *result);
+
+static int tool_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                           ompt_data_t *tool_data) {
+  return 1;
+}
+
+static int target_initialize(ompt_function_lookup_t lookup,
+                             int initial_device_num, ompt_data_t *tool_data) {
+  printf("ompt_set_frame_enter: %s\n",
+         lookup("ompt_set_frame_enter") ? "present" : "absent");
+  return 1;
+}
+
+static void ompt_finalize(ompt_data_t *tool_data) {}
+
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t result = {&tool_initialize, &ompt_finalize, 0};
+  return &result;
+}
+
+int main() {
+  static ompt_start_tool_result_t result = {&target_initialize, &ompt_finalize, 0};
+  // This entry point initializes OMPT before invoking the target connector.
+  ompt_libomp_connect(&result);
+  return 0;
+}
+
+// CHECK: ompt_set_frame_enter: absent

>From 08773ad3f32f6dbecef3cd41317334ba98c0e473 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 2 Apr 2026 07:27:58 -0500
Subject: [PATCH 02/11] [Offload] Add GenericProfilerTy abstraction and
 APITypes extensions

Introduce a GenericProfilerTy base class that decouples the plugin
layer from OMPT. This allows profiling/tracing backends to be
implemented independently of the plugin code.

Key changes:
- Add GenericProfiler.h/cpp with virtual hooks for device lifecycle
  events (init, deinit, loadBinary), kernel launch, data alloc/delete,
  kernel completion, and data transfer timing
- Add ProfTimerTy RAII timer for scoped alloc/delete measurements
- Add weak getProfilerToAttach() factory (overridable by OMPT)
- Add ExecAsync and ProfilerData fields to __tgt_async_info
- Replace direct OMPT callback invocations in PluginInterface with
  profiler hook calls (handleInit, handleDeinit, handleLoadBinary,
  handlePreKernelLaunch)
- Add getDeviceTimeStamp() virtual to GenericDeviceTy
- Add Profiler member and getProfiler() to GenericPluginTy
- Suppress ProfilerData during KLE upload to avoid spurious traces
- Add sync path in AsyncInfoWrapperTy::finalize for !ExecAsync

Made-with: Cursor
Co-Authored-By: Claude <noreply at anthropic.com>
---
 offload/include/Shared/APITypes.h             |   5 +
 offload/plugins-nextgen/common/CMakeLists.txt |   1 +
 .../common/include/GenericProfiler.h          | 182 ++++++++++++++++++
 .../common/include/PluginInterface.h          |  35 ++--
 .../common/src/GenericProfiler.cpp            |  35 ++++
 .../common/src/PluginInterface.cpp            |  72 +++----
 6 files changed, 268 insertions(+), 62 deletions(-)
 create mode 100644 offload/plugins-nextgen/common/include/GenericProfiler.h
 create mode 100644 offload/plugins-nextgen/common/src/GenericProfiler.cpp

diff --git a/offload/include/Shared/APITypes.h b/offload/include/Shared/APITypes.h
index 47d8c49bf7ef2..527aa8e617304 100644
--- a/offload/include/Shared/APITypes.h
+++ b/offload/include/Shared/APITypes.h
@@ -85,6 +85,11 @@ struct __tgt_async_info {
   /// ensure it is a valid location while the transfer to the device is
   /// happening.
   KernelLaunchEnvironmentTy KernelLaunchEnvironment;
+
+  /// Opaque handle for profiler-specific data (e.g., OMPT trace record info).
+  /// Owned by the profiler; the runtime threads this pointer through the plugin
+  /// layer to associate async operations with trace records.
+  void *ProfilerData = nullptr;
 };
 
 /// This struct contains all of the arguments to a target kernel region launch.
diff --git a/offload/plugins-nextgen/common/CMakeLists.txt b/offload/plugins-nextgen/common/CMakeLists.txt
index 6ad20796c3ca4..19ce102309a72 100644
--- a/offload/plugins-nextgen/common/CMakeLists.txt
+++ b/offload/plugins-nextgen/common/CMakeLists.txt
@@ -11,6 +11,7 @@ add_public_tablegen_target(PluginErrcodes)
 # don't want to export `PluginInterface` while `add_llvm_library` requires that.
 add_library(PluginCommon OBJECT
   src/PluginInterface.cpp
+  src/GenericProfiler.cpp
   src/GlobalHandler.cpp
   src/JIT.cpp
   src/RecordReplay.cpp
diff --git a/offload/plugins-nextgen/common/include/GenericProfiler.h b/offload/plugins-nextgen/common/include/GenericProfiler.h
new file mode 100644
index 0000000000000..ea3c3766554c8
--- /dev/null
+++ b/offload/plugins-nextgen/common/include/GenericProfiler.h
@@ -0,0 +1,182 @@
+//===- GenericProfiler.h - GenericProfiler interface for use in Plugins ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The GenericProfiler interface allows to implement profiler logic for various
+// backends, such as OMPT or other tracing mechanisms.
+// This enables the plugins to be agnostic of the actual high-level language
+// that is implemented.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OFFLOAD_PLUGINS_NEXTGEN_COMMON_INCLUDE_GENERICPROFILER_H
+#define OFFLOAD_PLUGINS_NEXTGEN_COMMON_INCLUDE_GENERICPROFILER_H
+
+#include "Shared/APITypes.h"
+
+#include <cstdint>
+#include <functional>
+#include <tuple>
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace plugin {
+
+struct GenericDeviceTy;
+struct GenericPluginTy;
+class GenericProfilerTy;
+
+template <typename FunT, typename... ArgsT, size_t... IdxSequence>
+void callViaIndexSeq(FunT F, GenericProfilerTy *P, uint64_t StartNanos,
+                     uint64_t EndNanos, std::tuple<ArgsT...> Args,
+                     std::index_sequence<IdxSequence...>) {
+  F(P, StartNanos, EndNanos, std::get<IdxSequence>(Args)...);
+}
+
+template <typename FunT, typename... ArgsT>
+void callViaUnpack(FunT F, GenericProfilerTy *P, uint64_t StartNanos,
+                   uint64_t EndNanos, std::tuple<ArgsT...> Tup) {
+  callViaIndexSeq(F, P, StartNanos, EndNanos, Tup,
+                  std::index_sequence_for<ArgsT...>{});
+}
+
+/// Abstraction layer to implement different profiler backends.
+///
+/// The plugins call into the GenericProfilerTy to handle the specific events
+/// with whatever specific backend was instantiated. For now, the supported
+/// backends are limited to an OMPT implementation.
+class GenericProfilerTy {
+public:
+  GenericProfilerTy() = default;
+  virtual ~GenericProfilerTy() = default;
+
+  /// Obtain a pointer to profiler-specific data, if any.
+  virtual void *getProfilerSpecificData() { return nullptr; }
+
+  virtual bool isProfilingEnabled() { return false; }
+
+  /// Set the factors which are used to interpolate the device clock compared to
+  /// the host clock. This follows a simple linear interpolation: Slope * <time>
+  /// + Offset.
+  void setTimeConversionFactors(double Slope, double Offset) {
+    HostToDeviceSlope = Slope;
+    HostToDeviceOffset = Offset;
+    setTimeConversionFactorsImpl(HostToDeviceSlope, HostToDeviceOffset);
+  }
+
+  /// Hook that is called when the plugin is initialized.
+  virtual void handleInit(GenericDeviceTy *Device, GenericPluginTy *Plugin) {}
+
+  /// Hook that is called when the plugin is de-initialized.
+  virtual void handleDeinit(GenericDeviceTy *Device, GenericPluginTy *Plugin) {}
+
+  /// Hook that is called when the device image is loaded.
+  virtual void handleLoadBinary(GenericDeviceTy *Device,
+                                GenericPluginTy *Plugin,
+                                const StringRef InputTgtImage) {}
+
+  /// Hook that is called when memory is allocated on the device.
+  virtual void handleDataAlloc(uint64_t StartNanos, uint64_t EndNanos,
+                               void *HostPtr, uint64_t Size, void *Data) {}
+
+  /// Hook that is called when memory is freed on the device.
+  virtual void handleDataDelete(uint64_t StartNanos, uint64_t EndNanos,
+                                void *TgtPtr, void *Data) {}
+
+  /// Hook that is called before launching a kernel.
+  virtual void handlePreKernelLaunch(GenericDeviceTy *Device,
+                                     uint32_t NumBlocks[3],
+                                     __tgt_async_info *AI) {}
+
+  /// Hook that is called when the kernel is finished to extract the specific
+  /// timing info for that kernel execution.
+  virtual void handleKernelCompletion(uint64_t StartNanos, uint64_t EndNanos,
+                                      void *Data) {}
+
+  /// Hook that is called when a data transfer happens to extract timing info
+  /// for that transfer.
+  virtual void handleDataTransfer(uint64_t StartNanos, uint64_t EndNanos,
+                                  void *Data) {}
+
+  /// Allow factors for time conversion between host and device.
+  virtual void setTimeConversionFactorsImpl(double Slope, double Offset) {}
+
+  /// RAII style timer that measures the elapsed time between construction and
+  /// destruction, then invokes a callback with the profiler, start/end times,
+  /// and any captured arguments.
+  template <typename FnT, typename... ArgsT> class ProfTimerTy {
+  public:
+    ProfTimerTy(FnT &&F, GenericProfilerTy *P, GenericDeviceTy *D, ArgsT... As)
+        : Fun(F), Prof(P), Dev(D), Args(As...) {
+      assert(Prof && "GenericProfilerTy is null");
+      assert(Dev && "GenericDeviceTy is null");
+      if (Prof)
+        StartTime = Prof->getDeviceTimeStamp(Dev);
+    }
+
+    ~ProfTimerTy() {
+      assert(Prof && "GenericProfilerTy is null");
+      assert(Dev && "GenericDeviceTy is null");
+      if (Prof) {
+        uint64_t EndTime = Prof->getDeviceTimeStamp(Dev);
+        callViaUnpack(Fun, Prof, StartTime, EndTime, Args);
+      }
+    }
+
+  private:
+    FnT Fun;
+    GenericProfilerTy *Prof;
+    GenericDeviceTy *Dev;
+    uint64_t StartTime = 0;
+    std::tuple<ArgsT...> Args;
+  };
+
+  template <typename FnT, typename... ArgsT>
+  ProfTimerTy(FnT &&, GenericProfilerTy *, ArgsT...)
+      -> ProfTimerTy<FnT, ArgsT...>;
+
+  template <typename FnT, typename... ArgsT> friend class ProfTimerTy;
+
+  /// Returns an RAII style timer, which will handle data allocation timing.
+  [[nodiscard]] auto getScopedDataAllocTimer(GenericDeviceTy *Dev,
+                                             void *HostPtr, uint64_t Size,
+                                             void *ProfData = nullptr) {
+    return ProfTimerTy(
+        [](GenericProfilerTy *P, auto... args) {
+          assert(P && "P was null");
+          P->handleDataAlloc(args...);
+        },
+        this, Dev, HostPtr, Size, ProfData);
+  }
+
+  /// Returns an RAII style timer, which will handle data deletion timing.
+  [[nodiscard]] auto getScopedDataDeleteTimer(GenericDeviceTy *Dev,
+                                              void *TgtPtr,
+                                              void *ProfData = nullptr) {
+    return ProfTimerTy(
+        [](GenericProfilerTy *P, auto... args) {
+          assert(P && "P was null");
+          P->handleDataDelete(args...);
+        },
+        this, Dev, TgtPtr, ProfData);
+  }
+
+protected:
+  double HostToDeviceSlope = 1.0;
+  double HostToDeviceOffset = .0;
+
+private:
+  /// Vendor-specific implementation to obtain device time.
+  uint64_t getDeviceTimeStamp(GenericDeviceTy *D);
+};
+} // namespace plugin
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+#endif // OFFLOAD_PLUGINS_NEXTGEN_COMMON_INCLUDE_GENERICPROFILER_H
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index b3675e5a8700f..ab2f3b2643648 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -29,6 +29,7 @@
 #include "Shared/Requirements.h"
 #include "Shared/Utils.h"
 
+#include "GenericProfiler.h"
 #include "GlobalHandler.h"
 #include "JIT.h"
 #include "MemoryManager.h"
@@ -57,6 +58,12 @@
 
 using namespace llvm::offload::debug;
 
+/// Factory function for creating a profiler. The default (weak) implementation
+/// returns a no-op GenericProfilerTy. When OMPT is enabled, a strong override
+/// returns an OmptProfilerTy instance.
+std::unique_ptr<llvm::omp::target::plugin::GenericProfilerTy>
+getProfilerToAttach();
+
 namespace llvm {
 namespace omp {
 namespace target {
@@ -1191,6 +1198,11 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
   uint32_t getDebugKind() const { return OMPX_DebugKind; }
   virtual uint64_t getClockFrequency() const { return CLOCKS_PER_SEC; }
 
+  /// Get a device-specific timestamp in nanoseconds, used by the profiler
+  /// for timing device operations. Subclasses should override this to provide
+  /// hardware-accurate timestamps (e.g., via HSA system info).
+  virtual uint64_t getDeviceTimeStamp() { return 0; }
+
   /// Get target compute unit kind (e.g., sm_80, or gfx908).
   virtual std::string getComputeUnitKind() const { return "unknown"; }
 
@@ -1448,16 +1460,6 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
   /// This is used to run the RPC server during task synchronization.
   RPCServerTy *RPCServer;
 
-#ifdef OMPT_SUPPORT
-  /// OMPT callback functions
-#define defineOmptCallback(Name, Type, Code) Name##_t Name##_fn = nullptr;
-  FOREACH_OMPT_DEVICE_EVENT(defineOmptCallback)
-#undef defineOmptCallback
-
-  /// Internal representation for OMPT device (initialize & finalize)
-  std::atomic<bool> OmptInitialized;
-#endif
-
   /// The total per-block native shared memory that a kernel may use.
   size_t MaxBlockSharedMemSize = 0;
 };
@@ -1469,7 +1471,8 @@ struct GenericPluginTy {
 
   /// Construct a plugin instance.
   GenericPluginTy(Triple::ArchType TA)
-      : GlobalHandler(nullptr), JIT(TA), RPCServer(nullptr) {}
+      : GlobalHandler(nullptr), JIT(TA), RPCServer(nullptr),
+        Profiler(getProfilerToAttach()) {}
 
   virtual ~GenericPluginTy() {}
 
@@ -1560,7 +1563,12 @@ struct GenericPluginTy {
   /// Tear down any target-specific doorbell resources.
   virtual Error deinitRPCDoorbell() { return Plugin::success(); }
 
-  /// Get a reference to the record and replay interface for the plugin.
+  /// Get a pointer to the profiler attached to this plugin.
+  GenericProfilerTy *getProfiler() {
+    assert(Profiler && "Profiler not initialized");
+    return Profiler.get();
+  }
+
   /// Initialize a device within the plugin.
   Error initDevice(int32_t DeviceId);
 
@@ -1842,6 +1850,9 @@ struct GenericPluginTy {
 
   /// The interface between the plugin and the GPU for host services.
   RPCServerTy *RPCServer;
+
+  /// The profiler backend attached to this plugin (e.g., OMPT).
+  std::unique_ptr<GenericProfilerTy> Profiler;
 };
 
 /// Auxiliary interface class for GenericDeviceResourceManagerTy. This class
diff --git a/offload/plugins-nextgen/common/src/GenericProfiler.cpp b/offload/plugins-nextgen/common/src/GenericProfiler.cpp
new file mode 100644
index 0000000000000..7269a8aaf8991
--- /dev/null
+++ b/offload/plugins-nextgen/common/src/GenericProfiler.cpp
@@ -0,0 +1,35 @@
+//===- GenericProfiler.cpp - GenericProfiler implementation ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#include "GenericProfiler.h"
+#include "PluginInterface.h"
+
+#include <cstdint>
+#include <memory>
+
+__attribute__((weak)) std::unique_ptr<llvm::omp::target::plugin::GenericProfilerTy>
+getProfilerToAttach() {
+  return std::make_unique<llvm::omp::target::plugin::GenericProfilerTy>();
+}
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace plugin {
+
+uint64_t GenericProfilerTy::getDeviceTimeStamp(GenericDeviceTy *D) {
+  if (D)
+    return D->getDeviceTimeStamp();
+  return 0;
+}
+} // namespace plugin
+} // namespace target
+} // namespace omp
+} // namespace llvm
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index e539cbfc55324..2c0d5bda2e362 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -21,10 +21,7 @@
 #include "Utils/ELF.h"
 #include "omptarget.h"
 
-#ifdef OMPT_SUPPORT
-#include "OpenMP/OMPT/Callback.h"
-#include "omp-tools.h"
-#endif
+#include "GenericProfiler.h"
 
 #include "llvm/Bitcode/BitcodeReader.h"
 #include "llvm/Frontend/OpenMP/OMPConstants.h"
@@ -47,7 +44,9 @@ using namespace llvm::offload::debug;
 AsyncInfoWrapperTy::AsyncInfoWrapperTy(GenericDeviceTy &Device,
                                        __tgt_async_info *AsyncInfoPtr)
     : Device(Device),
-      AsyncInfoPtr(AsyncInfoPtr ? AsyncInfoPtr : &LocalAsyncInfo) {}
+      AsyncInfoPtr(AsyncInfoPtr ? AsyncInfoPtr : &LocalAsyncInfo) {
+  this->AsyncInfoPtr->ProfilerData = nullptr;
+}
 
 Error AsyncInfoWrapperTy::synchronize() {
   assert(AsyncInfoPtr && "AsyncInfoWrapperTy already finalized");
@@ -168,9 +167,15 @@ GenericKernelTy::getKernelLaunchEnvironment(
        DPxPTR(&LocalKLE), DPxPTR(*AllocOrErr),
        sizeof(KernelLaunchEnvironmentTy));
 
+  // Temporarily suppress ProfilerData so the KLE upload is not traced as
+  // a user data operation.
+  __tgt_async_info *AI = AsyncInfoWrapper;
+  void *SavedProfilerData = AI->ProfilerData;
+  AI->ProfilerData = nullptr;
   auto Err = GenericDevice.dataSubmit(*AllocOrErr, &LocalKLE,
                                       sizeof(KernelLaunchEnvironmentTy),
                                       AsyncInfoWrapper);
+  AI->ProfilerData = SavedProfilerData;
   if (Err)
     return Err;
   return static_cast<KernelLaunchEnvironmentTy *>(*AllocOrErr);
@@ -331,6 +336,9 @@ Error GenericKernelTy::launch(GenericDeviceTy &GenericDevice, void **ArgPtrs,
     RRHandle = *RRHandleOrErr;
   }
 
+  GenericDevice.Plugin.getProfiler()->handlePreKernelLaunch(
+      &GenericDevice, EffectiveNumBlocks, AsyncInfoWrapper);
+
   if (auto Err = launchImpl(GenericDevice, EffectiveNumThreads,
                             EffectiveNumBlocks, DynBlockMemConf.NativeSize,
                             KernelArgs, LaunchParams, AsyncInfoWrapper))
@@ -508,22 +516,6 @@ GenericDeviceTy::GenericDeviceTy(GenericPluginTy &Plugin, int32_t DeviceId,
   // vendor (u)uid will become available later.
   setDeviceUidFromVendorUid(std::to_string(static_cast<uint64_t>(DeviceId)));
 
-#ifdef OMPT_SUPPORT
-  OmptInitialized.store(false);
-  // Bind the callbacks to this device's member functions
-#define bindOmptCallback(Name, Type, Code)                                     \
-  if (ompt::Initialized && ompt::lookupCallbackByCode) {                       \
-    ompt::lookupCallbackByCode((ompt_callbacks_t)(Code),                       \
-                               ((ompt_callback_t *)&(Name##_fn)));             \
-    ODBG(OLDT_Tool) << "OMPT: class bound " << #Name << "="                    \
-                    << ((void *)(uint64_t)Name##_fn);                          \
-  }
-
-  FOREACH_OMPT_DEVICE_EVENT(bindOmptCallback);
-#undef bindOmptCallback
-
-#endif
-
   // Envar that indicates whether mapped host buffers should be locked
   // automatically. The possible values are boolean (on/off) and a special:
   //   off:       Mapped host buffers are not locked.
@@ -555,17 +547,7 @@ Error GenericDeviceTy::init(GenericPluginTy &Plugin) {
   if (auto Err = initImpl(Plugin))
     return Err;
 
-#ifdef OMPT_SUPPORT
-  if (ompt::Initialized) {
-    bool ExpectedStatus = false;
-    if (OmptInitialized.compare_exchange_strong(ExpectedStatus, true))
-      performOmptCallback(device_initialize, Plugin.getUserId(DeviceId),
-                          /*type=*/getComputeUnitKind().c_str(),
-                          /*device=*/reinterpret_cast<ompt_device_t *>(this),
-                          /*lookup=*/ompt::lookupCallbackByName,
-                          /*documentation=*/nullptr);
-  }
-#endif
+  Plugin.getProfiler()->handleInit(this, &Plugin);
 
   // Read and reinitialize the envars that depend on the device initialization.
   // Notice these two envars may change the stack size and heap size of the
@@ -660,13 +642,7 @@ Error GenericDeviceTy::deinit(GenericPluginTy &Plugin) {
     RecordReplay = nullptr;
   }
 
-#ifdef OMPT_SUPPORT
-  if (ompt::Initialized) {
-    bool ExpectedStatus = true;
-    if (OmptInitialized.compare_exchange_strong(ExpectedStatus, false))
-      performOmptCallback(device_finalize, Plugin.getUserId(DeviceId));
-  }
-#endif
+  Plugin.getProfiler()->handleDeinit(this, &Plugin);
 
   return deinitImpl();
 }
@@ -713,17 +689,7 @@ Expected<DeviceImageTy *> GenericDeviceTy::loadBinary(GenericPluginTy &Plugin,
   if (auto Err = setupRPCServer(Plugin, *Image))
     return std::move(Err);
 
-#ifdef OMPT_SUPPORT
-  if (ompt::Initialized) {
-    size_t Bytes = InputTgtImage.size();
-    performOmptCallback(
-        device_load, Plugin.getUserId(DeviceId),
-        /*FileName=*/nullptr, /*FileOffset=*/0, /*VmaInFile=*/nullptr,
-        /*ImgSize=*/Bytes,
-        /*HostAddr=*/const_cast<unsigned char *>(InputTgtImage.bytes_begin()),
-        /*DeviceAddr=*/nullptr, /* FIXME: ModuleId */ 0);
-  }
-#endif
+  Plugin.getProfiler()->handleLoadBinary(this, &Plugin, InputTgtImage);
 
   // Call any global constructors present on the device.
   if (auto Err = callGlobalConstructors(Plugin, *Image))
@@ -1006,6 +972,9 @@ Error GenericDeviceTy::getDeviceMemorySize(uint64_t &DSize) {
 Expected<void *> GenericDeviceTy::dataAlloc(int64_t Size, void *HostPtr,
                                             TargetAllocTy Kind,
                                             size_t Alignment) {
+  auto ProfTimer =
+      Plugin.getProfiler()->getScopedDataAllocTimer(this, HostPtr, Size);
+
   void *Alloc = nullptr;
 
   // TODO Check alignment.
@@ -1073,6 +1042,9 @@ Expected<void *> GenericDeviceTy::dataAlloc(int64_t Size, void *HostPtr,
 }
 
 Error GenericDeviceTy::dataDelete(void *TgtPtr, TargetAllocTy Kind) {
+  auto ProfTimer =
+      Plugin.getProfiler()->getScopedDataDeleteTimer(this, TgtPtr);
+
   // Free is a noop when recording or replaying.
   if (RecordReplay && RecordReplay->isRecordingOrReplaying())
     return RecordReplay->deallocate(TgtPtr);

>From c5c95a0c8137fd91c3fb2b0bbed84565a7138b4d Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 2 Apr 2026 07:36:31 -0500
Subject: [PATCH 03/11] [Offload][AMDGPU] Wire HSA profiling into
 GenericProfiler abstraction

Add device profiling infrastructure to the AMDGPU plugin so that the
GenericProfiler can receive nanosecond-accurate kernel execution and
data transfer timestamps from the HSA runtime.

Key changes:
- Add ProfilingInfoTy struct to transport HSA profiling data
- Add timeKernelInNsAsync/timeDataTransferInNsAsync callbacks that
  extract dispatch/copy times from HSA signals and call
  handleKernelCompletion/handleDataTransfer on the profiler
- Add getOrNullProfilerSpecificData helper to extract ProfilerData
  from AsyncInfoWrapperTy
- Add getDeviceTimeStamp() override using hsa_system_get_info
- Add getSystemTimestampInNs() for HSA system timestamp queries
- Add schedProfilerKernelTiming/schedProfilerDataTransferTiming to
  StreamSlotTy for scheduling profiler callbacks on stream slots
- Thread ProfilerSpecificData through pushKernelLaunch,
  pushMemoryCopyH2DAsync, pushMemoryCopyD2HAsync, pushMemoryCopyD2DAsync
- Extract ProfilerSpecificData in dataSubmitImpl, dataRetrieveImpl,
  dataExchangeImpl, and launchImpl
- Add stub getDeviceTimeStamp() to CUDA plugin

Made-with: Cursor
Co-Authored-By: Claude <noreply at anthropic.com>
---
 .../amdgpu/dynamic_hsa/hsa.cpp                |   1 +
 .../amdgpu/dynamic_hsa/hsa_ext_amd.h          |   9 +
 offload/plugins-nextgen/amdgpu/src/rtl.cpp    | 204 +++++++++++++++++-
 offload/plugins-nextgen/cuda/src/rtl.cpp      |   3 +
 4 files changed, 209 insertions(+), 8 deletions(-)

diff --git a/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa.cpp b/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa.cpp
index 5c7ec186b0ceb..99202574bfecc 100644
--- a/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa.cpp
+++ b/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa.cpp
@@ -71,6 +71,7 @@ DLWRAP(hsa_amd_signal_create, 5)
 DLWRAP(hsa_amd_signal_async_handler, 5)
 DLWRAP(hsa_amd_pointer_info, 5)
 DLWRAP(hsa_amd_profiling_get_dispatch_time, 3)
+DLWRAP(hsa_amd_profiling_get_async_copy_time, 2)
 DLWRAP(hsa_amd_profiling_set_profiler_enabled, 2)
 DLWRAP(hsa_code_object_reader_create_from_memory, 3)
 DLWRAP(hsa_code_object_reader_destroy, 1)
diff --git a/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa_ext_amd.h b/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa_ext_amd.h
index d26f9248e27ef..35b3fc1a594cc 100644
--- a/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa_ext_amd.h
+++ b/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa_ext_amd.h
@@ -206,6 +206,15 @@ hsa_amd_profiling_get_dispatch_time(hsa_agent_t agent, hsa_signal_t signal,
 hsa_status_t hsa_amd_profiling_set_profiler_enabled(hsa_queue_t *queue,
                                                     int enable);
 
+typedef struct hsa_amd_profiling_async_copy_time_s {
+  uint64_t start;
+  uint64_t end;
+} hsa_amd_profiling_async_copy_time_t;
+
+hsa_status_t
+hsa_amd_profiling_get_async_copy_time(hsa_signal_t signal,
+                                      hsa_amd_profiling_async_copy_time_t *time);
+
 hsa_status_t hsa_amd_vmem_address_reserve(void **va, size_t size,
                                           uint64_t address, uint64_t flags);
 
diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index 29b5006f0d7b6..cfac10ffb9183 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -30,6 +30,7 @@
 #include "Shared/Utils.h"
 #include "Utils/ELF.h"
 
+#include "GenericProfiler.h"
 #include "GlobalHandler.h"
 #include "OffloadAPI.h"
 #include "OpenMP/OMPT/Callback.h"
@@ -95,6 +96,88 @@ struct AMDGPUEventManagerTy;
 struct AMDGPUDeviceImageTy;
 struct AMDGPUMemoryManagerTy;
 struct AMDGPUMemoryPoolTy;
+struct AMDGPUSignalTy;
+
+/// Use to transport information to profiler timing functions.
+struct ProfilingInfoTy {
+  GenericPluginTy *Plugin;
+  hsa_agent_t Agent;
+  AMDGPUSignalTy *Signal;
+  double TicksToTime;
+  void *ProfilerSpecificData;
+};
+
+static ProfilingInfoTy *getProfilingInfo(void *Data);
+
+static std::pair<uint64_t, uint64_t>
+getKernelStartAndEndTime(const ProfilingInfoTy *Args);
+
+static std::pair<uint64_t, uint64_t>
+getCopyStartAndEndTime(const ProfilingInfoTy *Args);
+
+static Error timeKernelInNsAsync(void *Data);
+
+static Error timeDataTransferInNsAsync(void *Data) {
+  auto Args = getProfilingInfo(Data);
+  auto [Start, End] = getCopyStartAndEndTime(Args);
+  Args->Plugin->getProfiler()->handleDataTransfer(Start, End,
+                                                  Args->ProfilerSpecificData);
+  return Plugin::success();
+}
+
+static void *
+getOrNullProfilerSpecificData(AsyncInfoWrapperTy &AsyncInfoWrapper) {
+  __tgt_async_info *AI = AsyncInfoWrapper;
+  return AI ? AI->ProfilerData : nullptr;
+}
+
+} // namespace plugin
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+static double setTicksToTime() {
+  uint64_t TicksFrequency = 1;
+  double TicksToTime = 1.0;
+
+  hsa_status_t Status =
+      hsa_system_get_info(HSA_SYSTEM_INFO_TIMESTAMP_FREQUENCY, &TicksFrequency);
+  if (Status == HSA_STATUS_SUCCESS)
+    TicksToTime = (double)1e9 / (double)TicksFrequency;
+
+  return TicksToTime;
+}
+
+static double TicksToTime = 1.0;
+
+static void setHSATicksToTimeConstant() { TicksToTime = setTicksToTime(); }
+
+/// Get the current HSA-based system timestamp in nanoseconds.
+/// Called by OmptTracing.cpp (from PluginOmpt) for device time queries.
+uint64_t getSystemTimestampInNs() {
+  uint64_t TimeStamp = 0;
+  hsa_status_t Status =
+      hsa_system_get_info(HSA_SYSTEM_INFO_TIMESTAMP, &TimeStamp);
+  if (Status != HSA_STATUS_SUCCESS)
+    return 0;
+  return TimeStamp;
+}
+
+/// Enable or disable HSA async copy profiling for OMPT device tracing.
+/// Called by OmptTracing.cpp when a tool activates/deactivates device tracing.
+/// Full HSA profiling integration (enabling per-copy timing signals) will be
+/// wired up in a follow-on commit.
+void setOmptAsyncCopyProfile(bool Enable) {}
+
+/// Enable or disable HSA queue kernel profiling for OMPT device tracing.
+/// Called by OmptTracing.cpp when a tool activates/deactivates device tracing.
+/// Full HSA queue profiling integration will be wired up in a follow-on commit.
+void setGlobalOmptKernelProfile(void *Device, int Enable) {}
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace plugin {
 
 namespace hsa_utils {
 
@@ -1066,6 +1149,7 @@ struct AMDGPUStreamTy {
       MemcpyArgsTy MemcpyArgs;
       ReleaseBufferArgsTy ReleaseBufferArgs;
       ReleaseSignalArgsTy ReleaseSignalArgs;
+      ProfilingInfoTy ProfilerArgs;
       void *CallbackArgs;
     };
 
@@ -1103,7 +1187,32 @@ struct AMDGPUStreamTy {
       return Plugin::success();
     }
 
-    /// Register a callback to be called on compleition
+    /// Schedule kernel timing measurement via the profiler on the slot.
+    Error schedProfilerKernelTiming(GenericDeviceTy *Device, hsa_agent_t Agent,
+                                    AMDGPUSignalTy *OutputSignal,
+                                    double TicksToTime,
+                                    void *ProfilerSpecificData) {
+      Callbacks.emplace_back(timeKernelInNsAsync);
+      ActionArgs.emplace_back().ProfilerArgs =
+          ProfilingInfoTy{&(Device->Plugin), Agent, OutputSignal, TicksToTime,
+                          ProfilerSpecificData};
+      return Plugin::success();
+    }
+
+    /// Schedule data transfer timing via the profiler on the slot.
+    Error schedProfilerDataTransferTiming(GenericDeviceTy *Device,
+                                          hsa_agent_t Agent,
+                                          AMDGPUSignalTy *OutputSignal,
+                                          double TicksToTime,
+                                          void *ProfilerSpecificData) {
+      Callbacks.emplace_back(timeDataTransferInNsAsync);
+      ActionArgs.emplace_back().ProfilerArgs =
+          ProfilingInfoTy{&(Device->Plugin), Agent, OutputSignal, TicksToTime,
+                          ProfilerSpecificData};
+      return Plugin::success();
+    }
+
+    /// Register a callback to be called on completion.
     Error schedCallback(AMDGPUStreamCallbackTy *Func, void *Data) {
       Callbacks.emplace_back(Func);
       ActionArgs.emplace_back().CallbackArgs = Data;
@@ -1128,6 +1237,12 @@ struct AMDGPUStreamTy {
         } else if (Callback == releaseSignalAction) {
           if (auto Err = releaseSignalAction(&ActionArg))
             return Err;
+        } else if (Callback == timeKernelInNsAsync) {
+          if (auto Err = timeKernelInNsAsync(&ActionArg))
+            return Err;
+        } else if (Callback == timeDataTransferInNsAsync) {
+          if (auto Err = timeDataTransferInNsAsync(&ActionArg))
+            return Err;
         } else if (Callback) {
           if (auto Err = Callback(ActionArg.CallbackArgs))
             return Err;
@@ -1399,7 +1514,8 @@ struct AMDGPUStreamTy {
   Error pushKernelLaunch(const AMDGPUKernelTy &Kernel, void *KernelArgs,
                          uint32_t NumThreads[3], uint32_t NumBlocks[3],
                          uint32_t GroupSize, uint64_t StackSize,
-                         AMDGPUMemoryManagerTy &MemoryManager) {
+                         AMDGPUMemoryManagerTy &MemoryManager,
+                         void *ProfilerSpecificData = nullptr) {
     if (Queue == nullptr)
       return Plugin::error(ErrorCode::INVALID_NULL_POINTER,
                            "target queue was nullptr");
@@ -1420,6 +1536,14 @@ struct AMDGPUStreamTy {
     if (auto Err = Slots[Curr].schedReleaseBuffer(KernelArgs, MemoryManager))
       return Err;
 
+#ifdef OMPT_SUPPORT
+    if (ProfilerSpecificData) {
+      if (auto Err = Slots[Curr].schedProfilerKernelTiming(
+              &Device, Agent, OutputSignal, TicksToTime, ProfilerSpecificData))
+        return Err;
+    }
+#endif
+
     // If we are running an RPC server we want to wake up the server thread
     // whenever there is a kernel running and let it sleep otherwise.
     if (Device.getRPCServer())
@@ -1482,7 +1606,8 @@ struct AMDGPUStreamTy {
   /// manager once the operation completes.
   Error pushMemoryCopyD2HAsync(void *Dst, const void *Src, void *Inter,
                                uint64_t CopySize,
-                               AMDGPUMemoryManagerTy &MemoryManager) {
+                               AMDGPUMemoryManagerTy &MemoryManager,
+                               void *ProfilerSpecificData = nullptr) {
     // Retrieve available signals for the operation's outputs.
     AMDGPUSignalTy *OutputSignals[2] = {};
     if (auto Err = SignalManager.getResources(/*Num=*/2, OutputSignals))
@@ -1545,6 +1670,7 @@ struct AMDGPUStreamTy {
   Error pushMemoryCopyH2DAsync(void *Dst, const void *Src, void *Inter,
                                uint64_t CopySize,
                                AMDGPUMemoryManagerTy &MemoryManager,
+                               void *ProfilerSpecificData = nullptr,
                                size_t NumTimes = 1) {
     // Retrieve available signals for the operation's outputs.
     AMDGPUSignalTy *OutputSignals[2] = {};
@@ -1620,7 +1746,8 @@ struct AMDGPUStreamTy {
 
   // AMDGPUDeviceTy is incomplete here, passing the underlying agent instead
   Error pushMemoryCopyD2DAsync(void *Dst, hsa_agent_t DstAgent, const void *Src,
-                               hsa_agent_t SrcAgent, uint64_t CopySize) {
+                               hsa_agent_t SrcAgent, uint64_t CopySize,
+                               void *ProfilerSpecificData = nullptr) {
     AMDGPUSignalTy *OutputSignal;
     if (auto Err = SignalManager.getResources(/*Num=*/1, &OutputSignal))
       return Err;
@@ -2286,6 +2413,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     if (auto Err = initMemoryPools())
       return Err;
 
+    setHSATicksToTimeConstant();
+
     char GPUName[64];
     if (auto Err = getDeviceAttr(HSA_AGENT_INFO_NAME, GPUName))
       return Err;
@@ -2630,6 +2759,9 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
   /// Returns the clock frequency for the given AMDGPU device.
   uint64_t getClockFrequency() const override { return ClockFrequency; }
 
+  /// Returns the current HSA system timestamp for profiling.
+  uint64_t getDeviceTimeStamp() override { return getSystemTimestampInNs(); }
+
   /// Returns the HSA system timestamp frequency. Zero means unavailable.
   uint64_t getSystemTimestampFrequency() const {
     return SystemTimestampFrequency;
@@ -2839,6 +2971,9 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     AMDGPUStreamTy *Stream = nullptr;
     void *PinnedPtr = nullptr;
 
+    auto ProfilerSpecificData =
+        getOrNullProfilerSpecificData(AsyncInfoWrapper);
+
     // Use one-step asynchronous operation when host memory is already pinned.
     if (void *PinnedPtr =
             PinnedAllocs.getDeviceAccessiblePtrFromPinnedBuffer(HstPtr)) {
@@ -2889,7 +3024,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
       return Err;
 
     return Stream->pushMemoryCopyH2DAsync(TgtPtr, HstPtr, PinnedPtr, Size,
-                                          PinnedMemoryManager);
+                                          PinnedMemoryManager,
+                                          ProfilerSpecificData);
   }
 
   /// Retrieve data from the device (device to host transfer).
@@ -2898,6 +3034,9 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     AMDGPUStreamTy *Stream = nullptr;
     void *PinnedPtr = nullptr;
 
+    auto ProfilerSpecificData =
+        getOrNullProfilerSpecificData(AsyncInfoWrapper);
+
     // Use one-step asynchronous operation when host memory is already pinned.
     if (void *PinnedPtr =
             PinnedAllocs.getDeviceAccessiblePtrFromPinnedBuffer(HstPtr)) {
@@ -2949,7 +3088,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
       return Err;
 
     return Stream->pushMemoryCopyD2HAsync(HstPtr, TgtPtr, PinnedPtr, Size,
-                                          PinnedMemoryManager);
+                                          PinnedMemoryManager,
+                                          ProfilerSpecificData);
   }
 
   Error dataMemcpyImpl(void *DstPtr, const void *SrcPtr, int64_t Size,
@@ -2985,6 +3125,9 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
                          AsyncInfoWrapperTy &AsyncInfoWrapper) override {
     AMDGPUDeviceTy &DstDevice = static_cast<AMDGPUDeviceTy &>(DstGenericDevice);
 
+    auto ProfilerSpecificData =
+        getOrNullProfilerSpecificData(AsyncInfoWrapper);
+
     // For large transfers use synchronous behavior.
     if (Size >= OMPX_MaxAsyncCopyBytes) {
       if (AsyncInfoWrapper.hasQueue())
@@ -3013,7 +3156,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
       return Plugin::success();
 
     return Stream->pushMemoryCopyD2DAsync(DstPtr, DstDevice.getAgent(), SrcPtr,
-                                          getAgent(), (uint64_t)Size);
+                                          getAgent(), (uint64_t)Size,
+                                          ProfilerSpecificData);
   }
 
   /// Insert a data fence between previous data operations and the following
@@ -3091,6 +3235,7 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
 
     return Stream->pushMemoryCopyH2DAsync(TgtPtr, PatternPtr, PinnedPtr,
                                           PatternSize, PinnedMemoryManager,
+                                          /*ProfilerSpecificData=*/nullptr,
                                           Size / PatternSize);
   }
 
@@ -4371,10 +4516,12 @@ Error AMDGPUKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
   // HSA requires the group segment size to include both static and dynamic.
   uint32_t TotalBlockMemSize = getStaticBlockMemSize() + DynBlockMemSize;
 
+  auto ProfilerSpecificData = getOrNullProfilerSpecificData(AsyncInfoWrapper);
+
   // Push the kernel launch into the stream.
   return Stream->pushKernelLaunch(*this, AllArgs, NumThreads, NumBlocks,
                                   TotalBlockMemSize, StackSize,
-                                  ArgsMemoryManager);
+                                  ArgsMemoryManager, ProfilerSpecificData);
 }
 
 Error AMDGPUKernelTy::printLaunchInfoDetails(GenericDeviceTy &GenericDevice,
@@ -4560,6 +4707,47 @@ void AMDGPUQueueTy::callbackError(hsa_status_t Status, hsa_queue_t *Source,
   FATAL_MESSAGE(1, "%s", toString(std::move(Err)).data());
 }
 
+/// Implementation of profiling helper functions.
+static ProfilingInfoTy *getProfilingInfo(void *Data) {
+  return reinterpret_cast<ProfilingInfoTy *>(Data);
+}
+
+static std::pair<uint64_t, uint64_t>
+getKernelStartAndEndTime(const ProfilingInfoTy *Args) {
+  hsa_amd_profiling_dispatch_time_t Time = {};
+  hsa_status_t Status =
+      hsa_amd_profiling_get_dispatch_time(Args->Agent, Args->Signal->get(),
+                                          &Time);
+  if (Status != HSA_STATUS_SUCCESS)
+    return {0, 0};
+  return {static_cast<uint64_t>(Time.start * Args->TicksToTime),
+          static_cast<uint64_t>(Time.end * Args->TicksToTime)};
+}
+
+static std::pair<uint64_t, uint64_t>
+getCopyStartAndEndTime(const ProfilingInfoTy *Args) {
+  hsa_amd_profiling_async_copy_time_t Time = {};
+  hsa_status_t Status =
+      hsa_amd_profiling_get_async_copy_time(Args->Signal->get(), &Time);
+  if (Status != HSA_STATUS_SUCCESS)
+    return {0, 0};
+  return {static_cast<uint64_t>(Time.start * Args->TicksToTime),
+          static_cast<uint64_t>(Time.end * Args->TicksToTime)};
+}
+
+static Error timeKernelInNsAsync(void *Data) {
+  assert(Data && "Invalid data pointer");
+  auto ProfilerInfo = getProfilingInfo(Data);
+  assert(ProfilerInfo && "Invalid profiling info");
+  assert(ProfilerInfo->ProfilerSpecificData &&
+         "Invalid ProfilerSpecificData");
+
+  auto [StartTime, EndTime] = getKernelStartAndEndTime(ProfilerInfo);
+  ProfilerInfo->Plugin->getProfiler()->handleKernelCompletion(
+      StartTime, EndTime, ProfilerInfo->ProfilerSpecificData);
+  return Plugin::success();
+}
+
 } // namespace plugin
 } // namespace target
 } // namespace omp
diff --git a/offload/plugins-nextgen/cuda/src/rtl.cpp b/offload/plugins-nextgen/cuda/src/rtl.cpp
index 997dba577f3ce..4bb2ed1107049 100644
--- a/offload/plugins-nextgen/cuda/src/rtl.cpp
+++ b/offload/plugins-nextgen/cuda/src/rtl.cpp
@@ -1405,6 +1405,9 @@ struct CUDADeviceTy : public GenericDeviceTy {
   /// Returns the clock frequency for the given NVPTX device.
   uint64_t getClockFrequency() const override { return 1000000000; }
 
+  /// Device timestamp stub for CUDA - full profiling is a future extension.
+  uint64_t getDeviceTimeStamp() override { return 0; }
+
 private:
   using CUDAStreamManagerTy = GenericDeviceResourceManagerTy<CUDAStreamRef>;
   using CUDAEventManagerTy = GenericDeviceResourceManagerTy<CUDAEventRef>;

>From 1b7ec2a222956bca50bba5ac5c7cca382ea9959c Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 2 Apr 2026 07:40:01 -0500
Subject: [PATCH 04/11] [Offload][OMPT] Add OmptProfilerTy, common types, and
 plugin tracing shim

Introduce the OMPT-specific profiler backend and shared type
definitions for the device tracing feature.

New files:
- OmptCommonDefs.h: Shared macros (FOREACH_OMPT_DEVICE_TRACING_FN,
  OMPT_IF_TRACING_ENABLED, performOmptCallback), function pointer
  typedefs for libomptarget_ompt_* entry points
- OmptEventInfoTy.h: Small struct carrying NumTeams and TraceRecord
  pointer for async profiler data handoff
- OmptDeviceTracing.h: Plugin-side API surface for device tracing
  state management, device ID mapping, clock correlation, and
  dynamic symbol loading via ensureFuncPtrLoaded template
- OmptProfiler.h/cpp: OmptProfilerTy inheriting GenericProfilerTy
  with OMPT callback bindings and profiler-specific data allocation.
  Provides strong override of getProfilerToAttach() factory
- OmptTracing.cpp (plugin side): Shim that dynamically loads
  libomptarget_ompt_* symbols, implements ompt_set_trace_ompt,
  ompt_start_trace, ompt_stop_trace, ompt_flush_trace,
  ompt_advance_buffer_cursor, ompt_get_record_ompt,
  ompt_get_device_time, ompt_translate_time

Modified:
- common/CMakeLists.txt: Add PluginOmpt static library target

Made-with: Cursor
Co-Authored-By: Claude <noreply at anthropic.com>
---
 offload/include/OpenMP/OMPT/OmptCommonDefs.h  | 127 +++++++
 offload/include/OpenMP/OMPT/OmptEventInfoTy.h |  39 +++
 offload/libomptarget/CMakeLists.txt           |  16 +
 offload/plugins-nextgen/common/CMakeLists.txt |  28 ++
 .../common/OMPT/OmptDeviceTracing.h           | 133 ++++++++
 .../common/OMPT/OmptProfiler.cpp              | 163 +++++++++
 .../common/OMPT/OmptProfiler.h                | 166 ++++++++++
 .../common/OMPT/OmptTracing.cpp               | 313 ++++++++++++++++++
 8 files changed, 985 insertions(+)
 create mode 100644 offload/include/OpenMP/OMPT/OmptCommonDefs.h
 create mode 100644 offload/include/OpenMP/OMPT/OmptEventInfoTy.h
 create mode 100644 offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h
 create mode 100644 offload/plugins-nextgen/common/OMPT/OmptProfiler.cpp
 create mode 100644 offload/plugins-nextgen/common/OMPT/OmptProfiler.h
 create mode 100644 offload/plugins-nextgen/common/OMPT/OmptTracing.cpp

diff --git a/offload/include/OpenMP/OMPT/OmptCommonDefs.h b/offload/include/OpenMP/OMPT/OmptCommonDefs.h
new file mode 100644
index 0000000000000..5391658e80262
--- /dev/null
+++ b/offload/include/OpenMP/OMPT/OmptCommonDefs.h
@@ -0,0 +1,127 @@
+//===------ OmptCommonDefs.h - Common definitions for OMPT --*- C++ -*-----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Common defines and typedefs for OMPT callback and tracing functionality.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OFFLOAD_INCLUDE_OMPTCOMMONDEFS_H
+#define OFFLOAD_INCLUDE_OMPTCOMMONDEFS_H
+
+#ifdef OMPT_SUPPORT
+
+#include "omp-tools.h"
+
+#pragma push_macro("DEBUG_PREFIX")
+#undef DEBUG_PREFIX
+#define DEBUG_PREFIX "OMPT"
+
+#define FUNCPTR_TO_PTR(x) ((void *)(uint64_t)x)
+
+#define FOREACH_OMPT_TARGET_CALLBACK(macro)                                    \
+  FOREACH_OMPT_DEVICE_EVENT(macro)                                             \
+  FOREACH_OMPT_NOEMI_EVENT(macro)                                              \
+  FOREACH_OMPT_EMI_EVENT(macro)
+
+// Common device tracing functions
+#define FOREACH_OMPT_DEVICE_TRACING_FN_COMMON(macro)                           \
+  macro(ompt_set_trace_ompt) macro(ompt_start_trace) macro(ompt_flush_trace)   \
+      macro(ompt_stop_trace) macro(ompt_advance_buffer_cursor)                 \
+          macro(ompt_get_record_type)
+
+// Supported device tracing entry points
+#define FOREACH_OMPT_DEVICE_TRACING_FN(macro)                                  \
+  FOREACH_OMPT_DEVICE_TRACING_FN_COMMON(macro)                                 \
+  macro(ompt_get_record_ompt) macro(ompt_get_device_time)                      \
+      macro(ompt_translate_time)
+
+// Device tracing functionalities, which are also e.g. coupled to mutexes
+#define FOREACH_OMPT_DEVICE_TRACING_FN_IMPLEMENTAIONS(macro)                   \
+  FOREACH_OMPT_DEVICE_TRACING_FN_COMMON(macro)                                 \
+  macro(ompt_set_timestamp) macro(ompt_set_granted_teams)
+
+#define OMPT_API_ROUTINE static
+
+#define OMPT_CALLBACK_AVAILABLE(fn) (llvm::omp::target::ompt::Initialized && fn)
+
+#define OMPT_IF_BUILT(stmt) stmt
+
+#define OMPT_IF_ENABLED(stmts)                                                 \
+  do {                                                                         \
+    if (llvm::omp::target::ompt::Initialized) {                                \
+      stmts                                                                    \
+    }                                                                          \
+  } while (0)
+
+#define OMPT_IF_TRACING_ENABLED(stmts)                                         \
+  do {                                                                         \
+    if (llvm::omp::target::ompt::TracingActive) {                              \
+      stmts                                                                    \
+    }                                                                          \
+  } while (0)
+
+#define OMPT_FRAME_FLAGS (ompt_frame_runtime | OMPT_FRAME_POSITION_DEFAULT)
+
+#if (__PPC64__ | __arm__)
+#define OMPT_GET_FRAME_ADDRESS(level) __builtin_frame_address(level)
+#define OMPT_FRAME_POSITION_DEFAULT ompt_frame_cfa
+#else
+#define OMPT_GET_FRAME_ADDRESS(level) __builtin_frame_address(level)
+#define OMPT_FRAME_POSITION_DEFAULT ompt_frame_framepointer
+#endif
+
+#define OMPT_PTR_UNKNOWN ((void *)0)
+
+#define performIfOmptInitialized(stmt)                                         \
+  do {                                                                         \
+    if (llvm::omp::target::ompt::Initialized) {                                \
+      stmt;                                                                    \
+    }                                                                          \
+  } while (0)
+
+#define performOmptCallback(CallbackName, ...)                                 \
+  do {                                                                         \
+    if (ompt_callback_##CallbackName##_fn)                                     \
+      ompt_callback_##CallbackName##_fn(__VA_ARGS__);                          \
+  } while (0)
+
+typedef ompt_set_result_t (*libomptarget_ompt_set_trace_ompt_t)(
+    int Device, unsigned int Enable, unsigned int EventTy);
+typedef int (*libomptarget_ompt_start_trace_t)(int,
+                                               ompt_callback_buffer_request_t,
+                                               ompt_callback_buffer_complete_t);
+typedef int (*libomptarget_ompt_flush_trace_t)(int);
+typedef int (*libomptarget_ompt_stop_trace_t)(int);
+typedef int (*libomptarget_ompt_advance_buffer_cursor_t)(
+    ompt_device_t *, ompt_buffer_t *, size_t, ompt_buffer_cursor_t,
+    ompt_buffer_cursor_t *);
+typedef ompt_get_record_ompt_t libomptarget_ompt_get_record_ompt_t;
+typedef ompt_device_time_t (*libomptarget_ompt_get_device_time_t)(
+    ompt_device_t *);
+typedef ompt_translate_time_t libomptarget_ompt_translate_time_t;
+typedef ompt_device_time_t (*libomptarget_ompt_get_device_time_t)(
+    ompt_device_t *);
+typedef ompt_record_t (*libomptarget_ompt_get_record_type_t)(
+    ompt_buffer_t *, ompt_buffer_cursor_t);
+typedef void (*libomptarget_ompt_set_timestamp_t)(uint64_t start, uint64_t end);
+typedef void (*libomptarget_ompt_set_granted_teams_t)(uint32_t);
+
+/// Function type def used for maintaining unique target region, target
+/// operations ids
+typedef uint64_t (*IdInterfaceTy)();
+
+#pragma pop_macro("DEBUG_PREFIX")
+
+#else
+#define performIfOmptInitialized(stmt)
+#define OMPT_IF_BUILT(stmt)
+#define OMPT_IF_ENABLED(stmts)
+#define OMPT_IF_TRACING_ENABLED(stmts)
+#endif // OMPT_SUPPORT
+
+#endif // OFFLOAD_INCLUDE_OMPTCOMMONDEFS_H
diff --git a/offload/include/OpenMP/OMPT/OmptEventInfoTy.h b/offload/include/OpenMP/OMPT/OmptEventInfoTy.h
new file mode 100644
index 0000000000000..7124b3a3ff501
--- /dev/null
+++ b/offload/include/OpenMP/OMPT/OmptEventInfoTy.h
@@ -0,0 +1,39 @@
+//===- OmptEventInfoTy.h - OMPT specific trace record data ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Data structure used to communicate OMPT specific profiler data from the
+// high-level libomptarget into the vendor-specific plugins
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OFFLOAD_INCLUDE_OPENMP_OMPT_OMPTEVENTINFOTY_H
+#define OFFLOAD_INCLUDE_OPENMP_OMPT_OMPTEVENTINFOTY_H
+
+#include "Shared/Debug.h"
+
+struct ompt_record_ompt_t;
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace ompt {
+
+/// Holds info needed to fill asynchronous trace records
+struct OmptEventInfoTy {
+  /// The granted number of teams at runtime
+  uint64_t NumTeams;
+  /// Pointer to the actual buffer storage location
+  ompt_record_ompt_t *TraceRecord;
+};
+
+} // namespace ompt
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+#endif // OFFLOAD_INCLUDE_OPENMP_OMPT_OMPTEVENTINFOTY_H
diff --git a/offload/libomptarget/CMakeLists.txt b/offload/libomptarget/CMakeLists.txt
index 8e6314b2a6eae..7909d55eff5c3 100644
--- a/offload/libomptarget/CMakeLists.txt
+++ b/offload/libomptarget/CMakeLists.txt
@@ -47,6 +47,22 @@ endforeach()
 
 target_compile_options(omptarget PRIVATE ${offload_compile_flags})
 target_link_options(omptarget PRIVATE ${offload_link_flags})
+if (OMPT_TARGET_DEFAULT AND LIBOMPTARGET_OMPT_SUPPORT)
+  # OmptProfiler.cpp provides the strong getProfilerToAttach() that routes
+  # plugin device events through OmptProfilerTy. It must be compiled directly
+  # into omptarget so its strong definition unconditionally overrides the weak
+  # fallback in GenericProfiler.cpp (which is embedded into each plugin archive
+  # via the PluginCommon OBJECT library and would otherwise win the link).
+  target_sources(omptarget PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}/../plugins-nextgen/common/OMPT/OmptProfiler.cpp
+  )
+  target_include_directories(omptarget PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}/../plugins-nextgen/common/OMPT
+    ${CMAKE_CURRENT_SOURCE_DIR}/../plugins-nextgen/common/include
+  )
+  # PluginOmpt provides OmptTracing.cpp (plugin-side GPU tracing).
+  target_link_libraries(omptarget PRIVATE PluginOmpt)
+endif()
 
 # libomptarget.so needs to be aware of where the plugins live as they
 # are now separated in the build directory.
diff --git a/offload/plugins-nextgen/common/CMakeLists.txt b/offload/plugins-nextgen/common/CMakeLists.txt
index 19ce102309a72..c3dcb0f53d218 100644
--- a/offload/plugins-nextgen/common/CMakeLists.txt
+++ b/offload/plugins-nextgen/common/CMakeLists.txt
@@ -37,6 +37,34 @@ endif()
 include(FindLibcCommonUtils)
 target_link_libraries(PluginCommon PRIVATE llvm-libc-common-utilities)
 
+if (OMPT_TARGET_DEFAULT AND LIBOMPTARGET_OMPT_SUPPORT)
+  # OmptTracing.cpp calls plugin-specific symbols (HSA profiling etc.) and
+  # is linked per-plugin. OmptProfiler.cpp is added directly to omptarget
+  # in offload/libomptarget/CMakeLists.txt to ensure its strong
+  # getProfilerToAttach() override is unconditionally included.
+  add_library(PluginOmpt STATIC OMPT/OmptTracing.cpp)
+  target_include_directories(PluginOmpt PUBLIC
+    OMPT
+    ${CMAKE_CURRENT_SOURCE_DIR}/include
+    ${CMAKE_CURRENT_BINARY_DIR}/include
+    ${LIBOMPTARGET_LLVM_INCLUDE_DIRS}
+    ${LIBOMPTARGET_BINARY_INCLUDE_DIR}
+    ${LIBOMPTARGET_INCLUDE_DIR}
+  )
+  target_compile_options(PluginOmpt PUBLIC ${offload_compile_flags} -fPIC)
+  target_link_options(PluginOmpt PUBLIC ${offload_link_flags})
+  target_compile_definitions(PluginOmpt PRIVATE
+    TARGET_NAME="Profiler"
+    DEBUG_PREFIX="OMPT"
+  )
+  add_dependencies(PluginOmpt PluginErrcodes)
+endif()
+
+if (TARGET flang_rt.runtime.static)
+  target_link_libraries(PluginCommon PRIVATE flang_rt.runtime.static)
+  target_compile_definitions(PluginCommon PRIVATE OFFLOAD_HAS_FLANG_RT)
+endif()
+
 # Define the TARGET_NAME and DEBUG_PREFIX.
 target_compile_definitions(PluginCommon PRIVATE
   TARGET_NAME=PluginInterface
diff --git a/offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h b/offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h
new file mode 100644
index 0000000000000..5744a784825da
--- /dev/null
+++ b/offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h
@@ -0,0 +1,133 @@
+//===- OmptDeviceTracing.h - Target independent OMPT callbacks --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Interface used by target-independent runtimes to coordinate registration and
+// invocation of OMPT tracing functionality.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_COMMON_OMPTDEVICETRACING_H
+#define OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_COMMON_OMPTDEVICETRACING_H
+
+#ifdef OMPT_SUPPORT
+
+#include "OpenMP/OMPT/OmptCommonDefs.h"
+
+#include "llvm/Support/DynamicLibrary.h"
+
+#include <map>
+#include <memory>
+
+#pragma push_macro("DEBUG_PREFIX")
+#undef DEBUG_PREFIX
+#define DEBUG_PREFIX "OMPT"
+
+extern void setOmptAsyncCopyProfile(bool Enable);
+extern void setGlobalOmptKernelProfile(void *Device, int Enable);
+extern uint64_t getSystemTimestampInNs();
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace ompt {
+
+// Declare OMPT device tracing function entry points
+#define declareOmptTracingFn(Name) extern libomptarget_##Name##_t Name##_fn;
+FOREACH_OMPT_DEVICE_TRACING_FN_IMPLEMENTAIONS(declareOmptTracingFn)
+#undef declareOmptTracingFn
+
+// Declare OMPT device tracing function mutexes
+#define declareOmptTracingFnMutex(Name) extern std::mutex Name##_mutex;
+FOREACH_OMPT_DEVICE_TRACING_FN_IMPLEMENTAIONS(declareOmptTracingFnMutex)
+#undef declareOmptTracingFnMutex
+
+extern std::mutex DeviceIdWritingMutex;
+
+/// Activate tracing on the given device
+void enableDeviceTracing(int DeviceId);
+
+/// Deactivate tracing on the given device
+void disableDeviceTracing(int DeviceId);
+
+/// Set 'start' and 'stop' in trace records
+void setOmptTimestamp(uint64_t StartTime, uint64_t EndTime);
+
+/// Set the linear function correlation between host and device clocks
+void setOmptHostToDeviceRate(double Slope, double Offset);
+
+/// Set / store the number of granted teams in trace records
+void setOmptGrantedNumTeams(uint64_t NumTeams);
+
+/// Lookup the given device pointer and return its RTL device ID
+int getDeviceId(ompt_device_t *Device);
+
+/// Map the given device pointer to the given DeviceId
+void setDeviceId(ompt_device_t *Device, int32_t DeviceId);
+
+/// Rempve the given device pointer from the current mapping
+void removeDeviceId(ompt_device_t *Device);
+
+/// Check whether the provided device is currently traced.
+bool isTracedDevice(int32_t DeviceId);
+
+/// Provide name based lookup for the device tracing functions
+extern ompt_interface_fn_t
+lookupDeviceTracingFn(const char *InterfaceFunctionName);
+
+/// Host to device linear clock correlation
+extern double HostToDeviceSlope;
+
+/// Host to device constant clock offset
+extern double HostToDeviceOffset;
+
+/// Mapping of device pointers to their corresponding RTL device ID
+extern std::map<ompt_device_t *, int32_t> Devices;
+
+/// Mapping of RTL device IDs to their currently enabled tracing event types.
+/// Note: Event type '0' (bit position) indicates if this device is traced.
+extern std::map<int32_t, uint64_t> TracedDevices;
+
+/// OMPT global tracing status. Indicates if at least one device is traced.
+extern bool TracingActive;
+
+/// Parent library pointer
+extern std::shared_ptr<llvm::sys::DynamicLibrary> ParentLibrary;
+
+/// Get the parent library by pointer. If it is not already set, it will set the
+/// parent library pointer.
+std::shared_ptr<llvm::sys::DynamicLibrary> getParentLibrary();
+
+/// Set the parent library by filename
+void setParentLibrary(const char *Filename);
+
+/// Search for FuncName inside the parent library and assign to FuncPtr.
+/// IMPORTANT: This function assumes that the *caller* holds the respective lock
+/// for FuncPtr.
+template <typename FT>
+void ensureFuncPtrLoaded(const std::string &FuncName, FT *FuncPtr) {
+  if (*FuncPtr == nullptr) {
+    if ((ParentLibrary == nullptr && getParentLibrary() == nullptr) ||
+        !ParentLibrary->isValid())
+      return;
+    void *SymbolPtr = ParentLibrary->getAddressOfSymbol(FuncName.c_str());
+    if (SymbolPtr == nullptr)
+      return;
+    *FuncPtr = reinterpret_cast<FT>(SymbolPtr);
+  }
+}
+
+} // namespace ompt
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+#pragma pop_macro("DEBUG_PREFIX")
+
+#endif // OMPT_SUPPORT
+
+#endif // OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_COMMON_OMPTDEVICETRACING_H
diff --git a/offload/plugins-nextgen/common/OMPT/OmptProfiler.cpp b/offload/plugins-nextgen/common/OMPT/OmptProfiler.cpp
new file mode 100644
index 0000000000000..4b74492352c44
--- /dev/null
+++ b/offload/plugins-nextgen/common/OMPT/OmptProfiler.cpp
@@ -0,0 +1,163 @@
+//===- OmptProfiler.cpp - OMPT impl of GenericProfilerTy --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementation of OmptProfilerTy
+//
+//===----------------------------------------------------------------------===//
+
+#include "OmptProfiler.h"
+#include "OpenMP/OMPT/Interface.h"
+#include "PluginInterface.h"
+#include "Shared/Debug.h"
+
+#include <memory>
+
+using namespace llvm::omp::target;
+
+/// Strong override of getProfilerToAttach() that returns an OmptProfilerTy
+/// when OMPT is enabled. This overrides the weak default in GenericProfiler.cpp.
+std::unique_ptr<llvm::omp::target::plugin::GenericProfilerTy>
+getProfilerToAttach() {
+  return std::make_unique<llvm::omp::target::ompt::OmptProfilerTy>();
+}
+
+void ompt::OmptProfilerTy::handleInit(plugin::GenericDeviceTy *Device,
+                                      plugin::GenericPluginTy *Plugin) {
+  auto DeviceId = Device->getDeviceId();
+  auto DevicePtr = reinterpret_cast<ompt_device_t *>(Device);
+  ompt::setDeviceId(DevicePtr, Plugin->getUserId(DeviceId));
+
+  if (ompt::Initialized) {
+    bool ExpectedStatus = false;
+    if (OmptInitialized.compare_exchange_strong(ExpectedStatus, true))
+      performOmptCallback(device_initialize, Plugin->getUserId(DeviceId),
+                          /*type=*/Device->getComputeUnitKind().c_str(),
+                          /*device=*/DevicePtr,
+                          /*lookup=*/ompt::lookupDeviceTracingFn,
+                          /*documentation=*/nullptr);
+  }
+}
+
+void ompt::OmptProfilerTy::handleDeinit(
+    plugin::GenericDeviceTy *Device, target::plugin::GenericPluginTy *Plugin) {
+  auto DeviceId = Device->getDeviceId();
+
+  if (ompt::Initialized) {
+    bool ExpectedStatus = true;
+    if (OmptInitialized.compare_exchange_strong(ExpectedStatus, false))
+      performOmptCallback(device_finalize, Plugin->getUserId(DeviceId));
+  }
+  ompt::removeDeviceId(reinterpret_cast<ompt_device_t *>(Device));
+}
+
+void ompt::OmptProfilerTy::handleLoadBinary(plugin::GenericDeviceTy *Device,
+                                            plugin::GenericPluginTy *Plugin,
+                                            const StringRef InputTgtImage) {
+
+  if (!ompt::Initialized)
+    return;
+
+  auto DeviceId = Device->getDeviceId();
+  size_t Bytes = InputTgtImage.size();
+  performOmptCallback(
+      device_load, Plugin->getUserId(DeviceId),
+      /*FileName=*/nullptr, /*FileOffset=*/0, /*VmaInFile=*/nullptr,
+      /*ImgSize=*/Bytes,
+      /*HostAddr=*/const_cast<unsigned char *>(InputTgtImage.bytes_begin()),
+      /*DeviceAddr=*/nullptr, /* FIXME: ModuleId */ 0);
+}
+
+void ompt::OmptProfilerTy::handleDataAlloc(uint64_t StartNanos,
+                                           uint64_t EndNanos, void *HostPtr,
+                                           uint64_t Size, void *Data) {
+  ompt::setOmptTimestamp(StartNanos, EndNanos);
+}
+
+void ompt::OmptProfilerTy::handleDataDelete(uint64_t StartNanos,
+                                            uint64_t EndNanos, void *TgtPtr,
+                                            void *Data) {
+  ompt::setOmptTimestamp(StartNanos, EndNanos);
+}
+
+void ompt::OmptProfilerTy::handlePreKernelLaunch(
+    plugin::GenericDeviceTy *Device, uint32_t NumBlocks[3],
+    __tgt_async_info *AI) {
+  if (!ompt::isTracedDevice(
+          ompt::getDeviceId(reinterpret_cast<ompt_device_t *>(Device))))
+    return;
+
+  if (AI->ProfilerData == nullptr)
+    return;
+
+  auto ProfilerSpecificData =
+      reinterpret_cast<ompt::OmptEventInfoTy *>(AI->ProfilerData);
+  assert(ProfilerSpecificData && "Invalid ProfilerSpecificData");
+  // Set number of granted teams for OMPT
+  setOmptGrantedNumTeams(NumBlocks[0]);
+  ProfilerSpecificData->NumTeams = NumBlocks[0];
+}
+
+void ompt::OmptProfilerTy::handleKernelCompletion(uint64_t StartNanos,
+                                                  uint64_t EndNanos,
+                                                  void *Data) {
+
+  if (!isProfilingEnabled())
+    return;
+
+  /// Empty data means no tracing in OMPT
+  /// offload/include/OpenMP/OMPT/Interface.h line 492
+  if (!Data)
+    return;
+
+  ODBG(ODT_Tool) << "OMPT-Async: Time kernel for asynchronous execution: Start "
+                 << StartNanos << " End " << EndNanos;
+
+  auto OmptEventInfo = reinterpret_cast<ompt::OmptEventInfoTy *>(Data);
+  assert(OmptEventInfo && "Invalid OmptEventInfo");
+  assert(OmptEventInfo->TraceRecord && "Invalid TraceRecord");
+
+  ompt::RegionInterface.stopTargetSubmitTraceAsync(OmptEventInfo->TraceRecord,
+                                                   OmptEventInfo->NumTeams,
+                                                   StartNanos, EndNanos);
+
+  // Done processing, our responsibility to free the memory
+  freeProfilerDataEntry(OmptEventInfo);
+}
+
+void ompt::OmptProfilerTy::handleDataTransfer(uint64_t StartNanos,
+                                              uint64_t EndNanos, void *Data) {
+
+  if (!isProfilingEnabled())
+    return;
+
+  /// Empty data means no tracing in OMPT
+  /// offload/include/OpenMP/OMPT/Interface.h line 492
+  if (!Data)
+    return;
+
+  ODBG(ODT_Tool) << "OMPT-Async: Time data for asynchronous execution: Start "
+                 << StartNanos << " End " << EndNanos;
+
+  auto OmptEventInfo = reinterpret_cast<ompt::OmptEventInfoTy *>(Data);
+  assert(OmptEventInfo && "Invalid OmptEventInfo");
+  assert(OmptEventInfo->TraceRecord && "Invalid TraceRecord");
+
+  ompt::RegionInterface.stopTargetDataMovementTraceAsync(
+      OmptEventInfo->TraceRecord, StartNanos, EndNanos);
+
+  // Done processing, our responsibility to free the memory
+  freeProfilerDataEntry(OmptEventInfo);
+}
+
+bool ompt::OmptProfilerTy::isProfilingEnabled() { return ompt::TracingActive; }
+
+void ompt::OmptProfilerTy::setTimeConversionFactorsImpl(double Slope,
+                                                        double Offset) {
+  ODBG(ODT_Tool) << "Using Time Slope: " << Slope << " and Offset: " << Offset;
+  setOmptHostToDeviceRate(Slope, Offset);
+}
diff --git a/offload/plugins-nextgen/common/OMPT/OmptProfiler.h b/offload/plugins-nextgen/common/OMPT/OmptProfiler.h
new file mode 100644
index 0000000000000..9a12d0eda4a53
--- /dev/null
+++ b/offload/plugins-nextgen/common/OMPT/OmptProfiler.h
@@ -0,0 +1,166 @@
+//===- OmptProfiler.h - OMPT specific impl of GenericProfilerTy -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// OMPT specific implementation of the GenericProfilerTy class.
+// This class uses the already existing implementation of OMPT to invoke
+// callbacks and perform tracing.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OFFLOAD_PLUGINS_NEXTGEN_COMMON_OMPT_OMPTPROFILERTY_H
+#define OFFLOAD_PLUGINS_NEXTGEN_COMMON_OMPT_OMPTPROFILERTY_H
+
+#include "GenericProfiler.h"
+
+#include "OmptDeviceTracing.h"
+#include "OpenMP/OMPT/Callback.h"
+#include "OpenMP/OMPT/OmptEventInfoTy.h"
+#include "Shared/Debug.h"
+#include "omp-tools.h"
+
+#include <functional>
+#include <tuple>
+
+#pragma push_macro("DEBUG_PREFIX")
+#undef DEBUG_PREFIX
+#define DEBUG_PREFIX "OMPT"
+
+extern uint64_t getSystemTimestampInNs();
+
+using namespace llvm::omp::target::debug;
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace plugin {
+struct GenericDeviceTy;
+struct GenericPluginTy;
+class GenericProfilerTy;
+
+} // namespace plugin
+
+namespace ompt {
+
+// From Callback.h / Callback.cpp
+extern bool Initialized;
+
+/**
+ * Implements an OMPT backend for the Profiler interface used in the plugins.
+ *
+ * Forwards / Implements the different generic hooks with OMPT semantics.
+ */
+class OmptProfilerTy : public plugin::GenericProfilerTy {
+public:
+  /** Public members **/
+  OmptProfilerTy() {
+
+    OmptInitialized.store(false);
+    // Bind the callbacks to this device's member functions
+#define bindOmptCallback(Name, Type, Code)                                     \
+  if (ompt::Initialized && ompt::lookupCallbackByCode) {                       \
+    ompt::lookupCallbackByCode((ompt_callbacks_t)(Code),                       \
+                               ((ompt_callback_t *)&(Name##_fn)));             \
+    ODBG(ODT_Tool) << "class bound " << #Name                                  \
+                   << "=" << ((void *)(uint64_t)Name##_fn);                    \
+  }
+
+    FOREACH_OMPT_DEVICE_EVENT(bindOmptCallback);
+#undef bindOmptCallback
+
+#define bindOmptTracingFunction(FunctionName)                                  \
+  if (ompt::Initialized && ompt::lookupDeviceTracingFn) {                      \
+    FunctionName##_fn = ompt::lookupDeviceTracingFn(#FunctionName);            \
+    ODBG(ODT_Tool) << "device tracing fn bound " << #FunctionName              \
+                   << "=" << ((void *)(uint64_t)FunctionName##_fn);            \
+  }
+
+    FOREACH_OMPT_DEVICE_TRACING_FN_COMMON(bindOmptTracingFunction);
+#undef bindOmptTracingFunction
+  }
+
+  bool isProfilingEnabled() override;
+
+  void handleInit(plugin::GenericDeviceTy *Device,
+                  plugin::GenericPluginTy *Plugin) override;
+
+  void handleDeinit(plugin::GenericDeviceTy *Device,
+                    plugin::GenericPluginTy *Plugin) override;
+
+  void handleLoadBinary(plugin::GenericDeviceTy *Device,
+                        plugin::GenericPluginTy *Plugin,
+                        const StringRef InputTgtImage) override;
+
+  void handleDataAlloc(uint64_t StartNanos, uint64_t EndNanos, void *HostPtr,
+                       uint64_t Size, void *Data) override;
+  void handleDataDelete(uint64_t StartNanos, uint64_t EndNanos, void *TgtPtr,
+                        void *Data) override;
+
+  void handlePreKernelLaunch(plugin::GenericDeviceTy *Device,
+                             uint32_t NumBlocks[3],
+                             __tgt_async_info *AI) override;
+
+  void handleKernelCompletion(uint64_t StartNanos, uint64_t EndNanos,
+                              void *Data) override;
+
+  void handleDataTransfer(uint64_t StartNanos, uint64_t EndNanos,
+                          void *Data) override;
+
+  void setTimeConversionFactorsImpl(double Slope, double Offset) override;
+
+  void *getProfilerSpecificData() override {
+    // TODO: This is ID is not used currently
+    uint64_t Id = OmptProfDataId.fetch_add(1);
+    {
+      std::scoped_lock Lock(ProfilerDataMutex);
+      ProfilerData[Id] = std::make_unique<OmptEventInfoTy>();
+      return ProfilerData[Id].get();
+    }
+  }
+
+  void freeProfilerDataEntry(OmptEventInfoTy *DataPtr) {
+    std::scoped_lock Lock(ProfilerDataMutex);
+
+    for (auto &Entry : ProfilerData)
+      if (Entry.second.get() == DataPtr) {
+        ProfilerData.erase(Entry.first);
+        break;
+      }
+  }
+
+private:
+  /// Holds a unique ID for each allocation of OmptEventInfoTy
+  std::atomic<uint64_t> OmptProfDataId{0};
+
+  /// Holds memory used to store OMPT specific data and pass it down from
+  /// libomptarget into the plugins.
+  std::map<uint64_t, std::unique_ptr<OmptEventInfoTy>> ProfilerData;
+
+  /// Lock to guard STL ProfilerData map
+  std::mutex ProfilerDataMutex;
+
+  /// OMPT callback functions
+#define defineOmptCallback(Name, Type, Code) Name##_t Name##_fn = nullptr;
+  FOREACH_OMPT_DEVICE_EVENT(defineOmptCallback)
+#undef defineOmptCallback
+
+  /// OMPT device tracing functions
+#define defineOmptTracingFunction(Name) ompt_interface_fn_t Name##_fn = nullptr;
+  FOREACH_OMPT_DEVICE_TRACING_FN_COMMON(defineOmptTracingFunction);
+#undef defineOmptTracingFunction
+
+  /// Internal representation for OMPT device (initialize & finalize)
+  std::atomic<bool> OmptInitialized;
+};
+} // namespace ompt
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+#pragma pop_macro("DEBUG_PREFIX")
+
+#endif
diff --git a/offload/plugins-nextgen/common/OMPT/OmptTracing.cpp b/offload/plugins-nextgen/common/OMPT/OmptTracing.cpp
new file mode 100644
index 0000000000000..d5be9b8d9cd8a
--- /dev/null
+++ b/offload/plugins-nextgen/common/OMPT/OmptTracing.cpp
@@ -0,0 +1,313 @@
+//===-- OmptTracing.cpp - Target independent OpenMP target RTL --- C++ -*--===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementation of OMPT tracing interfaces for PluginInterface
+//
+//===----------------------------------------------------------------------===//
+
+#ifdef OMPT_SUPPORT
+
+#include "Shared/Debug.h"
+#include "OmptDeviceTracing.h"
+#include "omp-tools.h"
+
+#include "llvm/Support/DynamicLibrary.h"
+
+#include <atomic>
+#include <cassert>
+#include <cstdlib>
+#include <cstring>
+#include <memory>
+#include <mutex>
+
+#pragma push_macro("DEBUG_PREFIX")
+#undef DEBUG_PREFIX
+#define DEBUG_PREFIX "OMPT"
+
+// Define OMPT device tracing function entry points
+#define defineOmptTracingFn(Name)                                              \
+  libomptarget_##Name##_t llvm::omp::target::ompt::Name##_fn = nullptr;
+FOREACH_OMPT_DEVICE_TRACING_FN_IMPLEMENTAIONS(defineOmptTracingFn)
+#undef defineOmptTracingFn
+
+// Define OMPT device tracing function mutexes
+#define defineOmptTracingFnMutex(Name)                                         \
+  std::mutex llvm::omp::target::ompt::Name##_mutex;
+FOREACH_OMPT_DEVICE_TRACING_FN_IMPLEMENTAIONS(defineOmptTracingFnMutex)
+#undef defineOmptTracingFnMutex
+
+std::mutex llvm::omp::target::ompt::DeviceIdWritingMutex;
+
+using namespace llvm::omp::target::ompt;
+using namespace llvm::omp::target::debug;
+
+std::shared_ptr<llvm::sys::DynamicLibrary>
+    llvm::omp::target::ompt::ParentLibrary(nullptr);
+
+double llvm::omp::target::ompt::HostToDeviceSlope = .0;
+double llvm::omp::target::ompt::HostToDeviceOffset = .0;
+
+std::map<ompt_device_t *, int32_t> llvm::omp::target::ompt::Devices;
+
+std::shared_ptr<llvm::sys::DynamicLibrary>
+llvm::omp::target::ompt::getParentLibrary() {
+  static bool ParentLibraryAssigned = false;
+  if (!ParentLibraryAssigned) {
+    setParentLibrary("libomptarget.so");
+    ParentLibraryAssigned = true;
+  }
+  return ParentLibrary;
+}
+
+void llvm::omp::target::ompt::setParentLibrary(const char *Filename) {
+  if (ParentLibrary)
+    return;
+  std::string ErrorMsg;
+  ParentLibrary = std::make_shared<llvm::sys::DynamicLibrary>(
+      llvm::sys::DynamicLibrary::getPermanentLibrary(Filename, &ErrorMsg));
+  if ((ParentLibrary == nullptr) || (!ParentLibrary->isValid()))
+    REPORT() << "Failed to set parent library: " << ErrorMsg.c_str();
+}
+
+int llvm::omp::target::ompt::getDeviceId(ompt_device_t *Device) {
+  // Block other threads, which might trigger an erase (for the same device)
+  std::unique_lock<std::mutex> Lock(DeviceIdWritingMutex);
+  auto DeviceIterator = Devices.find(Device);
+  if (Device == nullptr || DeviceIterator == Devices.end()) {
+    REPORT() << "Failed to get ID for Device=" << Device;
+    return -1;
+  }
+  return DeviceIterator->second;
+}
+
+void llvm::omp::target::ompt::setDeviceId(ompt_device_t *Device,
+                                          int32_t DeviceId) {
+  assert(Device && "Mapping device ID to nullptr is not allowed");
+  if (Device == nullptr || DeviceId < 0) {
+    REPORT() << "Failed to set ID=%d for Device=" << DeviceId << Device;
+    return;
+  }
+  std::unique_lock<std::mutex> Lock(DeviceIdWritingMutex);
+  auto DeviceIterator = Devices.find(Device);
+  if (DeviceIterator != Devices.end()) {
+    auto CurrentDeviceId = DeviceIterator->second;
+    if (DeviceId == CurrentDeviceId) {
+      REPORT() << "Tried to duplicate OMPT Device= " << Device <<  " ID=" << DeviceId;
+    } else {
+      REPORT() << "Tried to overwrite OMPT Device=" << Device << " (ID=" << CurrentDeviceId << " with new ID=" << DeviceId;
+    }
+    return;
+  }
+  Devices.emplace(Device, DeviceId);
+}
+
+void llvm::omp::target::ompt::removeDeviceId(ompt_device_t *Device) {
+  int DeviceId = getDeviceId(Device);
+  if (DeviceId < 0) {
+    REPORT() << "Tried to remove Device= " << Device <<  " ID=" << DeviceId;
+    return;
+  }
+  std::unique_lock<std::mutex> Lock(DeviceIdWritingMutex);
+  Devices.erase(Device);
+  TracedDevices.erase(DeviceId);
+}
+
+OMPT_API_ROUTINE ompt_set_result_t ompt_set_trace_ompt(ompt_device_t *Device,
+                                                       unsigned int Enable,
+                                                       unsigned int EventTy) {
+  ODBG(ODT_Tool) << "Executing ompt_set_trace_ompt";
+
+  int DeviceId = getDeviceId(Device);
+  if (DeviceId < 0) {
+    REPORT() << "Failed to set trace events for Device=" << Device <<
+                 " (Unknown device) [Enable=" << Enable << " EventTy=" << EventTy;
+    return ompt_set_never;
+  }
+
+  std::unique_lock<std::mutex> Lock(ompt_set_trace_ompt_mutex);
+  ensureFuncPtrLoaded<libomptarget_ompt_set_trace_ompt_t>(
+      "libomptarget_ompt_set_trace_ompt", &ompt_set_trace_ompt_fn);
+  assert(ompt_set_trace_ompt_fn && "libomptarget_ompt_set_trace_ompt loaded");
+  return ompt_set_trace_ompt_fn(DeviceId, Enable, EventTy);
+}
+
+OMPT_API_ROUTINE int
+ompt_start_trace(ompt_device_t *Device, ompt_callback_buffer_request_t Request,
+                 ompt_callback_buffer_complete_t Complete) {
+  ODBG(ODT_Tool) << "Executing ompt_start_trace";
+
+  int DeviceId = getDeviceId(Device);
+  if (DeviceId < 0) {
+    REPORT() << "Failed to start trace for Device=" << Device << " (Unknown device";
+    // Indicate failure
+    return 0;
+  }
+
+  {
+    // Protect the function pointer
+    std::unique_lock<std::mutex> Lock(ompt_start_trace_mutex);
+
+    if (Request && Complete) {
+      llvm::omp::target::ompt::enableDeviceTracing(DeviceId);
+      // Enable asynchronous memory copy profiling
+      setOmptAsyncCopyProfile(/*Enable=*/true);
+      // Enable queue dispatch profiling
+      if (DeviceId >= 0)
+        setGlobalOmptKernelProfile(Device, /*Enable=*/1);
+      else
+        REPORT() << "May not enable kernel profiling for invalid device id=" <<
+               DeviceId;
+    }
+
+    // Call libomptarget specific function
+    ensureFuncPtrLoaded<libomptarget_ompt_start_trace_t>(
+        "libomptarget_ompt_start_trace", &ompt_start_trace_fn);
+    assert(ompt_start_trace_fn && "libomptarget_ompt_start_trace loaded");
+  }
+  return ompt_start_trace_fn(DeviceId, Request, Complete);
+}
+
+OMPT_API_ROUTINE int ompt_flush_trace(ompt_device_t *Device) {
+  ODBG(ODT_Tool) << "Executing ompt_flush_trace";
+
+  std::unique_lock<std::mutex> Lock(ompt_flush_trace_mutex);
+  ensureFuncPtrLoaded<libomptarget_ompt_flush_trace_t>(
+      "libomptarget_ompt_flush_trace", &ompt_flush_trace_fn);
+  assert(ompt_flush_trace_fn && "libomptarget_ompt_flush_trace loaded");
+  return ompt_flush_trace_fn(getDeviceId(Device));
+}
+
+OMPT_API_ROUTINE int ompt_stop_trace(ompt_device_t *Device) {
+  ODBG(ODT_Tool) << "Executing ompt_stop_trace";
+
+  int DeviceId = getDeviceId(Device);
+  if (DeviceId < 0) {
+    REPORT() << "Failed to stop trace for Device=" << Device << " (Unknown device)";
+    // Indicate failure
+    return 0;
+  }
+
+  {
+    // Protect the function pointer
+    std::unique_lock<std::mutex> Lock(ompt_stop_trace_mutex);
+    llvm::omp::target::ompt::disableDeviceTracing(DeviceId);
+    // Disable asynchronous memory copy profiling
+    setOmptAsyncCopyProfile(/*Enable=*/false);
+    // Disable queue dispatch profiling
+    if (DeviceId >= 0)
+      setGlobalOmptKernelProfile(Device, /*Enable=*/0);
+    else
+      REPORT() << "May not disable kernel profiling for invalid device id=" <<
+             DeviceId;
+    ensureFuncPtrLoaded<libomptarget_ompt_stop_trace_t>(
+        "libomptarget_ompt_stop_trace", &ompt_stop_trace_fn);
+    assert(ompt_stop_trace_fn && "libomptarget_ompt_stop_trace loaded");
+  }
+  return ompt_stop_trace_fn(DeviceId);
+}
+
+OMPT_API_ROUTINE ompt_record_ompt_t *
+ompt_get_record_ompt(ompt_buffer_t *Buffer, ompt_buffer_cursor_t CurrentPos) {
+  // TODO In debug mode, get the metadata associated with this buffer
+  // and assert that there are enough bytes for the current record
+
+  // Currently, no synchronization required since a disjoint set of
+  // trace records is handed over to a thread.
+
+  // Note that CurrentPos can be nullptr. In that case, we return
+  // nullptr. The tool has to handle that properly.
+  return (ompt_record_ompt_t *)CurrentPos;
+}
+
+OMPT_API_ROUTINE int ompt_advance_buffer_cursor(ompt_device_t *Device,
+                                                ompt_buffer_t *Buffer,
+                                                size_t Size,
+                                                ompt_buffer_cursor_t CurrentPos,
+                                                ompt_buffer_cursor_t *NextPos) {
+  // Note: The input parameter size is unused here. It refers to the
+  // bytes returned in the corresponding callback.
+  // Advance can be called concurrently, so synchronize setting the
+  // function pointer. The actual libomptarget function does not need
+  // to be synchronized since it must be working on logically disjoint
+  // buffers.
+  std::unique_lock<std::mutex> Lock(ompt_advance_buffer_cursor_mutex);
+  ensureFuncPtrLoaded<libomptarget_ompt_advance_buffer_cursor_t>(
+      "libomptarget_ompt_advance_buffer_cursor",
+      &ompt_advance_buffer_cursor_fn);
+  assert(ompt_advance_buffer_cursor_fn &&
+         "libomptarget_ompt_advance_buffer_cursor loaded");
+  return ompt_advance_buffer_cursor_fn(Device, Buffer, Size, CurrentPos,
+                                       NextPos);
+}
+
+OMPT_API_ROUTINE ompt_record_t
+ompt_get_record_type(ompt_buffer_t *Buffer, ompt_buffer_cursor_t CurrentPos) {
+  std::unique_lock<std::mutex> Lock(ompt_get_record_type_mutex);
+  ensureFuncPtrLoaded<libomptarget_ompt_get_record_type_t>(
+      "libomptarget_ompt_get_record_type", &ompt_get_record_type_fn);
+  assert(ompt_get_record_type_fn && "libomptarget_ompt_get_record_type loaded");
+  return ompt_get_record_type_fn(Buffer, CurrentPos);
+}
+
+OMPT_API_ROUTINE ompt_device_time_t
+ompt_get_device_time(ompt_device_t *Device) {
+  ODBG(ODT_Tool) << "Executing ompt_get_device_time";
+  return getSystemTimestampInNs();
+}
+
+OMPT_API_ROUTINE double ompt_translate_time(ompt_device_t *Device,
+                                            ompt_device_time_t DeviceTime) {
+  // Translate a device time to a meaningful timepoint in host time
+  // We do not need to account for clock-skew / drift. So simple linear
+  // translation using the host to device rate we obtained.
+  double TranslatedTime = DeviceTime * HostToDeviceSlope + HostToDeviceOffset;
+  ODBG(ODT_Tool) << "D2H translated time: " << TranslatedTime;
+
+  return TranslatedTime;
+}
+
+void llvm::omp::target::ompt::setOmptTimestamp(uint64_t StartTime,
+                                               uint64_t EndTime) {
+  std::unique_lock<std::mutex> Lock(ompt_set_timestamp_mutex);
+  ensureFuncPtrLoaded<libomptarget_ompt_set_timestamp_t>(
+      "libomptarget_ompt_set_timestamp", &ompt_set_timestamp_fn);
+  // No need to hold a lock
+  ompt_set_timestamp_fn(StartTime, EndTime);
+}
+
+void llvm::omp::target::ompt::setOmptHostToDeviceRate(double Slope,
+                                                      double Offset) {
+  HostToDeviceSlope = Slope;
+  HostToDeviceOffset = Offset;
+}
+
+void llvm::omp::target::ompt::setOmptGrantedNumTeams(uint64_t NumTeams) {
+  std::unique_lock<std::mutex> Lock(ompt_set_granted_teams_mutex);
+  ensureFuncPtrLoaded<libomptarget_ompt_set_granted_teams_t>(
+      "libomptarget_ompt_set_granted_teams", &ompt_set_granted_teams_fn);
+  // No need to hold a lock
+  ompt_set_granted_teams_fn(NumTeams);
+}
+
+ompt_interface_fn_t llvm::omp::target::ompt::lookupDeviceTracingFn(
+    const char *InterfaceFunctionName) {
+#define compareAgainst(AvailableFunction)                                      \
+  if (strcmp(InterfaceFunctionName, #AvailableFunction) == 0)                  \
+    return (ompt_interface_fn_t)AvailableFunction;
+
+  FOREACH_OMPT_DEVICE_TRACING_FN(compareAgainst);
+#undef compareAgainst
+
+  ODBG(ODT_Tool) << "Warning: Could not find requested function "
+                 << InterfaceFunctionName;
+  return (ompt_interface_fn_t) nullptr;
+}
+
+#pragma pop_macro("DEBUG_PREFIX")
+
+#endif // OMPT_SUPPORT

>From e853dad7fb444f965595e7b72be08a7fde91c6fc Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 2 Apr 2026 07:40:55 -0500
Subject: [PATCH 05/11] [Offload][OMPT] Add tracing extensions to Interface.h

Extend the OMPT Interface class with trace record creation methods and
the async-aware TracerInterfaceRAII for device tracing support.

Key changes to Interface.h:
- Add includes for OmptEventInfoTy.h, APITypes.h, GenericProfiler.h
- Add extern TracingActive flag and isTracingEnabled() declaration
- Add trace start/stop methods for all target operations:
  startTargetDataAllocTrace, stopTargetDataAllocTrace,
  startTargetDataSubmitTrace, startTargetDataDeleteTrace,
  startTargetDataRetrieveTrace, stopTargetDataMovementTraceAsync,
  startTargetSubmitTrace, stopTargetSubmitTraceAsync, and
  corresponding methods for enter/exit/update/target regions
- Add getTraceGenerators<>() template overloads parallel to
  existing getCallbacks<>()
- Add private helpers: setTraceRecordCommon, setTraceRecordTargetDataOp,
  setTraceRecordTargetKernel, setTraceRecordTarget, announceTargetRegion
- Add TracerInterfaceRAII class: async-aware RAII that checks
  isTracingEnabled, calls begin trace generator, allocates
  OmptEventInfoTy via profiler, sets AsyncInfo->ProfilerData
- Rename CallbackPairTy to FunctionPairTy in InterfaceRAII
- Make InvokeInterfaceFunction return a value
- Replace OMPT_IF_BUILT with OMPT_IF_TRACING_ENABLED

Key changes to Callback.h:
- Replace inline macro/typedef definitions with include of
  OmptCommonDefs.h (shared definitions moved there in PR3)

Made-with: Cursor
Co-Authored-By: Claude <noreply at anthropic.com>
---
 offload/include/OpenMP/OMPT/Callback.h  |  26 +--
 offload/include/OpenMP/OMPT/Interface.h | 255 ++++++++++++++++++++++--
 2 files changed, 243 insertions(+), 38 deletions(-)

diff --git a/offload/include/OpenMP/OMPT/Callback.h b/offload/include/OpenMP/OMPT/Callback.h
index 9d545c643223f..7aff68ced0d79 100644
--- a/offload/include/OpenMP/OMPT/Callback.h
+++ b/offload/include/OpenMP/OMPT/Callback.h
@@ -16,34 +16,12 @@
 
 #ifdef OMPT_SUPPORT
 
-#include "omp-tools.h"
+#include "OmptCommonDefs.h"
 
 #pragma push_macro("DEBUG_PREFIX")
 #undef DEBUG_PREFIX
 #define DEBUG_PREFIX "OMPT"
 
-#define FOREACH_OMPT_TARGET_CALLBACK(macro)                                    \
-  FOREACH_OMPT_DEVICE_EVENT(macro)                                             \
-  FOREACH_OMPT_NOEMI_EVENT(macro)                                              \
-  FOREACH_OMPT_EMI_EVENT(macro)
-
-#define performIfOmptInitialized(stmt)                                         \
-  do {                                                                         \
-    if (llvm::omp::target::ompt::Initialized) {                                \
-      stmt;                                                                    \
-    }                                                                          \
-  } while (0)
-
-#define performOmptCallback(CallbackName, ...)                                 \
-  do {                                                                         \
-    if (ompt_callback_##CallbackName##_fn)                                     \
-      ompt_callback_##CallbackName##_fn(__VA_ARGS__);                          \
-  } while (0)
-
-/// Function type def used for maintaining unique target region, target
-/// operations ids
-typedef uint64_t (*IdInterfaceTy)();
-
 namespace llvm {
 namespace omp {
 namespace target {
@@ -98,8 +76,6 @@ extern bool Initialized;
 
 #pragma pop_macro("DEBUG_PREFIX")
 
-#else
-#define performIfOmptInitialized(stmt)
 #endif // OMPT_SUPPORT
 
 #endif // OFFLOAD_INCLUDE_OPENMP_OMPT_CALLBACK_H
diff --git a/offload/include/OpenMP/OMPT/Interface.h b/offload/include/OpenMP/OMPT/Interface.h
index 6961641769b76..54e7564c21ad8 100644
--- a/offload/include/OpenMP/OMPT/Interface.h
+++ b/offload/include/OpenMP/OMPT/Interface.h
@@ -16,14 +16,21 @@
 // Only provide functionality if target OMPT support is enabled
 #ifdef OMPT_SUPPORT
 #include "Callback.h"
+#include "OmptEventInfoTy.h"
+#include "Shared/APITypes.h"
+#include "Shared/Debug.h"
 #include "omp-tools.h"
 
+#include "GenericProfiler.h"
+
 #include "llvm/Support/ErrorHandling.h"
 
 #include <functional>
 #include <tuple>
 
-#define OMPT_IF_BUILT(stmt) stmt
+#pragma push_macro("DEBUG_PREFIX")
+#undef DEBUG_PREFIX
+#define DEBUG_PREFIX "OMPT"
 
 /// Callbacks for target regions require task_data representing the
 /// encountering task.
@@ -42,9 +49,17 @@ namespace ompt {
 static ompt_get_task_data_t ompt_get_task_data_fn;
 static ompt_get_target_task_data_t ompt_get_target_task_data_fn;
 
+/// OMPT global tracing status. Indicates if at least one device is traced.
+extern bool TracingActive;
+
+/// Check if this device traces the given event type
+extern bool isTracingEnabled(int DeviceId, unsigned int EventTy);
+
 /// Used to maintain execution state for this thread
 class Interface {
 public:
+  // Target data callbacks
+
   /// Top-level function for invoking callback before device data allocation
   void beginTargetDataAlloc(int64_t DeviceId, void *HstPtrBegin,
                             void **TgtPtrBegin, size_t Size, void *Code);
@@ -142,6 +157,85 @@ class Interface {
   /// Top-level function for invoking callback after target construct
   void endTarget(int64_t DeviceId, void *Code);
 
+  // Target data tracing
+
+  /// Top-level function for starting trace before device data allocation
+  void startTargetDataAllocTrace(int64_t DeviceId, void *HstPtrBegin,
+                                 void **TgtPtrBegin, size_t Size, void *Code);
+
+  /// Top-level function for stopping trace after device data allocation
+  ompt_record_ompt_t *stopTargetDataAllocTrace(int64_t DeviceId,
+                                               void *HstPtrBegin,
+                                               void **TgtPtrBegin, size_t Size,
+                                               void *Code);
+
+  /// Top-level function for starting trace before data submit
+  ompt_record_ompt_t *startTargetDataSubmitTrace(int64_t SrcDeviceId,
+                                                 void *SrcPtrBegin,
+                                                 int64_t DstDeviceId,
+                                                 void *DstPtrBegin, size_t Size,
+                                                 void *Code);
+
+  /// Top-level function for starting trace before device data deallocation
+  void startTargetDataDeleteTrace(int64_t DeviceId, void *TgtPtrBegin,
+                                  void *Code);
+
+  /// Top-level function for stopping trace after device data deallocation
+  ompt_record_ompt_t *stopTargetDataDeleteTrace(int64_t DeviceId,
+                                                void *TgtPtrBegin, void *Code);
+
+  /// Top-level function for starting trace before data retrieve
+  ompt_record_ompt_t *startTargetDataRetrieveTrace(int64_t SrcDeviceId,
+                                                   void *SrcPtrBegin,
+                                                   int64_t DstDeviceId,
+                                                   void *DstPtrBegin,
+                                                   size_t Size, void *Code);
+
+  ompt_record_ompt_t *
+  stopTargetDataMovementTraceAsync(ompt_record_ompt_t *DataPtr,
+                                   uint64_t NanosStart, uint64_t NanosEnd);
+
+  /// Top-level function for starting trace before kernel dispatch
+  ompt_record_ompt_t *startTargetSubmitTrace(int64_t DeviceId,
+                                             unsigned int NumTeams = 1);
+
+  ompt_record_ompt_t *stopTargetSubmitTraceAsync(ompt_record_ompt_t *DataPtr,
+                                                 unsigned int NumTeams,
+                                                 uint64_t NanosStart,
+                                                 uint64_t NanosStop);
+
+  // Target region tracing
+
+  /// Top-level function for starting trace before target enter data
+  /// construct
+  ompt_record_ompt_t *startTargetDataEnterTrace(int64_t DeviceId, void *Code);
+
+  /// Top-level function for stopping trace after target enter data
+  /// construct
+  ompt_record_ompt_t *stopTargetDataEnterTrace(int64_t DeviceId, void *Code);
+
+  /// Top-level function for starting trace before target exit data
+  /// construct
+  ompt_record_ompt_t *startTargetDataExitTrace(int64_t DeviceId, void *Code);
+
+  /// Top-level function for stopping trace after target exit data
+  /// construct
+  ompt_record_ompt_t *stopTargetDataExitTrace(int64_t DeviceId, void *Code);
+
+  /// Top-level function for starting trace before target update construct
+  ompt_record_ompt_t *startTargetUpdateTrace(int64_t DeviceId, void *Code);
+
+  /// Top-level function for stopping trace after target update construct
+  ompt_record_ompt_t *stopTargetUpdateTrace(int64_t DeviceId, void *Code);
+
+  // Target kernel tracing
+
+  /// Top-level function for starting trace before target construct
+  ompt_record_ompt_t *startTargetTrace(int64_t DeviceId, void *Code);
+
+  /// Top-level function for stopping trace after target construct
+  ompt_record_ompt_t *stopTargetTrace(int64_t DeviceId, void *Code);
+
   // Callback getter: Target data operations
   template <ompt_target_data_op_t OpType> auto getCallbacks() {
     if constexpr (OpType == ompt_target_data_alloc ||
@@ -217,6 +311,69 @@ class Interface {
     llvm_unreachable("Unhandled target operation!");
   }
 
+  // Callback getter: Target data operations
+  template <ompt_target_data_op_t OpType> auto getTraceGenerators() {
+    if constexpr (OpType == ompt_target_data_alloc ||
+                  OpType == ompt_target_data_alloc_async)
+      return std::make_pair(std::mem_fn(&Interface::startTargetDataAllocTrace),
+                            std::mem_fn(&Interface::stopTargetDataAllocTrace));
+
+    if constexpr (OpType == ompt_target_data_delete ||
+                  OpType == ompt_target_data_delete_async)
+      return std::make_pair(std::mem_fn(&Interface::startTargetDataDeleteTrace),
+                            std::mem_fn(&Interface::stopTargetDataDeleteTrace));
+
+    if constexpr (OpType == ompt_target_data_transfer_to_device ||
+                  OpType == ompt_target_data_transfer_to_device_async)
+      return std::make_pair(
+          std::mem_fn(&Interface::startTargetDataSubmitTrace),
+          std::mem_fn(&Interface::stopTargetDataMovementTraceAsync));
+
+    if constexpr (OpType == ompt_target_data_transfer_from_device ||
+                  OpType == ompt_target_data_transfer_from_device_async)
+      return std::make_pair(
+          std::mem_fn(&Interface::startTargetDataRetrieveTrace),
+          std::mem_fn(&Interface::stopTargetDataMovementTraceAsync));
+
+    llvm_unreachable("Unhandled target data operation type!");
+  }
+
+  // Callback getter: Target region operations
+  template <ompt_target_t OpType> auto getTraceGenerators() {
+    if constexpr (OpType == ompt_target_enter_data ||
+                  OpType == ompt_target_enter_data_nowait)
+      return std::make_pair(std::mem_fn(&Interface::startTargetDataEnterTrace),
+                            std::mem_fn(&Interface::stopTargetDataEnterTrace));
+
+    if constexpr (OpType == ompt_target_exit_data ||
+                  OpType == ompt_target_exit_data_nowait)
+      return std::make_pair(std::mem_fn(&Interface::startTargetDataExitTrace),
+                            std::mem_fn(&Interface::stopTargetDataExitTrace));
+
+    if constexpr (OpType == ompt_target_update ||
+                  OpType == ompt_target_update_nowait)
+      return std::make_pair(std::mem_fn(&Interface::startTargetUpdateTrace),
+                            std::mem_fn(&Interface::stopTargetUpdateTrace));
+
+    if constexpr (OpType == ompt_target || OpType == ompt_target_nowait)
+      return std::make_pair(std::mem_fn(&Interface::startTargetTrace),
+                            std::mem_fn(&Interface::stopTargetTrace));
+
+    llvm_unreachable("Unknown target region operation type!");
+  }
+
+  // Callback getter: Kernel launch operation
+  template <ompt_callbacks_t OpType> auto getTraceGenerators() {
+    // We use 'ompt_callbacks_t', because no other enum is currently available
+    // to model a kernel launch / target submit operation.
+    if constexpr (OpType == ompt_callback_target_submit)
+      return std::make_pair(
+          std::mem_fn(&Interface::startTargetSubmitTrace),
+          std::mem_fn(&Interface::stopTargetSubmitTraceAsync));
+
+    llvm_unreachable("Unhandled target operation!");
+  }
+
   /// Setters for target region and target operation correlation ids
   void setTargetDataValue(uint64_t DataValue) { TargetData.value = DataValue; }
   void setTargetDataPtr(void *DataPtr) { TargetData.ptr = DataPtr; }
@@ -240,6 +397,9 @@ class Interface {
   /// Target task data representing the target task region
   ompt_data_t *TargetTaskData = nullptr;
 
+  /// Used for marking begin of a data operation
+  void announceTargetRegion(const char *RegionName);
+
   /// Used for marking begin of a data operation
   void beginTargetDataOperation();
 
@@ -251,6 +411,23 @@ class Interface {
 
   /// Used for marking end of a target region
   void endTargetRegion();
+
+  // Called by all trace generation routines
+  void setTraceRecordCommon(ompt_record_ompt_t *DataPtr,
+                            ompt_callbacks_t CallbackType);
+  // Type specific helpers
+  void setTraceRecordTargetDataOp(ompt_record_target_data_op_t *Record,
+                                  ompt_target_data_op_t DataOpType,
+                                  void *SrcAddr, int64_t SrcDeviceNum,
+                                  void *DstAddr, int64_t DstDeviceNum,
+                                  size_t Bytes, void *CodePtr);
+
+  void setTraceRecordTargetKernel(ompt_record_target_kernel_t *Record,
+                                  unsigned int NumTeams);
+
+  void setTraceRecordTarget(ompt_record_target_t *Record, int64_t DeviceId,
+                            ompt_target_t TargetKind,
+                            ompt_scope_endpoint_t Endpoint, void *CodePtr);
 };
 
 /// Thread local state for target region and associated metadata
@@ -262,14 +439,14 @@ extern thread_local Interface RegionInterface;
 extern thread_local void *ReturnAddress;
 
 template <typename FuncTy, typename ArgsTy, size_t... IndexSeq>
-void InvokeInterfaceFunction(FuncTy Func, ArgsTy Args,
+auto InvokeInterfaceFunction(FuncTy Func, ArgsTy Args,
                              std::index_sequence<IndexSeq...>) {
-  std::invoke(Func, RegionInterface, std::get<IndexSeq>(Args)...);
+  return std::invoke(Func, RegionInterface, std::get<IndexSeq>(Args)...);
 }
 
-template <typename CallbackPairTy, typename... ArgsTy> class InterfaceRAII {
+template <typename FunctionPairTy, typename... ArgsTy> class InterfaceRAII {
 public:
-  InterfaceRAII(CallbackPairTy Callbacks, ArgsTy... Args)
+  InterfaceRAII(FunctionPairTy Callbacks, ArgsTy... Args)
       : Arguments(Args...), beginFunction(std::get<0>(Callbacks)),
         endFunction(std::get<1>(Callbacks)) {
     performIfOmptInitialized(begin());
@@ -290,14 +467,66 @@ template <typename CallbackPairTy, typename... ArgsTy> class InterfaceRAII {
   }
 
   std::tuple<ArgsTy...> Arguments;
-  typename CallbackPairTy::first_type beginFunction;
-  typename CallbackPairTy::second_type endFunction;
+  typename FunctionPairTy::first_type beginFunction;
+  typename FunctionPairTy::second_type endFunction;
 };
 
 // InterfaceRAII's class template argument deduction guide
-template <typename CallbackPairTy, typename... ArgsTy>
-InterfaceRAII(CallbackPairTy Callbacks, ArgsTy... Args)
-    -> InterfaceRAII<CallbackPairTy, ArgsTy...>;
+template <typename FunctionPairTy, typename... ArgsTy>
+InterfaceRAII(FunctionPairTy Callbacks, ArgsTy... Args)
+    -> InterfaceRAII<FunctionPairTy, ArgsTy...>;
+
+/// Similar to the original InterfaceRAII this class is used for tracing and
+/// extends the original with async capabilities. That is: It takes an
+/// additional AsyncInfo reference as argument to populate the relevant fields.
+/// The AsyncInfoTy propagates the info into the RTL / plugins.
+/// TracedDeviceId represents the trace record's device affinity. EventType is
+/// the callback type that needs to be enabled via ompt_set_trace_ompt.
+template <typename FunctionPairTy, typename AsyncInfoTy, typename... ArgsTy>
+class TracerInterfaceRAII {
+public:
+  TracerInterfaceRAII(FunctionPairTy Callbacks, AsyncInfoTy &AsyncInfo,
+                      plugin::GenericProfilerTy *Prof, int TracedDeviceId,
+                      ompt_callbacks_t EventType, ArgsTy... Args)
+      : Arguments(Args...), beginFunction(std::get<0>(Callbacks)) {
+    __tgt_async_info *AI = AsyncInfo;
+    if (isTracingEnabled(TracedDeviceId, EventType)) {
+      auto Record = begin();
+
+      // The Profiler can allocate specific data to be used to pass information
+      // from here to lower parts of the runtime system.
+      // NOTE: It is the responsibility of the programmer to ensure type
+      // compatibility and correct usage of the data. The profiler, however,
+      // OWNS the pointer and frees it at an appropriate time.
+      OmptEventInfoTy *ProfilerData =
+          reinterpret_cast<OmptEventInfoTy *>(Prof->getProfilerSpecificData());
+      ProfilerData->TraceRecord = Record;
+      ProfilerData->NumTeams = 0;
+
+      // Allows to pass down into the plugins via AsyncInfoTy
+      AI->ProfilerData = ProfilerData;
+    } else {
+      // Actively prevent further tracing of this event
+      AI->ProfilerData = nullptr;
+    }
+  }
+
+private:
+  auto begin() {
+    auto IndexSequence =
+        std::make_index_sequence<std::tuple_size_v<decltype(Arguments)>>{};
+    return InvokeInterfaceFunction(beginFunction, Arguments, IndexSequence);
+  }
+
+  std::tuple<ArgsTy...> Arguments;
+  typename FunctionPairTy::first_type beginFunction;
+  /// No end-function here, since the end is called asynchronously from the
+  /// plugins, once the operation has completed.
+};
+
+template <typename FunctionPairTy, typename... ArgsTy>
+TracerInterfaceRAII(FunctionPairTy Callbacks, ArgsTy... Args)
+    -> TracerInterfaceRAII<FunctionPairTy, ArgsTy...>;
 
 /// Used to set and reset the thread-local return address. The RAII is expected
 /// to be created at a runtime entry point when the return address should be
@@ -335,8 +564,8 @@ class ReturnAddressSetterRAII {
 // The getter returns the address stored in the thread local variable.
 #define OMPT_GET_RETURN_ADDRESS llvm::omp::target::ompt::ReturnAddress
 
-#else
-#define OMPT_IF_BUILT(stmt)
-#endif
+#pragma pop_macro("DEBUG_PREFIX")
+
+#endif // OMPT_SUPPORT
 
 #endif // OFFLOAD_INCLUDE_OPENMP_OMPT_INTERFACE_H

>From 53756739228706e58252cbfb80a9dd89a7f0b5bc Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 2 Apr 2026 07:44:36 -0500
Subject: [PATCH 06/11] [Offload][OMPT] Add OmptTracingBufferMgr trace buffer
 infrastructure

Add the trace buffer management subsystem for OMPT device tracing. This
manages allocation, population, and flushing of trace record buffers
between OpenMP worker threads and helper threads.

New files:
- OmptTracingBuffer.h: OmptTracingBufferMgr class with Buffer metadata,
  FlushInfo, TraceRecord types, thread-local per-device buffer pointers,
  helper thread management, and cursor-based record allocation
- OmptTracingBuffer.cpp: Full implementation of buffer lifecycle:
  assignCursor (lock-free fast path for existing buffers, locked path
  for new allocations), triggerFlushOnBufferFull, driveCompletion
  (helper thread main loop), invokeCallbacks, flushBuffer (dispatches
  buffer-completion callbacks for ranges of ready records),
  flushAllBuffers, helper thread start/shutdown/flush-and-shutdown
- OmptTracing.h: Libomptarget-side declarations for tracing state,
  buffer management callback registration, device tracing control,
  timestamp/clock correlation, and thread-local trace record fields.
  Implementation deferred to subsequent PR.

Changes to existing files:
- PluginManager.h: Add OmptTracingBufferMgr* TraceRecordManager member
  with getTraceRecordManager() accessor, include OmptTracingBuffer.h
- libomptarget/CMakeLists.txt: Add OmptTracingBuffer.cpp to omptarget

Key design points:
- Per-thread buffer assignment avoids contention on the common path
- Helper threads process buffers in flush-id order via condition variables
- LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL / FLUSH_ON_SHUTDOWN envars
  control automatic flushing behavior
- Buffer states: init -> ready -> released track record lifecycle
- Flush states: waiting -> processing prevent concurrent helper access

Made-with: Cursor
---
 offload/include/OpenMP/OMPT/OmptTracing.h     | 154 ++++
 .../include/OpenMP/OMPT/OmptTracingBuffer.h   | 412 ++++++++++
 offload/include/PluginManager.h               |  10 +-
 offload/libomptarget/CMakeLists.txt           |   1 +
 .../OpenMP/OMPT/OmptTracingBuffer.cpp         | 763 ++++++++++++++++++
 5 files changed, 1339 insertions(+), 1 deletion(-)
 create mode 100644 offload/include/OpenMP/OMPT/OmptTracing.h
 create mode 100644 offload/include/OpenMP/OMPT/OmptTracingBuffer.h
 create mode 100644 offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp

diff --git a/offload/include/OpenMP/OMPT/OmptTracing.h b/offload/include/OpenMP/OMPT/OmptTracing.h
new file mode 100644
index 0000000000000..2a892582923d5
--- /dev/null
+++ b/offload/include/OpenMP/OMPT/OmptTracing.h
@@ -0,0 +1,154 @@
+//===---- OmptTracing.h - Target independent OMPT callbacks --*- C++ -*----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Interface used by target-independent runtimes to coordinate registration and
+// invocation of OMPT tracing functionality.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OPENMP_LIBOMPTARGET_INCLUDE_OMPTTRACING_H
+#define OPENMP_LIBOMPTARGET_INCLUDE_OMPTTRACING_H
+
+#ifdef OMPT_SUPPORT
+
+#include <unordered_map>
+
+#include "OmptCommonDefs.h"
+#include "OmptTracingBuffer.h"
+
+#pragma push_macro("DEBUG_PREFIX")
+#undef DEBUG_PREFIX
+#define DEBUG_PREFIX "OMPT"
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace ompt {
+
+/// After a timestamp has been read, reset it.
+void resetTimestamp(uint64_t *T);
+
+/// A tool may register unique buffer-request and buffer-completion
+/// callback functions for a device. The following are utility functions to
+/// manage those functions.
+
+/// Given a device-id, return the corresponding buffer-request callback
+/// function.
+ompt_callback_buffer_request_t getBufferRequestFn(int DeviceId);
+
+/// Give a device-id, return the corresponding buffer-completion callback
+/// function.
+ompt_callback_buffer_complete_t getBufferCompleteFn(int DeviceId);
+
+/// Given a device-id, set the corresponding buffer-request and
+/// buffer-completion callback functions.
+void setBufferManagementFns(int DeviceId, ompt_callback_buffer_request_t ReqFn,
+                            ompt_callback_buffer_complete_t CmpltFn);
+
+/// Given a device-id, remove the corresponding buffer-request and
+/// buffer-completion callback functions.
+void removeBufferManagementFns(int DeviceId);
+
+/// Is device tracing stopped for all devices?
+bool isAllDeviceTracingStopped();
+
+/// Invoke callback function for buffer request events
+void ompt_callback_buffer_request(int DeviceId, ompt_buffer_t **BufferPtr,
+                                  size_t *Bytes);
+
+/// Invoke callback function for buffer complete events
+void ompt_callback_buffer_complete(int DeviceId, ompt_buffer_t *Buffer,
+                                   size_t Bytes,
+                                   ompt_buffer_cursor_t BeginCursor,
+                                   int BufferOwned);
+
+/// Set 'start' and 'stop' for the current trace record
+void setOmptTimestamp(uint64_t StartTime, uint64_t EndTime);
+
+/// Set the linear function correlation between host and device clocks
+void setOmptHostToDeviceRate(double Slope, double Offset);
+
+/// Set / store the number of granted teams
+void setOmptGrantedNumTeams(uint64_t NumTeams);
+
+/// Check if (1) tracing is globally active (2) the given device is actively
+/// traced and (3) the given event type is traced on the device
+bool isTracingEnabled(int DeviceId, unsigned int EventTy);
+
+/// Check if the given device is actively traced
+bool isTracedDevice(int DeviceId);
+
+/// Check if the given device is monitoring the provided tracing type
+bool isTracingTypeEnabled(int DeviceId, unsigned int EventTy);
+
+/// Check if the given device is monitoring the provided tracing type 'group'
+/// Where group means we will check for both: EMI and non-EMI event types
+bool isTracingTypeGroupEnabled(int DeviceId, unsigned int EventTy);
+
+/// Set whether the given tracing type should be monitored (or not) on the
+/// device
+void setTracingTypeEnabled(uint64_t &TracedEventTy, bool Enable,
+                           unsigned int EventTy);
+
+/// Set / reset the given tracing types (EventTy = 0 corresponds to 'all')
+ompt_set_result_t setTraceEventTy(int DeviceId, unsigned int Enable,
+                                  unsigned int EventTy);
+
+/// Return thread id
+uint64_t getThreadId();
+
+/// See TracedDevices in OmptDeviceTracing.h
+extern std::map<int32_t, uint64_t> TracedDevices;
+/// Activate tracing on the given device
+void enableDeviceTracing(int DeviceId);
+/// Deactivate tracing on the given device
+void disableDeviceTracing(int DeviceId);
+
+/// Mutexes to serialize invocation of device registration and checks
+extern std::mutex DeviceAccessMutex;
+
+/// Mutexes to serialize invocation of device-independent entry points
+extern std::mutex TraceAccessMutex;
+extern std::mutex TraceControlMutex;
+
+/// Ensure serialization of calls to std::hash
+extern std::mutex TraceHashThreadMutex;
+
+/// Protect map from device-id to the corresponding buffer-request and
+/// buffer-completion callback functions.
+extern std::mutex BufferManagementFnMutex;
+
+/// Map from device-id to the corresponding buffer-request and buffer-completion
+/// callback functions.
+extern std::unordered_map<int, std::pair<ompt_callback_buffer_request_t,
+                                         ompt_callback_buffer_complete_t>>
+    BufferManagementFns;
+
+/// Thread local variables used by the plugin to communicate OMPT information
+/// that are then used to populate trace records. This method assumes a
+/// synchronous implementation, otherwise it won't work.
+extern thread_local uint32_t TraceRecordNumGrantedTeams;
+extern thread_local uint64_t TraceRecordStartTime;
+extern thread_local uint64_t TraceRecordStopTime;
+
+/// Thread local thread-id.
+extern thread_local uint64_t ThreadId;
+
+/// OMPT global tracing status. Indicates if at least one device is traced.
+extern bool TracingActive;
+
+} // namespace ompt
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+#pragma pop_macro("DEBUG_PREFIX")
+
+#endif // OMPT_SUPPORT
+
+#endif // OPENMP_LIBOMPTARGET_INCLUDE_OMPTTRACING_H
diff --git a/offload/include/OpenMP/OMPT/OmptTracingBuffer.h b/offload/include/OpenMP/OMPT/OmptTracingBuffer.h
new file mode 100644
index 0000000000000..5c9f4bf33dae8
--- /dev/null
+++ b/offload/include/OpenMP/OMPT/OmptTracingBuffer.h
@@ -0,0 +1,412 @@
+//===- OmptTracingBuffer.h - Target independent OpenMP target RTL -- C++ *-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Interface to be used for generating and flushing OMPT device trace records.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OPENMP_LIBOMPTARGET_OMPTTRACINGBUFFER_H
+#define OPENMP_LIBOMPTARGET_OMPTTRACINGBUFFER_H
+
+#ifdef OMPT_SUPPORT
+
+#include <atomic>
+#include <cassert>
+#include <condition_variable>
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <set>
+#include <thread>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#include <omp-tools.h>
+
+#include "Shared/EnvironmentVar.h"
+
+// Maximum number of devices supported in device tracing. No device tracing
+// will be performed for any device-id larger than 1023.
+#define MAX_NUM_DEVICES 1024
+
+// TODO Start with 1 helper thread and add dynamically if required
+// Number of helper threads must not execeed 32 since the
+// thread-wait-tracker is 32 bits in length.
+#define OMPT_NUM_HELPER_THREADS 1
+
+/*
+ * Buffer manager for trace records generated by OpenMP master and
+ * worker threads. During device init, a tool may register a
+ * buffer-request and a buffer-completion callback. The buffer-request
+ * callback should be used to allocate new buffers as required. The
+ * buffer-complete callback should be used to return trace records to
+ * the tool.
+ *
+ * In addition to trace records, this class manages the helper threads
+ * for dispatching a range of trace records to the tool.
+ */
+class OmptTracingBufferMgr {
+public:
+  /*
+   * A trace record (TR) holds the trace data. Its type
+   * can be ompt or native. Currently, only ompt type is implemented.
+   */
+
+  /*
+   * A TR can be in the following states:
+   * TR_init: initial state
+   * TR_ready: An OpenMP thread marks a TR ready when it is done
+   * populating the TR
+   * TR_released: A helper thread marks a TR released after it has
+   * completed returning the TR to the tool
+   */
+  enum TRStatus { TR_init, TR_ready, TR_released };
+
+  /*
+   * Metadata capturing the state of a buffer of trace records. Once a
+   * buffer is allocated by an OpenMP worker thread, trace records are
+   * carved out from that buffer by that same OpenMP thread alone. Thus
+   * the allocated buffer is thread-specific from the allocation/population
+   * standpoint. But it may be manipulated by helper threads.
+   *
+   * Id, DeviceId, Start, and TotalBytes are not changed once set.
+   * RemainingBytes could be written multiple times but only by the same
+   * thread. But Cursor and IsFull may be read/written by an OpenMP worker
+   * thread and read by helper threads. Hence, accesses of
+   * this 2nd set of locations need to be atomic or synchronized.
+   */
+  struct Buffer {
+    uint64_t Id;                // Unique identifier of the buffer
+    int64_t DeviceId;           // Device for which this buffer is allocated
+    void *Start;                // Start of allocated space for trace records
+    size_t TotalBytes;          // Total number of bytes in the allocated space
+    size_t RemainingBytes;      // Total number of unused bytes
+                                // corresponding to Cursor
+    std::atomic<void *> Cursor; // Address of the last trace record carved out
+    std::atomic<bool> IsFull;   // true if no more trace records can be
+                                // accomodated, otherwise false
+    Buffer(uint64_t BufId, int64_t DevId, void *S, size_t Bytes, size_t Rem,
+           void *C, bool F)
+        : Id(BufId), DeviceId(DevId), Start(S), TotalBytes(Bytes),
+          RemainingBytes(Rem), Cursor(C), IsFull(F) {}
+    Buffer() = delete;
+    Buffer(const Buffer &) = delete;
+    Buffer &operator=(const Buffer &) = delete;
+  };
+  using BufPtr = std::shared_ptr<Buffer>;
+
+private:
+  /// Envar to control whether a buffer should be flushed when it gets full.
+  BoolEnvar OMPX_FlushOnBufferFull;
+
+  /// Envar to control whether all buffers should be flushed during shutdown.
+  BoolEnvar OMPX_FlushOnShutdown;
+
+  // Internal variable for tracking threads to wait for flush
+  uint32_t ThreadFlushTracker;
+
+  // Internal variable for tracking threads shutting down
+  uint32_t ThreadShutdownTracker;
+
+  using MapId2Buf = std::map<uint64_t, BufPtr>;
+
+  // Map from id to corresponding buffer. The ids are assigned in
+  // increasing order of creation.
+  MapId2Buf Id2BufferMap;
+
+  // Trace record. We currently support OMPT data type only. The state
+  // (TRStatus type) is maintained inline in the trace record. The
+  // tool is expected to access only the OMPT record.
+  struct TraceRecord {
+    ompt_record_ompt_t TR;
+    std::atomic<TRStatus> TRState;
+  };
+
+  // Thread-specific array of pointers to a buffer. The buffer pointed to
+  // is the last one allocated by this thread for a given device. The ith
+  // element points to the buffer for the ith device. At most MAX_NUM_DEVICES
+  // devices are supported.
+  static thread_local BufPtr ArrayOfBufPtr[MAX_NUM_DEVICES];
+
+  /*
+   * A buffer is flushed when it fills up or when the tool invokes
+   * flush_trace. So it's possible that the same buffer may be flushed
+   * more than once. When a buffer is flushed the first time, a unique
+   * id (flush-id) is generated and assigned to that buffer. Even if
+   * it is flushed again, the previously assigned id is maintained for
+   * that buffer. This id is loosely used to determine the order in
+   * which the buffers are processed and the corresponding trace
+   * records released to the tool.
+   */
+
+  struct FlushInfo {
+    uint64_t FlushId;
+    void *FlushCursor;
+    BufPtr FlushBuf;
+    FlushInfo() = default;
+    FlushInfo(uint64_t Id, void *CR, BufPtr Buf)
+        : FlushId{Id}, FlushCursor{CR}, FlushBuf{Buf} {}
+  };
+
+  /*
+   * A buffer may be in the following states:
+   * Flush_waiting: when a buffer is flushed, either because it is
+   * full or because the tool invokes ompt_flush_trace
+   * Flush_processing: when a helper thread claims the waiting buffer
+   * and is in the process of dispatching buffer-completion callbacks
+   * on an associated range of trace records. If all trace records are
+   * not released, the state may be reset to Flush_waiting after the
+   * buffer-completion callbacks return
+   */
+  enum BufferFlushStatus { Flush_waiting, Flush_processing };
+  struct FlushMd {
+    void *FlushCursor;
+    BufPtr FlushBuf;
+    BufferFlushStatus FlushStatus;
+    FlushMd(void *CR, BufPtr Buf, BufferFlushStatus Status)
+        : FlushCursor{CR}, FlushBuf{Buf}, FlushStatus{Status} {}
+    FlushMd() = delete;
+  };
+
+  using MapId2Md = std::map<uint64_t, FlushMd>;
+
+  /*
+   * A map from a flush-id to metadata containing the current
+   * cursor. the corresponding buffer, and its flushed status. If a
+   * buffer is flushed multiple times, the cursor is updated to the
+   * furthest one
+   */
+  MapId2Md Id2FlushMdMap;
+
+  using UMapBufPtr2Id = std::unordered_map<BufPtr, uint64_t>;
+
+  // A hash map from a buffer address to the corresponding flush-id
+  UMapBufPtr2Id FlushBufPtr2IdMap;
+
+  using USetCursor = std::unordered_set<void *>;
+
+  USetCursor LastCursors;
+
+  using UMapThd2Id = std::unordered_map<std::thread::id, uint32_t>;
+
+  // A hash map from a helper thread id to an integer
+  UMapThd2Id HelperThreadIdMap;
+
+  // Mutex to protect Id2BufferMap and Cursor2BufMdMap
+  std::mutex BufferMgrMutex;
+
+  // Mutex to protect FlushBufPtr2IdMap and Id2FlushMdMap
+  std::mutex FlushMutex;
+
+  // Mutex to protect metadata tracking last cursors of buffer-completion
+  // callbacks
+  std::mutex LastCursorMutex;
+
+  // Condition variable used by helper thread to signal that flush is requested
+  std::condition_variable FlushCv;
+
+  // Condition variable used while waiting for flushing to complete
+  std::condition_variable ThreadFlushCv;
+
+  // Condition variable used while waiting for threads to shutdown
+  std::condition_variable ThreadShutdownCv;
+
+  // TODO Separate out the helper thread into its own class
+  std::vector<std::thread> CompletionThreads;
+
+  /// Called when a buffer \p Buf may be flushed with \p Cursor as the
+  /// last allocated trace record in the buffer.
+  /// triggerFlushOnBufferFull should be called without holding any lock.
+  void triggerFlushOnBufferFull(void *Cursor, BufPtr Buf);
+
+  // Called to dispatch buffer-completion callbacks for the trace records in
+  // this buffer
+  void flushBuffer(FlushInfo);
+
+  // Dispatch a buffer-completion callback with a range of trace records
+  void dispatchCallback(int64_t DeviceId, void *Buffer, void *FirstCursor,
+                        void *LastCursor);
+
+  // Add a last cursor
+  void addLastCursor(void *Cursor) {
+    std::unique_lock<std::mutex> Lock(LastCursorMutex);
+    LastCursors.emplace(Cursor);
+  }
+
+  // Remove a last cursor
+  void removeLastCursor(void *Cursor) {
+    std::unique_lock<std::mutex> Lock(LastCursorMutex);
+    assert(LastCursors.find(Cursor) != LastCursors.end());
+    LastCursors.erase(Cursor);
+  }
+
+  // Given a trace record pointer, initialize its metadata
+  void initTraceRecordMetaData(void *Rec);
+
+  // Given a device-id, get/set a pointer to the last allocated buffer metadata.
+  BufPtr getDeviceSpecificBuffer(int64_t DevId);
+  void setDeviceSpecificBuffer(int64_t DevId, BufPtr Buf);
+
+  // Reserve a candidate buffer for flushing, preventing other helper threads
+  // from accessing it
+  FlushInfo findAndReserveFlushedBuf(uint64_t FlushId);
+
+  // Unreserve a buffer so that other helper threads can process it
+  void unreserveFlushedBuf(const FlushInfo &);
+
+  // All done with this buffer, so the buffer and its metadata can be removed
+  void destroyFlushedBuf(const FlushInfo &);
+
+  // Add a new buffer by an OpenMP thread so that a helper thread can process it
+  uint64_t addNewFlushEntry(BufPtr Buf, void *Cursor);
+
+  // Get the next trace record
+  void *getNextTR(void *TR);
+
+  // Given a buffer, return the latest cursor
+  void *getBufferCursor(BufPtr);
+
+  // Is no more space remaining for trace records in this buffer?
+  bool isBufferFull(const FlushInfo &);
+
+  // Have all trace records in this buffer been returned to the tool?
+  bool isBufferOwned(const FlushInfo &);
+
+  // Dispatch a buffer-completion callback and indicate that the buffer can be
+  // deallocated
+  void dispatchBufferOwnedCallback(const FlushInfo &);
+
+  // Main entry point for a helper thread
+  void driveCompletion();
+
+  // Examine the flushed buffers and dispatch buffer-completion callbacks
+  void invokeCallbacks();
+
+  // The caller does not hold a lock while calling this method
+  void waitForFlushCompletion();
+
+  // Given a thread number, set the corresponding bit in the flush
+  // tracker. The caller must hold the flush lock.
+  void setThreadFlush(uint32_t ThreadNum) {
+    ThreadFlushTracker |= (1 << ThreadNum);
+  }
+
+  // Reset this thread's flush bit. The caller must hold the flush lock
+  void resetThisThreadFlush() {
+    std::thread::id ID = std::this_thread::get_id();
+    assert(HelperThreadIdMap.find(ID) != HelperThreadIdMap.end());
+    ThreadFlushTracker &= ~(1 << HelperThreadIdMap[ID]);
+  }
+
+  // Given a thread number, set the corresponding bit in the shutdown
+  // tracker. The caller must hold the flush lock.
+  void setThreadShutdown(uint32_t ThreadNum) {
+    ThreadShutdownTracker |= (1 << ThreadNum);
+  }
+
+  // Reset this thread's shutdown bit. The caller must hold the flush
+  // lock
+  void resetThisThreadShutdown() {
+    std::thread::id ID = std::this_thread::get_id();
+    assert(HelperThreadIdMap.find(ID) != HelperThreadIdMap.end());
+    ThreadShutdownTracker &= ~(1 << HelperThreadIdMap[ID]);
+  }
+
+  // Return true if this thread's flush bit is set. The caller must
+  // hold the flush lock
+  bool isThisThreadFlushWaitedUpon() {
+    std::thread::id ID = std::this_thread::get_id();
+    assert(HelperThreadIdMap.find(ID) != HelperThreadIdMap.end());
+    return (ThreadFlushTracker & (1 << HelperThreadIdMap[ID])) != 0;
+  }
+
+  // Return true if this thread's shutdown bit is set. The caller must
+  // hold the flush lock
+  bool isThisThreadShutdownWaitedUpon() {
+    std::thread::id ID = std::this_thread::get_id();
+    assert(HelperThreadIdMap.find(ID) != HelperThreadIdMap.end());
+    return (ThreadShutdownTracker & (1 << HelperThreadIdMap[ID])) != 0;
+  }
+
+  // The caller must not hold the flush lock
+  bool amIHelperThread() {
+    std::unique_lock<std::mutex> flush_lock(FlushMutex);
+    if (HelperThreadIdMap.find(std::this_thread::get_id()) !=
+        HelperThreadIdMap.end())
+      return true;
+    return false;
+  }
+
+  // The caller must not hold the flush lock
+  bool areHelperThreadsAvailable();
+
+  // The caller must hold the appropriate lock
+  void init();
+
+  // The caller must hold the flush lock
+  void createHelperThreads();
+
+  // The caller must hold the flush lock
+  void destroyHelperThreads();
+
+public:
+  OmptTracingBufferMgr()
+      : OMPX_FlushOnBufferFull("LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL", true),
+        OMPX_FlushOnShutdown("LIBOMPTARGET_OMPT_FLUSH_ON_SHUTDOWN", true) {
+    // no need to hold locks for init() since object is getting constructed
+    // here.
+    init();
+  }
+
+  OmptTracingBufferMgr(const OmptTracingBufferMgr &) = delete;
+  OmptTracingBufferMgr &operator=(const OmptTracingBufferMgr &) = delete;
+
+  // The caller must not hold the flush lock
+  void startHelperThreads();
+
+  // The caller must not hold the flush lock. The helper threads are shut down
+  // without flushing any outstanding trace records.
+  void shutdownHelperThreads();
+
+  // The caller must not hold the flush lock. The helper threads are shut down
+  // after flushing all outstanding trace records for all devices.
+  void flushAndShutdownHelperThreads();
+
+  // Assign a cursor for a new trace record. This will assign a trace record
+  // for the provided device-id, allocating a new buffer if required.
+  void *assignCursor(ompt_callbacks_t Type, int64_t DeviceId);
+
+  // Get the size of a trace record
+  size_t getTRSize() { return sizeof(TraceRecord); }
+
+  // Get the status of a trace record. This function does not acquire
+  // a lock. If locking is required, the caller must hold a lock.
+  TRStatus getTRStatus(void *Rec);
+
+  // Set the status of a trace record. This function does not acquire
+  // a lock. If locking is required, the caller must hold a lock.
+  void setTRStatus(void *Rec, TRStatus);
+
+  // Is this a last cursor of a buffer completion callback?
+  bool isLastCursor(void *Cursor) {
+    std::unique_lock<std::mutex> Lock(LastCursorMutex);
+    return LastCursors.find(Cursor) != LastCursors.end();
+  }
+
+  // Called for flushing outstanding buffers for the provided device-id.
+  int flushAllBuffers(int DeviceId);
+};
+
+#else
+class OmptTracingBufferMgr {};
+#endif // OMPT_SUPPORT
+
+#endif // OPENMP_LIBOMPTARGET_OMPTTRACINGBUFFER_H
diff --git a/offload/include/PluginManager.h b/offload/include/PluginManager.h
index 6c6fdebe76dff..972afd962cbc1 100644
--- a/offload/include/PluginManager.h
+++ b/offload/include/PluginManager.h
@@ -13,6 +13,7 @@
 #ifndef OMPTARGET_PLUGIN_MANAGER_H
 #define OMPTARGET_PLUGIN_MANAGER_H
 
+#include "OpenMP/OMPT/OmptTracingBuffer.h"
 #include "PluginInterface.h"
 
 #include "DeviceImage.h"
@@ -48,7 +49,7 @@ struct PluginManager {
   /// Exclusive accessor type for the device container.
   using ExclusiveDevicesAccessorTy = Accessor<DeviceContainerTy>;
 
-  PluginManager() {}
+  PluginManager() : TraceRecordManager(nullptr) {}
 
   void init();
 
@@ -150,6 +151,11 @@ struct PluginManager {
     return count;
   }
 
+  auto getTraceRecordManager() const {
+    assert(TraceRecordManager && "Trace record manager not initialized");
+    return TraceRecordManager;
+  }
+
 private:
   bool RTLsLoaded = false;
   llvm::SmallVector<__tgt_bin_desc *> DelayedBinDesc;
@@ -176,6 +182,8 @@ struct PluginManager {
   /// Devices associated with plugins, accesses to the container are exclusive.
   ProtectedObj<DeviceContainerTy> Devices;
 
+  OmptTracingBufferMgr *TraceRecordManager;
+
   /// References to upgraded legacy offloading entries.
   std::list<llvm::SmallVector<llvm::offloading::EntryTy, 0>> LegacyEntries;
   std::list<llvm::SmallVector<__tgt_device_image, 0>> LegacyImages;
diff --git a/offload/libomptarget/CMakeLists.txt b/offload/libomptarget/CMakeLists.txt
index 7909d55eff5c3..3301c01ac90a3 100644
--- a/offload/libomptarget/CMakeLists.txt
+++ b/offload/libomptarget/CMakeLists.txt
@@ -17,6 +17,7 @@ add_library(omptarget SHARED
   OpenMP/Mapping.cpp
   OpenMP/InteropAPI.cpp
   OpenMP/OMPT/Callback.cpp
+  OpenMP/OMPT/OmptTracingBuffer.cpp
 
   KernelLanguage/API.cpp
 )
diff --git a/offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp b/offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp
new file mode 100644
index 0000000000000..2d4b32e3554ac
--- /dev/null
+++ b/offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp
@@ -0,0 +1,763 @@
+//=== OmptTracingBuffer.cpp - Target independent OpenMP target RTL -- C++ -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementation of OMPT device trace record generation and flushing.
+//
+//===----------------------------------------------------------------------===//
+
+#ifdef OMPT_SUPPORT
+
+#include "OpenMP/OMPT/OmptTracingBuffer.h"
+#include "OpenMP/OMPT/OmptTracing.h"
+#include "Shared/Debug.h"
+
+#include <atomic>
+#include <cassert>
+#include <cstdlib>
+#include <limits>
+
+using namespace llvm::omp::target::debug;
+
+// When set to true, helper threads terminate their work
+static bool DoneTracing{false};
+
+// Unique buffer id in creation order
+static std::atomic<uint64_t> BufId{0};
+
+// Unique id in buffer flush order
+static std::atomic<uint64_t> FlushId{0};
+
+thread_local OmptTracingBufferMgr::BufPtr
+    OmptTracingBufferMgr::ArrayOfBufPtr[MAX_NUM_DEVICES];
+
+static uint64_t get_and_inc_buf_id() { return BufId++; }
+
+static uint64_t get_and_inc_flush_id() { return FlushId++; }
+static uint64_t get_flush_id() { return FlushId; }
+
+/*
+ * Used by OpenMP threads for assigning space for a trace record. If
+ * there is no space in the last buffer allocated by this thread, the
+ * last buffer is marked full and scheduled for flushing. Otherwise,
+ * space is assigned for a trace record and the new cursor returned.
+ * Since the memory allocated by a thread is used by that thread alone
+ * for creating trace records, a lock need not be held. In the less
+ * common branch when memory is allocated, a lock needs to be acquired
+ * for updating shared metadata. The common path of allocating a trace
+ * record from an existing buffer proceeds without locking.
+ */
+void *OmptTracingBufferMgr::assignCursor(ompt_callbacks_t Type,
+                                         int64_t DeviceId) {
+  // The caller should handle nullptr by not tracing for this event.
+  if (DeviceId < 0 || DeviceId > MAX_NUM_DEVICES - 1)
+    return nullptr;
+
+  size_t RecSize = getTRSize();
+
+  // If the buffer fills up, it will be scheduled for flushing with the
+  // following cursor.
+  void *ToBeFlushedCursor = nullptr;
+  BufPtr ToBeFlushedBuf = nullptr;
+
+  // Thread local buffer pointer should be non-null once an allocation
+  // has been done by this thread.
+  BufPtr DeviceBuf = getDeviceSpecificBuffer(DeviceId);
+  if (DeviceBuf != nullptr) {
+    assert(DeviceBuf->DeviceId == DeviceId && "Unexpected device id in buffer");
+    void *OldCursor = DeviceBuf->Cursor.load(std::memory_order_acquire);
+    // Try to assign a trace record from the last allocated buffer
+    if (RecSize <= DeviceBuf->RemainingBytes) {
+      assert((char *)DeviceBuf->Start + DeviceBuf->TotalBytes -
+                 DeviceBuf->RemainingBytes ==
+             (char *)OldCursor + RecSize);
+      DeviceBuf->RemainingBytes -= RecSize;
+
+      // Note the trace record status must be initialized before setting
+      // the cursor, ensuring that a helper thread always sees an initialized
+      // trace record status.
+      void *NewCursor = (char *)OldCursor + RecSize;
+      initTraceRecordMetaData(NewCursor);
+      DeviceBuf->Cursor.store(NewCursor, std::memory_order_release);
+
+      ODBG(ODT_Tool) << "Thread " << llvm::omp::target::ompt::getThreadId()
+                     << ": Assigned " << RecSize << " bytes at " << NewCursor
+                     << " in existing buffer " << DeviceBuf->Start
+                     << " for device " << DeviceId;
+      return NewCursor;
+    } else {
+      ToBeFlushedCursor = OldCursor;
+      ToBeFlushedBuf = DeviceBuf;
+
+      // Mark that no space is present for any more trace records.
+      // The following is atomic but there is no logical order between when
+      // it is set here and when it is checked by a helper thread. That works
+      // because the helper thread uses this info to decide whether a buffer
+      // can be scheduled for removal. In the worst case, the buffer will be
+      // removed late.
+      DeviceBuf->IsFull.store(true, std::memory_order_release);
+    }
+  }
+  void *NewBuffer = nullptr;
+  size_t TotalBytes = 0;
+  // TODO Move the buffer allocation to a helper thread
+  llvm::omp::target::ompt::ompt_callback_buffer_request(DeviceId, &NewBuffer,
+                                                        &TotalBytes);
+
+  // The caller should handle nullptr by not tracing for this event.
+  if (NewBuffer == nullptr || TotalBytes < RecSize)
+    return nullptr;
+
+  uint64_t NewBufId = get_and_inc_buf_id();
+  auto new_buf = std::make_shared<Buffer>(
+      NewBufId, DeviceId, /*Start=*/NewBuffer, TotalBytes,
+      /*RemainingBytes=*/TotalBytes - RecSize,
+      /*Cursor=*/NewBuffer,
+      /*IsFull=*/false);
+
+  // Initialize trace record status before publishing it to helper threads.
+  initTraceRecordMetaData(new_buf->Cursor.load(std::memory_order_acquire));
+  setDeviceSpecificBuffer(DeviceId, new_buf);
+
+  // Make this trace record visible to helper threads by adding to shared
+  // metadata.
+  std::unique_lock<std::mutex> lck(BufferMgrMutex);
+  assert(Id2BufferMap.find(NewBufId) == Id2BufferMap.end());
+  Id2BufferMap[NewBufId] = new_buf;
+  lck.unlock();
+
+  // Schedule the full buffer for flushing till the corresponding cursor.
+  if (OMPX_FlushOnBufferFull && ToBeFlushedCursor)
+    triggerFlushOnBufferFull(ToBeFlushedCursor, ToBeFlushedBuf);
+
+  ODBG(ODT_Tool) << "Thread " << llvm::omp::target::ompt::getThreadId()
+                 << ": Assigned " << RecSize << " bytes at " << NewBuffer
+                 << " in new buffer with id " << NewBufId << " for device "
+                 << DeviceId;
+  return NewBuffer;
+}
+
+/*
+ * Called by an OpenMP thread when a buffer fills up and should be
+ * flushed. This function assigns a new FlushId to the buffer, adds
+ * to the flush-related metadata and wakes up a helper thread to
+ * dispatch a buffer-completion callback. This function should be
+ * called without holding any lock.
+ * Note lock order: buf_lock -> flush_lock
+ */
+void OmptTracingBufferMgr::triggerFlushOnBufferFull(void *cursor, BufPtr Buf) {
+  std::unique_lock<std::mutex> buf_lock(BufferMgrMutex);
+
+  // Between calling this function and this check, a flush-all may have
+  // delivered this buffer to the tool and deleted it. So the buffer
+  // may not exist.
+  if (Id2BufferMap.find(Buf->Id) == Id2BufferMap.end())
+    return;
+
+  // Cannot assert that the state of the cursor is ready since a
+  // different thread may be in the process of populating it. If it
+  // remains in init state when the range of trace records is
+  // determined for dispatching the buffer-completion callback, it
+  // will not be included.
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  uint64_t flush_id;
+  auto flush_itr = FlushBufPtr2IdMap.find(Buf);
+  if (flush_itr == FlushBufPtr2IdMap.end()) {
+    // This buffer has not been flushed yet
+    addNewFlushEntry(Buf, cursor);
+  } else {
+    // This buffer has been flushed before
+    flush_id = flush_itr->second;
+    auto flush_md_itr = Id2FlushMdMap.find(flush_id);
+    assert(flush_md_itr != Id2FlushMdMap.end());
+    flush_md_itr->second.FlushCursor = cursor; // update the cursor
+    // Do not update the flush status since it may be under processing
+    // by another thread
+    ODBG(ODT_Tool) << "Updated id " << flush_id << " cursor " << cursor
+                   << " buf " << flush_md_itr->second.FlushBuf->Start;
+  }
+  flush_lock.unlock();
+  buf_lock.unlock();
+
+  // Wake up a helper thread to invoke the buffer-completion callback
+  FlushCv.notify_one();
+}
+
+// This is the driver routine for the completion thread
+void OmptTracingBufferMgr::driveCompletion() {
+  while (true) {
+    bool should_signal_workers = false;
+    std::unique_lock<std::mutex> flush_lock(FlushMutex);
+    if (DoneTracing) {
+      // An upper layer serializes flush_trace and stop_trace. In
+      // addition, before DoneTracing is set, a flush is performed as
+      // part of stop_trace. So assert that no flush is in progress.
+      assert(ThreadFlushTracker == 0);
+      break;
+    }
+    FlushCv.wait(flush_lock, [this] {
+      return DoneTracing ||
+             (!Id2FlushMdMap.empty() &&
+              llvm::omp::target::ompt::TracingActive) ||
+             isThisThreadFlushWaitedUpon();
+    });
+    if (isThisThreadFlushWaitedUpon()) {
+      resetThisThreadFlush();
+      if (ThreadFlushTracker == 0)
+        should_signal_workers = true;
+    }
+    flush_lock.unlock();
+
+    invokeCallbacks();
+
+    if (should_signal_workers)
+      ThreadFlushCv.notify_all();
+
+    // There is a scenario where a buffer was processed but not full
+    // or owned, so it was put back in waiting state. So this thread
+    // would not wait but keep on looping without having any actual
+    // work until new trace records are added and this thread
+    // signaled. Hence, this thread yields.
+    std::this_thread::yield();
+  }
+  bool is_last_helper = false;
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  assert(DoneTracing && "Helper thread exiting but not yet done");
+  assert(isThisThreadShutdownWaitedUpon() &&
+         "Helper thread exiting but not waited upon");
+  resetThisThreadShutdown();
+  if (ThreadShutdownTracker == 0)
+    is_last_helper = true;
+  flush_lock.unlock();
+  if (is_last_helper)
+    ThreadShutdownCv.notify_all();
+
+  // Note that some trace records may have been written but not
+  // delivered to the tool. If flush/stop APIs are not called by the
+  // tool, those trace records may never be delivered to the tool and
+  // the corresponding buffers not reclaimed. TODO Explore whether
+  // this cleanup must be done.
+}
+
+/*
+ * Called by a buffer-completion helper thread. This function examines
+ * the flushed buffers in flush order and dispatches
+ * callbacks. Lock holding is minimized by reserving a buffer,
+ * processing it, and then unreserving it if there are more trace
+ * records to flush later. If all trace records are flushed, a
+ * callback is dispatched informing the tool that the buffer can be
+ * deallocated. If the buffer can be deallocated, all metadata is
+ * destroyed.
+ * Note that this function must be called without holding any locks.
+ */
+void OmptTracingBufferMgr::invokeCallbacks() {
+  ODBG(ODT_Tool) << "Looking for callbacks to invoke";
+  auto max_id = std::numeric_limits<uint64_t>::max();
+  auto curr_id = max_id;
+  auto end_id = get_flush_id();
+  ODBG(ODT_Tool) << "End id is " << end_id;
+  while (true) {
+    // Set the status of the flushed buffer to in-processing so that
+    // another helper thread does not process it concurrently. An
+    // OpenMP worker thread may, however, populate a trace record in a
+    // reserved buffer concurrently.
+    FlushInfo flush_info = findAndReserveFlushedBuf(curr_id);
+
+    // no entry found, nothing to process
+    if (curr_id == max_id && flush_info.FlushCursor == nullptr)
+      return;
+
+    if (flush_info.FlushCursor != nullptr) {
+      // increment curr_id to get the candidate for the next iteration
+      curr_id = flush_info.FlushId + 1;
+    } else {
+      assert(curr_id != max_id && "Cannot increment max id");
+      ++curr_id;
+    }
+
+    ODBG(ODT_Tool) << "Next id will be " << curr_id;
+
+    if (flush_info.FlushCursor == nullptr) {
+      // This buffer must have been processed already
+      if (curr_id < end_id)
+        continue;
+      else
+        return; // nothing else to process
+    }
+
+    ODBG(ODT_Tool) << "Buf " << flush_info.FlushBuf->Start
+             << " Cursor " << flush_info.FlushCursor
+             << " Id " << flush_info.FlushId
+             << " will be flushed";
+
+    // Examine the status of the trace records and dispatch
+    // buffer-completion callbacks as appropriate.
+    flushBuffer(flush_info);
+
+    // TODO optimize to set buffer-owned in the same pass above.
+    // Currently, this is the only way a buffer is deallocated
+    if (isBufferFull(flush_info)) {
+      // All trace records have been delivered to the tool
+      if (isBufferOwned(flush_info)) {
+        // erase element from buffer and flush maps
+        destroyFlushedBuf(flush_info);
+
+        // dispatch callback with a null range and have the tool
+        // deallocate the buffer
+        dispatchBufferOwnedCallback(flush_info);
+      } else {
+        unreserveFlushedBuf(flush_info);
+      }
+    } else {
+      unreserveFlushedBuf(flush_info);
+    }
+    if (curr_id >= end_id)
+      return;
+  }
+}
+
+/*
+ * This function is called on a buffer that is already reserved by
+ * this thread. Buffer-completion callbacks are dispatched for every
+ * range of trace records that are ready.
+ * This routine must be called without holding locks
+ */
+void OmptTracingBufferMgr::flushBuffer(FlushInfo flush_info) {
+  assert(flush_info.FlushBuf && "Cannot flush an empty buffer");
+  assert(flush_info.FlushCursor && "Cannot flush upto a null cursor");
+
+  void *curr_tr = flush_info.FlushBuf->Start;
+  void *last_tr = flush_info.FlushCursor;
+  // Compute a range [first_cursor,last_cursor] to flush
+  void *first_cursor = nullptr;
+  void *last_cursor = nullptr;
+  while (curr_tr <= last_tr) {
+    TRStatus tr_status = getTRStatus(curr_tr);
+    if (tr_status == TR_init || tr_status == TR_released) {
+      if (first_cursor == nullptr) {
+        // This TR won't be part of a range
+        assert(last_cursor == nullptr &&
+               "Begin/last cursors mutually inconsistent");
+      } else {
+        // End the current interval
+        dispatchCallback(flush_info.FlushBuf->DeviceId,
+                         flush_info.FlushBuf->Start, first_cursor, last_cursor);
+        first_cursor = last_cursor = nullptr;
+      }
+    } else {
+      assert(tr_status == TR_ready && "Unknown trace record status");
+      setTRStatus(curr_tr, TR_released);
+      if (first_cursor == nullptr)
+        first_cursor = curr_tr;
+      last_cursor = curr_tr;
+    }
+    curr_tr = getNextTR(curr_tr);
+  }
+  if (first_cursor != nullptr) {
+    assert(last_cursor != nullptr);
+    dispatchCallback(flush_info.FlushBuf->DeviceId, flush_info.FlushBuf->Start,
+                     first_cursor, last_cursor);
+  }
+}
+
+// Given a range of trace records, dispatch a buffer-completion callback
+void OmptTracingBufferMgr::dispatchCallback(int64_t DeviceId, void *Buffer,
+                                            void *FirstCursor,
+                                            void *LastCursor) {
+  assert(FirstCursor != nullptr && LastCursor != nullptr &&
+         "Callback with nullptr");
+  addLastCursor(LastCursor);
+
+  // This is best effort.
+  // There is a small window when the buffer-completion callback may
+  // be invoked even after tracing has been disabled.
+  // Note that we don't want to hold a lock when dispatching the callback.
+  if (llvm::omp::target::ompt::isTracedDevice(DeviceId)) {
+    ODBG(ODT_Tool) << "Dispatch callback w/ range (inclusive) to be flushed: "
+                   << FirstCursor << " -> " << LastCursor;
+    llvm::omp::target::ompt::ompt_callback_buffer_complete(
+        DeviceId, Buffer,
+        /* bytes returned in this callback */
+        (char *)getNextTR(LastCursor) - (char *)FirstCursor,
+        (ompt_buffer_cursor_t)FirstCursor, false /* buffer_owned */);
+  }
+
+  removeLastCursor(LastCursor);
+}
+
+// Dispatch a buffer-completion callback with buffer_owned set so that
+// the tool can deallocate the buffer
+void OmptTracingBufferMgr::dispatchBufferOwnedCallback(
+    const FlushInfo &flush_info) {
+  // This is best effort.
+  // There is a small window when the buffer-completion callback may
+  // be invoked even after tracing has been disabled.
+  // Note that we don't want to hold a lock when dispatching the callback.
+  if (llvm::omp::target::ompt::isTracedDevice(flush_info.FlushBuf->DeviceId)) {
+    ODBG(ODT_Tool) << "Dispatch callback with buffer "
+             << flush_info.FlushBuf->Start << " owned";
+    llvm::omp::target::ompt::ompt_callback_buffer_complete(
+        flush_info.FlushBuf->DeviceId, flush_info.FlushBuf->Start, 0,
+        (ompt_buffer_cursor_t)0, true /* buffer owned */);
+  }
+}
+
+void OmptTracingBufferMgr::initTraceRecordMetaData(void *Rec) {
+  setTRStatus(Rec, TR_init);
+}
+
+OmptTracingBufferMgr::BufPtr
+OmptTracingBufferMgr::getDeviceSpecificBuffer(int64_t DeviceId) {
+  if (DeviceId < 0 || DeviceId > MAX_NUM_DEVICES - 1) {
+    REPORT() << "getDeviceSpecificBuffer: Device id " << DeviceId
+             << " invalid or exceeds supported max: "
+             << MAX_NUM_DEVICES - 1;
+    return nullptr;
+  }
+  return ArrayOfBufPtr[DeviceId];
+}
+
+void OmptTracingBufferMgr::setDeviceSpecificBuffer(int64_t DeviceId,
+                                                   BufPtr Buf) {
+  if (DeviceId < 0 || DeviceId > MAX_NUM_DEVICES - 1) {
+    REPORT() << "setDeviceSpecificBuffer: Device id " << DeviceId
+             << " invalid or exceeds supported max: "
+             << MAX_NUM_DEVICES - 1;
+    return;
+  }
+  ArrayOfBufPtr[DeviceId] = Buf;
+}
+
+void OmptTracingBufferMgr::setTRStatus(void *Rec, TRStatus Status) {
+  TraceRecord *TR = static_cast<TraceRecord *>(Rec);
+  TR->TRState.store(Status, std::memory_order_release);
+}
+
+OmptTracingBufferMgr::TRStatus OmptTracingBufferMgr::getTRStatus(void *Rec) {
+  return static_cast<TraceRecord *>(Rec)->TRState.load(
+      std::memory_order_acquire);
+}
+
+void *OmptTracingBufferMgr::getNextTR(void *TR) {
+  size_t RecSize = getTRSize();
+  // warning: no overflow check done
+  return (char *)TR + RecSize;
+}
+
+bool OmptTracingBufferMgr::isBufferFull(const FlushInfo &flush_info) {
+  std::unique_lock<std::mutex> buf_lock(BufferMgrMutex);
+  return flush_info.FlushBuf->IsFull;
+}
+
+void *OmptTracingBufferMgr::getBufferCursor(BufPtr buf) {
+  return buf->Cursor.load(std::memory_order_acquire);
+}
+
+/*
+ * Traverse all the trace records of a buffer and return true if all
+ * of them have been released to the tool, otherwise return false
+ */
+bool OmptTracingBufferMgr::isBufferOwned(const FlushInfo &flush_info) {
+  assert(isBufferFull(flush_info) && "Compute buffer-owned when it is full");
+  void *curr_tr = flush_info.FlushBuf->Start;
+  // Since the buffer is full, the cursor must be the last valid
+  // TR. Note that this may be more up-to-date than the cursor in the
+  // flush_info. Use the last valid TR to avoid dropping trace records
+  void *last_tr = getBufferCursor(flush_info.FlushBuf);
+  while (curr_tr <= last_tr) {
+    if (getTRStatus(curr_tr) != TR_released)
+      return false;
+    curr_tr = getNextTR(curr_tr);
+  }
+  return true;
+}
+
+/*
+ * A buffer must be reserved by a thread before it can be processed
+ * and callbacks dispatched for that buffer. Reservation is done by
+ * setting the status to in-processing.
+ *
+ * If a buffer is found in the flush metadata for the given id and it
+ * is not in in-processing mode, reserve it by setting its mode to
+ * in-processing and return the corresponding flush metadata. If the
+ * given id is set to max, return the first waiting buffer in the
+ * list of buffers to be flushed.
+ */
+OmptTracingBufferMgr::FlushInfo
+OmptTracingBufferMgr::findAndReserveFlushedBuf(uint64_t FlushId) {
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  MapId2Md::iterator flush_itr;
+  if (FlushId == std::numeric_limits<uint64_t>::max()) {
+    // Reserve the first waiting buffer and return it
+    if (Id2FlushMdMap.empty())
+      return FlushInfo();
+    for (flush_itr = Id2FlushMdMap.begin(); flush_itr != Id2FlushMdMap.end();
+         ++flush_itr) {
+      // Reserve only if waiting
+      if (flush_itr->second.FlushStatus == Flush_waiting)
+        break;
+    }
+    if (flush_itr == Id2FlushMdMap.end())
+      return FlushInfo();
+  } else {
+    flush_itr = Id2FlushMdMap.find(FlushId);
+    if (flush_itr == Id2FlushMdMap.end() ||
+        flush_itr->second.FlushStatus == Flush_processing)
+      return FlushInfo();
+  }
+  assert(flush_itr->second.FlushStatus == Flush_waiting);
+  flush_itr->second.FlushStatus = Flush_processing;
+  // Update the metadata cursor since more trace records may have been
+  // generated.
+  flush_itr->second.FlushCursor =
+      flush_itr->second.FlushBuf->Cursor.load(std::memory_order_acquire);
+
+  FlushInfo flush_info(flush_itr->first, flush_itr->second.FlushCursor,
+                       flush_itr->second.FlushBuf);
+  ODBG(ODT_Tool) << "Reserved buffer: flush_id:" << flush_itr->first
+                 << ", cursor:" << flush_itr->second.FlushCursor
+                 << ", buf:" << flush_itr->second.FlushBuf->Start;
+  return flush_info;
+}
+
+/*
+ * Given a buffer, verify that it is in processing state and set its
+ * status to waiting, removing the reservation. The same thread that
+ * reserved it should be unreserving it but currently there is no such
+ * check.
+ */
+void OmptTracingBufferMgr::unreserveFlushedBuf(const FlushInfo &flush_info) {
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  auto itr = Id2FlushMdMap.find(flush_info.FlushId);
+  assert(itr != Id2FlushMdMap.end() &&
+         itr->second.FlushStatus == Flush_processing);
+  itr->second.FlushStatus = Flush_waiting;
+  ODBG(ODT_Tool) << "Unreserved buffer: flush_id:" << flush_info.FlushId
+                 << ", cursor:" << flush_info.FlushCursor
+                 << ", buf:" << flush_info.FlushBuf->Start;
+}
+
+/*
+ * This function must be called after all of the trace records in the
+ * buffer have been released to the tool. The buffer is removed from
+ * all metadata maps.
+ * Note lock order: buf_lock -> flush_lock
+ */
+void OmptTracingBufferMgr::destroyFlushedBuf(const FlushInfo &flush_info) {
+  ODBG(ODT_Tool) << "Destroying buffer: flush_id:" << flush_info.FlushId
+                 << ", cursor:" << flush_info.FlushCursor
+                 << ", buf:" << flush_info.FlushBuf->Start;
+
+  BufPtr buf = flush_info.FlushBuf;
+
+  std::unique_lock<std::mutex> buf_lock(BufferMgrMutex);
+  Id2BufferMap.erase(buf->Id);
+
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  auto flush_itr = Id2FlushMdMap.find(flush_info.FlushId);
+  assert(flush_itr != Id2FlushMdMap.end());
+  assert(flush_itr->second.FlushBuf == buf);
+  Id2FlushMdMap.erase(flush_itr);
+  FlushBufPtr2IdMap.erase(buf);
+}
+
+/*
+ * Generate a new flush id and add the buffer to the flush metadata
+ * maps. This function must be called while holding the flush lock.
+ */
+uint64_t OmptTracingBufferMgr::addNewFlushEntry(BufPtr Buf, void *Cursor) {
+  assert(FlushBufPtr2IdMap.find(Buf) == FlushBufPtr2IdMap.end());
+  uint64_t FlushId = get_and_inc_flush_id();
+  FlushBufPtr2IdMap.emplace(Buf, FlushId);
+  assert(Id2FlushMdMap.find(FlushId) == Id2FlushMdMap.end());
+  Id2FlushMdMap.emplace(FlushId, FlushMd(Cursor, Buf, Flush_waiting));
+
+  ODBG(ODT_Tool) << "Added new flush id "
+           << FlushId << " cursor "
+           << Cursor << " buf " << Buf->Start;
+
+  return FlushId;
+}
+
+/*
+ * Called by ompt_flush_trace and ompt_stop_trace. Traverse the
+ * existing buffers in creation order and flush all the ready TRs
+ */
+int OmptTracingBufferMgr::flushAllBuffers(int DeviceId) {
+  ODBG(ODT_Tool) << "Flushing buffers for device " << DeviceId << " :: START";
+  // Overloading MAX_NUM_DEVICES to mean all devices.
+  if (DeviceId < 0 || DeviceId > MAX_NUM_DEVICES)
+    return 0; // failed to flush
+
+  if (!areHelperThreadsAvailable())
+    return 0; // failed to flush
+
+  // If flush is called from a helper thread, just bail out
+  if (amIHelperThread())
+    return 0; // failed to flush
+
+  // To avoid holding the mutex for too long, get the ids of the first
+  // and the last TRs under lock, and then go through that range,
+  // holding the mutex for an individual TR
+  std::unique_lock<std::mutex> buf_lock(BufferMgrMutex);
+  if (Id2BufferMap.empty())
+    return 1; // no trace records to flush
+  uint64_t curr_buf_id = Id2BufferMap.begin()->first;
+  uint64_t last_buf_id = Id2BufferMap.rbegin()->first;
+  buf_lock.unlock();
+
+  while (curr_buf_id <= last_buf_id) {
+    std::unique_lock<std::mutex> buf_lock(BufferMgrMutex);
+    // Another thread may have deleted this buffer by now
+    auto buf_itr = Id2BufferMap.find(curr_buf_id);
+    if (buf_itr == Id2BufferMap.end()) {
+      ++curr_buf_id;
+      continue;
+    }
+    BufPtr curr_buf = buf_itr->second;
+
+    // If the device-id does not match, skip it. A device-id of MAX_NUM_DEVICES
+    // indicates flushing for all devices.
+    if (DeviceId != MAX_NUM_DEVICES && curr_buf->DeviceId != DeviceId) {
+      ++curr_buf_id;
+      continue;
+    }
+
+    // If this buffer is in the flush-map, skip it. It is either in
+    // process by another thread or will be processed
+    std::unique_lock<std::mutex> flush_lock(FlushMutex);
+    auto flush_itr = FlushBufPtr2IdMap.find(curr_buf);
+    if (flush_itr != FlushBufPtr2IdMap.end()) {
+      ++curr_buf_id;
+      continue;
+    }
+    // This buffer has not been flushed yet
+    void *CurrBufCursor = getBufferCursor(curr_buf);
+    uint64_t flush_id = addNewFlushEntry(curr_buf, CurrBufCursor);
+    (void)flush_id; // Silence warning.
+    ODBG(ODT_Tool) << "flushAllBuffers: Added new id "
+             << flush_id << " cursor " << CurrBufCursor
+             << " buf " << curr_buf->Start;
+
+    flush_lock.unlock();
+    buf_lock.unlock();
+
+    ++curr_buf_id;
+  }
+
+  ODBG(ODT_Tool) << "Flushing buffers for device " << DeviceId << " :: WAIT";
+
+  // This is best effort. It is possible that some trace records are
+  // not flushed when the wait is done.
+  waitForFlushCompletion();
+
+  ODBG(ODT_Tool) << "Flushing buffers for device " << DeviceId << " :: STOP";
+
+  return 1; // success
+}
+
+void OmptTracingBufferMgr::waitForFlushCompletion() {
+  {
+    std::unique_lock<std::mutex> flush_lock(FlushMutex);
+    // Setting the flush bit for a given helper thread indicates that the worker
+    // thread is ready for the helper thread to do some work.
+    for (uint32_t i = 0; i < OMPT_NUM_HELPER_THREADS; ++i)
+      setThreadFlush(i);
+  }
+
+  // Wake up all helper threads to invoke buffer-completion callbacks.
+  FlushCv.notify_all();
+
+  // Now wait for all helper threads  to complete flushing.
+  {
+    std::unique_lock<std::mutex> flush_lock(FlushMutex);
+    ThreadFlushCv.wait(flush_lock, [this] { return ThreadFlushTracker == 0; });
+  }
+}
+
+void OmptTracingBufferMgr::init() {
+  for (int i = 0; i < MAX_NUM_DEVICES; ++i)
+    ArrayOfBufPtr[i] = nullptr;
+  ThreadFlushTracker = 0;
+  ThreadShutdownTracker = 0;
+  DoneTracing = false; // TODO make it a class member
+}
+
+void OmptTracingBufferMgr::startHelperThreads() {
+  // All helper threads are stopped while holding FlushMutex. So if
+  // any helper thread is present, just return. This takes care of
+  // repeated calls to start-trace.
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  if (!HelperThreadIdMap.empty()) {
+    assert(!DoneTracing && "Helper threads exist but tracing is done");
+    return;
+  }
+  init();
+  createHelperThreads();
+}
+
+bool OmptTracingBufferMgr::areHelperThreadsAvailable() {
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  if (DoneTracing // If another thread called stop, assume there are no threads
+      || HelperThreadIdMap.empty() // Threads were never started
+  ) {
+    // Don't assert on HelperThreadIdMap since shutdown by another
+    // thread may be in progress
+    return false;
+  }
+  return true;
+}
+
+void OmptTracingBufferMgr::shutdownHelperThreads() {
+  if (!areHelperThreadsAvailable())
+    return;
+
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  // If I am destroying the threads, then at least one thread must be present
+  assert(!CompletionThreads.empty());
+  assert(!HelperThreadIdMap.empty());
+  assert(ThreadShutdownTracker == 0);
+
+  // Set the done flag which helper threads will look at
+  DoneTracing = true;
+  // Wait to make sure all helper threads exit
+  for (uint32_t i = 0; i < OMPT_NUM_HELPER_THREADS; ++i)
+    setThreadShutdown(i);
+  // Signal indicating that DoneTracing is set
+  FlushCv.notify_all();
+  ThreadShutdownCv.wait(flush_lock,
+                        [this] { return ThreadShutdownTracker == 0; });
+
+  // Now destroy all the helper threads
+  destroyHelperThreads();
+}
+
+void OmptTracingBufferMgr::flushAndShutdownHelperThreads() {
+  std::unique_lock<std::mutex> Lock(llvm::omp::target::ompt::TraceControlMutex);
+  // Flush buffers for all devices.
+  if (OMPX_FlushOnShutdown)
+    flushAllBuffers(MAX_NUM_DEVICES);
+  else
+    waitForFlushCompletion(); // Dont initiate but wait for outstanding flushes.
+  shutdownHelperThreads();
+}
+
+void OmptTracingBufferMgr::createHelperThreads() {
+  for (uint32_t i = 0; i < OMPT_NUM_HELPER_THREADS; ++i) {
+    CompletionThreads.emplace_back(
+        std::thread(&OmptTracingBufferMgr::driveCompletion, this));
+    HelperThreadIdMap[CompletionThreads.back().get_id()] = i;
+  }
+}
+
+void OmptTracingBufferMgr::destroyHelperThreads() {
+  for (auto &thd : CompletionThreads)
+    thd.join();
+  CompletionThreads.clear();
+  HelperThreadIdMap.clear();
+}
+#endif

>From c080980ffe552662f5a2359a31067b6d37495614 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 2 Apr 2026 07:52:40 -0500
Subject: [PATCH 07/11] [Offload][OMPT] Add tracing orchestration and
 libomptarget integration

Wire the OMPT device tracing subsystem into libomptarget, completing the tracing pipeline from record production through buffer management to tool delivery. Include the synchronous fallback and initialization fixes in the integration that requires them.

Co-Authored-By: Claude <noreply at anthropic.com>
---
 offload/libomptarget/CMakeLists.txt           |   1 +
 offload/libomptarget/OpenMP/OMPT/Callback.cpp |  12 +
 .../libomptarget/OpenMP/OMPT/OmptTracing.cpp  | 885 ++++++++++++++++++
 offload/libomptarget/PluginManager.cpp        |  22 +
 offload/libomptarget/device.cpp               |  58 +-
 offload/libomptarget/exports                  |   8 +
 offload/libomptarget/interface.cpp            |  44 +-
 offload/libomptarget/omptarget.cpp            |   6 +-
 8 files changed, 1008 insertions(+), 28 deletions(-)
 create mode 100644 offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp

diff --git a/offload/libomptarget/CMakeLists.txt b/offload/libomptarget/CMakeLists.txt
index 3301c01ac90a3..7f2ff7329a536 100644
--- a/offload/libomptarget/CMakeLists.txt
+++ b/offload/libomptarget/CMakeLists.txt
@@ -17,6 +17,7 @@ add_library(omptarget SHARED
   OpenMP/Mapping.cpp
   OpenMP/InteropAPI.cpp
   OpenMP/OMPT/Callback.cpp
+  OpenMP/OMPT/OmptTracing.cpp
   OpenMP/OMPT/OmptTracingBuffer.cpp
 
   KernelLanguage/API.cpp
diff --git a/offload/libomptarget/OpenMP/OMPT/Callback.cpp b/offload/libomptarget/OpenMP/OMPT/Callback.cpp
index 1e03f1455d1b2..071276b3903ca 100644
--- a/offload/libomptarget/OpenMP/OMPT/Callback.cpp
+++ b/offload/libomptarget/OpenMP/OMPT/Callback.cpp
@@ -12,9 +12,13 @@
 
 #ifdef OMPT_SUPPORT
 
+#include <atomic>
+#include <cassert>
 #include <cstdlib>
 #include <cstring>
 #include <memory>
+#include <mutex>
+#include <thread>
 
 #include "Shared/Debug.h"
 
@@ -23,7 +27,9 @@
 #include "OpenMP/OMPT/Interface.h"
 
 #include "llvm/Support/DynamicLibrary.h"
+#include "llvm/Support/ErrorHandling.h"
 
+#pragma push_macro("DEBUG_PREFIX")
 #undef DEBUG_PREFIX
 #define DEBUG_PREFIX "OMPT"
 
@@ -441,6 +447,11 @@ void Interface::endTarget(int64_t DeviceId, void *Code) {
   endTargetRegion();
 }
 
+void Interface::announceTargetRegion(const char *RegionName) {
+  ODBG(ODT_Tool) << "in Interface::target_region_" << RegionName
+                 << " target_id=" << TargetData.value;
+}
+
 void Interface::beginTargetDataOperation() {
   ODBG(ODT_Tool) << "in ompt_target_region_begin (TargetRegionId = "
                  << TargetData.value << ")";
@@ -562,4 +573,5 @@ void llvm::omp::target::ompt::connectLibrary() {
   ODBG(ODT_Tool) << "Exiting connectLibrary";
 }
 
+#pragma pop_macro("DEBUG_PREFIX")
 #endif // OMPT_SUPPORT
diff --git a/offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp b/offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp
new file mode 100644
index 0000000000000..a74b10137f2c3
--- /dev/null
+++ b/offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp
@@ -0,0 +1,885 @@
+//===-- OmptTracing.cpp - Target independent OpenMP target RTL --- C++ -*--===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementation of OMPT tracing interfaces for target independent layer
+//
+//===----------------------------------------------------------------------===//
+
+#ifdef OMPT_SUPPORT
+
+#include "OpenMP/OMPT/OmptTracing.h"
+#include "OpenMP/OMPT/Callback.h"
+#include "OpenMP/OMPT/Interface.h"
+#include "OpenMP/OMPT/OmptTracingBuffer.h"
+#include "PluginManager.h"
+#include "Shared/Debug.h"
+#include "omp-tools.h"
+
+#include "llvm/Support/DynamicLibrary.h"
+
+#include <atomic>
+#include <cassert>
+#include <cstdlib>
+#include <cstring>
+#include <memory>
+#include <mutex>
+#include <thread>
+
+#pragma push_macro("DEBUG_PREFIX")
+#undef DEBUG_PREFIX
+#define DEBUG_PREFIX "OMPT"
+
+using namespace llvm::omp::target::ompt;
+
+std::mutex llvm::omp::target::ompt::DeviceAccessMutex;
+std::mutex llvm::omp::target::ompt::TraceAccessMutex;
+std::mutex llvm::omp::target::ompt::TraceControlMutex;
+std::mutex llvm::omp::target::ompt::TraceHashThreadMutex;
+std::mutex llvm::omp::target::ompt::BufferManagementFnMutex;
+
+std::unordered_map<int /*DeviceId*/, std::pair<ompt_callback_buffer_request_t,
+                                               ompt_callback_buffer_complete_t>>
+    llvm::omp::target::ompt::BufferManagementFns;
+
+thread_local uint32_t llvm::omp::target::ompt::TraceRecordNumGrantedTeams = 0;
+thread_local uint64_t llvm::omp::target::ompt::TraceRecordStartTime = 0;
+thread_local uint64_t llvm::omp::target::ompt::TraceRecordStopTime = 0;
+thread_local uint64_t llvm::omp::target::ompt::ThreadId =
+    std::numeric_limits<uint64_t>::max();
+
+std::map<int32_t, uint64_t> llvm::omp::target::ompt::TracedDevices;
+
+bool llvm::omp::target::ompt::TracingActive = false;
+
+void llvm::omp::target::ompt::resetTimestamp(uint64_t *T) { *T = 0; }
+
+ompt_callback_buffer_request_t
+llvm::omp::target::ompt::getBufferRequestFn(int DeviceId) {
+  std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
+  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
+  if (BufferMgrItr == BufferManagementFns.end()) {
+    return nullptr;
+  }
+  return BufferMgrItr->second.first;
+}
+
+ompt_callback_buffer_complete_t
+llvm::omp::target::ompt::getBufferCompleteFn(int DeviceId) {
+  std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
+  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
+  if (BufferMgrItr == BufferManagementFns.end()) {
+    return nullptr;
+  }
+  return BufferMgrItr->second.second;
+}
+
+void llvm::omp::target::ompt::setBufferManagementFns(
+    int DeviceId, ompt_callback_buffer_request_t ReqFn,
+    ompt_callback_buffer_complete_t CmpltFn) {
+  std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
+  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
+  if (BufferMgrItr != BufferManagementFns.end()) {
+    REPORT() << "Buffer request and complete functions already exist for device  "
+             << DeviceId  << "ignoring ...";
+    return;
+  }
+  BufferManagementFns[DeviceId] = std::make_pair(ReqFn, CmpltFn);
+}
+
+void llvm::omp::target::ompt::removeBufferManagementFns(int DeviceId) {
+  std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
+  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
+  if (BufferMgrItr == BufferManagementFns.end()) {
+    REPORT() << "Buffer request and complete functions don't exist for device  "
+             << DeviceId  << "ignoring ...";
+    return;
+  }
+  BufferManagementFns.erase(BufferMgrItr);
+}
+
+bool llvm::omp::target::ompt::isAllDeviceTracingStopped() {
+  std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
+  return BufferManagementFns.empty();
+}
+
+void llvm::omp::target::ompt::ompt_callback_buffer_request(
+    int DeviceId, ompt_buffer_t **BufferPtr, size_t *Bytes) {
+  if (auto Fn = getBufferRequestFn(DeviceId))
+    Fn(DeviceId, BufferPtr, Bytes);
+}
+
+void llvm::omp::target::ompt::ompt_callback_buffer_complete(
+    int DeviceId, ompt_buffer_t *Buffer, size_t Bytes,
+    ompt_buffer_cursor_t BeginCursor, int BufferOwned) {
+  if (auto Fn = getBufferCompleteFn(DeviceId))
+    Fn(DeviceId, Buffer, Bytes, BeginCursor, BufferOwned);
+}
+
+inline void setDeviceTracing(uint64_t &TracingTypes) {
+  // Set bit 0 to indicate generally enabled device tracing.
+  TracingTypes |= 1UL;
+}
+
+inline void resetDeviceTracing(uint64_t &TracingTypes) {
+  // Reset bit 0 to indicate generally disabled device tracing.
+  TracingTypes &= ~(1UL);
+}
+
+inline bool checkDeviceTracingState(const uint64_t &TracingTypes) {
+  // Return state of bit 0 to indicate if device is actively traced.
+  return TracingTypes & 1UL;
+}
+
+void llvm::omp::target::ompt::enableDeviceTracing(int DeviceId) {
+  std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
+  auto Device = TracedDevices.find(DeviceId);
+  if (Device == TracedDevices.end()) {
+    uint64_t TracingTypes{0};
+    setDeviceTracing(TracingTypes);
+    TracedDevices.emplace(DeviceId, TracingTypes);
+  } else
+    setDeviceTracing(Device->second);
+  // In any case: at least one device is traced
+  TracingActive = true;
+}
+
+void llvm::omp::target::ompt::disableDeviceTracing(int DeviceId) {
+  std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
+  auto Device = TracedDevices.find(DeviceId);
+  if (Device == TracedDevices.end()) {
+    uint64_t TracingTypes{0};
+    resetDeviceTracing(TracingTypes);
+    TracedDevices.emplace(DeviceId, TracingTypes);
+  } else
+    resetDeviceTracing(Device->second);
+
+  // Check for actively traced devices
+  for (auto &Dev : TracedDevices)
+    if (checkDeviceTracingState(Dev.second))
+      return;
+
+  // If no device is currently traced: set global tracing flag to false
+  TracingActive = false;
+}
+
+bool llvm::omp::target::ompt::isTracingEnabled(int DeviceId,
+                                               unsigned int EventTy) {
+  return TracingActive && isTracedDevice(DeviceId) &&
+         isTracingTypeGroupEnabled(DeviceId, EventTy);
+}
+
+bool llvm::omp::target::ompt::isTracedDevice(int DeviceId) {
+  std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
+  auto Device = TracedDevices.find(DeviceId);
+  if (Device != TracedDevices.end())
+    return checkDeviceTracingState(Device->second);
+
+  return false;
+}
+
+bool llvm::omp::target::ompt::isTracingTypeEnabled(int DeviceId,
+                                                   unsigned int EventTy) {
+  std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
+  // Make sure we do not shift more than std::numeric_limits<uint64_t>::digits
+  assert(EventTy < 64 && "Shift limit exceeded: EventTy must be less than 64");
+  auto Device = TracedDevices.find(DeviceId);
+  if (Device != TracedDevices.end() && EventTy < 64)
+    return (Device->second & (1UL << EventTy));
+  return false;
+}
+
+bool llvm::omp::target::ompt::isTracingTypeGroupEnabled(int DeviceId,
+                                                        unsigned int EventTy) {
+  std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
+  // Make sure we do not shift more than std::numeric_limits<uint64_t>::digits
+  assert(EventTy < 64 && "Shift limit exceeded: EventTy must be less than 64");
+  auto Device = TracedDevices.find(DeviceId);
+  if (Device != TracedDevices.end() && EventTy < 64) {
+    auto TracedEvents = Device->second;
+    switch (EventTy) {
+    case ompt_callbacks_t::ompt_callback_target:
+    case ompt_callbacks_t::ompt_callback_target_emi:
+      return ((TracedEvents & (1UL << ompt_callback_target))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_emi)));
+    case ompt_callbacks_t::ompt_callback_target_data_op:
+    case ompt_callbacks_t::ompt_callback_target_data_op_emi:
+      return ((TracedEvents & (1UL << ompt_callback_target_data_op))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_data_op_emi)));
+    case ompt_callbacks_t::ompt_callback_target_submit:
+    case ompt_callbacks_t::ompt_callback_target_submit_emi:
+      return ((TracedEvents & (1UL << ompt_callback_target_submit))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_submit_emi)));
+    // Special case: EventTy == 0 -> Check all EventTy
+    case 0:
+      return ((TracedEvents & (1UL << ompt_callback_target))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_emi))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_data_op))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_data_op_emi))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_submit))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_submit_emi)));
+    }
+  }
+  return false;
+}
+
+void llvm::omp::target::ompt::setTracingTypeEnabled(uint64_t &TracedEventTy,
+                                                    bool Enable,
+                                                    unsigned int EventTy) {
+  // Make sure we do not shift more than std::numeric_limits<uint64_t>::digits
+  assert(EventTy < 64 && "Shift limit exceeded: EventTy must be less than 64");
+  if (EventTy < 64) {
+    if (Enable)
+      TracedEventTy |= (1UL << EventTy);
+    else
+      TracedEventTy &= ~(1UL << EventTy);
+  }
+}
+
+ompt_set_result_t
+llvm::omp::target::ompt::setTraceEventTy(int DeviceId, unsigned int Enable,
+                                         unsigned int EventTy) {
+  if (DeviceId < 0) {
+    REPORT() << "Failed to set trace event type for DeviceId=" << DeviceId;
+    return ompt_set_never;
+  }
+
+  ODBG(debug::ODT_Tool) << "Executing setTraceEventTy: DeviceId=" << DeviceId
+                 << " Enable=" << Enable << " EventTy=" << EventTy;
+
+  std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
+  if (TracedDevices.find(DeviceId) == TracedDevices.end())
+    TracedDevices.emplace(DeviceId, 0UL);
+
+  auto &TracedEventTy = TracedDevices[DeviceId];
+  bool Enabled = Enable > 0;
+  if (EventTy == 0) {
+    // Set / reset all supported types
+    setTracingTypeEnabled(TracedEventTy, Enabled,
+                          ompt_callbacks_t::ompt_callback_target);
+    setTracingTypeEnabled(TracedEventTy, Enabled,
+                          ompt_callbacks_t::ompt_callback_target_data_op);
+    setTracingTypeEnabled(TracedEventTy, Enabled,
+                          ompt_callbacks_t::ompt_callback_target_submit);
+    setTracingTypeEnabled(TracedEventTy, Enabled,
+                          ompt_callbacks_t::ompt_callback_target_emi);
+    setTracingTypeEnabled(TracedEventTy, Enabled,
+                          ompt_callbacks_t::ompt_callback_target_data_op_emi);
+    setTracingTypeEnabled(TracedEventTy, Enabled,
+                          ompt_callbacks_t::ompt_callback_target_submit_emi);
+
+    if (Enabled) {
+      // Event subset is enabled
+      return ompt_set_sometimes;
+    } else {
+      // All events are disabled
+      return ompt_set_always;
+    }
+  }
+
+  switch (EventTy) {
+  case ompt_callbacks_t::ompt_callback_target:
+  case ompt_callbacks_t::ompt_callback_target_data_op:
+  case ompt_callbacks_t::ompt_callback_target_submit:
+  case ompt_callbacks_t::ompt_callback_target_emi:
+  case ompt_callbacks_t::ompt_callback_target_data_op_emi:
+  case ompt_callbacks_t::ompt_callback_target_submit_emi: {
+    setTracingTypeEnabled(TracedEventTy, Enabled, EventTy);
+    return ompt_set_always;
+  }
+  default: {
+    if (Enabled) {
+      // Unimplemented
+      return ompt_set_never;
+    } else {
+      // Always disabled anyways
+      return ompt_set_always;
+    }
+  }
+  }
+}
+
+uint64_t llvm::omp::target::ompt::getThreadId() {
+  // Grab the value from thread local storage, if valid.
+  if (ThreadId != std::numeric_limits<uint64_t>::max())
+    return ThreadId;
+  // Otherwise set it, protecting the hash with a lock.
+  std::unique_lock<std::mutex> Lock(TraceHashThreadMutex);
+  ThreadId = std::hash<std::thread::id>()(std::this_thread::get_id());
+  return ThreadId;
+}
+
+void Interface::setTraceRecordCommon(ompt_record_ompt_t *DataPtr,
+                                     ompt_callbacks_t CallbackType) {
+  DataPtr->type = CallbackType;
+
+  if (CallbackType == ompt_callback_target)
+    DataPtr->time = 0; // Currently, no consumer, so no need to set it
+  else {
+    DataPtr->time = TraceRecordStartTime;
+    resetTimestamp(&TraceRecordStartTime);
+  }
+
+  DataPtr->thread_id = getThreadId();
+  DataPtr->target_id = TargetData.value;
+}
+
+void Interface::setTraceRecordTargetDataOp(ompt_record_target_data_op_t *Record,
+                                           ompt_target_data_op_t DataOpType,
+                                           void *SrcAddr, int64_t SrcDeviceNum,
+                                           void *DstAddr, int64_t DstDeviceNum,
+                                           size_t Bytes, void *CodePtr) {
+  Record->host_op_id = HostOpId;
+  Record->optype = DataOpType;
+  Record->src_addr = SrcAddr;
+  Record->src_device_num = SrcDeviceNum;
+  Record->dest_addr = DstAddr;
+  Record->dest_device_num = DstDeviceNum;
+  Record->bytes = Bytes;
+
+  Record->end_time = TraceRecordStopTime;
+  resetTimestamp(&TraceRecordStopTime);
+
+  Record->codeptr_ra = CodePtr;
+}
+
+void Interface::setTraceRecordTargetKernel(ompt_record_target_kernel_t *Record,
+                                           unsigned int NumTeams) {
+  Record->host_op_id = HostOpId;
+  Record->requested_num_teams = NumTeams;
+  Record->granted_num_teams = TraceRecordNumGrantedTeams;
+
+  Record->end_time = TraceRecordStopTime;
+  resetTimestamp(&TraceRecordStopTime);
+}
+
+void Interface::setTraceRecordTarget(ompt_record_target_t *Record,
+                                     int64_t DeviceId, ompt_target_t TargetKind,
+                                     ompt_scope_endpoint_t Endpoint,
+                                     void *CodePtr) {
+  Record->kind = TargetKind;
+  Record->endpoint = Endpoint;
+  Record->device_num = DeviceId;
+  assert(TaskData);
+  Record->task_id = TaskData->value;
+  Record->target_id = TargetData.value;
+  Record->codeptr_ra = CodePtr;
+}
+
+void Interface::startTargetDataAllocTrace(int64_t DeviceId, void *HstPtrBegin,
+                                          void **TgtPtrBegin, size_t Size,
+                                          void *Code) {}
+
+ompt_record_ompt_t *Interface::stopTargetDataAllocTrace(int64_t DeviceId,
+                                                        void *HstPtrBegin,
+                                                        void **TgtPtrBegin,
+                                                        size_t Size,
+                                                        void *Code) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target_data_op))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr = (ompt_record_ompt_t *)TRM->assignCursor(
+      ompt_callback_target_data_op, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target_data_op);
+  setTraceRecordTargetDataOp(&DataPtr->record.target_data_op,
+                             ompt_target_data_alloc, HstPtrBegin,
+                             /*SrcDeviceNum=*/omp_get_initial_device(),
+                             *TgtPtrBegin, DeviceId, Size, Code);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Generated trace record: " << DataPtr
+                 << " (ompt_target_data_alloc)";
+  return DataPtr;
+}
+
+void Interface::startTargetDataDeleteTrace(int64_t DeviceId, void *TgtPtrBegin,
+                                           void *Code) {}
+
+ompt_record_ompt_t *Interface::stopTargetDataDeleteTrace(int64_t DeviceId,
+                                                         void *TgtPtrBegin,
+                                                         void *Code) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target_data_op))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr = (ompt_record_ompt_t *)TRM->assignCursor(
+      ompt_callback_target_data_op, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target_data_op);
+  setTraceRecordTargetDataOp(&DataPtr->record.target_data_op,
+                             ompt_target_data_delete, TgtPtrBegin, DeviceId,
+                             /*DstAddr=*/nullptr,
+                             /*DstDeviceNum=*/-1, /*Bytes=*/0, Code);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Generated trace record: " << DataPtr
+                 << " (ompt_target_data_delete)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *
+Interface::startTargetDataSubmitTrace(int64_t SrcDeviceId, void *SrcPtrBegin,
+                                      int64_t DstDeviceId, void *DstPtrBegin,
+                                      size_t Size, void *Code) {
+  if (!isTracingEnabled(DstDeviceId, ompt_callback_target_data_op))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr = (ompt_record_ompt_t *)TRM->assignCursor(
+      ompt_callback_target_data_op, DstDeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target_data_op);
+  DataPtr->time = 0; // Set to sanity value and let "stop" function fix it
+
+  // Set some of the data-op specific fields here
+  setTraceRecordTargetDataOp(&DataPtr->record.target_data_op,
+                             ompt_target_data_transfer_to_device, SrcPtrBegin,
+                             SrcDeviceId, DstPtrBegin, DstDeviceId, Size, Code);
+
+  ODBG(debug::ODT_Tool) << "OMPT-Async: Returning data trace record buf ptr " << DataPtr;
+  return DataPtr;
+}
+
+ompt_record_ompt_t *
+Interface::startTargetDataRetrieveTrace(int64_t SrcDeviceId, void *SrcPtrBegin,
+                                        int64_t DstDeviceId, void *DstPtrBegin,
+                                        size_t Size, void *Code) {
+  if (!isTracingEnabled(SrcDeviceId, ompt_callback_target_data_op))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr = (ompt_record_ompt_t *)TRM->assignCursor(
+      ompt_callback_target_data_op, SrcDeviceId);
+
+  if (!DataPtr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target_data_op);
+  DataPtr->time = 0; // Set to sanity value and let "stop" function fix it
+
+  // Set some of the data-op specific fields here
+  setTraceRecordTargetDataOp(&DataPtr->record.target_data_op,
+                             ompt_target_data_transfer_from_device, SrcPtrBegin,
+                             SrcDeviceId, DstPtrBegin, DstDeviceId, Size, Code);
+
+  ODBG(debug::ODT_Tool) << "OMPT-Async: Returning data trace record buf ptr " << DataPtr;
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::stopTargetDataMovementTraceAsync(
+    ompt_record_ompt_t *DataPtr, uint64_t NanosStart, uint64_t NanosEnd) {
+  // Finalize the data that comes from the plugin.
+  DataPtr->time = NanosStart;
+  auto Record = static_cast<ompt_record_target_data_op_t *>(
+      &DataPtr->record.target_data_op);
+  Record->end_time = NanosEnd;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "OMPT-Async: Completed target_data trace record " << DataPtr;
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::startTargetSubmitTrace(int64_t DeviceId,
+                                                      unsigned int NumTeams) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target_submit))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr = (ompt_record_ompt_t *)TRM->assignCursor(
+      ompt_callback_target_submit, DeviceId);
+
+  // Set all known entries and leave remaining to the stop function
+  setTraceRecordCommon(DataPtr, ompt_callback_target_submit);
+  DataPtr->time = 0; // Set to sanity value and let "stop" function fix it
+  // Kernel specific things
+  DataPtr->record.target_kernel.requested_num_teams = NumTeams;
+  DataPtr->record.target_kernel.host_op_id = getHostOpId();
+
+  // May be null if event is not traced
+  ODBG(debug::ODT_Tool) << "OMPT-Async: Returning kernel trace record buf ptr " << DataPtr;
+  return DataPtr;
+}
+
+ompt_record_ompt_t *
+Interface::stopTargetSubmitTraceAsync(ompt_record_ompt_t *DataPtr,
+                                      unsigned int NumTeams,
+                                      uint64_t NanosStart, uint64_t NanosStop) {
+  // Common fields
+  DataPtr->time = NanosStart;
+  // Submit specific
+  DataPtr->record.target_kernel.end_time = NanosStop;
+  DataPtr->record.target_kernel.granted_num_teams = NumTeams;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  // Ready Record
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "OMPT-Async: Completed trace record buf ptr " << DataPtr;
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::startTargetDataEnterTrace(int64_t DeviceId,
+                                                         void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId,
+                       ompt_target_enter_data, ompt_scope_begin, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Returning trace record buf ptr " << DataPtr
+                 << " (ompt_target_enter_data)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::stopTargetDataEnterTrace(int64_t DeviceId,
+                                                        void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId,
+                       ompt_target_enter_data, ompt_scope_end, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Generated trace record " << DataPtr
+                 << " (ompt_target_enter_data)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::startTargetDataExitTrace(int64_t DeviceId,
+                                                        void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId, ompt_target_exit_data,
+                       ompt_scope_begin, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Returning trace record buf ptr " << DataPtr
+                 << " (ompt_target_exit_data)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::stopTargetDataExitTrace(int64_t DeviceId,
+                                                       void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId, ompt_target_exit_data,
+                       ompt_scope_end, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Generated trace record " << DataPtr
+                 << " (ompt_target_exit_data)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::startTargetUpdateTrace(int64_t DeviceId,
+                                                      void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId, ompt_target_update,
+                       ompt_scope_begin, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Returning trace record buf ptr " << DataPtr
+                 << " (ompt_target_update)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::stopTargetUpdateTrace(int64_t DeviceId,
+                                                     void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId, ompt_target_update,
+                       ompt_scope_end, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Generated trace record " << DataPtr
+                 << " (ompt_target_update)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::startTargetTrace(int64_t DeviceId,
+                                                void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId, ompt_target,
+                       ompt_scope_begin, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Returning trace record buf ptr " << DataPtr
+                 << " (ompt_target)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::stopTargetTrace(int64_t DeviceId,
+                                               void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId, ompt_target,
+                       ompt_scope_end, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+
+  ODBG(debug::ODT_Tool) << "Generated trace record " << DataPtr
+                 << " (ompt_target)";
+  return DataPtr;
+}
+
+extern "C" {
+// Device-independent entry point for ompt_set_trace_ompt
+ompt_set_result_t libomptarget_ompt_set_trace_ompt(int DeviceId,
+                                                   unsigned int Enable,
+                                                   unsigned int EventTy) {
+  std::unique_lock<std::mutex> Lock(TraceAccessMutex);
+  return llvm::omp::target::ompt::setTraceEventTy(DeviceId, Enable, EventTy);
+}
+
+// Device-independent entry point for ompt_start_trace
+int libomptarget_ompt_start_trace(int DeviceId,
+                                  ompt_callback_buffer_request_t Request,
+                                  ompt_callback_buffer_complete_t Complete) {
+  if (!PM) {
+    REPORT() << "Failed to start trace for DeviceId="
+             << DeviceId << " (invalid plugin manager)";
+    // Indicate failure
+    return 0;
+  }
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  std::unique_lock<std::mutex> Lock(TraceControlMutex);
+  if (Request && Complete) {
+    // Set buffer related functions
+    llvm::omp::target::ompt::setBufferManagementFns(DeviceId, Request,
+                                                    Complete);
+    llvm::omp::target::ompt::enableDeviceTracing(DeviceId);
+    TRM->startHelperThreads();
+    // Success
+    return 1;
+  }
+  // Failure
+  return 0;
+}
+
+// Device-independent entry point for ompt_flush_trace
+int libomptarget_ompt_flush_trace(int DeviceId) {
+  if (!PM) {
+    REPORT() << "Failed to flush trace for DeviceId="
+             << DeviceId << " (invalid plugin manager)";
+    // Indicate failure
+    return 0;
+  }
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  std::unique_lock<std::mutex> Lock(TraceControlMutex);
+  return TRM->flushAllBuffers(DeviceId);
+}
+
+// Device independent entry point for ompt_stop_trace
+int libomptarget_ompt_stop_trace(int DeviceId) {
+  if (!PM) {
+    REPORT() << "Failed to stop trace for DeviceId="
+             << DeviceId << " (invalid plugin manager)";
+    // Indicate failure
+    return 0;
+  }
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  std::unique_lock<std::mutex> Lock(TraceControlMutex);
+
+  // Schedule flushing of trace records for this device
+  int Status = TRM->flushAllBuffers(DeviceId);
+
+  // De-register this device so that no more traces are collected
+  // or delivered for this device until an ompt_start_trace is
+  // invoked for this device.
+  removeBufferManagementFns(DeviceId);
+
+  // If no device is being traced, shut down the helper threads. A
+  // subsequent ompt_start_trace will start up the helper threads.
+  if (isAllDeviceTracingStopped()) {
+    // TODO shutdown should perhaps return a status
+    TRM->shutdownHelperThreads();
+    llvm::omp::target::ompt::disableDeviceTracing(DeviceId);
+  }
+  return Status;
+}
+
+// Device independent entry point for ompt_advance_buffer_cursor
+// Note: The input parameter size is unused here. It refers to the
+// bytes returned in the corresponding callback.
+int libomptarget_ompt_advance_buffer_cursor(ompt_device_t *Device,
+                                            ompt_buffer_t *Buffer, size_t Size,
+                                            ompt_buffer_cursor_t CurrentPos,
+                                            ompt_buffer_cursor_t *NextPos) {
+  if (!PM) {
+    REPORT() << "Failed to advance buffer cursor for Device="
+             << Device << " (invalid plugin manager)";
+    // Indicate failure
+    return false;
+  }
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  char *TraceRecord = (char *)CurrentPos;
+  // Don't assert if CurrentPos is null, just indicate end of buffer
+  if (TraceRecord == nullptr || TRM->isLastCursor(TraceRecord)) {
+    *NextPos = 0;
+    return false;
+  }
+  // TODO In debug mode, assert that the metadata points to the
+  // input parameter buffer
+
+  size_t TRSize = TRM->getTRSize();
+  *NextPos = (ompt_buffer_cursor_t)(TraceRecord + TRSize);
+  ODBG(llvm::omp::target::debug::ODT_Tool) << "Advanced buffer pointer by "
+                 << TRSize << " bytes to "
+                 << TraceRecord + TRSize;
+  return true;
+}
+
+// This function is invoked before the kernel launch. So, when the trace record
+// is populated after kernel completion, TraceRecordNumGrantedTeams is already
+// updated.
+void libomptarget_ompt_set_granted_teams(uint32_t NumTeams) {
+  TraceRecordNumGrantedTeams = NumTeams;
+}
+
+// Assume a synchronous implementation and set thread local variables to track
+// timestamps. The thread local variables can then be used to populate trace
+// records.
+void libomptarget_ompt_set_timestamp(uint64_t Start, uint64_t Stop) {
+  TraceRecordStartTime = Start;
+  TraceRecordStopTime = Stop;
+}
+
+// Device-independent entry point to query for the trace format used.
+// Currently, only OMPT format is supported.
+ompt_record_t
+libomptarget_ompt_get_record_type(ompt_buffer_t *Buffer,
+                                  ompt_buffer_cursor_t CurrentPos) {
+  // TODO: When different OMPT trace buffer formats supported, this needs to be
+  // fixed.
+  return ompt_record_t::ompt_record_ompt;
+}
+} // extern "C"
+
+#pragma pop_macro("DEBUG_PREFIX")
+
+#endif // OMPT_SUPPORT
diff --git a/offload/libomptarget/PluginManager.cpp b/offload/libomptarget/PluginManager.cpp
index 41b653a60adfd..33c35b4f8db17 100644
--- a/offload/libomptarget/PluginManager.cpp
+++ b/offload/libomptarget/PluginManager.cpp
@@ -11,6 +11,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "PluginManager.h"
+#include "OpenMP/OMPT/Callback.h"
+#include "OpenMP/OMPT/OmptCommonDefs.h"
+#include "OpenMP/OMPT/OmptTracing.h"
 #include "OffloadPolicy.h"
 #include "Shared/Debug.h"
 #include "Shared/Profile.h"
@@ -47,6 +50,12 @@ void PluginManager::init() {
   } while (false);
 #include "Shared/Targets.def"
 
+#ifdef OMPT_SUPPORT
+  assert(TraceRecordManager == nullptr &&
+         "Expected trace record manager to be null");
+  TraceRecordManager = new OmptTracingBufferMgr();
+#endif
+
   ODBG(ODT_Init) << "RTLs loaded!";
 }
 
@@ -54,6 +63,16 @@ void PluginManager::deinit() {
   TIMESCOPE();
   ODBG(ODT_Deinit) << "Unloading RTLs...";
 
+#ifdef OMPT_SUPPORT
+  // When offloading is disabled, the TRM is not initialized.
+  if (!OffloadPolicy::isOffloadDisabled()) {
+    assert(TraceRecordManager != nullptr &&
+           "Trace record manager should have been non-null");
+    delete TraceRecordManager;
+    TraceRecordManager = nullptr;
+  }
+#endif
+
   for (auto &Plugin : Plugins) {
     if (!Plugin->is_initialized())
       continue;
@@ -323,6 +342,9 @@ int target(ident_t *Loc, DeviceTy &Device, void *HostPtr,
 void PluginManager::unregisterLib(__tgt_bin_desc *Desc) {
   ODBG(ODT_Deinit) << "Unloading target library!";
 
+  OMPT_IF_TRACING_ENABLED(
+      PM->getTraceRecordManager()->flushAndShutdownHelperThreads(););
+
   Desc = upgradeLegacyEntries(Desc);
 
   PM->RTLsMtx.lock();
diff --git a/offload/libomptarget/device.cpp b/offload/libomptarget/device.cpp
index 18339e9afe975..770b74ca35597 100644
--- a/offload/libomptarget/device.cpp
+++ b/offload/libomptarget/device.cpp
@@ -15,6 +15,8 @@
 #include "OpenMP/Mapping.h"
 #include "OpenMP/OMPT/Callback.h"
 #include "OpenMP/OMPT/Interface.h"
+#include "OpenMP/OMPT/OmptCommonDefs.h"
+#include "OpenMP/OMPT/OmptTracing.h"
 #include "PluginManager.h"
 #include "Shared/APITypes.h"
 #include "Shared/Debug.h"
@@ -34,7 +36,8 @@
 #include <thread>
 
 #ifdef OMPT_SUPPORT
-using namespace llvm::omp::target::ompt;
+using namespace llvm::omp::target;
+using namespace ompt;
 #endif
 
 using namespace llvm::omp::target::plugin;
@@ -249,10 +252,15 @@ DeviceTy::loadBinary(__tgt_device_image *Img) {
 void *DeviceTy::allocData(int64_t Size, void *HstPtr, int32_t Kind) {
   /// RAII to establish tool anchors before and after data allocation
   void *TargetPtr = nullptr;
-  OMPT_IF_BUILT(InterfaceRAII TargetDataAllocRAII(
-                    RegionInterface.getCallbacks<ompt_target_data_alloc>(),
-                    DeviceID, HstPtr, &TargetPtr, Size,
-                    /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
+  OMPT_IF_BUILT(
+      InterfaceRAII TargetDataAllocRAII(
+          RegionInterface.getCallbacks<ompt_target_data_alloc>(), DeviceID,
+          HstPtr, &TargetPtr, Size,
+          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+      InterfaceRAII TargetDataAllocTraceRAII(
+          RegionInterface.getTraceGenerators<ompt_target_data_alloc>(),
+          RTLDeviceID, HstPtr, &TargetPtr, Size,
+          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
 
   TargetPtr = RTL->data_alloc(RTLDeviceID, Size, HstPtr, Kind);
   return TargetPtr;
@@ -260,11 +268,15 @@ void *DeviceTy::allocData(int64_t Size, void *HstPtr, int32_t Kind) {
 
 int32_t DeviceTy::deleteData(void *TgtAllocBegin, int32_t Kind) {
   /// RAII to establish tool anchors before and after data deletion
-  OMPT_IF_BUILT(InterfaceRAII TargetDataDeleteRAII(
-                    RegionInterface.getCallbacks<ompt_target_data_delete>(),
-                    DeviceID, TgtAllocBegin,
-                    /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
-
+  OMPT_IF_BUILT(
+      InterfaceRAII TargetDataDeleteRAII(
+          RegionInterface.getCallbacks<ompt_target_data_delete>(), DeviceID,
+          TgtAllocBegin,
+          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+      InterfaceRAII TargetDataDeleteTraceRAII(
+          RegionInterface.getTraceGenerators<ompt_target_data_delete>(),
+          DeviceID, TgtAllocBegin,
+          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
   return RTL->data_delete(RTLDeviceID, TgtAllocBegin, Kind);
 }
 
@@ -281,6 +293,13 @@ int32_t DeviceTy::submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size,
       InterfaceRAII TargetDataSubmitRAII(
           RegionInterface.getCallbacks<ompt_target_data_transfer_to_device>(),
           omp_initial_device, HstPtrBegin, DeviceID, TgtPtrBegin, Size,
+          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+      TracerInterfaceRAII TargetDataSubmitTraceRAII(
+          RegionInterface
+              .getTraceGenerators<ompt_target_data_transfer_to_device>(),
+          AsyncInfo, RTL->getProfiler(), /*TracedDeviceId=*/DeviceID,
+          /*EventType=*/ompt_callback_target_data_op, omp_initial_device,
+          HstPtrBegin, DeviceID, TgtPtrBegin, Size,
           /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
 
   return RTL->data_submit_async(RTLDeviceID, TgtPtrBegin, HstPtrBegin, Size,
@@ -301,6 +320,13 @@ int32_t DeviceTy::retrieveData(void *HstPtrBegin, void *TgtPtrBegin,
       InterfaceRAII TargetDataRetrieveRAII(
           RegionInterface.getCallbacks<ompt_target_data_transfer_from_device>(),
           DeviceID, TgtPtrBegin, omp_initial_device, HstPtrBegin, Size,
+          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+      TracerInterfaceRAII TargetDataRetrieveTraceRAII(
+          RegionInterface
+              .getTraceGenerators<ompt_target_data_transfer_from_device>(),
+          AsyncInfo, RTL->getProfiler(), /*TracedDeviceId=*/DeviceID,
+          /*EventType=*/ompt_callback_target_data_op, DeviceID, TgtPtrBegin,
+          omp_initial_device, HstPtrBegin, Size,
           /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
 
   return RTL->data_retrieve_async(RTLDeviceID, HstPtrBegin, TgtPtrBegin, Size,
@@ -320,11 +346,15 @@ int32_t DeviceTy::dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr,
       InterfaceRAII TargetDataExchangeRAII(
           RegionInterface.getCallbacks<ompt_target_data_transfer_from_device>(),
           RTLDeviceID, SrcPtr, DstDev.RTLDeviceID, DstPtr, Size,
+          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+      TracerInterfaceRAII TargetDataExchangeTraceRAII(
+          RegionInterface
+              .getTraceGenerators<ompt_target_data_transfer_from_device>(),
+          AsyncInfo, RTL->getProfiler(), /*TracedDeviceId=*/RTLDeviceID,
+          /*EventType=*/ompt_callback_target_data_op, RTLDeviceID, SrcPtr,
+          DstDev.RTLDeviceID, DstPtr, Size,
           /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
-  if (!AsyncInfo) {
-    return RTL->data_exchange(RTLDeviceID, SrcPtr, DstDev.RTLDeviceID, DstPtr,
-                              Size);
-  }
+
   return RTL->data_exchange_async(RTLDeviceID, SrcPtr, DstDev.RTLDeviceID,
                                   DstPtr, Size, AsyncInfo);
 }
diff --git a/offload/libomptarget/exports b/offload/libomptarget/exports
index 1831c43cc5f29..6e404ed059096 100644
--- a/offload/libomptarget/exports
+++ b/offload/libomptarget/exports
@@ -67,6 +67,14 @@ VERS1.0 {
     llvm_omp_target_unlock_mem;
     __tgt_set_info_flag;
     __tgt_print_device_info;
+    libomptarget_ompt_set_trace_ompt;
+    libomptarget_ompt_start_trace;
+    libomptarget_ompt_flush_trace;
+    libomptarget_ompt_stop_trace;
+    libomptarget_ompt_set_granted_teams;
+    libomptarget_ompt_set_timestamp;
+    libomptarget_ompt_advance_buffer_cursor;
+    libomptarget_ompt_get_record_type;
     omp_get_interop_ptr;
     omp_get_interop_str;
     omp_get_interop_int;
diff --git a/offload/libomptarget/interface.cpp b/offload/libomptarget/interface.cpp
index 5d7d948711b99..1e04d10dab8e4 100644
--- a/offload/libomptarget/interface.cpp
+++ b/offload/libomptarget/interface.cpp
@@ -14,6 +14,7 @@
 #include "OpenMP/OMPT/Interface.h"
 #include "OffloadPolicy.h"
 #include "OpenMP/OMPT/Callback.h"
+#include "OpenMP/OMPT/OmptCommonDefs.h"
 #include "OpenMP/omp.h"
 #include "PluginManager.h"
 #include "omptarget.h"
@@ -153,19 +154,30 @@ targetData(ident_t *Loc, int64_t DeviceId, int32_t ArgNum, void **ArgsBase,
   AsyncInfoTy &AsyncInfo = TargetAsyncInfo;
 
   /// RAII to establish tool anchors before and after data begin / end / update
-  OMPT_IF_BUILT(assert((TargetDataFunction == targetDataBegin ||
-                        TargetDataFunction == targetDataEnd ||
-                        TargetDataFunction == targetDataUpdate) &&
-                       "Encountered unexpected TargetDataFunction during "
-                       "execution of targetData");
-                auto CallbackFunctions =
-                    (TargetDataFunction == targetDataBegin)
-                        ? RegionInterface.getCallbacks<ompt_target_enter_data>()
-                    : (TargetDataFunction == targetDataEnd)
-                        ? RegionInterface.getCallbacks<ompt_target_exit_data>()
-                        : RegionInterface.getCallbacks<ompt_target_update>();
-                InterfaceRAII TargetDataRAII(CallbackFunctions, DeviceId,
-                                             OMPT_GET_RETURN_ADDRESS);)
+  OMPT_IF_BUILT(
+      assert((TargetDataFunction == targetDataBegin ||
+              TargetDataFunction == targetDataEnd ||
+              TargetDataFunction == targetDataUpdate) &&
+             "Encountered unexpected TargetDataFunction during "
+             "execution of targetData");
+      auto CallbackFunctions =
+          (TargetDataFunction == targetDataBegin)
+              ? RegionInterface.getCallbacks<ompt_target_enter_data>()
+          : (TargetDataFunction == targetDataEnd)
+              ? RegionInterface.getCallbacks<ompt_target_exit_data>()
+              : RegionInterface.getCallbacks<ompt_target_update>();
+
+      auto TraceGenerators =
+          (TargetDataFunction == targetDataBegin)
+              ? RegionInterface.getTraceGenerators<ompt_target_enter_data>()
+          : (TargetDataFunction == targetDataEnd)
+              ? RegionInterface.getTraceGenerators<ompt_target_exit_data>()
+              : RegionInterface.getTraceGenerators<ompt_target_update>();
+
+      InterfaceRAII TargetDataRAII(CallbackFunctions, DeviceId,
+                                   /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+      InterfaceRAII TargetDataTraceRAII(TraceGenerators, DeviceId,
+                                        /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
 
   int Rc = OFFLOAD_SUCCESS;
 
@@ -433,6 +445,9 @@ static inline int targetKernel(ident_t *Loc, int64_t DeviceId, int32_t NumTeams,
   /// RAII to establish tool anchors before and after target region
   OMPT_IF_BUILT(InterfaceRAII TargetRAII(
                     RegionInterface.getCallbacks<ompt_target>(), DeviceId,
+                    /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+                InterfaceRAII TargetTraceRAII(
+                    RegionInterface.getTraceGenerators<ompt_target>(), DeviceId,
                     /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
 
   int Rc = OFFLOAD_SUCCESS;
@@ -542,6 +557,9 @@ EXTERN int __tgt_target_kernel_replay(
   /// RAII to establish tool anchors before and after target region
   OMPT_IF_BUILT(InterfaceRAII TargetRAII(
                     RegionInterface.getCallbacks<ompt_target>(), DeviceId,
+                    /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+                InterfaceRAII TargetTraceRAII(
+                    RegionInterface.getTraceGenerators<ompt_target>(), DeviceId,
                     /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
 
   AsyncInfoTy AsyncInfo(*DeviceOrErr);
diff --git a/offload/libomptarget/omptarget.cpp b/offload/libomptarget/omptarget.cpp
index 24169e68a29c7..187e7b5351d4b 100644
--- a/offload/libomptarget/omptarget.cpp
+++ b/offload/libomptarget/omptarget.cpp
@@ -2363,8 +2363,12 @@ int target(ident_t *Loc, DeviceTy &Device, void *HostPtr,
     // No need to guard this with OMPT_IF_BUILT
     InterfaceRAII TargetSubmitRAII(
         RegionInterface.getCallbacks<ompt_callback_target_submit>(), NumTeams);
-#endif
 
+    TracerInterfaceRAII TargetTraceRAII(
+        RegionInterface.getTraceGenerators<ompt_callback_target_submit>(),
+        AsyncInfo, Device.RTL->getProfiler(), /*TracedDeviceId=*/DeviceId,
+        /*EventType=*/ompt_callback_target_submit, DeviceId, NumTeams);
+#endif
     Ret = Device.launchKernel(TgtEntryPtr, TgtArgs.data(), TgtOffsets.data(),
                               KernelArgs, nullptr, AsyncInfo);
   }

>From e74fcb23fbdcfdc1816f8eeabb8aa11da12596aa Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Mon, 13 Apr 2026 06:26:35 -0500
Subject: [PATCH 08/11] [Offload][OMPT] Fix global destructor warnings in OMPT
 tracing globals

Replace five globals with non-trivial destructors with function-local
statics, following the LLVM canonical pattern (see Signals.cpp). This
resolves -Werror=global-constructors failures that were triggered by
the destructor of std::map, std::unordered_map, std::shared_ptr, and
a thread_local array of shared_ptr.

Affected globals:
- ParentLibrary (shared_ptr) in plugin OmptTracing.cpp -> file-local
  getParentLibraryStorage() helper
- Devices (map) in plugin OmptTracing.cpp -> getDevices()
- BufferManagementFns (unordered_map) in libomptarget OmptTracing.cpp
  -> file-local getBufferMgmt() helper
- TracedDevices (map) in libomptarget OmptTracing.cpp -> getTracedDevices()
- ArrayOfBufPtr (thread_local shared_ptr array) in OmptTracingBuffer.cpp
  -> getArrayOfBufPtr()

extern declarations are replaced with getter function declarations in
the corresponding headers (OmptDeviceTracing.h, OmptTracing.h,
OmptTracingBuffer.h). All use-sites are updated accordingly.

Made-with: Cursor
---
 offload/include/OpenMP/OMPT/OmptTracing.h     | 11 +--
 .../include/OpenMP/OMPT/OmptTracingBuffer.h   |  5 +-
 .../libomptarget/OpenMP/OMPT/OmptTracing.cpp  | 72 ++++++++++---------
 .../OpenMP/OMPT/OmptTracingBuffer.cpp         | 12 ++--
 .../common/OMPT/OmptDeviceTracing.h           | 20 +++---
 .../common/OMPT/OmptTracing.cpp               | 41 ++++++-----
 6 files changed, 85 insertions(+), 76 deletions(-)

diff --git a/offload/include/OpenMP/OMPT/OmptTracing.h b/offload/include/OpenMP/OMPT/OmptTracing.h
index 2a892582923d5..4c8c049a9ae48 100644
--- a/offload/include/OpenMP/OMPT/OmptTracing.h
+++ b/offload/include/OpenMP/OMPT/OmptTracing.h
@@ -102,8 +102,9 @@ ompt_set_result_t setTraceEventTy(int DeviceId, unsigned int Enable,
 /// Return thread id
 uint64_t getThreadId();
 
-/// See TracedDevices in OmptDeviceTracing.h
-extern std::map<int32_t, uint64_t> TracedDevices;
+/// Access the map of RTL device IDs to their currently enabled tracing event
+/// types. See also OmptDeviceTracing.h.
+std::map<int32_t, uint64_t> &getTracedDevices();
 /// Activate tracing on the given device
 void enableDeviceTracing(int DeviceId);
 /// Deactivate tracing on the given device
@@ -123,12 +124,6 @@ extern std::mutex TraceHashThreadMutex;
 /// buffer-completion callback functions.
 extern std::mutex BufferManagementFnMutex;
 
-/// Map from device-id to the corresponding buffer-request and buffer-completion
-/// callback functions.
-extern std::unordered_map<int, std::pair<ompt_callback_buffer_request_t,
-                                         ompt_callback_buffer_complete_t>>
-    BufferManagementFns;
-
 /// Thread local variables used by the plugin to communicate OMPT information
 /// that are then used to populate trace records. This method assumes a
 /// synchronous implementation, otherwise it won't work.
diff --git a/offload/include/OpenMP/OMPT/OmptTracingBuffer.h b/offload/include/OpenMP/OMPT/OmptTracingBuffer.h
index 5c9f4bf33dae8..a9b7f8d569fcb 100644
--- a/offload/include/OpenMP/OMPT/OmptTracingBuffer.h
+++ b/offload/include/OpenMP/OMPT/OmptTracingBuffer.h
@@ -133,8 +133,9 @@ class OmptTracingBufferMgr {
   // Thread-specific array of pointers to a buffer. The buffer pointed to
   // is the last one allocated by this thread for a given device. The ith
   // element points to the buffer for the ith device. At most MAX_NUM_DEVICES
-  // devices are supported.
-  static thread_local BufPtr ArrayOfBufPtr[MAX_NUM_DEVICES];
+  // devices are supported. Stored as a function-local thread_local to avoid
+  // a global destructor.
+  static BufPtr *getArrayOfBufPtr();
 
   /*
    * A buffer is flushed when it fills up or when the tool invokes
diff --git a/offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp b/offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp
index a74b10137f2c3..1045801842e90 100644
--- a/offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp
+++ b/offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp
@@ -42,17 +42,25 @@ std::mutex llvm::omp::target::ompt::TraceControlMutex;
 std::mutex llvm::omp::target::ompt::TraceHashThreadMutex;
 std::mutex llvm::omp::target::ompt::BufferManagementFnMutex;
 
-std::unordered_map<int /*DeviceId*/, std::pair<ompt_callback_buffer_request_t,
-                                               ompt_callback_buffer_complete_t>>
-    llvm::omp::target::ompt::BufferManagementFns;
-
 thread_local uint32_t llvm::omp::target::ompt::TraceRecordNumGrantedTeams = 0;
 thread_local uint64_t llvm::omp::target::ompt::TraceRecordStartTime = 0;
 thread_local uint64_t llvm::omp::target::ompt::TraceRecordStopTime = 0;
 thread_local uint64_t llvm::omp::target::ompt::ThreadId =
     std::numeric_limits<uint64_t>::max();
 
-std::map<int32_t, uint64_t> llvm::omp::target::ompt::TracedDevices;
+// File-local helper returning a reference to the buffer-management map stored
+// as a function-local static, avoiding a global destructor.
+static auto &getBufferMgmt() {
+  static std::unordered_map<int, std::pair<ompt_callback_buffer_request_t,
+                                           ompt_callback_buffer_complete_t>>
+      M;
+  return M;
+}
+
+std::map<int32_t, uint64_t> &llvm::omp::target::ompt::getTracedDevices() {
+  static std::map<int32_t, uint64_t> TracedDevices;
+  return TracedDevices;
+}
 
 bool llvm::omp::target::ompt::TracingActive = false;
 
@@ -61,8 +69,8 @@ void llvm::omp::target::ompt::resetTimestamp(uint64_t *T) { *T = 0; }
 ompt_callback_buffer_request_t
 llvm::omp::target::ompt::getBufferRequestFn(int DeviceId) {
   std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
-  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
-  if (BufferMgrItr == BufferManagementFns.end()) {
+  auto BufferMgrItr = getBufferMgmt().find(DeviceId);
+  if (BufferMgrItr == getBufferMgmt().end()) {
     return nullptr;
   }
   return BufferMgrItr->second.first;
@@ -71,8 +79,8 @@ llvm::omp::target::ompt::getBufferRequestFn(int DeviceId) {
 ompt_callback_buffer_complete_t
 llvm::omp::target::ompt::getBufferCompleteFn(int DeviceId) {
   std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
-  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
-  if (BufferMgrItr == BufferManagementFns.end()) {
+  auto BufferMgrItr = getBufferMgmt().find(DeviceId);
+  if (BufferMgrItr == getBufferMgmt().end()) {
     return nullptr;
   }
   return BufferMgrItr->second.second;
@@ -82,29 +90,29 @@ void llvm::omp::target::ompt::setBufferManagementFns(
     int DeviceId, ompt_callback_buffer_request_t ReqFn,
     ompt_callback_buffer_complete_t CmpltFn) {
   std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
-  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
-  if (BufferMgrItr != BufferManagementFns.end()) {
+  auto BufferMgrItr = getBufferMgmt().find(DeviceId);
+  if (BufferMgrItr != getBufferMgmt().end()) {
     REPORT() << "Buffer request and complete functions already exist for device  "
              << DeviceId  << "ignoring ...";
     return;
   }
-  BufferManagementFns[DeviceId] = std::make_pair(ReqFn, CmpltFn);
+  getBufferMgmt()[DeviceId] = std::make_pair(ReqFn, CmpltFn);
 }
 
 void llvm::omp::target::ompt::removeBufferManagementFns(int DeviceId) {
   std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
-  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
-  if (BufferMgrItr == BufferManagementFns.end()) {
+  auto BufferMgrItr = getBufferMgmt().find(DeviceId);
+  if (BufferMgrItr == getBufferMgmt().end()) {
     REPORT() << "Buffer request and complete functions don't exist for device  "
              << DeviceId  << "ignoring ...";
     return;
   }
-  BufferManagementFns.erase(BufferMgrItr);
+  getBufferMgmt().erase(BufferMgrItr);
 }
 
 bool llvm::omp::target::ompt::isAllDeviceTracingStopped() {
   std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
-  return BufferManagementFns.empty();
+  return getBufferMgmt().empty();
 }
 
 void llvm::omp::target::ompt::ompt_callback_buffer_request(
@@ -137,11 +145,11 @@ inline bool checkDeviceTracingState(const uint64_t &TracingTypes) {
 
 void llvm::omp::target::ompt::enableDeviceTracing(int DeviceId) {
   std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
-  auto Device = TracedDevices.find(DeviceId);
-  if (Device == TracedDevices.end()) {
+  auto Device = getTracedDevices().find(DeviceId);
+  if (Device == getTracedDevices().end()) {
     uint64_t TracingTypes{0};
     setDeviceTracing(TracingTypes);
-    TracedDevices.emplace(DeviceId, TracingTypes);
+    getTracedDevices().emplace(DeviceId, TracingTypes);
   } else
     setDeviceTracing(Device->second);
   // In any case: at least one device is traced
@@ -150,16 +158,16 @@ void llvm::omp::target::ompt::enableDeviceTracing(int DeviceId) {
 
 void llvm::omp::target::ompt::disableDeviceTracing(int DeviceId) {
   std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
-  auto Device = TracedDevices.find(DeviceId);
-  if (Device == TracedDevices.end()) {
+  auto Device = getTracedDevices().find(DeviceId);
+  if (Device == getTracedDevices().end()) {
     uint64_t TracingTypes{0};
     resetDeviceTracing(TracingTypes);
-    TracedDevices.emplace(DeviceId, TracingTypes);
+    getTracedDevices().emplace(DeviceId, TracingTypes);
   } else
     resetDeviceTracing(Device->second);
 
   // Check for actively traced devices
-  for (auto &Dev : TracedDevices)
+  for (auto &Dev : getTracedDevices())
     if (checkDeviceTracingState(Dev.second))
       return;
 
@@ -175,8 +183,8 @@ bool llvm::omp::target::ompt::isTracingEnabled(int DeviceId,
 
 bool llvm::omp::target::ompt::isTracedDevice(int DeviceId) {
   std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
-  auto Device = TracedDevices.find(DeviceId);
-  if (Device != TracedDevices.end())
+  auto Device = getTracedDevices().find(DeviceId);
+  if (Device != getTracedDevices().end())
     return checkDeviceTracingState(Device->second);
 
   return false;
@@ -187,8 +195,8 @@ bool llvm::omp::target::ompt::isTracingTypeEnabled(int DeviceId,
   std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
   // Make sure we do not shift more than std::numeric_limits<uint64_t>::digits
   assert(EventTy < 64 && "Shift limit exceeded: EventTy must be less than 64");
-  auto Device = TracedDevices.find(DeviceId);
-  if (Device != TracedDevices.end() && EventTy < 64)
+  auto Device = getTracedDevices().find(DeviceId);
+  if (Device != getTracedDevices().end() && EventTy < 64)
     return (Device->second & (1UL << EventTy));
   return false;
 }
@@ -198,8 +206,8 @@ bool llvm::omp::target::ompt::isTracingTypeGroupEnabled(int DeviceId,
   std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
   // Make sure we do not shift more than std::numeric_limits<uint64_t>::digits
   assert(EventTy < 64 && "Shift limit exceeded: EventTy must be less than 64");
-  auto Device = TracedDevices.find(DeviceId);
-  if (Device != TracedDevices.end() && EventTy < 64) {
+  auto Device = getTracedDevices().find(DeviceId);
+  if (Device != getTracedDevices().end() && EventTy < 64) {
     auto TracedEvents = Device->second;
     switch (EventTy) {
     case ompt_callbacks_t::ompt_callback_target:
@@ -252,10 +260,10 @@ llvm::omp::target::ompt::setTraceEventTy(int DeviceId, unsigned int Enable,
                  << " Enable=" << Enable << " EventTy=" << EventTy;
 
   std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
-  if (TracedDevices.find(DeviceId) == TracedDevices.end())
-    TracedDevices.emplace(DeviceId, 0UL);
+  if (getTracedDevices().find(DeviceId) == getTracedDevices().end())
+    getTracedDevices().emplace(DeviceId, 0UL);
 
-  auto &TracedEventTy = TracedDevices[DeviceId];
+  auto &TracedEventTy = getTracedDevices()[DeviceId];
   bool Enabled = Enable > 0;
   if (EventTy == 0) {
     // Set / reset all supported types
diff --git a/offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp b/offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp
index 2d4b32e3554ac..cf93b5f968ac7 100644
--- a/offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp
+++ b/offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp
@@ -32,8 +32,10 @@ static std::atomic<uint64_t> BufId{0};
 // Unique id in buffer flush order
 static std::atomic<uint64_t> FlushId{0};
 
-thread_local OmptTracingBufferMgr::BufPtr
-    OmptTracingBufferMgr::ArrayOfBufPtr[MAX_NUM_DEVICES];
+OmptTracingBufferMgr::BufPtr *OmptTracingBufferMgr::getArrayOfBufPtr() {
+  thread_local BufPtr Array[MAX_NUM_DEVICES]{};
+  return Array;
+}
 
 static uint64_t get_and_inc_buf_id() { return BufId++; }
 
@@ -418,7 +420,7 @@ OmptTracingBufferMgr::getDeviceSpecificBuffer(int64_t DeviceId) {
              << MAX_NUM_DEVICES - 1;
     return nullptr;
   }
-  return ArrayOfBufPtr[DeviceId];
+  return getArrayOfBufPtr()[DeviceId];
 }
 
 void OmptTracingBufferMgr::setDeviceSpecificBuffer(int64_t DeviceId,
@@ -429,7 +431,7 @@ void OmptTracingBufferMgr::setDeviceSpecificBuffer(int64_t DeviceId,
              << MAX_NUM_DEVICES - 1;
     return;
   }
-  ArrayOfBufPtr[DeviceId] = Buf;
+  getArrayOfBufPtr()[DeviceId] = Buf;
 }
 
 void OmptTracingBufferMgr::setTRStatus(void *Rec, TRStatus Status) {
@@ -681,7 +683,7 @@ void OmptTracingBufferMgr::waitForFlushCompletion() {
 
 void OmptTracingBufferMgr::init() {
   for (int i = 0; i < MAX_NUM_DEVICES; ++i)
-    ArrayOfBufPtr[i] = nullptr;
+    getArrayOfBufPtr()[i] = nullptr;
   ThreadFlushTracker = 0;
   ThreadShutdownTracker = 0;
   DoneTracing = false; // TODO make it a class member
diff --git a/offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h b/offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h
index 5744a784825da..ba9e1dcd6ccc3 100644
--- a/offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h
+++ b/offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h
@@ -85,19 +85,17 @@ extern double HostToDeviceSlope;
 /// Host to device constant clock offset
 extern double HostToDeviceOffset;
 
-/// Mapping of device pointers to their corresponding RTL device ID
-extern std::map<ompt_device_t *, int32_t> Devices;
+/// Access the map of device pointers to their corresponding RTL device ID
+std::map<ompt_device_t *, int32_t> &getDevices();
 
-/// Mapping of RTL device IDs to their currently enabled tracing event types.
-/// Note: Event type '0' (bit position) indicates if this device is traced.
-extern std::map<int32_t, uint64_t> TracedDevices;
+/// Access the map of RTL device IDs to their currently enabled tracing event
+/// types. Note: Event type '0' (bit position) indicates if this device is
+/// traced.
+std::map<int32_t, uint64_t> &getTracedDevices();
 
 /// OMPT global tracing status. Indicates if at least one device is traced.
 extern bool TracingActive;
 
-/// Parent library pointer
-extern std::shared_ptr<llvm::sys::DynamicLibrary> ParentLibrary;
-
 /// Get the parent library by pointer. If it is not already set, it will set the
 /// parent library pointer.
 std::shared_ptr<llvm::sys::DynamicLibrary> getParentLibrary();
@@ -111,10 +109,10 @@ void setParentLibrary(const char *Filename);
 template <typename FT>
 void ensureFuncPtrLoaded(const std::string &FuncName, FT *FuncPtr) {
   if (*FuncPtr == nullptr) {
-    if ((ParentLibrary == nullptr && getParentLibrary() == nullptr) ||
-        !ParentLibrary->isValid())
+    auto Lib = getParentLibrary();
+    if (!Lib || !Lib->isValid())
       return;
-    void *SymbolPtr = ParentLibrary->getAddressOfSymbol(FuncName.c_str());
+    void *SymbolPtr = Lib->getAddressOfSymbol(FuncName.c_str());
     if (SymbolPtr == nullptr)
       return;
     *FuncPtr = reinterpret_cast<FT>(SymbolPtr);
diff --git a/offload/plugins-nextgen/common/OMPT/OmptTracing.cpp b/offload/plugins-nextgen/common/OMPT/OmptTracing.cpp
index d5be9b8d9cd8a..6e900143c4012 100644
--- a/offload/plugins-nextgen/common/OMPT/OmptTracing.cpp
+++ b/offload/plugins-nextgen/common/OMPT/OmptTracing.cpp
@@ -46,39 +46,44 @@ std::mutex llvm::omp::target::ompt::DeviceIdWritingMutex;
 using namespace llvm::omp::target::ompt;
 using namespace llvm::omp::target::debug;
 
-std::shared_ptr<llvm::sys::DynamicLibrary>
-    llvm::omp::target::ompt::ParentLibrary(nullptr);
-
 double llvm::omp::target::ompt::HostToDeviceSlope = .0;
 double llvm::omp::target::ompt::HostToDeviceOffset = .0;
 
-std::map<ompt_device_t *, int32_t> llvm::omp::target::ompt::Devices;
+// File-local helper to hold the parent library shared_ptr as a function-local
+// static, avoiding a global destructor.
+static std::shared_ptr<llvm::sys::DynamicLibrary> &getParentLibraryStorage() {
+  static std::shared_ptr<llvm::sys::DynamicLibrary> Lib(nullptr);
+  return Lib;
+}
 
 std::shared_ptr<llvm::sys::DynamicLibrary>
 llvm::omp::target::ompt::getParentLibrary() {
-  static bool ParentLibraryAssigned = false;
-  if (!ParentLibraryAssigned) {
+  if (!getParentLibraryStorage()) {
     setParentLibrary("libomptarget.so");
-    ParentLibraryAssigned = true;
   }
-  return ParentLibrary;
+  return getParentLibraryStorage();
 }
 
 void llvm::omp::target::ompt::setParentLibrary(const char *Filename) {
-  if (ParentLibrary)
+  if (getParentLibraryStorage())
     return;
   std::string ErrorMsg;
-  ParentLibrary = std::make_shared<llvm::sys::DynamicLibrary>(
+  getParentLibraryStorage() = std::make_shared<llvm::sys::DynamicLibrary>(
       llvm::sys::DynamicLibrary::getPermanentLibrary(Filename, &ErrorMsg));
-  if ((ParentLibrary == nullptr) || (!ParentLibrary->isValid()))
+  if (!getParentLibraryStorage() || !getParentLibraryStorage()->isValid())
     REPORT() << "Failed to set parent library: " << ErrorMsg.c_str();
 }
 
+std::map<ompt_device_t *, int32_t> &llvm::omp::target::ompt::getDevices() {
+  static std::map<ompt_device_t *, int32_t> Devices;
+  return Devices;
+}
+
 int llvm::omp::target::ompt::getDeviceId(ompt_device_t *Device) {
   // Block other threads, which might trigger an erase (for the same device)
   std::unique_lock<std::mutex> Lock(DeviceIdWritingMutex);
-  auto DeviceIterator = Devices.find(Device);
-  if (Device == nullptr || DeviceIterator == Devices.end()) {
+  auto DeviceIterator = getDevices().find(Device);
+  if (Device == nullptr || DeviceIterator == getDevices().end()) {
     REPORT() << "Failed to get ID for Device=" << Device;
     return -1;
   }
@@ -93,8 +98,8 @@ void llvm::omp::target::ompt::setDeviceId(ompt_device_t *Device,
     return;
   }
   std::unique_lock<std::mutex> Lock(DeviceIdWritingMutex);
-  auto DeviceIterator = Devices.find(Device);
-  if (DeviceIterator != Devices.end()) {
+  auto DeviceIterator = getDevices().find(Device);
+  if (DeviceIterator != getDevices().end()) {
     auto CurrentDeviceId = DeviceIterator->second;
     if (DeviceId == CurrentDeviceId) {
       REPORT() << "Tried to duplicate OMPT Device= " << Device <<  " ID=" << DeviceId;
@@ -103,7 +108,7 @@ void llvm::omp::target::ompt::setDeviceId(ompt_device_t *Device,
     }
     return;
   }
-  Devices.emplace(Device, DeviceId);
+  getDevices().emplace(Device, DeviceId);
 }
 
 void llvm::omp::target::ompt::removeDeviceId(ompt_device_t *Device) {
@@ -113,8 +118,8 @@ void llvm::omp::target::ompt::removeDeviceId(ompt_device_t *Device) {
     return;
   }
   std::unique_lock<std::mutex> Lock(DeviceIdWritingMutex);
-  Devices.erase(Device);
-  TracedDevices.erase(DeviceId);
+  getDevices().erase(Device);
+  getTracedDevices().erase(DeviceId);
 }
 
 OMPT_API_ROUTINE ompt_set_result_t ompt_set_trace_ompt(ompt_device_t *Device,

>From 77d536c1a46189150efcab672b408f10631bfbc5 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 21 May 2026 08:43:39 -0500
Subject: [PATCH 09/11] [OpenMP] Add OMPT device tracing tests

---
 .../callbacks.h                               | 340 ++++++++++++++++++
 .../veccopy_ompt_selective_tracing_dataop.cpp |  66 ++++
 .../callbacks.h                               | 340 ++++++++++++++++++
 .../veccopy_ompt_selective_tracing_kernel.cpp |  70 ++++
 .../callbacks.h                               | 332 +++++++++++++++++
 .../veccopy-ompt-target-data-tracing-emi.cpp  | 146 ++++++++
 .../callbacks.h                               | 333 +++++++++++++++++
 .../veccopy-ompt-target-emi-tracing-dag.cpp   |  81 +++++
 .../callbacks.h                               | 332 +++++++++++++++++
 .../veccopy-ompt-target-emi-tracing.cpp       |  78 ++++
 .../callbacks.h                               | 332 +++++++++++++++++
 .../veccopy-ompt-target-emi-tracing.cpp       |  67 ++++
 .../callbacks.h                               | 332 +++++++++++++++++
 .../veccopy-ompt-target-emi-tracing.cpp       |  66 ++++
 .../veccopy-ompt-target-tracing/callbacks.h   | 292 +++++++++++++++
 .../veccopy-ompt-target-tracing.cpp           |  78 ++++
 16 files changed, 3285 insertions(+)
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/veccopy_ompt_selective_tracing_dataop.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/veccopy_ompt_selective_tracing_kernel.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/veccopy-ompt-target-data-tracing-emi.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/veccopy-ompt-target-emi-tracing-dag.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/veccopy-ompt-target-emi-tracing.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/veccopy-ompt-target-emi-tracing.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/veccopy-ompt-target-emi-tracing.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing/veccopy-ompt-target-tracing.cpp

diff --git a/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/callbacks.h
new file mode 100644
index 0000000000000..5cd611990d32b
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/callbacks.h
@@ -0,0 +1,340 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+  // Only enable device tracing for dataop events explicitly
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d filename:%s host_adddr:%p device_addr:%p"
+         " bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+  assert(DeviceMapPtr != nullptr && "DeviceMapPtr must be valid");
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d optype=%d target_task_data=%p "
+         "(0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) src=%p "
+         "src_device_num=%d dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, optype, target_task_data, target_task_data_value,
+         target_data, target_data->value, host_op_id, *host_op_id, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d req_num_teams=%d target_data=%p "
+         "(0x%lx) host_op_id=%p (0x%lx)\n",
+         endpoint, requested_num_teams, target_data, target_data->value,
+         host_op_id, *host_op_id);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+#if EMI
+  register_ompt_callback(ompt_callback_target_data_op_emi);
+  register_ompt_callback(ompt_callback_target_emi);
+  register_ompt_callback(ompt_callback_target_submit_emi);
+#else
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+#endif
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/veccopy_ompt_selective_tracing_dataop.cpp b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/veccopy_ompt_selective_tracing_dataop.cpp
new file mode 100644
index 0000000000000..c07bff4230688
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/veccopy_ompt_selective_tracing_dataop.cpp
@@ -0,0 +1,66 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
+/// CHECK: rec={{0x[0-9a-fA-F]+}} type=9
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=10
+/// CHECK: rec={{0x[0-9a-fA-F]+}} type=9
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
+
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
+/// CHECK: rec={{0x[0-9a-fA-F]+}} type=9
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=10
+/// CHECK: rec={{0x[0-9a-fA-F]+}} type=9
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
diff --git a/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/callbacks.h
new file mode 100644
index 0000000000000..9655d718c7c48
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/callbacks.h
@@ -0,0 +1,340 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+  // Only enable device tracing for kernel submit events
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d filename:%s host_adddr:%p device_addr:%p"
+         " bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+  assert(DeviceMapPtr != nullptr && "DeviceMapPtr must be valid");
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d optype=%d target_task_data=%p "
+         "(0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) src=%p "
+         "src_device_num=%d dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, optype, target_task_data, target_task_data_value,
+         target_data, target_data->value, host_op_id, *host_op_id, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d req_num_teams=%d target_data=%p "
+         "(0x%lx) host_op_id=%p (0x%lx)\n",
+         endpoint, requested_num_teams, target_data, target_data->value,
+         host_op_id, *host_op_id);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+#if EMI
+  register_ompt_callback(ompt_callback_target_data_op_emi);
+  register_ompt_callback(ompt_callback_target_emi);
+  register_ompt_callback(ompt_callback_target_submit_emi);
+#else
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+#endif
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/veccopy_ompt_selective_tracing_kernel.cpp b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/veccopy_ompt_selective_tracing_kernel.cpp
new file mode 100644
index 0000000000000..285f1f1011268
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/veccopy_ompt_selective_tracing_kernel.cpp
@@ -0,0 +1,70 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// XFAIL: *
+// clang-format on
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  for (auto Device : *DeviceMapPtr)
+    flush_trace(Device);
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=9
+/// CHECK: rec={{0x[0-9a-fA-F]+}} type=10
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=9
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
+
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=9
+/// CHECK: rec={{0x[0-9a-fA-F]+}} type=10
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=9
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/callbacks.h
new file mode 100644
index 0000000000000..d05f05a4a4501
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/callbacks.h
@@ -0,0 +1,332 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+#define EMI 1
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d filename:%s host_adddr:%p device_addr:%p"
+         " bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d optype=%d target_task_data=%p "
+         "(0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) src=%p "
+         "src_device_num=%d dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, optype, target_task_data, target_task_data_value,
+         target_data, target_data->value, host_op_id, *host_op_id, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d req_num_teams=%d target_data=%p "
+         "(0x%lx) host_op_id=%p (0x%lx)\n",
+         endpoint, requested_num_teams, target_data, target_data->value,
+         host_op_id, *host_op_id);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+#if EMI
+  register_ompt_callback(ompt_callback_target_data_op_emi);
+  register_ompt_callback(ompt_callback_target_emi);
+  register_ompt_callback(ompt_callback_target_submit_emi);
+#else
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+#endif
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/veccopy-ompt-target-data-tracing-emi.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/veccopy-ompt-target-data-tracing-emi.cpp
new file mode 100644
index 0000000000000..35a38f4eba739
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/veccopy-ompt-target-data-tracing-emi.cpp
@@ -0,0 +1,146 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+#define N 100000
+
+#pragma omp declare target
+int c[N];
+#pragma omp end declare target
+
+int main() {
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+  for (i = 0; i < N; i++)
+    c[i] = 0;
+
+#pragma omp target enter data map(to : a)
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+#pragma omp target exit data map(from : a)
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+#pragma omp target parallel for map(alloc : c)
+  {
+    for (int j = 0; j < N; j++)
+      c[j] = 2 * j + 1;
+  }
+#pragma omp target update from(c) nowait
+#pragma omp barrier
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+  int rc = 0;
+  for (i = 0; i < N; i++) {
+    if (a[i] != i) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+  }
+
+  for (i = 0; i < N; i++) {
+    if (c[i] != 2 * i + 1) {
+      rc++;
+      printf("Wrong value: c[%d]=%d\n", i, c[i]);
+    }
+  }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+// > OMPT device callback related checks below. <
+
+/// CHECK-NOT: Callback Target EMI:
+/// CHECK-NOT: device_num=-1
+
+/// CHECK: Callback Init:
+/// CHECK: Callback Load:
+
+/// CHECK-DAG: Callback Target EMI: kind=2 endpoint=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=2
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=2
+/// CHECK-DAG: Callback Target EMI: kind=2 endpoint=2
+
+/// CHECK-DAG: Callback Target EMI: kind=1 endpoint=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=2
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=2
+/// CHECK-DAG: Callback Submit EMI: endpoint=1 req_num_teams=1
+/// CHECK-DAG: Callback Submit EMI: endpoint=2 req_num_teams=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=3
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=3
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=4
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=4
+/// CHECK-DAG: Callback Target EMI: kind=1 endpoint=2
+/// CHECK-DAG: Callback Target EMI: kind=3 endpoint=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=3
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=3
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=4
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=4
+/// CHECK-DAG: Callback Target EMI: kind=3 endpoint=2
+
+/// CHECK-DAG: Callback Target EMI: kind=1 endpoint=1
+/// CHECK-DAG: Callback Submit EMI: endpoint=1 req_num_teams=1
+/// CHECK-DAG: Callback Submit EMI: endpoint=2 req_num_teams=1
+/// CHECK-DAG: Callback Target EMI: kind=1 endpoint=2
+/// CHECK-DAG: Callback Target EMI: kind=4 endpoint=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=3
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=3
+/// CHECK-DAG: Callback Target EMI: kind=4 endpoint=2
+
+/// CHECK-DAG: Success
+/// CHECK-DAG: Callback Fini:
+
+// > OMPT device tracing related checks below. <
+
+
+
+// Note: Split checks for record address and content. That way we do not imply
+//       any order. Records 01-06 and 12-17 occur interleaved and belong to the
+//       first target region. 07-11 occur interleaved with 18-22 and belong to
+//       the second target region.
+
+// Note: These addresses will only occur once. They are only captured to
+//       indicate their existence.
+
+/// CHECK-DAG: type=8 (Target task)
+/// CHECK-DAG: type=9 (Target data op)
+
+// Note: ADDRX_11 may not trigger a final callback.
+
+// Note: ADDRX_11 may not be deallocated.
+
+/// CHECK-NOT: host_op_id=0x0
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/callbacks.h
new file mode 100644
index 0000000000000..f5724fa7a6852
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/callbacks.h
@@ -0,0 +1,333 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// Enable / disable OMPT's 'External Monitoring Interface'
+#define EMI 1
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d filename:%s host_adddr:%p device_addr:%p"
+         " bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d optype=%d target_task_data=%p "
+         "(0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) src=%p "
+         "src_device_num=%d dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, optype, target_task_data, target_task_data_value,
+         target_data, target_data->value, host_op_id, *host_op_id, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d req_num_teams=%d target_data=%p "
+         "(0x%lx) host_op_id=%p (0x%lx)\n",
+         endpoint, requested_num_teams, target_data, target_data->value,
+         host_op_id, *host_op_id);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+#if EMI
+  register_ompt_callback(ompt_callback_target_data_op_emi);
+  register_ompt_callback(ompt_callback_target_emi);
+  register_ompt_callback(ompt_callback_target_submit_emi);
+#else
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+#endif
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/veccopy-ompt-target-emi-tracing-dag.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/veccopy-ompt-target-emi-tracing-dag.cpp
new file mode 100644
index 0000000000000..0fdae19017ab2
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/veccopy-ompt-target-emi-tracing-dag.cpp
@@ -0,0 +1,81 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+/// CHECK-NOT: host_op_id=0x0
+
+
+
+// Note: Split checks for record address and content. That way we do not imply
+//       any order. Records 01-06 and 12-17 occur interleaved and belong to the
+//       first target region. 07-11 occur interleaved with 18-22 and belong to
+//       the second target region.
+
+/// CHECK-DAG: type=8 (Target task)
+/// CHECK-DAG: type=9 (Target data op)
+
+// Note: These addresses will only occur once. They are only captured to
+//       indicate their existence.
+
+
+// Note: ADDRX_11 may not trigger a final callback.
+
+// Note: ADDRX_11 may not be deallocated.
+
+/// CHECK-DAG: Success
+
+/// CHECK-NOT: host_op_id=0x0
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/callbacks.h
new file mode 100644
index 0000000000000..d05f05a4a4501
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/callbacks.h
@@ -0,0 +1,332 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+#define EMI 1
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d filename:%s host_adddr:%p device_addr:%p"
+         " bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d optype=%d target_task_data=%p "
+         "(0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) src=%p "
+         "src_device_num=%d dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, optype, target_task_data, target_task_data_value,
+         target_data, target_data->value, host_op_id, *host_op_id, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d req_num_teams=%d target_data=%p "
+         "(0x%lx) host_op_id=%p (0x%lx)\n",
+         endpoint, requested_num_teams, target_data, target_data->value,
+         host_op_id, *host_op_id);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+#if EMI
+  register_ompt_callback(ompt_callback_target_data_op_emi);
+  register_ompt_callback(ompt_callback_target_emi);
+  register_ompt_callback(ompt_callback_target_submit_emi);
+#else
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+#endif
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/veccopy-ompt-target-emi-tracing.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/veccopy-ompt-target-emi-tracing.cpp
new file mode 100644
index 0000000000000..e682e9b20bfd8
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/veccopy-ompt-target-emi-tracing.cpp
@@ -0,0 +1,78 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-generic && env LIBOMPTARGET_OMPT_FLUSH_ON_SHUTDOWN=false LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL=false %libomptarget-run-generic | %fcheck-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+/*
+ * This test is run with LIBOMPTARGET_OMPT_FLUSH_ON_SHUTDOWN=false and
+ * LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL=false while doing explicit flushing
+ * using ompt_flush_trace. The intention is to check whether trace records are
+ * properly flushed when the program/tool uses the ompt_flush_trace API. There
+ * should be 23 trace records returned to the tool.
+ */
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+/// CHECK-NOT: host_op_id=0x0
+
+/// CHECK: rec=
+
+/// CHECK-DAG: Success
+
+/// The user calls flush before printing success, so 
+/// no more records should be returned here.
+
+/// CHECK-NOT: host_op_id=0x0
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/callbacks.h
new file mode 100644
index 0000000000000..d05f05a4a4501
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/callbacks.h
@@ -0,0 +1,332 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+#define EMI 1
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d filename:%s host_adddr:%p device_addr:%p"
+         " bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d optype=%d target_task_data=%p "
+         "(0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) src=%p "
+         "src_device_num=%d dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, optype, target_task_data, target_task_data_value,
+         target_data, target_data->value, host_op_id, *host_op_id, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d req_num_teams=%d target_data=%p "
+         "(0x%lx) host_op_id=%p (0x%lx)\n",
+         endpoint, requested_num_teams, target_data, target_data->value,
+         host_op_id, *host_op_id);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+#if EMI
+  register_ompt_callback(ompt_callback_target_data_op_emi);
+  register_ompt_callback(ompt_callback_target_emi);
+  register_ompt_callback(ompt_callback_target_submit_emi);
+#else
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+#endif
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/veccopy-ompt-target-emi-tracing.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/veccopy-ompt-target-emi-tracing.cpp
new file mode 100644
index 0000000000000..abf2426d2eee9
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/veccopy-ompt-target-emi-tracing.cpp
@@ -0,0 +1,67 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-generic && env LIBOMPTARGET_OMPT_FLUSH_ON_SHUTDOWN=false %libomptarget-run-generic | %fcheck-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+/*
+ * This test is run with LIBOMPTARGET_OMPT_FLUSH_ON_SHUTDOWN=false and
+ * ompt_flush_trace is not invoked by the user/tool. The intention is to check
+ * whether trace records are properly flushed as a buffer fills up. Currently,
+ * the buffer size in callbacks.h implies that every buffer holds 2 trace
+ * records. With this assumption, there should be 22 trace records. The last
+ * trace record is not flushed because the last buffer is not yet full.
+ */
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+/// CHECK-NOT: host_op_id=0x0
+
+/// CHECK: rec=
+/// CHECK-NOT: host_op_id=0x0
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/callbacks.h
new file mode 100644
index 0000000000000..d05f05a4a4501
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/callbacks.h
@@ -0,0 +1,332 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+#define EMI 1
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d filename:%s host_adddr:%p device_addr:%p"
+         " bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d optype=%d target_task_data=%p "
+         "(0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) src=%p "
+         "src_device_num=%d dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, optype, target_task_data, target_task_data_value,
+         target_data, target_data->value, host_op_id, *host_op_id, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d req_num_teams=%d target_data=%p "
+         "(0x%lx) host_op_id=%p (0x%lx)\n",
+         endpoint, requested_num_teams, target_data, target_data->value,
+         host_op_id, *host_op_id);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+#if EMI
+  register_ompt_callback(ompt_callback_target_data_op_emi);
+  register_ompt_callback(ompt_callback_target_emi);
+  register_ompt_callback(ompt_callback_target_submit_emi);
+#else
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+#endif
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/veccopy-ompt-target-emi-tracing.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/veccopy-ompt-target-emi-tracing.cpp
new file mode 100644
index 0000000000000..7f4e99b83e4e3
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/veccopy-ompt-target-emi-tracing.cpp
@@ -0,0 +1,66 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-generic && env LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL=false %libomptarget-run-generic | %fcheck-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+/*
+ * This test is run with LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL=false and
+ * ompt_flush_trace is not invoked by the user/tool. The intention is to check
+ * whether trace records are properly flushed on shutdown. 23 trace records
+ * should be flushed during shutdown.
+ */
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+/// CHECK-NOT: host_op_id=0x0
+
+/// CHECK: rec=
+
+/// CHECK-NOT: host_op_id=0x0
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-tracing/callbacks.h
new file mode 100644
index 0000000000000..cfb5fd4147ebf
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing/callbacks.h
@@ -0,0 +1,292 @@
+// Copyright © Advanced Micro Devices, Inc., or its affiliates.
+//
+// SPDX-License-Identifier:  MIT
+
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// Enable / disable OMPT's 'External Monitoring Interface'
+#define EMI 0
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE (4 * 1024)
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+static ompt_get_record_type_t ompt_get_record_type_fn = 0;
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+// Note: This callback must handle a null begin cursor. Currently,
+// ompt_get_record_ompt, print_record_ompt, and
+// ompt_advance_buffer_cursor handle a null cursor.
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+
+    if (ompt_get_record_type_fn(buffer, current) != ompt_record_ompt) {
+      printf("WARNING: received non-ompt type buffer object\n");
+    }
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n", device_num,
+         type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  ompt_get_record_type_fn =
+      (ompt_get_record_type_t)lookup("ompt_get_record_type");
+  if (!ompt_get_record_type_fn) {
+    printf("WARNING: No function ompt_get_record_type found in device "
+           "callbacks\n");
+  }
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Load: device_num:%d filename:%s host_adddr:%p device_addr:%p "
+         "bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: target_id=%lu host_op_id=%lu optype=%d src=%p "
+         "src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         target_id, host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: target_id=%lu kind=%d endpoint=%d device_num=%d "
+         "code=%p\n",
+         target_id, kind, endpoint, device_num, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing/veccopy-ompt-target-tracing.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-tracing/veccopy-ompt-target-tracing.cpp
new file mode 100644
index 0000000000000..d393bc63b2e14
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing/veccopy-ompt-target-tracing.cpp
@@ -0,0 +1,78 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+// Copyright © Advanced Micro Devices, Inc., or its affiliates.
+//
+// SPDX-License-Identifier:  MIT
+
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+// > OMPT device tracing related checks below. <
+
+// Note: This test will allocate one buffer, big enough to hold all trace
+//       records, hence there will be only one allocation.
+
+
+// Note: Split checks for record address and content. That way we do not imply
+//       any order. Records may / will occur interleaved.
+
+// Note: These addresses will only occur once. They are only captured to
+//       indicate their existence.
+
+/// CHECK-DAG: type=8 (Target task)
+/// CHECK-DAG: type=9 (Target data op)
+
+// Note: ADDRX_01 may not trigger a final callback.
+// Note: ADDRX_01 may not be deallocated.
+
+/// CHECK-NOT: host_op_id=0x0

>From a43f9b237e9d86bf72319b1511c048053f6989d0 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 21 May 2026 08:44:24 -0500
Subject: [PATCH 10/11] [Openmp] Add OMPT device tracing support tests

These tests concern functionality used by parts of the OMPT device
tracing implementation.
---
 .../veccopy-ompt-ctor-1/Inputs/callbacks.h    | 373 ++++++++++++++++++
 .../Inputs/veccopy-ctor-1.cpp                 |  37 ++
 .../tracing/veccopy-ompt-ctor-1/veccopy.cpp   |  52 +++
 .../callbacks.h                               | 286 ++++++++++++++
 .../veccopy-ompt-target-default-device.cpp    |  86 ++++
 .../veccopy-ompt-target-devices/callbacks.h   | 324 +++++++++++++++
 .../veccopy-ompt-target-devices.cpp           |  86 ++++
 .../callbacks.h                               | 166 ++++++++
 .../veccopy-ompt-target.c                     |  78 ++++
 .../callbacks.h                               | 160 ++++++++
 .../veccopy-ompt-target.c                     |  78 ++++
 11 files changed, 1726 insertions(+)
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/veccopy-ctor-1.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-ctor-1/veccopy.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-default-device/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-default-device/veccopy-ompt-target-default-device.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-devices/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-devices/veccopy-ompt-target-devices.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-translate-time/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-translate-time/veccopy-ompt-target.c
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/veccopy-ompt-target.c

diff --git a/offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/callbacks.h
new file mode 100644
index 0000000000000..d206c46567a3e
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/callbacks.h
@@ -0,0 +1,373 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+#define EMI 1
+
+// Tool related code below
+#include <omp-tools.h>
+
+ompt_id_t  next_op_id = 0x8000000000000001;
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t*> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL) return;
+  
+  printf("rec=%p type=%2d time=%lu thread_id=%lu target_id=%lu\n",
+	 rec, rec->type, rec->time, rec->thread_id, rec->target_id);
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi:
+    {
+      ompt_record_target_t target_rec = rec->record.target;
+      printf("\tRecord Target task: target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu codeptr=%p\n",
+	     target_rec.target_id, 
+	     target_rec.kind, target_rec.endpoint, target_rec.device_num,
+	     target_rec.task_id, target_rec.codeptr_ra);
+      break;
+    }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi:
+    {
+      ompt_record_target_data_op_t target_data_op_rec = rec->record.target_data_op;
+      printf("\t  Record Target data op: target_id=0x%lx host_op_id=0x%lx optype=%d src_addr=%p src_device=%d "
+	     "dest_addr=%p dest_device=%d bytes=%lu end_time=%lu duration=%lu ns codeptr=%p\n",
+             rec->target_id,
+	     target_data_op_rec.host_op_id, target_data_op_rec.optype,
+	     target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+	     target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+	     target_data_op_rec.bytes, target_data_op_rec.end_time,
+	     target_data_op_rec.end_time - rec->time,
+	     target_data_op_rec.codeptr_ra);
+      break;
+    }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi:
+    {
+      ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+      printf("\t  Record Target kernel:  target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u granted_num_teams=%u "
+	     "end_time=%lu duration=%lu ns\n",
+             rec->target_id,
+	     target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+	     target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+	     target_kernel_rec.end_time - rec->time);
+    break;
+    }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request (
+  int device_num,
+  ompt_buffer_t **buffer,
+  size_t *bytes
+) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes, *buffer);
+}
+
+static void on_ompt_callback_buffer_complete (
+  int device_num,
+  ompt_buffer_t *buffer,
+  size_t bytes, /* bytes returned in this callback */
+  ompt_buffer_cursor_t begin,
+  int buffer_owned
+) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n",
+	 device_num, buffer, bytes, (void*)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+					buffer,
+					bytes,
+					current,
+					&current);
+  }
+  if (buffer_owned) delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt) return ompt_set_error;
+
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+#endif
+  
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace) return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+	 "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+  
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+			  &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace) return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace) return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize
+(
+  int device_num,
+  const char *type,
+  ompt_device_t *device,
+  ompt_function_lookup_t lookup,
+  const char *documentation
+ ) {
+  printf("Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+	 device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t) lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t) lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t) lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t) lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t) lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor = (ompt_advance_buffer_cursor_t) lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_load
+    (
+     int device_num,
+     const char *filename,
+     int64_t offset_in_file,
+     void *vma_in_file,
+     size_t bytes,
+     void *host_addr,
+     void *device_addr,
+     uint64_t module_id
+     ) {
+  printf("Load: device_num:%d filename:%s host_adddr:%p device_addr:%p bytes:%lu\n",
+	 device_num, filename, host_addr, device_addr, bytes);
+}
+
+
+static void on_ompt_callback_target_data_op
+    (
+     ompt_id_t target_id,
+     ompt_id_t host_op_id,
+     ompt_target_data_op_t optype,
+     void *src_addr,
+     int src_device_num,
+     void *dest_addr,
+     int dest_device_num,
+     size_t bytes,
+     const void *codeptr_ra
+     ) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+	 "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+	 host_op_id, optype, src_addr, src_device_num,
+	 dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+
+static void on_ompt_callback_target_data_op_emi
+    (
+     ompt_scope_endpoint_t endpoint,
+     ompt_data_t *target_task_data,
+     ompt_data_t *target_data,
+     ompt_id_t *host_op_id,
+     ompt_target_data_op_t optype,
+     void *src_addr,
+     int src_device_num,
+     void *dest_addr,
+     int dest_device_num,
+     size_t bytes,
+     const void *codeptr_ra
+     ) {
+     if (endpoint == ompt_scope_begin) *host_op_id = next_op_id++;
+     // target_task_data may be null, avoid dereferencing it
+     uint64_t target_task_data_value = (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d target_task_data=%p (0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) optype=%d src=%p src_device_num=%d "
+	 "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+	 endpoint, 
+         target_task_data, target_task_data_value, 
+         target_data, target_data->value, 
+         host_op_id, *host_op_id, 
+         optype, src_addr, src_device_num,
+	 dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+
+static void on_ompt_callback_target
+    (
+     ompt_target_t kind,
+     ompt_scope_endpoint_t endpoint,
+     int device_num,
+     ompt_data_t *task_data,
+     ompt_id_t target_id,
+     const void *codeptr_ra
+     ) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu code=%p\n",
+	 kind, endpoint, device_num, target_id, codeptr_ra);
+}
+
+
+static void on_ompt_callback_target_emi
+    (
+     ompt_target_t kind,
+     ompt_scope_endpoint_t endpoint,
+     int device_num,
+     ompt_data_t *task_data,
+     ompt_data_t *target_task_data,
+     ompt_data_t *target_data,
+     const void *codeptr_ra
+     ) {
+     if (endpoint == ompt_scope_begin) target_data->value = next_op_id++;
+     // target_task_data may be null, avoid dereferencing it
+     uint64_t target_task_data_value = (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p (0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+	 kind, endpoint, device_num, 
+         task_data, task_data->value, 
+         target_task_data, target_task_data_value, 
+         target_data, target_data->value, 
+         codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit
+    (
+     ompt_id_t target_id,
+     ompt_id_t host_op_id,
+     unsigned int requested_num_teams
+     ) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+     target_id, host_op_id, requested_num_teams);
+}
+
+
+static void on_ompt_callback_target_submit_emi
+    (
+     ompt_scope_endpoint_t endpoint,
+     ompt_data_t *target_data,
+     ompt_id_t *host_op_id,
+     unsigned int requested_num_teams
+     ) {
+  printf("  Callback Submit EMI: endpoint=%d target_data=%p (0x%lx) host_op_id=%p (0x%lx) req_num_teams=%d\n",
+     endpoint, 
+     target_data, target_data->value, 
+     host_op_id, *host_op_id, 
+     requested_num_teams);
+}
+
+// Init functions
+int ompt_initialize(
+  ompt_function_lookup_t lookup,
+  int initial_device_num,
+  ompt_data_t *tool_data)
+{
+  ompt_set_callback = (ompt_set_callback_t) lookup("ompt_set_callback");
+
+  ompt_set_callback(ompt_callback_device_initialize,
+		    (ompt_callback_t)&on_ompt_callback_device_initialize);
+  ompt_set_callback(ompt_callback_device_load,
+		    (ompt_callback_t)&on_ompt_callback_device_load);
+#if EMI
+  ompt_set_callback(ompt_callback_target_submit_emi,
+		    (ompt_callback_t)&on_ompt_callback_target_submit_emi);
+  ompt_set_callback(ompt_callback_target_data_op_emi,
+		    (ompt_callback_t)&on_ompt_callback_target_data_op_emi);
+  ompt_set_callback(ompt_callback_target_emi,
+		    (ompt_callback_t)&on_ompt_callback_target_emi);
+#else
+  ompt_set_callback(ompt_callback_target_submit,
+		    (ompt_callback_t)&on_ompt_callback_target_submit);
+  ompt_set_callback(ompt_callback_target_data_op,
+		    (ompt_callback_t)&on_ompt_callback_target_data_op);
+  ompt_set_callback(ompt_callback_target,
+		    (ompt_callback_t)&on_ompt_callback_target);
+#endif
+  return 1; //success
+}
+
+void ompt_finalize(ompt_data_t *tool_data)
+{
+}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(
+  unsigned int omp_version,
+  const char *runtime_version)
+{
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,&ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
+
+
diff --git a/offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/veccopy-ctor-1.cpp b/offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/veccopy-ctor-1.cpp
new file mode 100644
index 0000000000000..c944d0b5831fa
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/veccopy-ctor-1.cpp
@@ -0,0 +1,37 @@
+#include <stdio.h>
+#include <omp.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int status;
+
+__attribute__((constructor(101))) void veccopy_init() {
+  int N = 10;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i=0; i<N; i++)
+    a[i]=0;
+
+  for (i=0; i<N; i++)
+    b[i]=i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j< N; j++)
+      a[j]=b[j];
+  }
+
+  if (a[5] != b[5])
+    status = 1;
+  else
+    status = 0;
+}
+
+
diff --git a/offload/test/ompt/tracing/veccopy-ompt-ctor-1/veccopy.cpp b/offload/test/ompt/tracing/veccopy-ompt-ctor-1/veccopy.cpp
new file mode 100644
index 0000000000000..3d6533ad45f0e
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-ctor-1/veccopy.cpp
@@ -0,0 +1,52 @@
+// clang-format off
+// RUN: %clangxx-generic %s %S/Inputs/veccopy-ctor-1.cpp -o %t -Xoffload-linker -lompdevice && %libomptarget-run-generic | %fcheck-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+/*
+ * Test whether OMPT support works for OpenMP directives 
+ * in a global constructor without using a shared library. 
+ * This currently succeeds.
+ */
+
+#include <stdio.h>
+#include <omp.h>
+
+extern int status;
+
+int main()
+{
+  int N = 10;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i=0; i<N; i++)
+    a[i]=0;
+
+  for (i=0; i<N; i++)
+    b[i]=i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j< N; j++)
+      a[j]=b[j];
+  }
+
+  int rc = 0;
+  for (i=0; i<N; i++)
+    if (a[i] != b[i] ) {
+      rc++;
+      printf ("Wrong varlue: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc && !status)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+/// CHECK: Record Target task
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-default-device/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-default-device/callbacks.h
new file mode 100644
index 0000000000000..5df9985ced0c3
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-default-device/callbacks.h
@@ -0,0 +1,286 @@
+#include <cassert>
+#include <memory>
+#include <unordered_set>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// Enable / disable OMPT's 'External Monitoring Interface'
+#define EMI 0
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+static ompt_get_record_type_t ompt_get_record_type_fn = 0;
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request_default_device(
+    int device_num, ompt_buffer_t **buffer, size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback for default "
+         "device %d\n",
+         *bytes, *buffer, device_num);
+}
+
+// Note: This callback must handle a null begin cursor. Currently,
+// ompt_get_record_ompt, print_record_ompt, and
+// ompt_advance_buffer_cursor handle a null cursor.
+static void on_ompt_callback_buffer_complete_default_device(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback for default device: \
+  device_num=%d, buffer=%p, num_bytes=%lu, begin=%p, buffer_owned=%d\n",
+         device_num, buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+
+    if (ompt_get_record_type_fn(buffer, current) != ompt_record_ompt) {
+      printf("WARNING: received non-ompt type buffer object\n");
+    }
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+  ompt_set_trace_ompt(Device, /*enable=*/1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, /*enable=*/1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, /*enable=*/1, ompt_callback_target_submit);
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+  if (device_num != omp_get_default_device())
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device,
+                          &on_ompt_callback_buffer_request_default_device,
+                          &on_ompt_callback_buffer_complete_default_device);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n", device_num,
+         type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  ompt_get_record_type_fn =
+      (ompt_get_record_type_t)lookup("ompt_get_record_type");
+  if (!ompt_get_record_type_fn) {
+    printf("WARNING: No function ompt_get_record_type found in device "
+           "callbacks\n");
+  }
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Load: device_num:%d filename:%s host_adddr:%p device_addr:%p "
+         "bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: target_id=%lu host_op_id=%lu optype=%d src=%p "
+         "src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         target_id, host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: target_id=%lu kind=%d endpoint=%d device_num=%d "
+         "code=%p\n",
+         target_id, kind, endpoint, device_num, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-default-device/veccopy-ompt-target-default-device.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-default-device/veccopy-ompt-target-default-device.cpp
new file mode 100644
index 0000000000000..ccf11f88b9cb3
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-default-device/veccopy-ompt-target-default-device.cpp
@@ -0,0 +1,86 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+#include <cassert>
+#include <omp.h>
+#include <stdio.h>
+
+// This test starts device tracing on the default device only (see
+// start_trace in callbacks.h). However, if more devices are
+// available, it calls flush and stop on the other devices as
+// well. The intention is to check correct runtime behavior if a tool
+// invokes flush or stop on a device that was not started. The runtime
+// should just return without doing anything.
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+    // Warm up
+#pragma omp target
+  {}
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+  for (int dev = 0; dev < omp_get_num_devices(); ++dev) {
+#pragma omp target teams distribute parallel for device(dev)
+    {
+      for (int j = 0; j < N; j++)
+        a[j] = b[j];
+    }
+  }
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+// > OMPT device tracing related checks below. <
+
+
+
+// Note: Split checks for record address and content. That way we do not imply
+//       any order. Records may / will occur interleaved.
+
+// Note: These addresses will only occur once. They are only captured to
+//       indicate their existence.
+
+/// CHECK-DAG: type=8 (Target task)
+/// CHECK-DAG: type=9 (Target data op)
+
+// Note: ADDRX_07 may not trigger a final callback.
+
+// Note: ADDRX_07 may not be deallocated.
+
+/// CHECK-NOT: host_op_id=0x0
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-devices/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-devices/callbacks.h
new file mode 100644
index 0000000000000..0625b1a7b1522
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-devices/callbacks.h
@@ -0,0 +1,324 @@
+#include <cassert>
+#include <memory>
+#include <unordered_set>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// Enable / disable OMPT's 'External Monitoring Interface'
+#define EMI 0
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+static ompt_get_record_type_t ompt_get_record_type_fn = 0;
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request_default_device(
+    int device_num, ompt_buffer_t **buffer, size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback for default "
+         "device %d\n",
+         *bytes, *buffer, device_num);
+}
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback for device %d\n",
+         *bytes, *buffer, device_num);
+}
+
+// Note: This callback must handle a null begin cursor. Currently,
+// ompt_get_record_ompt, print_record_ompt, and
+// ompt_advance_buffer_cursor handle a null cursor.
+static void on_ompt_callback_buffer_complete_default_device(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback for default device: \
+  device_num=%d, buffer=%p, num_bytes=%lu, begin=%p, buffer_owned=%d\n",
+         device_num, buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+
+    if (ompt_get_record_type_fn(buffer, current) != ompt_record_ompt) {
+      printf("WARNING: received non-ompt type buffer object\n");
+    }
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+// Note: This callback must handle a null begin cursor. Currently,
+// ompt_get_record_ompt, print_record_ompt, and
+// ompt_advance_buffer_cursor handle a null cursor.
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: \
+  device_num=%d, buffer=%p, num_bytes=%lu, begin=%p, buffer_owned=%d\n",
+         device_num, buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+
+    if (ompt_get_record_type_fn(buffer, current) != ompt_record_ompt) {
+      printf("WARNING: received non-ompt type buffer object\n");
+    }
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+  ompt_set_trace_ompt(Device, /*enable=*/1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, /*enable=*/1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, /*enable=*/1, ompt_callback_target_submit);
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  if (device_num == omp_get_default_device())
+    return ompt_start_trace(Device,
+                            &on_ompt_callback_buffer_request_default_device,
+                            &on_ompt_callback_buffer_complete_default_device);
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n", device_num,
+         type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  ompt_get_record_type_fn =
+      (ompt_get_record_type_t)lookup("ompt_get_record_type");
+  if (!ompt_get_record_type_fn) {
+    printf("WARNING: No function ompt_get_record_type found in device "
+           "callbacks\n");
+  }
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Load: device_num:%d filename:%s host_adddr:%p device_addr:%p "
+         "bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: target_id=%lu host_op_id=%lu optype=%d src=%p "
+         "src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         target_id, host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: target_id=%lu kind=%d endpoint=%d device_num=%d "
+         "code=%p\n",
+         target_id, kind, endpoint, device_num, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-devices/veccopy-ompt-target-devices.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-devices/veccopy-ompt-target-devices.cpp
new file mode 100644
index 0000000000000..7805e8c89de40
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-devices/veccopy-ompt-target-devices.cpp
@@ -0,0 +1,86 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+#include <cassert>
+#include <omp.h>
+#include <stdio.h>
+
+// This test starts device tracing on all available devices (see
+// start_trace in callbacks.h). It subsequently calls flush and stop
+// on all devices like a tool would normally do.
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+    // Warm up
+#pragma omp target
+  {}
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+  for (int dev = 0; dev < omp_get_num_devices(); ++dev) {
+#pragma omp target teams distribute parallel for device(dev)
+    {
+      for (int j = 0; j < N; j++)
+        a[j] = b[j];
+    }
+  }
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+// > OMPT device tracing related checks below. <
+
+// Note: Since this test may (theoretically) run on an arbitrary number of
+//       devices, we may only check for the minimal set of records.
+
+
+
+// Note: Split checks for record address and content. That way we do not imply
+//       any order. Records may / will occur interleaved.
+
+// Note: These addresses will only occur once. They are only captured to
+//       indicate their existence.
+
+/// CHECK-DAG: type=8 (Target task)
+/// CHECK-DAG: type=9 (Target data op)
+
+// Note: ADDRX_07 may not trigger a final callback.
+
+// Note: ADDRX_07 may not be deallocated.
+
+/// CHECK-NOT: host_op_id=0x0
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-translate-time/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-translate-time/callbacks.h
new file mode 100644
index 0000000000000..5ce283c1a16b3
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-translate-time/callbacks.h
@@ -0,0 +1,166 @@
+#include <assert.h>
+#include <math.h>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_translate_time_t ompt_translate_time_fn = 0;
+static ompt_get_device_time_t ompt_get_device_time_fn = 0;
+
+// OMPT callbacks
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize
+(
+  int device_num,
+  const char *type,
+  ompt_device_t *device,
+  ompt_function_lookup_t lookup,
+  const char *documentation
+ ) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+	 device_num, type, device, lookup, documentation);
+
+  ompt_device_time_t device_time = 0;
+
+  ompt_translate_time_fn = (ompt_translate_time_t) lookup("ompt_translate_time");
+  ompt_get_device_time_fn = (ompt_get_device_time_t) lookup("ompt_get_device_time");
+
+  if (ompt_get_device_time_fn) {
+    uint64_t time = ompt_get_device_time_fn(NULL);
+    printf("The device time can be queried: %lu\n", time);
+    device_time = time;
+  } else {
+    printf("Could not determine the device time\n");
+  }
+
+  if (ompt_translate_time_fn) {
+    assert(device_time > 0 && "Obtain device time was successfull");
+    printf("Translate time available\n");
+    double omp_wtime = omp_get_wtime();
+    device_time = ompt_get_device_time_fn(NULL);
+    double omp_device_wtime = ompt_translate_time_fn(NULL, device_time);
+    printf("OMPT: Host time: %f, Device time: %f\n", omp_wtime, omp_device_wtime);
+  } else {
+    printf("Could not translate time.\n");
+  }
+}
+
+static void on_ompt_callback_device_finalize
+(
+  int device_num
+ ) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load
+    (
+     int device_num,
+     const char *filename,
+     int64_t offset_in_file,
+     void *vma_in_file,
+     size_t bytes,
+     void *host_addr,
+     void *device_addr,
+     uint64_t module_id
+     ) {
+  printf("Callback Load: device_num:%d module_id:%lu filename:%s host_adddr:%p device_addr:%p bytes:%lu\n",
+	 device_num, module_id, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op
+    (
+     ompt_id_t target_id,
+     ompt_id_t host_op_id,
+     ompt_target_data_op_t optype,
+     void *src_addr,
+     int src_device_num,
+     void *dest_addr,
+     int dest_device_num,
+     size_t bytes,
+     const void *codeptr_ra
+     ) {
+  assert(codeptr_ra != 0);
+  // Both src and dest must not be null
+  assert(src_addr != 0 || dest_addr != 0);
+  printf("  Callback DataOp: target_id=%lu host_op_id=%lu optype=%d src=%p src_device_num=%d "
+	 "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+	 target_id, host_op_id, optype, src_addr, src_device_num,
+	 dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target
+    (
+     ompt_target_t kind,
+     ompt_scope_endpoint_t endpoint,
+     int device_num,
+     ompt_data_t *task_data,
+     ompt_id_t target_id,
+     const void *codeptr_ra
+     ) {
+  assert(codeptr_ra != 0);
+  printf("Callback Target: target_id=%lu kind=%d endpoint=%d device_num=%d code=%p\n",
+	 target_id, kind, endpoint, device_num, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit
+    (
+     ompt_id_t target_id,
+     ompt_id_t host_op_id,
+     unsigned int requested_num_teams
+     ) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+     target_id, host_op_id, requested_num_teams);
+}
+
+// Init functions
+int ompt_initialize(
+  ompt_function_lookup_t lookup,
+  int initial_device_num,
+  ompt_data_t *tool_data)
+{
+  ompt_set_callback = (ompt_set_callback_t) lookup("ompt_set_callback");
+
+  if (!ompt_set_callback) return 0; // failed
+  
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+
+
+
+  return 1; //success
+}
+
+void ompt_finalize(ompt_data_t *tool_data)
+{
+}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(
+  unsigned int omp_version,
+  const char *runtime_version)
+{
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,&ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-translate-time/veccopy-ompt-target.c b/offload/test/ompt/tracing/veccopy-ompt-target-translate-time/veccopy-ompt-target.c
new file mode 100644
index 0000000000000..211da6aaacd33
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-translate-time/veccopy-ompt-target.c
@@ -0,0 +1,78 @@
+// clang-format off
+// RUN: %libomptarget-compile-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+#include <stdio.h>
+#include <omp.h>
+
+#include "callbacks.h"
+
+int main()
+{
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i=0; i<N; i++)
+    a[i]=0;
+
+  for (i=0; i<N; i++)
+    b[i]=i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j< N; j++)
+      a[j]=b[j];
+  }
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j< N; j++)
+      a[j]=b[j];
+  }
+
+  int rc = 0;
+  for (i=0; i<N; i++)
+    if (a[i] != b[i] ) {
+      rc++;
+      printf ("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+  /// CHECK: Callback Init:
+  /// CHECK: The device time can be queried: [[X:[0-9]+]] 
+  /// CHECK: Translate time available
+  /// CHECK: Callback Load:
+  /// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+  /// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+  /// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2
+
+  /// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+  /// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+  /// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2
+  /// CHECK: Callback Fini:
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/callbacks.h
new file mode 100644
index 0000000000000..655e230275314
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/callbacks.h
@@ -0,0 +1,160 @@
+#include <assert.h>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_get_record_type_t ompt_get_record_type_fn = 0;
+static ompt_get_device_time_t ompt_get_device_time_fn = 0;
+
+// OMPT callbacks
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize
+(
+  int device_num,
+  const char *type,
+  ompt_device_t *device,
+  ompt_function_lookup_t lookup,
+  const char *documentation
+ ) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+	 device_num, type, device, lookup, documentation);
+
+  ompt_get_record_type_fn = (ompt_get_record_type_t) lookup("ompt_get_record_type");
+  ompt_get_device_time_fn = (ompt_get_device_time_t) lookup("ompt_get_device_time");
+
+  if (ompt_get_record_type_fn) {
+    ompt_buffer_cursor_t buf;
+    // FIXME: For now, we pass a NULL buffer
+    ompt_record_t rec_type = ompt_get_record_type_fn(NULL, buf);
+    printf("Record Type: %s\n", ((rec_type == ompt_record_ompt) ? "OMPT" : "Unknown"));
+  } else {
+    printf("Could not determine Record Type");
+  }
+
+  if (ompt_get_device_time_fn) {
+    uint64_t time = ompt_get_device_time_fn(NULL);
+    printf("The device time can be queried: %lu\n", time);
+  } else {
+    printf("Could not determine the device time \n");
+  }
+}
+
+static void on_ompt_callback_device_finalize
+(
+  int device_num
+ ) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load
+    (
+     int device_num,
+     const char *filename,
+     int64_t offset_in_file,
+     void *vma_in_file,
+     size_t bytes,
+     void *host_addr,
+     void *device_addr,
+     uint64_t module_id
+     ) {
+  printf("Callback Load: device_num:%d module_id:%lu filename:%s host_adddr:%p device_addr:%p bytes:%lu\n",
+	 device_num, module_id, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op
+    (
+     ompt_id_t target_id,
+     ompt_id_t host_op_id,
+     ompt_target_data_op_t optype,
+     void *src_addr,
+     int src_device_num,
+     void *dest_addr,
+     int dest_device_num,
+     size_t bytes,
+     const void *codeptr_ra
+     ) {
+  assert(codeptr_ra != 0);
+  // Both src and dest must not be null
+  assert(src_addr != 0 || dest_addr != 0);
+  printf("  Callback DataOp: target_id=%lu host_op_id=%lu optype=%d src=%p src_device_num=%d "
+	 "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+	 target_id, host_op_id, optype, src_addr, src_device_num,
+	 dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target
+    (
+     ompt_target_t kind,
+     ompt_scope_endpoint_t endpoint,
+     int device_num,
+     ompt_data_t *task_data,
+     ompt_id_t target_id,
+     const void *codeptr_ra
+     ) {
+  assert(codeptr_ra != 0);
+  printf("Callback Target: target_id=%lu kind=%d endpoint=%d device_num=%d code=%p\n",
+	 target_id, kind, endpoint, device_num, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit
+    (
+     ompt_id_t target_id,
+     ompt_id_t host_op_id,
+     unsigned int requested_num_teams
+     ) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+     target_id, host_op_id, requested_num_teams);
+}
+
+// Init functions
+int ompt_initialize(
+  ompt_function_lookup_t lookup,
+  int initial_device_num,
+  ompt_data_t *tool_data)
+{
+  ompt_set_callback = (ompt_set_callback_t) lookup("ompt_set_callback");
+
+  if (!ompt_set_callback) return 0; // failed
+  
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+
+
+
+  return 1; //success
+}
+
+void ompt_finalize(ompt_data_t *tool_data)
+{
+}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(
+  unsigned int omp_version,
+  const char *runtime_version)
+{
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,&ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/veccopy-ompt-target.c b/offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/veccopy-ompt-target.c
new file mode 100644
index 0000000000000..22374a691b8a3
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/veccopy-ompt-target.c
@@ -0,0 +1,78 @@
+// clang-format off
+// RUN: %libomptarget-compile-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+#include <stdio.h>
+#include <omp.h>
+
+#include "callbacks.h"
+
+int main()
+{
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i=0; i<N; i++)
+    a[i]=0;
+
+  for (i=0; i<N; i++)
+    b[i]=i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j< N; j++)
+      a[j]=b[j];
+  }
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j< N; j++)
+      a[j]=b[j];
+  }
+
+  int rc = 0;
+  for (i=0; i<N; i++)
+    if (a[i] != b[i] ) {
+      rc++;
+      printf ("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+  /// CHECK: Callback Init:
+  /// CHECK: Record Type: OMPT
+  /// CHECK: The device time can be queried: [[X:[0-9]+]] 
+  /// CHECK: Callback Load:
+  /// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+  /// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+  /// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2
+
+  /// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+  /// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+  /// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+  /// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2
+  /// CHECK: Callback Fini:

>From 57a884e8994390345b2ade665c3fa81dadf5f0f4 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 21 May 2026 09:11:16 -0500
Subject: [PATCH 11/11] [OMPT] Add test for synchronous path w/ OMPT

---
 .../large_mapping_sync_tracing.cpp            | 81 +++++++++++++++++++
 1 file changed, 81 insertions(+)
 create mode 100644 offload/test/ompt/tracing/large-mapping-sync-tracing/large_mapping_sync_tracing.cpp

diff --git a/offload/test/ompt/tracing/large-mapping-sync-tracing/large_mapping_sync_tracing.cpp b/offload/test/ompt/tracing/large-mapping-sync-tracing/large_mapping_sync_tracing.cpp
new file mode 100644
index 0000000000000..44939536d30f3
--- /dev/null
+++ b/offload/test/ompt/tracing/large-mapping-sync-tracing/large_mapping_sync_tracing.cpp
@@ -0,0 +1,81 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-generic && env LIBOMPTARGET_AMDGPU_MAX_ASYNC_COPY_BYTES=1 LIBOMPTARGET_OMPT_FLUSH_ON_SHUTDOWN=false LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL=false %libomptarget-run-generic | %fcheck-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+
+// Exercises OMPT device tracing for AMDGPU's synchronous large-copy path.
+// The transfer size is intentionally above the plugin's default async-copy
+// threshold, while the RUN line also lowers the threshold to keep the test
+// independent of future default changes.
+
+#include <omp.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "../veccopy-ompt-target-data-tracing-emi/callbacks.h"
+
+#define COPY_BYTES (50 * 1024 * 1024)
+
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  if (omp_get_num_devices() == 0) {
+    printf("Success\n");
+    return 0;
+  }
+
+  const size_t N = COPY_BYTES / sizeof(int);
+  int *Input = (int *)malloc(COPY_BYTES);
+  int *Output = (int *)malloc(COPY_BYTES);
+
+  if (!Input || !Output) {
+    free(Input);
+    free(Output);
+    printf("Failure\n");
+    return 1;
+  }
+
+  for (size_t I = 0; I < N; ++I) {
+    Input[I] = (int)(I & 0x7fff);
+    Output[I] = 0;
+  }
+
+#pragma omp target teams distribute parallel for map(to : Input[0 : N])        \
+    map(from : Output[0 : N])
+  for (size_t I = 0; I < N; ++I)
+    Output[I] = Input[I] + 1;
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+  int Failed = 0;
+  for (size_t I = 0; I < N; ++I) {
+    if (Output[I] != Input[I] + 1) {
+      Failed = 1;
+      break;
+    }
+  }
+
+  free(Input);
+  free(Output);
+
+  printf("%s\n", Failed ? "Failure" : "Success");
+  return Failed;
+}
+
+// clang-format off
+
+/// CHECK: Callback Init:
+/// CHECK: Callback Load:
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=2 {{.*}} bytes=52428800
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=2 {{.*}} bytes=52428800
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=3 {{.*}} bytes=52428800
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=3 {{.*}} bytes=52428800
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=4
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=4
+
+/// CHECK-DAG: type=9 (Target data op) {{.*}} optype=1 {{.*}} bytes=52428800
+/// CHECK: Success


From openmp-commits at lists.llvm.org  Wed Aug  5 01:29:11 2026
From: openmp-commits at lists.llvm.org (=?UTF-8?Q?Jan_Andr=C3=A9_Reuter?= via Openmp-commits)
Date: Wed, 05 Aug 2026 01:29:11 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OMPT] Upstream device tracing
	implementation (PR #200165)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/200165@github.com>
Message-ID: <6a72f457.abe55a58.0b77.8883@mx.google.com>


Thyre wrote:

With https://github.com/llvm/llvm-project/pull/201127 dropped, `[OpenMP][OMPT] Test libomptarget connector lookup` can probably also be removed, right? `ompt_set_frame_enter` doesn't exist anymore. This now basically tests if a non-implemented lookup returns a null pointer.

I'm surprised though that there isn't a test for that scenario yet.

https://github.com/llvm/llvm-project/pull/200165

From openmp-commits at lists.llvm.org  Wed Aug  5 01:45:53 2026
From: openmp-commits at lists.llvm.org (Jan Patrick Lehr via Openmp-commits)
Date: Wed, 05 Aug 2026 01:45:53 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OMPT] Upstream device tracing
	implementation (PR #200165)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/200165@github.com>
Message-ID: <6a72f841.d681cab9.19d854.7850@mx.google.com>


jplehr wrote:

> With #201127 dropped, `[OpenMP][OMPT] Test libomptarget connector lookup` can probably also be removed, right? `ompt_set_frame_enter` doesn't exist anymore. This now basically tests if a non-implemented lookup returns a null pointer.
> 
> I'm surprised though that there isn't a test for that scenario yet.

That is the intention when I'm opening up individual PRs. I had added that test so that I can make sure any residue is gone in that larger piece of changes and have not bothered to remove it from this draft.

https://github.com/llvm/llvm-project/pull/200165

From openmp-commits at lists.llvm.org  Wed Aug  5 01:46:43 2026
From: openmp-commits at lists.llvm.org (Amit Tiwari via Openmp-commits)
Date: Wed, 05 Aug 2026 01:46:43 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [Clang][OpenMP] Add
 `#pragma omp flatten` loop directive + `depth` clause (PR #206977)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206977@github.com>
Message-ID: <6a72f873.510f576d.25a50.de0f@mx.google.com>


https://github.com/loopacino updated https://github.com/llvm/llvm-project/pull/206977

>From 0fb5bd42f1e127faea963c359b1e3c69cb969c60 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 1 Jul 2026 09:06:58 -0400
Subject: [PATCH 01/13] add OMPFlattenDirective ast node

---
 clang/include/clang/AST/RecursiveASTVisitor.h |  3 +
 clang/include/clang/AST/StmtOpenMP.h          | 75 ++++++++++++++++++-
 clang/include/clang/Basic/StmtNodes.td        |  1 +
 clang/lib/AST/StmtOpenMP.cpp                  | 20 +++++
 clang/lib/AST/StmtPrinter.cpp                 |  5 ++
 clang/lib/AST/StmtProfile.cpp                 |  4 +
 clang/lib/Basic/OpenMPKinds.cpp               |  2 +-
 7 files changed, 108 insertions(+), 2 deletions(-)

diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index b000a34043696..e425f7a31f59c 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -3227,6 +3227,9 @@ DEF_TRAVERSE_STMT(OMPFuseDirective,
 DEF_TRAVERSE_STMT(OMPInterchangeDirective,
                   { TRY_TO(TraverseOMPExecutableDirective(S)); })
 
+DEF_TRAVERSE_STMT(OMPFlattenDirective,
+                  { TRY_TO(TraverseOMPExecutableDirective(S)); })
+
 DEF_TRAVERSE_STMT(OMPSplitDirective,
                   { TRY_TO(TraverseOMPExecutableDirective(S)); })
 
diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index dbc76e7df8ecd..1420331f37c1f 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -1041,7 +1041,7 @@ class OMPCanonicalLoopNestTransformationDirective
     Stmt::StmtClass C = T->getStmtClass();
     return C == OMPTileDirectiveClass || C == OMPUnrollDirectiveClass ||
            C == OMPReverseDirectiveClass || C == OMPInterchangeDirectiveClass ||
-           C == OMPStripeDirectiveClass;
+           C == OMPStripeDirectiveClass || C == OMPFlattenDirectiveClass;
   }
 };
 
@@ -5959,6 +5959,79 @@ class OMPInterchangeDirective final
   }
 };
 
+/// Represents the '#pragma omp flatten' loop transformation directive.
+///
+/// \code{c}
+///   #pragma omp flatten
+///   for (int i = 0; i < m; ++i)
+///     for (int j = 0; j < n; ++j)
+///       ..
+/// \endcode
+class OMPFlattenDirective final
+    : public OMPCanonicalLoopNestTransformationDirective {
+  friend class ASTStmtReader;
+  friend class OMPExecutableDirective;
+
+  /// Offsets of child members.
+  enum {
+    PreInitsOffset = 0,
+    TransformedStmtOffset,
+  };
+
+  explicit OMPFlattenDirective(SourceLocation StartLoc, SourceLocation EndLoc,
+                               unsigned NumLoops)
+      : OMPCanonicalLoopNestTransformationDirective(
+            OMPFlattenDirectiveClass, llvm::omp::OMPD_flatten, StartLoc, EndLoc,
+            NumLoops) {}
+
+  void setPreInits(Stmt *PreInits) {
+    Data->getChildren()[PreInitsOffset] = PreInits;
+  }
+
+  void setTransformedStmt(Stmt *S) {
+    Data->getChildren()[TransformedStmtOffset] = S;
+  }
+
+public:
+  /// Create a new AST node representation for '#pragma omp flatten'.
+  ///
+  /// \param C         Context of the AST.
+  /// \param StartLoc  Location of the introducer (e.g. the 'omp' token).
+  /// \param EndLoc    Location of the directive's end (e.g. the tok::eod).
+  /// \param Clauses   The directive's clauses.
+  /// \param NumLoops  Number of affected loops (the flatten depth, currently
+  ///                  always 2).
+  /// \param AssociatedStmt  The outermost associated loop.
+  /// \param TransformedStmt The flattened loop, or nullptr in dependent
+  ///                        contexts.
+  /// \param PreInits  Helper preinits statements for the loop nest.
+  static OMPFlattenDirective *
+  Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation EndLoc,
+         ArrayRef<OMPClause *> Clauses, unsigned NumLoops, Stmt *AssociatedStmt,
+         Stmt *TransformedStmt, Stmt *PreInits);
+
+  /// Build an empty '#pragma omp flatten' AST node for deserialization.
+  ///
+  /// \param C          Context of the AST.
+  /// \param NumClauses Number of clauses to allocate.
+  /// \param NumLoops   Number of associated loops to allocate.
+  static OMPFlattenDirective *
+  CreateEmpty(const ASTContext &C, unsigned NumClauses, unsigned NumLoops);
+
+  /// Gets the flattened loop after the transformation. This is the de-sugared
+  /// replacement or nullptr in dependent contexts.
+  Stmt *getTransformedStmt() const {
+    return Data->getChildren()[TransformedStmtOffset];
+  }
+
+  /// Return preinits statement.
+  Stmt *getPreInits() const { return Data->getChildren()[PreInitsOffset]; }
+
+  static bool classof(const Stmt *T) {
+    return T->getStmtClass() == OMPFlattenDirectiveClass;
+  }
+};
+
 /// The base class for all transformation directives of canonical loop
 /// sequences (currently only 'fuse')
 class OMPCanonicalLoopSequenceTransformationDirective
diff --git a/clang/include/clang/Basic/StmtNodes.td b/clang/include/clang/Basic/StmtNodes.td
index e166894ea024b..0d7a0c1490a1e 100644
--- a/clang/include/clang/Basic/StmtNodes.td
+++ b/clang/include/clang/Basic/StmtNodes.td
@@ -247,6 +247,7 @@ def OMPReverseDirective : StmtNode<OMPCanonicalLoopNestTransformationDirective>;
 def OMPSplitDirective : StmtNode<OMPCanonicalLoopNestTransformationDirective>;
 def OMPInterchangeDirective
     : StmtNode<OMPCanonicalLoopNestTransformationDirective>;
+def OMPFlattenDirective : StmtNode<OMPCanonicalLoopNestTransformationDirective>;
 def OMPCanonicalLoopSequenceTransformationDirective
     : StmtNode<OMPExecutableDirective, 1>;
 def OMPFuseDirective
diff --git a/clang/lib/AST/StmtOpenMP.cpp b/clang/lib/AST/StmtOpenMP.cpp
index 9d6b315effb41..477604ef3dafb 100644
--- a/clang/lib/AST/StmtOpenMP.cpp
+++ b/clang/lib/AST/StmtOpenMP.cpp
@@ -552,6 +552,26 @@ OMPInterchangeDirective::CreateEmpty(const ASTContext &C, unsigned NumClauses,
       SourceLocation(), SourceLocation(), NumLoops);
 }
 
+OMPFlattenDirective *OMPFlattenDirective::Create(
+    const ASTContext &C, SourceLocation StartLoc, SourceLocation EndLoc,
+    ArrayRef<OMPClause *> Clauses, unsigned NumLoops, Stmt *AssociatedStmt,
+    Stmt *TransformedStmt, Stmt *PreInits) {
+  OMPFlattenDirective *Dir = createDirective<OMPFlattenDirective>(
+      C, Clauses, AssociatedStmt, TransformedStmtOffset + 1, StartLoc, EndLoc,
+      NumLoops);
+  Dir->setTransformedStmt(TransformedStmt);
+  Dir->setPreInits(PreInits);
+  return Dir;
+}
+
+OMPFlattenDirective *OMPFlattenDirective::CreateEmpty(const ASTContext &C,
+                                                      unsigned NumClauses,
+                                                      unsigned NumLoops) {
+  return createEmptyDirective<OMPFlattenDirective>(
+      C, NumClauses, /*HasAssociatedStmt=*/true, TransformedStmtOffset + 1,
+      SourceLocation(), SourceLocation(), NumLoops);
+}
+
 OMPSplitDirective *
 OMPSplitDirective::Create(const ASTContext &C, SourceLocation StartLoc,
                           SourceLocation EndLoc, ArrayRef<OMPClause *> Clauses,
diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp
index 6c3294573e9d4..0bacb79ed86e3 100644
--- a/clang/lib/AST/StmtPrinter.cpp
+++ b/clang/lib/AST/StmtPrinter.cpp
@@ -800,6 +800,11 @@ void StmtPrinter::VisitOMPInterchangeDirective(OMPInterchangeDirective *Node) {
   PrintOMPExecutableDirective(Node);
 }
 
+void StmtPrinter::VisitOMPFlattenDirective(OMPFlattenDirective *Node) {
+  Indent() << "#pragma omp flatten";
+  PrintOMPExecutableDirective(Node);
+}
+
 void StmtPrinter::VisitOMPSplitDirective(OMPSplitDirective *Node) {
   Indent() << "#pragma omp split";
   PrintOMPExecutableDirective(Node);
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index 90eab530e0c2e..de7ed322d7424 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -1071,6 +1071,10 @@ void StmtProfiler::VisitOMPInterchangeDirective(
   VisitOMPCanonicalLoopNestTransformationDirective(S);
 }
 
+void StmtProfiler::VisitOMPFlattenDirective(const OMPFlattenDirective *S) {
+  VisitOMPCanonicalLoopNestTransformationDirective(S);
+}
+
 void StmtProfiler::VisitOMPSplitDirective(const OMPSplitDirective *S) {
   VisitOMPCanonicalLoopNestTransformationDirective(S);
 }
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index 675d86349c933..57e0347f8ea77 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -818,7 +818,7 @@ bool clang::isOpenMPCanonicalLoopNestTransformationDirective(
     OpenMPDirectiveKind DKind) {
   return DKind == OMPD_tile || DKind == OMPD_unroll || DKind == OMPD_reverse ||
          DKind == OMPD_split || DKind == OMPD_interchange ||
-         DKind == OMPD_stripe;
+         DKind == OMPD_stripe || DKind == OMPD_flatten;
 }
 
 bool clang::isOpenMPCanonicalLoopSequenceTransformationDirective(

>From 933007b6333584fb358adc4f89a61db1ae15d4cb Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:02:27 -0400
Subject: [PATCH 02/13] add OMPDepthClause ast node

---
 clang/include/clang/AST/OpenMPClause.h        | 68 +++++++++++++++++++
 clang/include/clang/AST/RecursiveASTVisitor.h |  6 ++
 clang/lib/AST/OpenMPClause.cpp                | 25 +++++++
 clang/lib/AST/StmtProfile.cpp                 |  5 ++
 clang/lib/Parse/ParseOpenMP.cpp               |  1 +
 llvm/include/llvm/Frontend/OpenMP/OMP.td      |  1 +
 6 files changed, 106 insertions(+)

diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 8ceafc4669297..50d1364a9726d 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -1380,6 +1380,74 @@ class OMPPartialClause final : public OMPClause {
   }
 };
 
+/// This represents the 'depth' clause on the '#pragma omp flatten' (and
+/// '#pragma omp fuse') loop-transformation directives.
+///
+/// \code
+/// #pragma omp flatten depth(3)
+/// \endcode
+/// In this example the 'flatten' directive has a 'depth' clause whose argument
+/// '3' specifies how many perfectly nested loops are combined into one.
+/// The argument must be a positive integer constant expression that evaluates
+/// to at most the loop nest depth of the associated loop nest.
+class OMPDepthClause final : public OMPClause {
+  friend class OMPClauseReader;
+
+  /// Location of '('.
+  SourceLocation LParenLoc;
+
+  /// The depth expression (number of loops to flatten).
+  Stmt *Depth = nullptr;
+
+  /// Build an empty clause.
+  explicit OMPDepthClause() : OMPClause(llvm::omp::OMPC_depth, {}, {}) {}
+
+  /// Set the depth expression.
+  void setDepth(Expr *E) { Depth = E; }
+
+  /// Sets the location of '('.
+  void setLParenLoc(SourceLocation Loc) { LParenLoc = Loc; }
+
+public:
+  /// Build an AST node for a 'depth' clause.
+  ///
+  /// \param C         Context of the AST.
+  /// \param StartLoc  Location of the 'depth' identifier.
+  /// \param LParenLoc Location of '('.
+  /// \param EndLoc    Location of ')'.
+  /// \param Depth     The depth expression.
+  static OMPDepthClause *Create(const ASTContext &C, SourceLocation StartLoc,
+                                SourceLocation LParenLoc, SourceLocation EndLoc,
+                                Expr *Depth);
+
+  /// Build an empty 'depth' AST node for deserialization.
+  ///
+  /// \param C     Context of the AST.
+  static OMPDepthClause *CreateEmpty(const ASTContext &C);
+
+  /// Returns the location of '('.
+  SourceLocation getLParenLoc() const { return LParenLoc; }
+
+  /// Returns the depth expression or nullptr if not set.
+  Expr *getDepth() const { return cast_or_null<Expr>(Depth); }
+
+  child_range children() { return child_range(&Depth, &Depth + 1); }
+  const_child_range children() const {
+    return const_child_range(&Depth, &Depth + 1);
+  }
+
+  child_range used_children() {
+    return child_range(child_iterator(), child_iterator());
+  }
+  const_child_range used_children() const {
+    return const_child_range(const_child_iterator(), const_child_iterator());
+  }
+
+  static bool classof(const OMPClause *T) {
+    return T->getClauseKind() == llvm::omp::OMPC_depth;
+  }
+};
+
 /// This represents 'collapse' clause in the '#pragma omp ...'
 /// directive.
 ///
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index e425f7a31f59c..da55e97fdfcb5 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -3568,6 +3568,12 @@ bool RecursiveASTVisitor<Derived>::VisitOMPPartialClause(OMPPartialClause *C) {
   return true;
 }
 
+template <typename Derived>
+bool RecursiveASTVisitor<Derived>::VisitOMPDepthClause(OMPDepthClause *C) {
+  TRY_TO(TraverseStmt(C->getDepth()));
+  return true;
+}
+
 template <typename Derived>
 bool
 RecursiveASTVisitor<Derived>::VisitOMPCollapseClause(OMPCollapseClause *C) {
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index ed00e80144c25..7b2f67a31cad9 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -1057,6 +1057,22 @@ OMPPartialClause *OMPPartialClause::CreateEmpty(const ASTContext &C) {
   return new (C) OMPPartialClause();
 }
 
+OMPDepthClause *OMPDepthClause::Create(const ASTContext &C,
+                                       SourceLocation StartLoc,
+                                       SourceLocation LParenLoc,
+                                       SourceLocation EndLoc, Expr *Depth) {
+  OMPDepthClause *Clause = CreateEmpty(C);
+  Clause->setLocStart(StartLoc);
+  Clause->setLParenLoc(LParenLoc);
+  Clause->setLocEnd(EndLoc);
+  Clause->setDepth(Depth);
+  return Clause;
+}
+
+OMPDepthClause *OMPDepthClause::CreateEmpty(const ASTContext &C) {
+  return new (C) OMPDepthClause();
+}
+
 OMPLoopRangeClause *
 OMPLoopRangeClause::Create(const ASTContext &C, SourceLocation StartLoc,
                            SourceLocation LParenLoc, SourceLocation FirstLoc,
@@ -2069,6 +2085,15 @@ void OMPClausePrinter::VisitOMPPermutationClause(OMPPermutationClause *Node) {
 
 void OMPClausePrinter::VisitOMPFullClause(OMPFullClause *Node) { OS << "full"; }
 
+void OMPClausePrinter::VisitOMPDepthClause(OMPDepthClause *Node) {
+  OS << "depth";
+  if (Expr *Depth = Node->getDepth()) {
+    OS << '(';
+    Depth->printPretty(OS, nullptr, Policy, 0);
+    OS << ')';
+  }
+}
+
 void OMPClausePrinter::VisitOMPPartialClause(OMPPartialClause *Node) {
   OS << "partial";
 
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index de7ed322d7424..838d6d02f7446 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -521,6 +521,11 @@ void OMPClauseProfiler::VisitOMPPartialClause(const OMPPartialClause *C) {
     Profiler->VisitExpr(Factor);
 }
 
+void OMPClauseProfiler::VisitOMPDepthClause(const OMPDepthClause *C) {
+  if (const Expr *Depth = C->getDepth())
+    Profiler->VisitExpr(Depth);
+}
+
 void OMPClauseProfiler::VisitOMPLoopRangeClause(const OMPLoopRangeClause *C) {
   if (const Expr *First = C->getFirst())
     Profiler->VisitExpr(First);
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index af52295df2d8b..952a517205381 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -3247,6 +3247,7 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
   case OMPC_nocontext:
   case OMPC_filter:
   case OMPC_partial:
+  case OMPC_depth:
   case OMPC_align:
   case OMPC_message:
   case OMPC_ompx_dyn_cgroup_mem:
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index 679a944fc4358..d49c5ee1a0db3 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -161,6 +161,7 @@ def OMPC_Depobj : Clause<[Spelling<"depobj">]> {
   let isImplicit = true;
 }
 def OMPC_Depth : Clause<[Spelling<"depth">]> {
+  let clangClass = "OMPDepthClause";
   let flangClass = "ScalarIntConstantExpr";
 }
 def OMPC_Destroy : Clause<[Spelling<"destroy">]> {

>From b63024e1e342cd168ef128a3f6a24909df2a82db Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:04:44 -0400
Subject: [PATCH 03/13] template-transform support for flatten directive &
 depth clause

---
 clang/lib/Sema/TreeTransform.h | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 3b99ff4bb9e23..865622ad50f1c 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -1804,6 +1804,14 @@ class TreeTransform {
                                                        LParenLoc, EndLoc);
   }
 
+  /// Build a new OpenMP 'depth' clause.
+  OMPClause *RebuildOMPDepthClause(Expr *Depth, SourceLocation StartLoc,
+                                   SourceLocation LParenLoc,
+                                   SourceLocation EndLoc) {
+    return getSema().OpenMP().ActOnOpenMPDepthClause(Depth, StartLoc, LParenLoc,
+                                                     EndLoc);
+  }
+
   OMPClause *
   RebuildOMPLoopRangeClause(Expr *First, Expr *Count, SourceLocation StartLoc,
                             SourceLocation LParenLoc, SourceLocation FirstLoc,
@@ -9791,6 +9799,17 @@ TreeTransform<Derived>::TransformOMPSplitDirective(OMPSplitDirective *D) {
   return Res;
 }
 
+template <typename Derived>
+StmtResult
+TreeTransform<Derived>::TransformOMPFlattenDirective(OMPFlattenDirective *D) {
+  DeclarationNameInfo DirName;
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      D->getDirectiveKind(), DirName, nullptr, D->getBeginLoc());
+  StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
+  return Res;
+}
+
 template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPFuseDirective(OMPFuseDirective *D) {
@@ -10717,6 +10736,20 @@ TreeTransform<Derived>::TransformOMPPartialClause(OMPPartialClause *C) {
                                  C->getEndLoc());
 }
 
+template <typename Derived>
+OMPClause *TreeTransform<Derived>::TransformOMPDepthClause(OMPDepthClause *C) {
+  ExprResult T = getDerived().TransformExpr(C->getDepth());
+  if (T.isInvalid())
+    return nullptr;
+  Expr *Depth = T.get();
+  bool Changed = Depth != C->getDepth();
+
+  if (!Changed && !getDerived().AlwaysRebuild())
+    return C;
+  return RebuildOMPDepthClause(Depth, C->getBeginLoc(), C->getLParenLoc(),
+                               C->getEndLoc());
+}
+
 template <typename Derived>
 OMPClause *
 TreeTransform<Derived>::TransformOMPLoopRangeClause(OMPLoopRangeClause *C) {

>From deafb222a7d251069c68a44fa267e58e750f1064 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:08:12 -0400
Subject: [PATCH 04/13] serialize flatten

---
 clang/include/clang/Serialization/ASTBitCodes.h |  1 +
 clang/lib/Serialization/ASTReaderStmt.cpp       | 11 +++++++++++
 clang/lib/Serialization/ASTWriterStmt.cpp       |  5 +++++
 3 files changed, 17 insertions(+)

diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h
index 279380de2f7fe..93a0e0a454b68 100644
--- a/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/clang/include/clang/Serialization/ASTBitCodes.h
@@ -1970,6 +1970,7 @@ enum StmtCode {
   STMT_OMP_REVERSE_DIRECTIVE,
   STMT_OMP_SPLIT_DIRECTIVE,
   STMT_OMP_INTERCHANGE_DIRECTIVE,
+  STMT_OMP_FLATTEN_DIRECTIVE,
   STMT_OMP_FUSE_DIRECTIVE,
   STMT_OMP_FOR_DIRECTIVE,
   STMT_OMP_FOR_SIMD_DIRECTIVE,
diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp
index 02ccf8d4d41c2..8c2debd6cdf32 100644
--- a/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -2537,6 +2537,10 @@ void ASTStmtReader::VisitOMPInterchangeDirective(OMPInterchangeDirective *D) {
   VisitOMPCanonicalLoopNestTransformationDirective(D);
 }
 
+void ASTStmtReader::VisitOMPFlattenDirective(OMPFlattenDirective *D) {
+  VisitOMPCanonicalLoopNestTransformationDirective(D);
+}
+
 void ASTStmtReader::VisitOMPSplitDirective(OMPSplitDirective *D) {
   VisitOMPCanonicalLoopNestTransformationDirective(D);
 }
@@ -3719,6 +3723,13 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) {
       break;
     }
 
+    case STMT_OMP_FLATTEN_DIRECTIVE: {
+      unsigned NumLoops = Record[ASTStmtReader::NumStmtFields];
+      unsigned NumClauses = Record[ASTStmtReader::NumStmtFields + 1];
+      S = OMPFlattenDirective::CreateEmpty(Context, NumClauses, NumLoops);
+      break;
+    }
+
     case STMT_OMP_FOR_DIRECTIVE: {
       unsigned CollapsedNum = Record[ASTStmtReader::NumStmtFields];
       unsigned NumClauses = Record[ASTStmtReader::NumStmtFields + 1];
diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp
index 4e9eadd730a56..dfe28a9b915e3 100644
--- a/clang/lib/Serialization/ASTWriterStmt.cpp
+++ b/clang/lib/Serialization/ASTWriterStmt.cpp
@@ -2547,6 +2547,11 @@ void ASTStmtWriter::VisitOMPInterchangeDirective(OMPInterchangeDirective *D) {
   Code = serialization::STMT_OMP_INTERCHANGE_DIRECTIVE;
 }
 
+void ASTStmtWriter::VisitOMPFlattenDirective(OMPFlattenDirective *D) {
+  VisitOMPCanonicalLoopNestTransformationDirective(D);
+  Code = serialization::STMT_OMP_FLATTEN_DIRECTIVE;
+}
+
 void ASTStmtWriter::VisitOMPSplitDirective(OMPSplitDirective *D) {
   VisitOMPCanonicalLoopNestTransformationDirective(D);
   Code = serialization::STMT_OMP_SPLIT_DIRECTIVE;

>From 3e2dfeb99230057b7cb9c484182ae33befbe1e41 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:09:03 -0400
Subject: [PATCH 05/13] serialize depth

---
 clang/lib/Serialization/ASTReader.cpp | 8 ++++++++
 clang/lib/Serialization/ASTWriter.cpp | 5 +++++
 2 files changed, 13 insertions(+)

diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index f8a6a38bb9b5c..4f51d79a52105 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -11495,6 +11495,9 @@ OMPClause *OMPClauseReader::readClause() {
   case llvm::omp::OMPC_partial:
     C = OMPPartialClause::CreateEmpty(Context);
     break;
+  case llvm::omp::OMPC_depth:
+    C = OMPDepthClause::CreateEmpty(Context);
+    break;
   case llvm::omp::OMPC_looprange:
     C = OMPLoopRangeClause::CreateEmpty(Context);
     break;
@@ -11924,6 +11927,11 @@ void OMPClauseReader::VisitOMPPartialClause(OMPPartialClause *C) {
   C->setLParenLoc(Record.readSourceLocation());
 }
 
+void OMPClauseReader::VisitOMPDepthClause(OMPDepthClause *C) {
+  C->setDepth(Record.readSubExpr());
+  C->setLParenLoc(Record.readSourceLocation());
+}
+
 void OMPClauseReader::VisitOMPLoopRangeClause(OMPLoopRangeClause *C) {
   C->setFirst(Record.readSubExpr());
   C->setCount(Record.readSubExpr());
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 357a7f7e95fa0..cbfeaa9e6638a 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -8106,6 +8106,11 @@ void OMPClauseWriter::VisitOMPPartialClause(OMPPartialClause *C) {
   Record.AddSourceLocation(C->getLParenLoc());
 }
 
+void OMPClauseWriter::VisitOMPDepthClause(OMPDepthClause *C) {
+  Record.AddStmt(C->getDepth());
+  Record.AddSourceLocation(C->getLParenLoc());
+}
+
 void OMPClauseWriter::VisitOMPLoopRangeClause(OMPLoopRangeClause *C) {
   Record.AddStmt(C->getFirst());
   Record.AddStmt(C->getCount());

>From 4ae746ca1eeaf11db10e859572a4500029d01cce Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:17:29 -0400
Subject: [PATCH 06/13] add analysis traversal support for flatten directive

---
 clang/lib/Sema/SemaExceptionSpec.cpp         | 1 +
 clang/lib/StaticAnalyzer/Core/ExprEngine.cpp | 1 +
 2 files changed, 2 insertions(+)

diff --git a/clang/lib/Sema/SemaExceptionSpec.cpp b/clang/lib/Sema/SemaExceptionSpec.cpp
index 40d530a1f3925..995b5b6010978 100644
--- a/clang/lib/Sema/SemaExceptionSpec.cpp
+++ b/clang/lib/Sema/SemaExceptionSpec.cpp
@@ -1508,6 +1508,7 @@ CanThrowResult Sema::canThrow(const Stmt *S) {
   case Stmt::OMPUnrollDirectiveClass:
   case Stmt::OMPReverseDirectiveClass:
   case Stmt::OMPInterchangeDirectiveClass:
+  case Stmt::OMPFlattenDirectiveClass:
   case Stmt::OMPSplitDirectiveClass:
   case Stmt::OMPFuseDirectiveClass:
   case Stmt::OMPSingleDirectiveClass:
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
index cfb294736ee02..3f666be00cc24 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
@@ -1783,6 +1783,7 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
     case Stmt::OMPStripeDirectiveClass:
     case Stmt::OMPTileDirectiveClass:
     case Stmt::OMPInterchangeDirectiveClass:
+    case Stmt::OMPFlattenDirectiveClass:
     case Stmt::OMPSplitDirectiveClass:
     case Stmt::OMPFuseDirectiveClass:
     case Stmt::OMPInteropDirectiveClass:

>From 415eba88ad0af63c552b4c5c2eabd660bcb29c02 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:21:25 -0400
Subject: [PATCH 07/13] codegen flatten directive

---
 clang/lib/CodeGen/CGStmt.cpp        | 3 +++
 clang/lib/CodeGen/CGStmtOpenMP.cpp  | 8 ++++++++
 clang/lib/CodeGen/CodeGenFunction.h | 1 +
 3 files changed, 12 insertions(+)

diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index 232094777f233..27f48cb63b553 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -236,6 +236,9 @@ void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {
   case Stmt::OMPInterchangeDirectiveClass:
     EmitOMPInterchangeDirective(cast<OMPInterchangeDirective>(*S));
     break;
+  case Stmt::OMPFlattenDirectiveClass:
+    EmitOMPFlattenDirective(cast<OMPFlattenDirective>(*S));
+    break;
   case Stmt::OMPFuseDirectiveClass:
     EmitOMPFuseDirective(cast<OMPFuseDirective>(*S));
     break;
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 95fd6694437fe..25da3430a23fe 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -218,6 +218,8 @@ class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
     } else if (const auto *Interchange =
                    dyn_cast<OMPInterchangeDirective>(&S)) {
       PreInits = Interchange->getPreInits();
+    } else if (const auto *Flatten = dyn_cast<OMPFlattenDirective>(&S)) {
+      PreInits = Flatten->getPreInits();
     } else {
       llvm_unreachable("Unknown loop-based directive kind.");
     }
@@ -3248,6 +3250,12 @@ void CodeGenFunction::EmitOMPInterchangeDirective(
   EmitStmt(S.getTransformedStmt());
 }
 
+void CodeGenFunction::EmitOMPFlattenDirective(const OMPFlattenDirective &S) {
+  // Emit the de-sugared statement.
+  OMPTransformDirectiveScopeRAII FlattenScope(*this, &S);
+  EmitStmt(S.getTransformedStmt());
+}
+
 void CodeGenFunction::EmitOMPFuseDirective(const OMPFuseDirective &S) {
   // Emit the de-sugared statement
   OMPTransformDirectiveScopeRAII FuseScope(*this, &S);
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 6d0718c243812..eea0163422cd3 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -3934,6 +3934,7 @@ class CodeGenFunction : public CodeGenTypeCache {
   void EmitOMPReverseDirective(const OMPReverseDirective &S);
   void EmitOMPSplitDirective(const OMPSplitDirective &S);
   void EmitOMPInterchangeDirective(const OMPInterchangeDirective &S);
+  void EmitOMPFlattenDirective(const OMPFlattenDirective &S);
   void EmitOMPFuseDirective(const OMPFuseDirective &S);
   void EmitOMPForDirective(const OMPForDirective &S);
   void EmitOMPForSimdDirective(const OMPForSimdDirective &S);

>From 814e2e9fe9bf4e28e4557999fbc16beec144cc39 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:24:42 -0400
Subject: [PATCH 08/13] add NYI CIR stub

---
 clang/lib/CIR/CodeGen/CIRGenFunction.h     | 1 +
 clang/lib/CIR/CodeGen/CIRGenStmt.cpp       | 2 ++
 clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp | 5 +++++
 3 files changed, 8 insertions(+)

diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.h b/clang/lib/CIR/CodeGen/CIRGenFunction.h
index 322355fde3957..8374583ffd4d0 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.h
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.h
@@ -2559,6 +2559,7 @@ class CIRGenFunction : public CIRGenTypeCache {
   mlir::LogicalResult emitOMPSplitDirective(const OMPSplitDirective &s);
   mlir::LogicalResult
   emitOMPInterchangeDirective(const OMPInterchangeDirective &s);
+  mlir::LogicalResult emitOMPFlattenDirective(const OMPFlattenDirective &s);
   mlir::LogicalResult emitOMPAssumeDirective(const OMPAssumeDirective &s);
   mlir::LogicalResult emitOMPMaskedDirective(const OMPMaskedDirective &s);
   mlir::LogicalResult emitOMPStripeDirective(const OMPStripeDirective &s);
diff --git a/clang/lib/CIR/CodeGen/CIRGenStmt.cpp b/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
index d3acac5801e74..33d425b22d61a 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
@@ -408,6 +408,8 @@ mlir::LogicalResult CIRGenFunction::emitStmt(const Stmt *s,
     return emitOMPSplitDirective(cast<OMPSplitDirective>(*s));
   case Stmt::OMPInterchangeDirectiveClass:
     return emitOMPInterchangeDirective(cast<OMPInterchangeDirective>(*s));
+  case Stmt::OMPFlattenDirectiveClass:
+    return emitOMPFlattenDirective(cast<OMPFlattenDirective>(*s));
   case Stmt::OMPAssumeDirectiveClass:
     return emitOMPAssumeDirective(cast<OMPAssumeDirective>(*s));
   case Stmt::OMPMaskedDirectiveClass:
diff --git a/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp b/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp
index 17a1fb8090f5c..4fffd17b25a16 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp
@@ -587,6 +587,11 @@ CIRGenFunction::emitOMPInterchangeDirective(const OMPInterchangeDirective &s) {
   return mlir::failure();
 }
 mlir::LogicalResult
+CIRGenFunction::emitOMPFlattenDirective(const OMPFlattenDirective &s) {
+  getCIRGenModule().errorNYI(s.getSourceRange(), "OpenMP OMPFlattenDirective");
+  return mlir::failure();
+}
+mlir::LogicalResult
 CIRGenFunction::emitOMPAssumeDirective(const OMPAssumeDirective &s) {
   getCIRGenModule().errorNYI(s.getSourceRange(), "OpenMP OMPAssumeDirective");
   return mlir::failure();

>From 0027ec6ab982a83766bd65f3e1bc45d9a671fc62 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:26:31 -0400
Subject: [PATCH 09/13] add libclang traversal support for depth clause

---
 clang/tools/libclang/CIndex.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index ac2fad38a1348..f8ab63e3e0dae 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2378,6 +2378,10 @@ void OMPClauseEnqueue::VisitOMPLoopRangeClause(const OMPLoopRangeClause *C) {
   Visitor->AddStmt(C->getCount());
 }
 
+void OMPClauseEnqueue::VisitOMPDepthClause(const OMPDepthClause *C) {
+  Visitor->AddStmt(C->getDepth());
+}
+
 void OMPClauseEnqueue::VisitOMPAllocatorClause(const OMPAllocatorClause *C) {
   Visitor->AddStmt(C->getAllocator());
 }

>From f5ac5f740c414310c8482f652f1bed936ae7cda3 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Fri, 10 Jul 2026 06:54:16 -0400
Subject: [PATCH 10/13] sema

---
 clang/include/clang/Sema/SemaOpenMP.h |  10 +
 clang/lib/Sema/SemaOpenMP.cpp         | 295 ++++++++++++++++++++++++++
 2 files changed, 305 insertions(+)

diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index 3621ce96b8724..7b086388416a0 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -470,6 +470,12 @@ class SemaOpenMP : public SemaBase {
                                              SourceLocation StartLoc,
                                              SourceLocation EndLoc);
 
+  /// Called on well-formed '#pragma omp flatten' after parsing of its
+  /// clauses and the associated statement.
+  StmtResult ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
+                                         Stmt *AStmt, SourceLocation StartLoc,
+                                         SourceLocation EndLoc);
+
   /// Called on well-formed '#pragma omp fuse' after parsing of its
   /// clauses and the associated statement.
   StmtResult ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
@@ -935,6 +941,10 @@ class SemaOpenMP : public SemaBase {
   OMPClause *ActOnOpenMPPartialClause(Expr *FactorExpr, SourceLocation StartLoc,
                                       SourceLocation LParenLoc,
                                       SourceLocation EndLoc);
+  /// Called on well-formed 'depth' clause.
+  OMPClause *ActOnOpenMPDepthClause(Expr *DepthExpr, SourceLocation StartLoc,
+                                    SourceLocation LParenLoc,
+                                    SourceLocation EndLoc);
   /// Called on well-formed 'collapse' clause.
   OMPClause *ActOnOpenMPCollapseClause(Expr *NumForLoops,
                                        SourceLocation StartLoc,
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 247a4a6ad9271..fcc256dec3c88 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -4629,6 +4629,7 @@ void SemaOpenMP::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind,
   case OMPD_reverse:
   case OMPD_split:
   case OMPD_interchange:
+  case OMPD_flatten:
   case OMPD_fuse:
   case OMPD_assume:
     break;
@@ -6475,6 +6476,10 @@ StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
     Res = ActOnOpenMPInterchangeDirective(ClausesWithImplicit, AStmt, StartLoc,
                                           EndLoc);
     break;
+  case OMPD_flatten:
+    Res = ActOnOpenMPFlattenDirective(ClausesWithImplicit, AStmt, StartLoc,
+                                      EndLoc);
+    break;
   case OMPD_fuse:
     Res =
         ActOnOpenMPFuseDirective(ClausesWithImplicit, AStmt, StartLoc, EndLoc);
@@ -16390,6 +16395,277 @@ StmtResult SemaOpenMP::ActOnOpenMPInterchangeDirective(
                                          buildPreInits(Context, PreInits));
 }
 
+/// Counts the perfectly nested canonical loop depth at \p AStmt. Returns
+/// std::nullopt if a loop-generating transformation prevents a static count.
+static std::optional<unsigned> getCanonicalLoopNestDepth(Stmt *AStmt) {
+  unsigned Depth = 0;
+  Stmt *CurStmt = AStmt ? AStmt->IgnoreContainers() : nullptr;
+  while (CurStmt) {
+    if (isa<OMPLoopTransformationDirective>(CurStmt))
+      return std::nullopt;
+    if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(CurStmt))
+      CurStmt = CanonLoop->getLoopStmt();
+    Stmt *Body = nullptr;
+    if (auto *For = dyn_cast<ForStmt>(CurStmt))
+      Body = For->getBody();
+    else if (auto *RangeFor = dyn_cast<CXXForRangeStmt>(CurStmt))
+      Body = RangeFor->getBody();
+    else
+      break;
+    ++Depth;
+    CurStmt = Body ? Body->IgnoreContainers() : nullptr;
+  }
+  return Depth;
+}
+
+StmtResult
+SemaOpenMP::ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
+                                        Stmt *AStmt, SourceLocation StartLoc,
+                                        SourceLocation EndLoc) {
+  ASTContext &Context = getASTContext();
+  DeclContext *CurContext = SemaRef.CurContext;
+  Scope *CurScope = SemaRef.getCurScope();
+
+  // Empty statement should only be possible if there already was an error.
+  if (!AStmt)
+    return StmtError();
+
+  // flatten without 'depth' clause combines two loops; 'depth(k)' selects k.
+  unsigned NumLoops = 2;
+  bool DepthIsValueDependent = false;
+  const auto *DepthClause =
+      OMPExecutableDirective::getSingleClause<OMPDepthClause>(Clauses);
+  if (DepthClause) {
+    Expr *DepthExpr = DepthClause->getDepth();
+    if (DepthExpr && DepthExpr->isValueDependent()) {
+      DepthIsValueDependent = true;
+    } else if (DepthExpr) {
+      Expr::EvalResult EvalResult;
+      if (DepthExpr->EvaluateAsInt(EvalResult, Context))
+        NumLoops = EvalResult.Val.getInt().getZExtValue();
+    }
+  }
+
+  // Report "expected k for loops, but found only n" when depth exceeds the
+  // perfect nest (same form as 'collapse'); skip if depth or nest is unknown.
+  if (DepthClause && !DepthIsValueDependent) {
+    if (std::optional<unsigned> NestDepth = getCanonicalLoopNestDepth(AStmt);
+        NestDepth && NumLoops > *NestDepth) {
+      Diag(AStmt->getBeginLoc(), diag::err_omp_not_for)
+          << /*expected N for loops form=*/1
+          << getOpenMPDirectiveName(OMPD_flatten) << NumLoops
+          << (*NestDepth > 0) << *NestDepth;
+      return StmtError();
+    }
+  }
+
+  // Defer when 'depth' is value-dependent (concrete k unknown until
+  // instantiation).
+  if (DepthIsValueDependent)
+    return OMPFlattenDirective::Create(Context, StartLoc, EndLoc, Clauses,
+                                       NumLoops, AStmt, nullptr, nullptr);
+
+  // Verify and diagnose loop nest.
+  SmallVector<OMPLoopBasedDirective::HelperExprs, 4> LoopHelpers(NumLoops);
+  Stmt *Body = nullptr;
+  SmallVector<SmallVector<Stmt *>, 4> OriginalInits;
+  if (!checkTransformableLoopNest(OMPD_flatten, AStmt, NumLoops, LoopHelpers,
+                                  Body, OriginalInits))
+    return StmtError();
+
+  // Delay flattening to when template is completely instantiated.
+  if (CurContext->isDependentContext())
+    return OMPFlattenDirective::Create(Context, StartLoc, EndLoc, Clauses,
+                                       NumLoops, AStmt, nullptr, nullptr);
+
+  assert(LoopHelpers.size() == NumLoops &&
+         "Expecting loop iteration space dimensionality to match number of "
+         "affected loops");
+  assert(OriginalInits.size() == NumLoops &&
+         "Expecting loop iteration space dimensionality to match number of "
+         "affected loops");
+
+  // Find the affected loops.
+  SmallVector<Stmt *> LoopStmts(NumLoops, nullptr);
+  collectLoopStmts(AStmt, LoopStmts);
+
+  // Collect pre-init statements in outer-to-inner order.
+  SmallVector<Stmt *> PreInits;
+  for (auto I : llvm::seq<int>(NumLoops)) {
+    OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
+    assert(LoopHelper.Counters.size() == 1 &&
+           "Single-dimensional loop iteration space expected");
+    addLoopPreInits(Context, LoopHelper, LoopStmts[I], OriginalInits[I],
+                    PreInits);
+  }
+
+  CaptureVars CopyTransformer(SemaRef);
+  auto MakeNumIterations = [&CopyTransformer,
+                            &LoopHelpers](unsigned I) -> Expr * {
+    return AssertSuccess(
+        CopyTransformer.TransformExpr(LoopHelpers[I].NumIterations));
+  };
+
+  OMPLoopBasedDirective::HelperExprs &OutermostHelper = LoopHelpers[0];
+  auto *OutermostCntVar = cast<DeclRefExpr>(OutermostHelper.Counters.front());
+  SourceLocation OrigVarLoc = OutermostCntVar->getExprLoc();
+  SourceLocation OrigVarLocBegin = OutermostCntVar->getBeginLoc();
+  SourceLocation OrigVarLocEnd = OutermostCntVar->getEndLoc();
+  SourceLocation CondLoc = OutermostHelper.Cond->getExprLoc();
+
+  // Product of trip counts; mirror 'collapse' IV-width selection to avoid
+  // overflow when several counts are multiplied.
+  auto BuildTripCount = [&](unsigned Bits) -> ExprResult {
+    ExprResult Product;
+    for (unsigned I = 0; I < NumLoops; ++I) {
+      ExprResult N = widenIterationCount(Bits, MakeNumIterations(I), SemaRef);
+      if (!N.isUsable())
+        return ExprError();
+      if (I == 0)
+        Product = N;
+      else
+        Product = SemaRef.BuildBinOp(CurScope, CondLoc, BO_Mul, Product.get(),
+                                     N.get());
+      if (!Product.isUsable())
+        return ExprError();
+    }
+    return Product;
+  };
+
+  bool AllCountsLessThan32Bits = true;
+  for (unsigned I = 0; I < NumLoops; ++I)
+    AllCountsLessThan32Bits &=
+        Context.getTypeSize(LoopHelpers[I].NumIterations->getType()) < 32;
+
+  ExprResult TripCount32 = BuildTripCount(/*Bits=*/32);
+  ExprResult TripCount64 = BuildTripCount(/*Bits=*/64);
+  if (!TripCount32.isUsable() || !TripCount64.isUsable())
+    return StmtError();
+
+  ExprResult TripCount = TripCount64;
+  if (Context.getTypeSize(TripCount32.get()->getType()) == 32 &&
+      (AllCountsLessThan32Bits || NumLoops == 1 ||
+       fitsInto(
+           /*Bits=*/32,
+           TripCount32.get()->getType()->hasSignedIntegerRepresentation(),
+           TripCount64.get(), SemaRef)))
+    TripCount = TripCount32;
+
+  QualType IVTy = TripCount.get()->getType();
+  uint64_t IVWidth = Context.getTypeSize(IVTy);
+
+  auto MakeNumIterationsInIVTy = [&](unsigned I) -> Expr * {
+    return AssertSuccess(SemaRef.PerformImplicitConversion(
+        MakeNumIterations(I), IVTy, AssignmentAction::Converting,
+        /*AllowExplicit=*/true));
+  };
+
+  // \code{.cpp}
+  //   for (auto .flatten.iv = 0; .flatten.iv < n0 * n1 * ...; ++.flatten.iv) {
+  //     .flatten.iv.0 = .flatten.iv / (n1 * ...);
+  //     i0 = ...;                                        // Updates[0]
+  //     .flatten.iv.1 = (.flatten.iv / ...) % n1;
+  //     i1 = ...;                                        // Updates[1]
+  //     ...
+  //     body(i0, i1, ...);
+  //   }
+  // \endcode
+  SmallString<64> FlattenedIVName(".flatten.iv");
+  VarDecl *FlattenedIVDecl = buildVarDecl(SemaRef, {}, IVTy, FlattenedIVName,
+                                          nullptr, OutermostCntVar);
+  auto MakeFlattenedRef = [&SemaRef = this->SemaRef, FlattenedIVDecl, IVTy,
+                           OrigVarLoc]() {
+    return buildDeclRefExpr(SemaRef, FlattenedIVDecl, IVTy, OrigVarLoc);
+  };
+
+  // For init-statement:
+  // \code{.cpp}
+  //   auto .flatten.iv = 0;
+  // \endcode
+  auto *Zero = IntegerLiteral::Create(Context, llvm::APInt::getZero(IVWidth),
+                                      IVTy, OrigVarLoc);
+  SemaRef.AddInitializerToDecl(FlattenedIVDecl, Zero, /*DirectInit=*/false);
+  StmtResult Init = new (Context)
+      DeclStmt(DeclGroupRef(FlattenedIVDecl), OrigVarLocBegin, OrigVarLocEnd);
+  if (!Init.isUsable())
+    return StmtError();
+
+  // For cond-expression:
+  // \code{.cpp}
+  //   .flatten.iv < n0 * n1 * ... * n(k-1)
+  // \endcode
+  ExprResult Cond = SemaRef.BuildBinOp(CurScope, CondLoc, BO_LT,
+                                       MakeFlattenedRef(), TripCount.get());
+  if (!Cond.isUsable())
+    return StmtError();
+
+  // For incr-statement:
+  // \code{.cpp}
+  //   ++.flatten.iv
+  // \endcode
+  ExprResult Incr =
+      SemaRef.BuildUnaryOp(CurScope, OutermostHelper.Inc->getExprLoc(),
+                           UO_PreInc, MakeFlattenedRef());
+  if (!Incr.isUsable())
+    return StmtError();
+
+  // Recover each logical iteration counter via mixed-radix div/mod; reuse the
+  // iteration variables from checkOpenMPLoop so Updates compute user counters.
+  SmallVector<Stmt *, 8> BodyStmts;
+  for (unsigned I = 0; I < NumLoops; ++I) {
+    OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
+    auto *IVRef = cast<DeclRefExpr>(LoopHelper.IterationVarRef);
+    auto *IVDecl = cast<VarDecl>(IVRef->getDecl());
+    std::string IVName = (".flatten.iv." + llvm::Twine(I)).str();
+    IVDecl->setDeclName(&SemaRef.PP.getIdentifierTable().get(IVName));
+
+    ExprResult Value = MakeFlattenedRef();
+    if (I + 1 < NumLoops) {
+      ExprResult Divisor = MakeNumIterationsInIVTy(I + 1);
+      for (unsigned J = I + 2; J < NumLoops; ++J) {
+        Divisor = SemaRef.BuildBinOp(CurScope, OrigVarLoc, BO_Mul,
+                                     Divisor.get(), MakeNumIterationsInIVTy(J));
+        if (!Divisor.isUsable())
+          return StmtError();
+      }
+      Value = SemaRef.BuildBinOp(CurScope, OrigVarLoc, BO_Div, Value.get(),
+                                 Divisor.get());
+      if (!Value.isUsable())
+        return StmtError();
+    }
+    if (I > 0) {
+      Value = SemaRef.BuildBinOp(CurScope, OrigVarLoc, BO_Rem, Value.get(),
+                                 MakeNumIterationsInIVTy(I));
+      if (!Value.isUsable())
+        return StmtError();
+    }
+
+    SemaRef.AddInitializerToDecl(IVDecl, Value.get(), /*DirectInit=*/false);
+    StmtResult IVStmt = new (Context)
+        DeclStmt(DeclGroupRef(IVDecl), OrigVarLocBegin, OrigVarLocEnd);
+    if (!IVStmt.isUsable())
+      return StmtError();
+
+    BodyStmts.push_back(IVStmt.get());
+    llvm::append_range(BodyStmts, LoopHelper.Updates);
+    if (auto *CXXFor = dyn_cast<CXXForRangeStmt>(LoopStmts[I]))
+      BodyStmts.push_back(CXXFor->getLoopVarStmt());
+  }
+  BodyStmts.push_back(Body);
+  auto *FlattenedBody =
+      CompoundStmt::Create(Context, BodyStmts, FPOptionsOverride(),
+                           Body->getBeginLoc(), Body->getEndLoc());
+
+  auto *FlattenedFor = new (Context) ForStmt(
+      Context, Init.get(), Cond.get(), nullptr, Incr.get(), FlattenedBody,
+      OutermostHelper.Init->getBeginLoc(), OutermostHelper.Init->getBeginLoc(),
+      OutermostHelper.Inc->getEndLoc());
+
+  return OMPFlattenDirective::Create(Context, StartLoc, EndLoc, Clauses,
+                                     NumLoops, AStmt, FlattenedFor,
+                                     buildPreInits(Context, PreInits));
+}
+
 StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
                                                 Stmt *AStmt,
                                                 SourceLocation StartLoc,
@@ -16920,6 +17196,9 @@ OMPClause *SemaOpenMP::ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind,
   case OMPC_partial:
     Res = ActOnOpenMPPartialClause(Expr, StartLoc, LParenLoc, EndLoc);
     break;
+  case OMPC_depth:
+    Res = ActOnOpenMPDepthClause(Expr, StartLoc, LParenLoc, EndLoc);
+    break;
   case OMPC_message:
     Res = ActOnOpenMPMessageClause(Expr, StartLoc, LParenLoc, EndLoc);
     break;
@@ -18241,6 +18520,22 @@ OMPClause *SemaOpenMP::ActOnOpenMPPartialClause(Expr *FactorExpr,
                                   FactorExpr);
 }
 
+OMPClause *SemaOpenMP::ActOnOpenMPDepthClause(Expr *DepthExpr,
+                                              SourceLocation StartLoc,
+                                              SourceLocation LParenLoc,
+                                              SourceLocation EndLoc) {
+  // The depth-expr must be a positive integer constant expression and
+  // not greater than the number of loops in the associated loop nest.
+  ExprResult DepthResult = VerifyPositiveIntegerConstantInClause(
+      DepthExpr, OMPC_depth, /*StrictlyPositive=*/true);
+  if (DepthResult.isInvalid())
+    return nullptr;
+  DepthExpr = DepthResult.get();
+
+  return OMPDepthClause::Create(getASTContext(), StartLoc, LParenLoc, EndLoc,
+                                DepthExpr);
+}
+
 OMPClause *SemaOpenMP::ActOnOpenMPLoopRangeClause(
     Expr *First, Expr *Count, SourceLocation StartLoc, SourceLocation LParenLoc,
     SourceLocation FirstLoc, SourceLocation CountLoc, SourceLocation EndLoc) {

>From 54be7347979638725cd4b6c421d11ec1cd618c43 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Mon, 27 Jul 2026 07:03:06 -0400
Subject: [PATCH 11/13] add libclang traversal support for flatten directive

---
 clang/tools/libclang/CIndex.cpp   | 7 +++++++
 clang/tools/libclang/CXCursor.cpp | 3 +++
 2 files changed, 10 insertions(+)

diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index f8ab63e3e0dae..9eb0ec8c21158 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2160,6 +2160,7 @@ class EnqueueVisitor : public ConstStmtVisitor<EnqueueVisitor, void>,
   void VisitOMPUnrollDirective(const OMPUnrollDirective *D);
   void VisitOMPReverseDirective(const OMPReverseDirective *D);
   void VisitOMPInterchangeDirective(const OMPInterchangeDirective *D);
+  void VisitOMPFlattenDirective(const OMPFlattenDirective *D);
   void VisitOMPCanonicalLoopSequenceTransformationDirective(
       const OMPCanonicalLoopSequenceTransformationDirective *D);
   void VisitOMPFuseDirective(const OMPFuseDirective *D);
@@ -3358,6 +3359,10 @@ void EnqueueVisitor::VisitOMPInterchangeDirective(
   VisitOMPCanonicalLoopNestTransformationDirective(D);
 }
 
+void EnqueueVisitor::VisitOMPFlattenDirective(const OMPFlattenDirective *D) {
+  VisitOMPCanonicalLoopNestTransformationDirective(D);
+}
+
 void EnqueueVisitor::VisitOMPCanonicalLoopSequenceTransformationDirective(
     const OMPCanonicalLoopSequenceTransformationDirective *D) {
   VisitOMPExecutableDirective(D);
@@ -6329,6 +6334,8 @@ CXString clang_getCursorKindSpelling(enum CXCursorKind Kind) {
     return cxstring::createRef("OMPReverseDirective");
   case CXCursor_OMPInterchangeDirective:
     return cxstring::createRef("OMPInterchangeDirective");
+  case CXCursor_OMPFlattenDirective:
+    return cxstring::createRef("OMPFlattenDirective");
   case CXCursor_OMPFuseDirective:
     return cxstring::createRef("OMPFuseDirective");
   case CXCursor_OMPSplitDirective:
diff --git a/clang/tools/libclang/CXCursor.cpp b/clang/tools/libclang/CXCursor.cpp
index b2feed578fc0f..97bbfd36558a9 100644
--- a/clang/tools/libclang/CXCursor.cpp
+++ b/clang/tools/libclang/CXCursor.cpp
@@ -703,6 +703,9 @@ CXCursor cxcursor::MakeCXCursor(const Stmt *S, const Decl *Parent,
   case Stmt::OMPInterchangeDirectiveClass:
     K = CXCursor_OMPInterchangeDirective;
     break;
+  case Stmt::OMPFlattenDirectiveClass:
+    K = CXCursor_OMPFlattenDirective;
+    break;
   case Stmt::OMPFuseDirectiveClass:
     K = CXCursor_OMPFuseDirective;
     break;

>From cdedb1045baf373937b0da858624a6ace14a3d21 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Mon, 27 Jul 2026 07:06:36 -0400
Subject: [PATCH 12/13] bindings

---
 clang/bindings/python/clang/cindex.py | 3 +++
 clang/include/clang-c/Index.h         | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/clang/bindings/python/clang/cindex.py b/clang/bindings/python/clang/cindex.py
index 24b737139dba8..ccad140e8c861 100644
--- a/clang/bindings/python/clang/cindex.py
+++ b/clang/bindings/python/clang/cindex.py
@@ -1456,6 +1456,9 @@ def is_unexposed(self):
     # OpenMP split directive.
     OMP_SPLIT_DIRECTIVE = 312
 
+    # OpenMP flatten directive.
+    OMP_FLATTEN_DIRECTIVE = 313
+
     # OpenACC Compute Construct.
     OPEN_ACC_COMPUTE_DIRECTIVE = 320
 
diff --git a/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h
index 8427236e0b444..f561fb5724897 100644
--- a/clang/include/clang-c/Index.h
+++ b/clang/include/clang-c/Index.h
@@ -2170,6 +2170,10 @@ enum CXCursorKind {
    */
   CXCursor_OMPSplitDirective = 312,
 
+  /** OpenMP flatten directive.
+   */
+  CXCursor_OMPFlattenDirective = 313,
+
   /** OpenACC Compute Construct.
    */
   CXCursor_OpenACCComputeConstruct = 320,

>From 94dee8fe1351f74760df0fb2d9903a291a5ccf66 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 5 Aug 2026 04:46:14 -0400
Subject: [PATCH 13/13] core tests-flatten and depth

---
 clang/test/OpenMP/flatten_codegen.cpp         |  65 ++++++++
 clang/test/OpenMP/flatten_depth_codegen.cpp   | 154 ++++++++++++++++++
 .../test/transform/flatten/depth-intfor.c     |  40 +++++
 .../runtime/test/transform/flatten/intfor.c   |  33 ++++
 .../test/transform/flatten/lit.local.cfg      |   5 +
 5 files changed, 297 insertions(+)
 create mode 100644 clang/test/OpenMP/flatten_codegen.cpp
 create mode 100644 clang/test/OpenMP/flatten_depth_codegen.cpp
 create mode 100644 openmp/runtime/test/transform/flatten/depth-intfor.c
 create mode 100644 openmp/runtime/test/transform/flatten/intfor.c
 create mode 100644 openmp/runtime/test/transform/flatten/lit.local.cfg

diff --git a/clang/test/OpenMP/flatten_codegen.cpp b/clang/test/OpenMP/flatten_codegen.cpp
new file mode 100644
index 0000000000000..f77b2fb00ce9b
--- /dev/null
+++ b/clang/test/OpenMP/flatten_codegen.cpp
@@ -0,0 +1,65 @@
+// Check code generation
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=60 -emit-llvm %s -o - | FileCheck %s
+
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+extern "C" void body(int, int);
+
+// CHECK-LABEL: define {{.*}}void @foo(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK:   store i64 0, ptr %.flatten.iv,
+// CHECK:   %[[COND_IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[MUL:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   icmp slt i64 %[[COND_IV]], %[[MUL]]
+// CHECK:   %[[DIV_IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[DIV:.+]] = sdiv i64 %[[DIV_IV]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   %[[REM_IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[REM:.+]] = srem i64 %[[REM_IV]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+// CHECK:   %[[I_VAL:.+]] = load i32, ptr %i,
+// CHECK:   %[[J_VAL:.+]] = load i32, ptr %j,
+// CHECK:   call void @body(i32{{.*}} %[[I_VAL]], i32{{.*}} %[[J_VAL]])
+// CHECK: for.inc:
+// CHECK:   %[[INC_IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[NEXT:.+]] = add nsw i64 %[[INC_IV]], 1
+// CHECK:   store i64 %[[NEXT]], ptr %.flatten.iv,
+// CHECK:   br label %for.cond
+extern "C" void foo(int n, int m) {
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+
+// CHECK-LABEL: define {{.*}}void @bar(
+// CHECK:   icmp slt i32 %{{.+}}, 63
+// CHECK:   sdiv i32 %{{.+}}, 9
+// CHECK:   srem i32 %{{.+}}, 9
+// CHECK:   call void @body(
+extern "C" void bar() {
+#pragma omp flatten
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+}
+
+// CHECK-LABEL: define {{.*}}void @baz(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   icmp slt i64 %{{.+}}, 10000000000
+// CHECK:   sdiv i64 %{{.+}}, 100000
+// CHECK:   srem i64 %{{.+}}, 100000
+// CHECK:   call void @body(
+extern "C" void baz() {
+#pragma omp flatten
+  for (int i = 0; i < 100000; ++i)
+    for (int j = 0; j < 100000; ++j)
+      body(i, j);
+}
+
+#endif
diff --git a/clang/test/OpenMP/flatten_depth_codegen.cpp b/clang/test/OpenMP/flatten_depth_codegen.cpp
new file mode 100644
index 0000000000000..77408fbcbdcf1
--- /dev/null
+++ b/clang/test/OpenMP/flatten_depth_codegen.cpp
@@ -0,0 +1,154 @@
+// Check code generation for the 'depth' clause on '#pragma omp flatten'.
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=61 -emit-llvm %s -o - | FileCheck %s
+
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+extern "C" void body(int, int, int);
+
+// CHECK-LABEL: define {{.*}}void @foo3(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK:   %.flatten.iv.2 = alloca i32
+// CHECK:   %[[CIV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[T01:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   %[[TALL:.+]] = mul nsw i64 %[[T01]], %{{.+}}
+// CHECK:   icmp slt i64 %[[CIV]], %[[TALL]]
+// CHECK:   %[[D0IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[D0M:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   %[[D0:.+]] = sdiv i64 %[[D0IV]], %[[D0M]]
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   %[[D1IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[D1:.+]] = sdiv i64 %[[D1IV]], %{{.+}}
+// CHECK:   %[[R1:.+]] = srem i64 %[[D1]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+// CHECK:   %[[R2IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[R2:.+]] = srem i64 %[[R2IV]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.2
+// CHECK:   call void @body(
+extern "C" void foo3(int n, int m, int p) {
+#pragma omp flatten depth(3)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        body(i, j, k);
+}
+
+// CHECK-LABEL: define {{.*}}void @foo2(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK:   %[[CIV2:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[M:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   icmp slt i64 %[[CIV2]], %[[M]]
+// CHECK:   sdiv i64 %{{.+}}, %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   srem i64 %{{.+}}, %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+extern "C" void foo2(int n, int m) {
+#pragma omp flatten depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j, 0);
+}
+
+// CHECK-LABEL: define {{.*}}void @foo1(
+// CHECK:   %.flatten.iv = alloca i32
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK: for.body:
+// CHECK:   %[[FV:.+]] = load i32, ptr %.flatten.iv,
+// CHECK:   store i32 %[[FV]], ptr %.flatten.iv.0,
+// CHECK-NOT: sdiv
+// CHECK-NOT: srem
+// CHECK:   call void @body(
+extern "C" void foo1(int n) {
+#pragma omp flatten depth(1)
+  for (int i = 0; i < n; ++i)
+    body(i, 0, 0);
+}
+
+// CHECK-LABEL: define {{.*}}void @foo2partial(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK-NOT: %.flatten.iv.2 = alloca
+// CHECK:   %[[CIV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[M:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   icmp slt i64 %[[CIV]], %[[M]]
+// CHECK:   sdiv i64 %{{.+}}, %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   srem i64 %{{.+}}, %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+// CHECK:   br label %for.cond{{.*}}
+// CHECK: for.cond{{.*}}:
+// CHECK:   icmp slt i32 %{{.+}}, 5
+extern "C" void foo2partial(int n, int m) {
+#pragma omp flatten depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < 5; ++k)
+        body(i, j, k);
+}
+
+// A value-dependent 'depth' (the argument is a non-type template parameter) is
+// deferred until instantiation, where the concrete depth drives the flattening.
+
+// CHECK-LABEL: define {{.*}}void @_Z11tmpl_depth3ILi3EEviii(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK:   %.flatten.iv.2 = alloca i32
+// CHECK:   %[[CIV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[T01:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   %[[TALL:.+]] = mul nsw i64 %[[T01]], %{{.+}}
+// CHECK:   icmp slt i64 %[[CIV]], %[[TALL]]
+// CHECK:   %[[D0IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[D0M:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   %[[D0:.+]] = sdiv i64 %[[D0IV]], %[[D0M]]
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   %[[D1IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[D1:.+]] = sdiv i64 %[[D1IV]], %{{.+}}
+// CHECK:   %[[R1:.+]] = srem i64 %[[D1]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+// CHECK:   %[[R2IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[R2:.+]] = srem i64 %[[R2IV]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.2
+// CHECK:   call void @body(
+template <int D>
+void tmpl_depth3(int n, int m, int p) {
+#pragma omp flatten depth(D)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        body(i, j, k);
+}
+
+// A value-dependent 'depth' that instantiates to depth(1) on a single loop is
+// well-formed: the shallower-than-default nest is accepted because validation is
+// deferred to instantiation. The single loop needs no div/mod recovery.
+
+// CHECK-LABEL: define {{.*}}void @_Z11tmpl_depth1ILi1EEvi(
+// CHECK:   %.flatten.iv = alloca i32
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK: for.body:
+// CHECK:   %[[FV:.+]] = load i32, ptr %.flatten.iv,
+// CHECK:   store i32 %[[FV]], ptr %.flatten.iv.0,
+// CHECK-NOT: sdiv
+// CHECK-NOT: srem
+// CHECK:   call void @body(
+template <int D>
+void tmpl_depth1(int n) {
+#pragma omp flatten depth(D)
+  for (int i = 0; i < n; ++i)
+    body(i, 0, 0);
+}
+
+void inst() {
+  tmpl_depth3<3>(1, 1, 1);
+  tmpl_depth1<1>(1);
+}
+
+#endif
diff --git a/openmp/runtime/test/transform/flatten/depth-intfor.c b/openmp/runtime/test/transform/flatten/depth-intfor.c
new file mode 100644
index 0000000000000..7211e02e25313
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/depth-intfor.c
@@ -0,0 +1,40 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+// 'depth(3)' fully flattens a three-deep perfectly nested loop into one loop.
+// This verifies that the flattened loop preserves the exact row-major
+// visitation order of the original nest, including non-unit steps and non-zero
+// start values.
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+int main() {
+  printf("do\n");
+#pragma omp flatten depth(3)
+  for (int i = 7; i < 11; i += 2)
+    for (int j = 1; j < 4; j += 1)
+      for (int k = 0; k < 2; ++k)
+        printf("i=%d j=%d k=%d\n", i, j, k);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: i=7 j=1 k=0
+// CHECK-NEXT: i=7 j=1 k=1
+// CHECK-NEXT: i=7 j=2 k=0
+// CHECK-NEXT: i=7 j=2 k=1
+// CHECK-NEXT: i=7 j=3 k=0
+// CHECK-NEXT: i=7 j=3 k=1
+// CHECK-NEXT: i=9 j=1 k=0
+// CHECK-NEXT: i=9 j=1 k=1
+// CHECK-NEXT: i=9 j=2 k=0
+// CHECK-NEXT: i=9 j=2 k=1
+// CHECK-NEXT: i=9 j=3 k=0
+// CHECK-NEXT: i=9 j=3 k=1
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/flatten/intfor.c b/openmp/runtime/test/transform/flatten/intfor.c
new file mode 100644
index 0000000000000..f937659fbebe1
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/intfor.c
@@ -0,0 +1,33 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+int main() {
+  printf("do\n");
+#pragma omp flatten
+  for (int i = 5; i < 11; i += 2)
+    for (int j = 1; j < 7; j += 2)
+      printf("i=%d j=%d\n", i, j);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// The flattened loop visits the original iteration space in the original
+// (row-major) order: for each outer iteration the inner loop runs fully.
+// CHECK:      do
+// CHECK-NEXT: i=5 j=1
+// CHECK-NEXT: i=5 j=3
+// CHECK-NEXT: i=5 j=5
+// CHECK-NEXT: i=7 j=1
+// CHECK-NEXT: i=7 j=3
+// CHECK-NEXT: i=7 j=5
+// CHECK-NEXT: i=9 j=1
+// CHECK-NEXT: i=9 j=3
+// CHECK-NEXT: i=9 j=5
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/flatten/lit.local.cfg b/openmp/runtime/test/transform/flatten/lit.local.cfg
new file mode 100644
index 0000000000000..e05bd627df1e9
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/lit.local.cfg
@@ -0,0 +1,5 @@
+# The flatten directive's depth clause requires OpenMP 6.1.
+for i, (pattern, replacement) in enumerate(config.substitutions):
+    if pattern == "%openmp_flags":
+        config.substitutions[i] = (pattern, replacement + " -fopenmp-version=61")
+        break


From openmp-commits at lists.llvm.org  Wed Aug  5 02:33:23 2026
From: openmp-commits at lists.llvm.org (Jan Patrick Lehr via Openmp-commits)
Date: Wed, 05 Aug 2026 02:33:23 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OMPT] Upstream device tracing
	implementation (PR #200165)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/200165@github.com>
Message-ID: <6a730363.0c05e68e.33df79.4e8b@mx.google.com>


https://github.com/jplehr updated https://github.com/llvm/llvm-project/pull/200165

>From 316fac5188c67c352129e13b4c3754edfd54ab49 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Tue, 4 Aug 2026 13:39:45 -0500
Subject: [PATCH 01/11] [OpenMP][OMPT] Test libomptarget connector lookup

Verify that the private target connector only exposes supported callbacks and does not retain the unwired frame-enter extension.

Co-Authored-By: Claude <noreply at anthropic.com>
---
 openmp/runtime/test/ompt/misc/target_lookup.c | 38 +++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 openmp/runtime/test/ompt/misc/target_lookup.c

diff --git a/openmp/runtime/test/ompt/misc/target_lookup.c b/openmp/runtime/test/ompt/misc/target_lookup.c
new file mode 100644
index 0000000000000..d062e759e1912
--- /dev/null
+++ b/openmp/runtime/test/ompt/misc/target_lookup.c
@@ -0,0 +1,38 @@
+// RUN: %libomp-compile-and-run | FileCheck %s
+// REQUIRES: ompt
+
+#include <omp-tools.h>
+#include <stdio.h>
+
+void ompt_libomp_connect(ompt_start_tool_result_t *result);
+
+static int tool_initialize(ompt_function_lookup_t lookup,
+                           int initial_device_num, ompt_data_t *tool_data) {
+  return 1;
+}
+
+static int target_initialize(ompt_function_lookup_t lookup,
+                             int initial_device_num, ompt_data_t *tool_data) {
+  printf("ompt_set_frame_enter: %s\n",
+         lookup("ompt_set_frame_enter") ? "present" : "absent");
+  return 1;
+}
+
+static void ompt_finalize(ompt_data_t *tool_data) {}
+
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t result = {&tool_initialize, &ompt_finalize,
+                                            0};
+  return &result;
+}
+
+int main() {
+  static ompt_start_tool_result_t result = {&target_initialize, &ompt_finalize,
+                                            0};
+  // This entry point initializes OMPT before invoking the target connector.
+  ompt_libomp_connect(&result);
+  return 0;
+}
+
+// CHECK: ompt_set_frame_enter: absent

>From bab7008924d0f50661ad8970b93197cfad1037d1 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 2 Apr 2026 07:27:58 -0500
Subject: [PATCH 02/11] [Offload] Add GenericProfilerTy abstraction and
 APITypes extensions

Introduce a GenericProfilerTy base class that decouples the plugin
layer from OMPT. This allows profiling/tracing backends to be
implemented independently of the plugin code.

Key changes:
- Add GenericProfiler.h/cpp with virtual hooks for device lifecycle
  events (init, deinit, loadBinary), kernel launch, data alloc/delete,
  kernel completion, and data transfer timing
- Add ProfTimerTy RAII timer for scoped alloc/delete measurements
- Add weak getProfilerToAttach() factory (overridable by OMPT)
- Add ExecAsync and ProfilerData fields to __tgt_async_info
- Replace direct OMPT callback invocations in PluginInterface with
  profiler hook calls (handleInit, handleDeinit, handleLoadBinary,
  handlePreKernelLaunch)
- Add getDeviceTimeStamp() virtual to GenericDeviceTy
- Add Profiler member and getProfiler() to GenericPluginTy
- Suppress ProfilerData during KLE upload to avoid spurious traces
- Add sync path in AsyncInfoWrapperTy::finalize for !ExecAsync

Made-with: Cursor
Co-Authored-By: Claude <noreply at anthropic.com>
---
 offload/include/Shared/APITypes.h             |   5 +
 offload/plugins-nextgen/common/CMakeLists.txt |   1 +
 .../common/include/GenericProfiler.h          | 182 ++++++++++++++++++
 .../common/include/PluginInterface.h          |  35 ++--
 .../common/src/GenericProfiler.cpp            |  36 ++++
 .../common/src/PluginInterface.cpp            |  71 ++-----
 6 files changed, 268 insertions(+), 62 deletions(-)
 create mode 100644 offload/plugins-nextgen/common/include/GenericProfiler.h
 create mode 100644 offload/plugins-nextgen/common/src/GenericProfiler.cpp

diff --git a/offload/include/Shared/APITypes.h b/offload/include/Shared/APITypes.h
index 47d8c49bf7ef2..527aa8e617304 100644
--- a/offload/include/Shared/APITypes.h
+++ b/offload/include/Shared/APITypes.h
@@ -85,6 +85,11 @@ struct __tgt_async_info {
   /// ensure it is a valid location while the transfer to the device is
   /// happening.
   KernelLaunchEnvironmentTy KernelLaunchEnvironment;
+
+  /// Opaque handle for profiler-specific data (e.g., OMPT trace record info).
+  /// Owned by the profiler; the runtime threads this pointer through the plugin
+  /// layer to associate async operations with trace records.
+  void *ProfilerData = nullptr;
 };
 
 /// This struct contains all of the arguments to a target kernel region launch.
diff --git a/offload/plugins-nextgen/common/CMakeLists.txt b/offload/plugins-nextgen/common/CMakeLists.txt
index 6ad20796c3ca4..19ce102309a72 100644
--- a/offload/plugins-nextgen/common/CMakeLists.txt
+++ b/offload/plugins-nextgen/common/CMakeLists.txt
@@ -11,6 +11,7 @@ add_public_tablegen_target(PluginErrcodes)
 # don't want to export `PluginInterface` while `add_llvm_library` requires that.
 add_library(PluginCommon OBJECT
   src/PluginInterface.cpp
+  src/GenericProfiler.cpp
   src/GlobalHandler.cpp
   src/JIT.cpp
   src/RecordReplay.cpp
diff --git a/offload/plugins-nextgen/common/include/GenericProfiler.h b/offload/plugins-nextgen/common/include/GenericProfiler.h
new file mode 100644
index 0000000000000..ea3c3766554c8
--- /dev/null
+++ b/offload/plugins-nextgen/common/include/GenericProfiler.h
@@ -0,0 +1,182 @@
+//===- GenericProfiler.h - GenericProfiler interface for use in Plugins ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The GenericProfiler interface allows to implement profiler logic for various
+// backends, such as OMPT or other tracing mechanisms.
+// This enables the plugins to be agnostic of the actual high-level language
+// that is implemented.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OFFLOAD_PLUGINS_NEXTGEN_COMMON_INCLUDE_GENERICPROFILER_H
+#define OFFLOAD_PLUGINS_NEXTGEN_COMMON_INCLUDE_GENERICPROFILER_H
+
+#include "Shared/APITypes.h"
+
+#include <cstdint>
+#include <functional>
+#include <tuple>
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace plugin {
+
+struct GenericDeviceTy;
+struct GenericPluginTy;
+class GenericProfilerTy;
+
+template <typename FunT, typename... ArgsT, size_t... IdxSequence>
+void callViaIndexSeq(FunT F, GenericProfilerTy *P, uint64_t StartNanos,
+                     uint64_t EndNanos, std::tuple<ArgsT...> Args,
+                     std::index_sequence<IdxSequence...>) {
+  F(P, StartNanos, EndNanos, std::get<IdxSequence>(Args)...);
+}
+
+template <typename FunT, typename... ArgsT>
+void callViaUnpack(FunT F, GenericProfilerTy *P, uint64_t StartNanos,
+                   uint64_t EndNanos, std::tuple<ArgsT...> Tup) {
+  callViaIndexSeq(F, P, StartNanos, EndNanos, Tup,
+                  std::index_sequence_for<ArgsT...>{});
+}
+
+/// Abstraction layer to implement different profiler backends.
+///
+/// The plugins call into the GenericProfilerTy to handle the specific events
+/// with whatever specific backend was instantiated. For now, the supported
+/// backends are limited to an OMPT implementation.
+class GenericProfilerTy {
+public:
+  GenericProfilerTy() = default;
+  virtual ~GenericProfilerTy() = default;
+
+  /// Obtain a pointer to profiler-specific data, if any.
+  virtual void *getProfilerSpecificData() { return nullptr; }
+
+  virtual bool isProfilingEnabled() { return false; }
+
+  /// Set the factors which are used to interpolate the device clock compared to
+  /// the host clock. This follows a simple linear interpolation: Slope * <time>
+  /// + Offset.
+  void setTimeConversionFactors(double Slope, double Offset) {
+    HostToDeviceSlope = Slope;
+    HostToDeviceOffset = Offset;
+    setTimeConversionFactorsImpl(HostToDeviceSlope, HostToDeviceOffset);
+  }
+
+  /// Hook that is called when the plugin is initialized.
+  virtual void handleInit(GenericDeviceTy *Device, GenericPluginTy *Plugin) {}
+
+  /// Hook that is called when the plugin is de-initialized.
+  virtual void handleDeinit(GenericDeviceTy *Device, GenericPluginTy *Plugin) {}
+
+  /// Hook that is called when the device image is loaded.
+  virtual void handleLoadBinary(GenericDeviceTy *Device,
+                                GenericPluginTy *Plugin,
+                                const StringRef InputTgtImage) {}
+
+  /// Hook that is called when memory is allocated on the device.
+  virtual void handleDataAlloc(uint64_t StartNanos, uint64_t EndNanos,
+                               void *HostPtr, uint64_t Size, void *Data) {}
+
+  /// Hook that is called when memory is freed on the device.
+  virtual void handleDataDelete(uint64_t StartNanos, uint64_t EndNanos,
+                                void *TgtPtr, void *Data) {}
+
+  /// Hook that is called before launching a kernel.
+  virtual void handlePreKernelLaunch(GenericDeviceTy *Device,
+                                     uint32_t NumBlocks[3],
+                                     __tgt_async_info *AI) {}
+
+  /// Hook that is called when the kernel is finished to extract the specific
+  /// timing info for that kernel execution.
+  virtual void handleKernelCompletion(uint64_t StartNanos, uint64_t EndNanos,
+                                      void *Data) {}
+
+  /// Hook that is called when a data transfer happens to extract timing info
+  /// for that transfer.
+  virtual void handleDataTransfer(uint64_t StartNanos, uint64_t EndNanos,
+                                  void *Data) {}
+
+  /// Allow factors for time conversion between host and device.
+  virtual void setTimeConversionFactorsImpl(double Slope, double Offset) {}
+
+  /// RAII style timer that measures the elapsed time between construction and
+  /// destruction, then invokes a callback with the profiler, start/end times,
+  /// and any captured arguments.
+  template <typename FnT, typename... ArgsT> class ProfTimerTy {
+  public:
+    ProfTimerTy(FnT &&F, GenericProfilerTy *P, GenericDeviceTy *D, ArgsT... As)
+        : Fun(F), Prof(P), Dev(D), Args(As...) {
+      assert(Prof && "GenericProfilerTy is null");
+      assert(Dev && "GenericDeviceTy is null");
+      if (Prof)
+        StartTime = Prof->getDeviceTimeStamp(Dev);
+    }
+
+    ~ProfTimerTy() {
+      assert(Prof && "GenericProfilerTy is null");
+      assert(Dev && "GenericDeviceTy is null");
+      if (Prof) {
+        uint64_t EndTime = Prof->getDeviceTimeStamp(Dev);
+        callViaUnpack(Fun, Prof, StartTime, EndTime, Args);
+      }
+    }
+
+  private:
+    FnT Fun;
+    GenericProfilerTy *Prof;
+    GenericDeviceTy *Dev;
+    uint64_t StartTime = 0;
+    std::tuple<ArgsT...> Args;
+  };
+
+  template <typename FnT, typename... ArgsT>
+  ProfTimerTy(FnT &&, GenericProfilerTy *, ArgsT...)
+      -> ProfTimerTy<FnT, ArgsT...>;
+
+  template <typename FnT, typename... ArgsT> friend class ProfTimerTy;
+
+  /// Returns an RAII style timer, which will handle data allocation timing.
+  [[nodiscard]] auto getScopedDataAllocTimer(GenericDeviceTy *Dev,
+                                             void *HostPtr, uint64_t Size,
+                                             void *ProfData = nullptr) {
+    return ProfTimerTy(
+        [](GenericProfilerTy *P, auto... args) {
+          assert(P && "P was null");
+          P->handleDataAlloc(args...);
+        },
+        this, Dev, HostPtr, Size, ProfData);
+  }
+
+  /// Returns an RAII style timer, which will handle data deletion timing.
+  [[nodiscard]] auto getScopedDataDeleteTimer(GenericDeviceTy *Dev,
+                                              void *TgtPtr,
+                                              void *ProfData = nullptr) {
+    return ProfTimerTy(
+        [](GenericProfilerTy *P, auto... args) {
+          assert(P && "P was null");
+          P->handleDataDelete(args...);
+        },
+        this, Dev, TgtPtr, ProfData);
+  }
+
+protected:
+  double HostToDeviceSlope = 1.0;
+  double HostToDeviceOffset = .0;
+
+private:
+  /// Vendor-specific implementation to obtain device time.
+  uint64_t getDeviceTimeStamp(GenericDeviceTy *D);
+};
+} // namespace plugin
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+#endif // OFFLOAD_PLUGINS_NEXTGEN_COMMON_INCLUDE_GENERICPROFILER_H
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index b3675e5a8700f..ab2f3b2643648 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -29,6 +29,7 @@
 #include "Shared/Requirements.h"
 #include "Shared/Utils.h"
 
+#include "GenericProfiler.h"
 #include "GlobalHandler.h"
 #include "JIT.h"
 #include "MemoryManager.h"
@@ -57,6 +58,12 @@
 
 using namespace llvm::offload::debug;
 
+/// Factory function for creating a profiler. The default (weak) implementation
+/// returns a no-op GenericProfilerTy. When OMPT is enabled, a strong override
+/// returns an OmptProfilerTy instance.
+std::unique_ptr<llvm::omp::target::plugin::GenericProfilerTy>
+getProfilerToAttach();
+
 namespace llvm {
 namespace omp {
 namespace target {
@@ -1191,6 +1198,11 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
   uint32_t getDebugKind() const { return OMPX_DebugKind; }
   virtual uint64_t getClockFrequency() const { return CLOCKS_PER_SEC; }
 
+  /// Get a device-specific timestamp in nanoseconds, used by the profiler
+  /// for timing device operations. Subclasses should override this to provide
+  /// hardware-accurate timestamps (e.g., via HSA system info).
+  virtual uint64_t getDeviceTimeStamp() { return 0; }
+
   /// Get target compute unit kind (e.g., sm_80, or gfx908).
   virtual std::string getComputeUnitKind() const { return "unknown"; }
 
@@ -1448,16 +1460,6 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
   /// This is used to run the RPC server during task synchronization.
   RPCServerTy *RPCServer;
 
-#ifdef OMPT_SUPPORT
-  /// OMPT callback functions
-#define defineOmptCallback(Name, Type, Code) Name##_t Name##_fn = nullptr;
-  FOREACH_OMPT_DEVICE_EVENT(defineOmptCallback)
-#undef defineOmptCallback
-
-  /// Internal representation for OMPT device (initialize & finalize)
-  std::atomic<bool> OmptInitialized;
-#endif
-
   /// The total per-block native shared memory that a kernel may use.
   size_t MaxBlockSharedMemSize = 0;
 };
@@ -1469,7 +1471,8 @@ struct GenericPluginTy {
 
   /// Construct a plugin instance.
   GenericPluginTy(Triple::ArchType TA)
-      : GlobalHandler(nullptr), JIT(TA), RPCServer(nullptr) {}
+      : GlobalHandler(nullptr), JIT(TA), RPCServer(nullptr),
+        Profiler(getProfilerToAttach()) {}
 
   virtual ~GenericPluginTy() {}
 
@@ -1560,7 +1563,12 @@ struct GenericPluginTy {
   /// Tear down any target-specific doorbell resources.
   virtual Error deinitRPCDoorbell() { return Plugin::success(); }
 
-  /// Get a reference to the record and replay interface for the plugin.
+  /// Get a pointer to the profiler attached to this plugin.
+  GenericProfilerTy *getProfiler() {
+    assert(Profiler && "Profiler not initialized");
+    return Profiler.get();
+  }
+
   /// Initialize a device within the plugin.
   Error initDevice(int32_t DeviceId);
 
@@ -1842,6 +1850,9 @@ struct GenericPluginTy {
 
   /// The interface between the plugin and the GPU for host services.
   RPCServerTy *RPCServer;
+
+  /// The profiler backend attached to this plugin (e.g., OMPT).
+  std::unique_ptr<GenericProfilerTy> Profiler;
 };
 
 /// Auxiliary interface class for GenericDeviceResourceManagerTy. This class
diff --git a/offload/plugins-nextgen/common/src/GenericProfiler.cpp b/offload/plugins-nextgen/common/src/GenericProfiler.cpp
new file mode 100644
index 0000000000000..350a0cb5e378a
--- /dev/null
+++ b/offload/plugins-nextgen/common/src/GenericProfiler.cpp
@@ -0,0 +1,36 @@
+//===- GenericProfiler.cpp - GenericProfiler implementation ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#include "GenericProfiler.h"
+#include "PluginInterface.h"
+
+#include <cstdint>
+#include <memory>
+
+__attribute__((weak))
+std::unique_ptr<llvm::omp::target::plugin::GenericProfilerTy>
+getProfilerToAttach() {
+  return std::make_unique<llvm::omp::target::plugin::GenericProfilerTy>();
+}
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace plugin {
+
+uint64_t GenericProfilerTy::getDeviceTimeStamp(GenericDeviceTy *D) {
+  if (D)
+    return D->getDeviceTimeStamp();
+  return 0;
+}
+} // namespace plugin
+} // namespace target
+} // namespace omp
+} // namespace llvm
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index e539cbfc55324..44e1b281385c2 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -21,10 +21,7 @@
 #include "Utils/ELF.h"
 #include "omptarget.h"
 
-#ifdef OMPT_SUPPORT
-#include "OpenMP/OMPT/Callback.h"
-#include "omp-tools.h"
-#endif
+#include "GenericProfiler.h"
 
 #include "llvm/Bitcode/BitcodeReader.h"
 #include "llvm/Frontend/OpenMP/OMPConstants.h"
@@ -47,7 +44,9 @@ using namespace llvm::offload::debug;
 AsyncInfoWrapperTy::AsyncInfoWrapperTy(GenericDeviceTy &Device,
                                        __tgt_async_info *AsyncInfoPtr)
     : Device(Device),
-      AsyncInfoPtr(AsyncInfoPtr ? AsyncInfoPtr : &LocalAsyncInfo) {}
+      AsyncInfoPtr(AsyncInfoPtr ? AsyncInfoPtr : &LocalAsyncInfo) {
+  this->AsyncInfoPtr->ProfilerData = nullptr;
+}
 
 Error AsyncInfoWrapperTy::synchronize() {
   assert(AsyncInfoPtr && "AsyncInfoWrapperTy already finalized");
@@ -168,9 +167,15 @@ GenericKernelTy::getKernelLaunchEnvironment(
        DPxPTR(&LocalKLE), DPxPTR(*AllocOrErr),
        sizeof(KernelLaunchEnvironmentTy));
 
+  // Temporarily suppress ProfilerData so the KLE upload is not traced as
+  // a user data operation.
+  __tgt_async_info *AI = AsyncInfoWrapper;
+  void *SavedProfilerData = AI->ProfilerData;
+  AI->ProfilerData = nullptr;
   auto Err = GenericDevice.dataSubmit(*AllocOrErr, &LocalKLE,
                                       sizeof(KernelLaunchEnvironmentTy),
                                       AsyncInfoWrapper);
+  AI->ProfilerData = SavedProfilerData;
   if (Err)
     return Err;
   return static_cast<KernelLaunchEnvironmentTy *>(*AllocOrErr);
@@ -331,6 +336,9 @@ Error GenericKernelTy::launch(GenericDeviceTy &GenericDevice, void **ArgPtrs,
     RRHandle = *RRHandleOrErr;
   }
 
+  GenericDevice.Plugin.getProfiler()->handlePreKernelLaunch(
+      &GenericDevice, EffectiveNumBlocks, AsyncInfoWrapper);
+
   if (auto Err = launchImpl(GenericDevice, EffectiveNumThreads,
                             EffectiveNumBlocks, DynBlockMemConf.NativeSize,
                             KernelArgs, LaunchParams, AsyncInfoWrapper))
@@ -508,22 +516,6 @@ GenericDeviceTy::GenericDeviceTy(GenericPluginTy &Plugin, int32_t DeviceId,
   // vendor (u)uid will become available later.
   setDeviceUidFromVendorUid(std::to_string(static_cast<uint64_t>(DeviceId)));
 
-#ifdef OMPT_SUPPORT
-  OmptInitialized.store(false);
-  // Bind the callbacks to this device's member functions
-#define bindOmptCallback(Name, Type, Code)                                     \
-  if (ompt::Initialized && ompt::lookupCallbackByCode) {                       \
-    ompt::lookupCallbackByCode((ompt_callbacks_t)(Code),                       \
-                               ((ompt_callback_t *)&(Name##_fn)));             \
-    ODBG(OLDT_Tool) << "OMPT: class bound " << #Name << "="                    \
-                    << ((void *)(uint64_t)Name##_fn);                          \
-  }
-
-  FOREACH_OMPT_DEVICE_EVENT(bindOmptCallback);
-#undef bindOmptCallback
-
-#endif
-
   // Envar that indicates whether mapped host buffers should be locked
   // automatically. The possible values are boolean (on/off) and a special:
   //   off:       Mapped host buffers are not locked.
@@ -555,17 +547,7 @@ Error GenericDeviceTy::init(GenericPluginTy &Plugin) {
   if (auto Err = initImpl(Plugin))
     return Err;
 
-#ifdef OMPT_SUPPORT
-  if (ompt::Initialized) {
-    bool ExpectedStatus = false;
-    if (OmptInitialized.compare_exchange_strong(ExpectedStatus, true))
-      performOmptCallback(device_initialize, Plugin.getUserId(DeviceId),
-                          /*type=*/getComputeUnitKind().c_str(),
-                          /*device=*/reinterpret_cast<ompt_device_t *>(this),
-                          /*lookup=*/ompt::lookupCallbackByName,
-                          /*documentation=*/nullptr);
-  }
-#endif
+  Plugin.getProfiler()->handleInit(this, &Plugin);
 
   // Read and reinitialize the envars that depend on the device initialization.
   // Notice these two envars may change the stack size and heap size of the
@@ -660,13 +642,7 @@ Error GenericDeviceTy::deinit(GenericPluginTy &Plugin) {
     RecordReplay = nullptr;
   }
 
-#ifdef OMPT_SUPPORT
-  if (ompt::Initialized) {
-    bool ExpectedStatus = true;
-    if (OmptInitialized.compare_exchange_strong(ExpectedStatus, false))
-      performOmptCallback(device_finalize, Plugin.getUserId(DeviceId));
-  }
-#endif
+  Plugin.getProfiler()->handleDeinit(this, &Plugin);
 
   return deinitImpl();
 }
@@ -713,17 +689,7 @@ Expected<DeviceImageTy *> GenericDeviceTy::loadBinary(GenericPluginTy &Plugin,
   if (auto Err = setupRPCServer(Plugin, *Image))
     return std::move(Err);
 
-#ifdef OMPT_SUPPORT
-  if (ompt::Initialized) {
-    size_t Bytes = InputTgtImage.size();
-    performOmptCallback(
-        device_load, Plugin.getUserId(DeviceId),
-        /*FileName=*/nullptr, /*FileOffset=*/0, /*VmaInFile=*/nullptr,
-        /*ImgSize=*/Bytes,
-        /*HostAddr=*/const_cast<unsigned char *>(InputTgtImage.bytes_begin()),
-        /*DeviceAddr=*/nullptr, /* FIXME: ModuleId */ 0);
-  }
-#endif
+  Plugin.getProfiler()->handleLoadBinary(this, &Plugin, InputTgtImage);
 
   // Call any global constructors present on the device.
   if (auto Err = callGlobalConstructors(Plugin, *Image))
@@ -1006,6 +972,9 @@ Error GenericDeviceTy::getDeviceMemorySize(uint64_t &DSize) {
 Expected<void *> GenericDeviceTy::dataAlloc(int64_t Size, void *HostPtr,
                                             TargetAllocTy Kind,
                                             size_t Alignment) {
+  auto ProfTimer =
+      Plugin.getProfiler()->getScopedDataAllocTimer(this, HostPtr, Size);
+
   void *Alloc = nullptr;
 
   // TODO Check alignment.
@@ -1073,6 +1042,8 @@ Expected<void *> GenericDeviceTy::dataAlloc(int64_t Size, void *HostPtr,
 }
 
 Error GenericDeviceTy::dataDelete(void *TgtPtr, TargetAllocTy Kind) {
+  auto ProfTimer = Plugin.getProfiler()->getScopedDataDeleteTimer(this, TgtPtr);
+
   // Free is a noop when recording or replaying.
   if (RecordReplay && RecordReplay->isRecordingOrReplaying())
     return RecordReplay->deallocate(TgtPtr);

>From 69fadaf5a7b7c1946fed0873bb00ba5a34ebcbde Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 2 Apr 2026 07:36:31 -0500
Subject: [PATCH 03/11] [Offload][AMDGPU] Wire HSA profiling into
 GenericProfiler abstraction

Add device profiling infrastructure to the AMDGPU plugin so that the
GenericProfiler can receive nanosecond-accurate kernel execution and
data transfer timestamps from the HSA runtime.

Key changes:
- Add ProfilingInfoTy struct to transport HSA profiling data
- Add timeKernelInNsAsync/timeDataTransferInNsAsync callbacks that
  extract dispatch/copy times from HSA signals and call
  handleKernelCompletion/handleDataTransfer on the profiler
- Add getOrNullProfilerSpecificData helper to extract ProfilerData
  from AsyncInfoWrapperTy
- Add getDeviceTimeStamp() override using hsa_system_get_info
- Add getSystemTimestampInNs() for HSA system timestamp queries
- Add schedProfilerKernelTiming/schedProfilerDataTransferTiming to
  StreamSlotTy for scheduling profiler callbacks on stream slots
- Thread ProfilerSpecificData through pushKernelLaunch,
  pushMemoryCopyH2DAsync, pushMemoryCopyD2HAsync, pushMemoryCopyD2DAsync
- Extract ProfilerSpecificData in dataSubmitImpl, dataRetrieveImpl,
  dataExchangeImpl, and launchImpl
- Add stub getDeviceTimeStamp() to CUDA plugin

Made-with: Cursor
Co-Authored-By: Claude <noreply at anthropic.com>
---
 .../amdgpu/dynamic_hsa/hsa.cpp                |   1 +
 .../amdgpu/dynamic_hsa/hsa_ext_amd.h          |   8 +
 offload/plugins-nextgen/amdgpu/src/rtl.cpp    | 204 +++++++++++++++++-
 offload/plugins-nextgen/cuda/src/rtl.cpp      |   3 +
 4 files changed, 205 insertions(+), 11 deletions(-)

diff --git a/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa.cpp b/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa.cpp
index 5c7ec186b0ceb..99202574bfecc 100644
--- a/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa.cpp
+++ b/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa.cpp
@@ -71,6 +71,7 @@ DLWRAP(hsa_amd_signal_create, 5)
 DLWRAP(hsa_amd_signal_async_handler, 5)
 DLWRAP(hsa_amd_pointer_info, 5)
 DLWRAP(hsa_amd_profiling_get_dispatch_time, 3)
+DLWRAP(hsa_amd_profiling_get_async_copy_time, 2)
 DLWRAP(hsa_amd_profiling_set_profiler_enabled, 2)
 DLWRAP(hsa_code_object_reader_create_from_memory, 3)
 DLWRAP(hsa_code_object_reader_destroy, 1)
diff --git a/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa_ext_amd.h b/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa_ext_amd.h
index d26f9248e27ef..9fa740c664625 100644
--- a/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa_ext_amd.h
+++ b/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa_ext_amd.h
@@ -206,6 +206,14 @@ hsa_amd_profiling_get_dispatch_time(hsa_agent_t agent, hsa_signal_t signal,
 hsa_status_t hsa_amd_profiling_set_profiler_enabled(hsa_queue_t *queue,
                                                     int enable);
 
+typedef struct hsa_amd_profiling_async_copy_time_s {
+  uint64_t start;
+  uint64_t end;
+} hsa_amd_profiling_async_copy_time_t;
+
+hsa_status_t hsa_amd_profiling_get_async_copy_time(
+    hsa_signal_t signal, hsa_amd_profiling_async_copy_time_t *time);
+
 hsa_status_t hsa_amd_vmem_address_reserve(void **va, size_t size,
                                           uint64_t address, uint64_t flags);
 
diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index 29b5006f0d7b6..854e13839be86 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -30,6 +30,7 @@
 #include "Shared/Utils.h"
 #include "Utils/ELF.h"
 
+#include "GenericProfiler.h"
 #include "GlobalHandler.h"
 #include "OffloadAPI.h"
 #include "OpenMP/OMPT/Callback.h"
@@ -95,6 +96,88 @@ struct AMDGPUEventManagerTy;
 struct AMDGPUDeviceImageTy;
 struct AMDGPUMemoryManagerTy;
 struct AMDGPUMemoryPoolTy;
+struct AMDGPUSignalTy;
+
+/// Use to transport information to profiler timing functions.
+struct ProfilingInfoTy {
+  GenericPluginTy *Plugin;
+  hsa_agent_t Agent;
+  AMDGPUSignalTy *Signal;
+  double TicksToTime;
+  void *ProfilerSpecificData;
+};
+
+static ProfilingInfoTy *getProfilingInfo(void *Data);
+
+static std::pair<uint64_t, uint64_t>
+getKernelStartAndEndTime(const ProfilingInfoTy *Args);
+
+static std::pair<uint64_t, uint64_t>
+getCopyStartAndEndTime(const ProfilingInfoTy *Args);
+
+static Error timeKernelInNsAsync(void *Data);
+
+static Error timeDataTransferInNsAsync(void *Data) {
+  auto Args = getProfilingInfo(Data);
+  auto [Start, End] = getCopyStartAndEndTime(Args);
+  Args->Plugin->getProfiler()->handleDataTransfer(Start, End,
+                                                  Args->ProfilerSpecificData);
+  return Plugin::success();
+}
+
+static void *
+getOrNullProfilerSpecificData(AsyncInfoWrapperTy &AsyncInfoWrapper) {
+  __tgt_async_info *AI = AsyncInfoWrapper;
+  return AI ? AI->ProfilerData : nullptr;
+}
+
+} // namespace plugin
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+static double setTicksToTime() {
+  uint64_t TicksFrequency = 1;
+  double TicksToTime = 1.0;
+
+  hsa_status_t Status =
+      hsa_system_get_info(HSA_SYSTEM_INFO_TIMESTAMP_FREQUENCY, &TicksFrequency);
+  if (Status == HSA_STATUS_SUCCESS)
+    TicksToTime = (double)1e9 / (double)TicksFrequency;
+
+  return TicksToTime;
+}
+
+static double TicksToTime = 1.0;
+
+static void setHSATicksToTimeConstant() { TicksToTime = setTicksToTime(); }
+
+/// Get the current HSA-based system timestamp in nanoseconds.
+/// Called by OmptTracing.cpp (from PluginOmpt) for device time queries.
+uint64_t getSystemTimestampInNs() {
+  uint64_t TimeStamp = 0;
+  hsa_status_t Status =
+      hsa_system_get_info(HSA_SYSTEM_INFO_TIMESTAMP, &TimeStamp);
+  if (Status != HSA_STATUS_SUCCESS)
+    return 0;
+  return TimeStamp;
+}
+
+/// Enable or disable HSA async copy profiling for OMPT device tracing.
+/// Called by OmptTracing.cpp when a tool activates/deactivates device tracing.
+/// Full HSA profiling integration (enabling per-copy timing signals) will be
+/// wired up in a follow-on commit.
+void setOmptAsyncCopyProfile(bool Enable) {}
+
+/// Enable or disable HSA queue kernel profiling for OMPT device tracing.
+/// Called by OmptTracing.cpp when a tool activates/deactivates device tracing.
+/// Full HSA queue profiling integration will be wired up in a follow-on commit.
+void setGlobalOmptKernelProfile(void *Device, int Enable) {}
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace plugin {
 
 namespace hsa_utils {
 
@@ -1066,6 +1149,7 @@ struct AMDGPUStreamTy {
       MemcpyArgsTy MemcpyArgs;
       ReleaseBufferArgsTy ReleaseBufferArgs;
       ReleaseSignalArgsTy ReleaseSignalArgs;
+      ProfilingInfoTy ProfilerArgs;
       void *CallbackArgs;
     };
 
@@ -1103,7 +1187,32 @@ struct AMDGPUStreamTy {
       return Plugin::success();
     }
 
-    /// Register a callback to be called on compleition
+    /// Schedule kernel timing measurement via the profiler on the slot.
+    Error schedProfilerKernelTiming(GenericDeviceTy *Device, hsa_agent_t Agent,
+                                    AMDGPUSignalTy *OutputSignal,
+                                    double TicksToTime,
+                                    void *ProfilerSpecificData) {
+      Callbacks.emplace_back(timeKernelInNsAsync);
+      ActionArgs.emplace_back().ProfilerArgs =
+          ProfilingInfoTy{&(Device->Plugin), Agent, OutputSignal, TicksToTime,
+                          ProfilerSpecificData};
+      return Plugin::success();
+    }
+
+    /// Schedule data transfer timing via the profiler on the slot.
+    Error schedProfilerDataTransferTiming(GenericDeviceTy *Device,
+                                          hsa_agent_t Agent,
+                                          AMDGPUSignalTy *OutputSignal,
+                                          double TicksToTime,
+                                          void *ProfilerSpecificData) {
+      Callbacks.emplace_back(timeDataTransferInNsAsync);
+      ActionArgs.emplace_back().ProfilerArgs =
+          ProfilingInfoTy{&(Device->Plugin), Agent, OutputSignal, TicksToTime,
+                          ProfilerSpecificData};
+      return Plugin::success();
+    }
+
+    /// Register a callback to be called on completion.
     Error schedCallback(AMDGPUStreamCallbackTy *Func, void *Data) {
       Callbacks.emplace_back(Func);
       ActionArgs.emplace_back().CallbackArgs = Data;
@@ -1128,6 +1237,12 @@ struct AMDGPUStreamTy {
         } else if (Callback == releaseSignalAction) {
           if (auto Err = releaseSignalAction(&ActionArg))
             return Err;
+        } else if (Callback == timeKernelInNsAsync) {
+          if (auto Err = timeKernelInNsAsync(&ActionArg))
+            return Err;
+        } else if (Callback == timeDataTransferInNsAsync) {
+          if (auto Err = timeDataTransferInNsAsync(&ActionArg))
+            return Err;
         } else if (Callback) {
           if (auto Err = Callback(ActionArg.CallbackArgs))
             return Err;
@@ -1399,7 +1514,8 @@ struct AMDGPUStreamTy {
   Error pushKernelLaunch(const AMDGPUKernelTy &Kernel, void *KernelArgs,
                          uint32_t NumThreads[3], uint32_t NumBlocks[3],
                          uint32_t GroupSize, uint64_t StackSize,
-                         AMDGPUMemoryManagerTy &MemoryManager) {
+                         AMDGPUMemoryManagerTy &MemoryManager,
+                         void *ProfilerSpecificData = nullptr) {
     if (Queue == nullptr)
       return Plugin::error(ErrorCode::INVALID_NULL_POINTER,
                            "target queue was nullptr");
@@ -1420,6 +1536,14 @@ struct AMDGPUStreamTy {
     if (auto Err = Slots[Curr].schedReleaseBuffer(KernelArgs, MemoryManager))
       return Err;
 
+#ifdef OMPT_SUPPORT
+    if (ProfilerSpecificData) {
+      if (auto Err = Slots[Curr].schedProfilerKernelTiming(
+              &Device, Agent, OutputSignal, TicksToTime, ProfilerSpecificData))
+        return Err;
+    }
+#endif
+
     // If we are running an RPC server we want to wake up the server thread
     // whenever there is a kernel running and let it sleep otherwise.
     if (Device.getRPCServer())
@@ -1482,7 +1606,8 @@ struct AMDGPUStreamTy {
   /// manager once the operation completes.
   Error pushMemoryCopyD2HAsync(void *Dst, const void *Src, void *Inter,
                                uint64_t CopySize,
-                               AMDGPUMemoryManagerTy &MemoryManager) {
+                               AMDGPUMemoryManagerTy &MemoryManager,
+                               void *ProfilerSpecificData = nullptr) {
     // Retrieve available signals for the operation's outputs.
     AMDGPUSignalTy *OutputSignals[2] = {};
     if (auto Err = SignalManager.getResources(/*Num=*/2, OutputSignals))
@@ -1545,6 +1670,7 @@ struct AMDGPUStreamTy {
   Error pushMemoryCopyH2DAsync(void *Dst, const void *Src, void *Inter,
                                uint64_t CopySize,
                                AMDGPUMemoryManagerTy &MemoryManager,
+                               void *ProfilerSpecificData = nullptr,
                                size_t NumTimes = 1) {
     // Retrieve available signals for the operation's outputs.
     AMDGPUSignalTy *OutputSignals[2] = {};
@@ -1620,7 +1746,8 @@ struct AMDGPUStreamTy {
 
   // AMDGPUDeviceTy is incomplete here, passing the underlying agent instead
   Error pushMemoryCopyD2DAsync(void *Dst, hsa_agent_t DstAgent, const void *Src,
-                               hsa_agent_t SrcAgent, uint64_t CopySize) {
+                               hsa_agent_t SrcAgent, uint64_t CopySize,
+                               void *ProfilerSpecificData = nullptr) {
     AMDGPUSignalTy *OutputSignal;
     if (auto Err = SignalManager.getResources(/*Num=*/1, &OutputSignal))
       return Err;
@@ -2286,6 +2413,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     if (auto Err = initMemoryPools())
       return Err;
 
+    setHSATicksToTimeConstant();
+
     char GPUName[64];
     if (auto Err = getDeviceAttr(HSA_AGENT_INFO_NAME, GPUName))
       return Err;
@@ -2630,6 +2759,9 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
   /// Returns the clock frequency for the given AMDGPU device.
   uint64_t getClockFrequency() const override { return ClockFrequency; }
 
+  /// Returns the current HSA system timestamp for profiling.
+  uint64_t getDeviceTimeStamp() override { return getSystemTimestampInNs(); }
+
   /// Returns the HSA system timestamp frequency. Zero means unavailable.
   uint64_t getSystemTimestampFrequency() const {
     return SystemTimestampFrequency;
@@ -2839,6 +2971,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     AMDGPUStreamTy *Stream = nullptr;
     void *PinnedPtr = nullptr;
 
+    auto ProfilerSpecificData = getOrNullProfilerSpecificData(AsyncInfoWrapper);
+
     // Use one-step asynchronous operation when host memory is already pinned.
     if (void *PinnedPtr =
             PinnedAllocs.getDeviceAccessiblePtrFromPinnedBuffer(HstPtr)) {
@@ -2889,7 +3023,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
       return Err;
 
     return Stream->pushMemoryCopyH2DAsync(TgtPtr, HstPtr, PinnedPtr, Size,
-                                          PinnedMemoryManager);
+                                          PinnedMemoryManager,
+                                          ProfilerSpecificData);
   }
 
   /// Retrieve data from the device (device to host transfer).
@@ -2898,6 +3033,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     AMDGPUStreamTy *Stream = nullptr;
     void *PinnedPtr = nullptr;
 
+    auto ProfilerSpecificData = getOrNullProfilerSpecificData(AsyncInfoWrapper);
+
     // Use one-step asynchronous operation when host memory is already pinned.
     if (void *PinnedPtr =
             PinnedAllocs.getDeviceAccessiblePtrFromPinnedBuffer(HstPtr)) {
@@ -2949,7 +3086,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
       return Err;
 
     return Stream->pushMemoryCopyD2HAsync(HstPtr, TgtPtr, PinnedPtr, Size,
-                                          PinnedMemoryManager);
+                                          PinnedMemoryManager,
+                                          ProfilerSpecificData);
   }
 
   Error dataMemcpyImpl(void *DstPtr, const void *SrcPtr, int64_t Size,
@@ -2985,6 +3123,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
                          AsyncInfoWrapperTy &AsyncInfoWrapper) override {
     AMDGPUDeviceTy &DstDevice = static_cast<AMDGPUDeviceTy &>(DstGenericDevice);
 
+    auto ProfilerSpecificData = getOrNullProfilerSpecificData(AsyncInfoWrapper);
+
     // For large transfers use synchronous behavior.
     if (Size >= OMPX_MaxAsyncCopyBytes) {
       if (AsyncInfoWrapper.hasQueue())
@@ -3013,7 +3153,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
       return Plugin::success();
 
     return Stream->pushMemoryCopyD2DAsync(DstPtr, DstDevice.getAgent(), SrcPtr,
-                                          getAgent(), (uint64_t)Size);
+                                          getAgent(), (uint64_t)Size,
+                                          ProfilerSpecificData);
   }
 
   /// Insert a data fence between previous data operations and the following
@@ -3089,9 +3230,9 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     if (auto Err = getStream(AsyncInfoWrapper, Stream))
       return Err;
 
-    return Stream->pushMemoryCopyH2DAsync(TgtPtr, PatternPtr, PinnedPtr,
-                                          PatternSize, PinnedMemoryManager,
-                                          Size / PatternSize);
+    return Stream->pushMemoryCopyH2DAsync(
+        TgtPtr, PatternPtr, PinnedPtr, PatternSize, PinnedMemoryManager,
+        /*ProfilerSpecificData=*/nullptr, Size / PatternSize);
   }
 
   /// Initialize the async info
@@ -4371,10 +4512,12 @@ Error AMDGPUKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
   // HSA requires the group segment size to include both static and dynamic.
   uint32_t TotalBlockMemSize = getStaticBlockMemSize() + DynBlockMemSize;
 
+  auto ProfilerSpecificData = getOrNullProfilerSpecificData(AsyncInfoWrapper);
+
   // Push the kernel launch into the stream.
   return Stream->pushKernelLaunch(*this, AllArgs, NumThreads, NumBlocks,
                                   TotalBlockMemSize, StackSize,
-                                  ArgsMemoryManager);
+                                  ArgsMemoryManager, ProfilerSpecificData);
 }
 
 Error AMDGPUKernelTy::printLaunchInfoDetails(GenericDeviceTy &GenericDevice,
@@ -4560,6 +4703,45 @@ void AMDGPUQueueTy::callbackError(hsa_status_t Status, hsa_queue_t *Source,
   FATAL_MESSAGE(1, "%s", toString(std::move(Err)).data());
 }
 
+/// Implementation of profiling helper functions.
+static ProfilingInfoTy *getProfilingInfo(void *Data) {
+  return reinterpret_cast<ProfilingInfoTy *>(Data);
+}
+
+static std::pair<uint64_t, uint64_t>
+getKernelStartAndEndTime(const ProfilingInfoTy *Args) {
+  hsa_amd_profiling_dispatch_time_t Time = {};
+  hsa_status_t Status = hsa_amd_profiling_get_dispatch_time(
+      Args->Agent, Args->Signal->get(), &Time);
+  if (Status != HSA_STATUS_SUCCESS)
+    return {0, 0};
+  return {static_cast<uint64_t>(Time.start * Args->TicksToTime),
+          static_cast<uint64_t>(Time.end * Args->TicksToTime)};
+}
+
+static std::pair<uint64_t, uint64_t>
+getCopyStartAndEndTime(const ProfilingInfoTy *Args) {
+  hsa_amd_profiling_async_copy_time_t Time = {};
+  hsa_status_t Status =
+      hsa_amd_profiling_get_async_copy_time(Args->Signal->get(), &Time);
+  if (Status != HSA_STATUS_SUCCESS)
+    return {0, 0};
+  return {static_cast<uint64_t>(Time.start * Args->TicksToTime),
+          static_cast<uint64_t>(Time.end * Args->TicksToTime)};
+}
+
+static Error timeKernelInNsAsync(void *Data) {
+  assert(Data && "Invalid data pointer");
+  auto ProfilerInfo = getProfilingInfo(Data);
+  assert(ProfilerInfo && "Invalid profiling info");
+  assert(ProfilerInfo->ProfilerSpecificData && "Invalid ProfilerSpecificData");
+
+  auto [StartTime, EndTime] = getKernelStartAndEndTime(ProfilerInfo);
+  ProfilerInfo->Plugin->getProfiler()->handleKernelCompletion(
+      StartTime, EndTime, ProfilerInfo->ProfilerSpecificData);
+  return Plugin::success();
+}
+
 } // namespace plugin
 } // namespace target
 } // namespace omp
diff --git a/offload/plugins-nextgen/cuda/src/rtl.cpp b/offload/plugins-nextgen/cuda/src/rtl.cpp
index 997dba577f3ce..4bb2ed1107049 100644
--- a/offload/plugins-nextgen/cuda/src/rtl.cpp
+++ b/offload/plugins-nextgen/cuda/src/rtl.cpp
@@ -1405,6 +1405,9 @@ struct CUDADeviceTy : public GenericDeviceTy {
   /// Returns the clock frequency for the given NVPTX device.
   uint64_t getClockFrequency() const override { return 1000000000; }
 
+  /// Device timestamp stub for CUDA - full profiling is a future extension.
+  uint64_t getDeviceTimeStamp() override { return 0; }
+
 private:
   using CUDAStreamManagerTy = GenericDeviceResourceManagerTy<CUDAStreamRef>;
   using CUDAEventManagerTy = GenericDeviceResourceManagerTy<CUDAEventRef>;

>From d5fcc0adf7e42d2cfbe9564cb9e8bf9b5e84ebcf Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 2 Apr 2026 07:40:01 -0500
Subject: [PATCH 04/11] [Offload][OMPT] Add OmptProfilerTy, common types, and
 plugin tracing shim

Introduce the OMPT-specific profiler backend and shared type
definitions for the device tracing feature.

New files:
- OmptCommonDefs.h: Shared macros (FOREACH_OMPT_DEVICE_TRACING_FN,
  OMPT_IF_TRACING_ENABLED, performOmptCallback), function pointer
  typedefs for libomptarget_ompt_* entry points
- OmptEventInfoTy.h: Small struct carrying NumTeams and TraceRecord
  pointer for async profiler data handoff
- OmptDeviceTracing.h: Plugin-side API surface for device tracing
  state management, device ID mapping, clock correlation, and
  dynamic symbol loading via ensureFuncPtrLoaded template
- OmptProfiler.h/cpp: OmptProfilerTy inheriting GenericProfilerTy
  with OMPT callback bindings and profiler-specific data allocation.
  Provides strong override of getProfilerToAttach() factory
- OmptTracing.cpp (plugin side): Shim that dynamically loads
  libomptarget_ompt_* symbols, implements ompt_set_trace_ompt,
  ompt_start_trace, ompt_stop_trace, ompt_flush_trace,
  ompt_advance_buffer_cursor, ompt_get_record_ompt,
  ompt_get_device_time, ompt_translate_time

Modified:
- common/CMakeLists.txt: Add PluginOmpt static library target

Made-with: Cursor
Co-Authored-By: Claude <noreply at anthropic.com>
---
 offload/include/OpenMP/OMPT/OmptCommonDefs.h  | 127 +++++++
 offload/include/OpenMP/OMPT/OmptEventInfoTy.h |  39 +++
 offload/libomptarget/CMakeLists.txt           |  16 +
 offload/plugins-nextgen/common/CMakeLists.txt |  28 ++
 .../common/OMPT/OmptDeviceTracing.h           | 133 ++++++++
 .../common/OMPT/OmptProfiler.cpp              | 164 +++++++++
 .../common/OMPT/OmptProfiler.h                | 166 +++++++++
 .../common/OMPT/OmptTracing.cpp               | 318 ++++++++++++++++++
 8 files changed, 991 insertions(+)
 create mode 100644 offload/include/OpenMP/OMPT/OmptCommonDefs.h
 create mode 100644 offload/include/OpenMP/OMPT/OmptEventInfoTy.h
 create mode 100644 offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h
 create mode 100644 offload/plugins-nextgen/common/OMPT/OmptProfiler.cpp
 create mode 100644 offload/plugins-nextgen/common/OMPT/OmptProfiler.h
 create mode 100644 offload/plugins-nextgen/common/OMPT/OmptTracing.cpp

diff --git a/offload/include/OpenMP/OMPT/OmptCommonDefs.h b/offload/include/OpenMP/OMPT/OmptCommonDefs.h
new file mode 100644
index 0000000000000..5391658e80262
--- /dev/null
+++ b/offload/include/OpenMP/OMPT/OmptCommonDefs.h
@@ -0,0 +1,127 @@
+//===------ OmptCommonDefs.h - Common definitions for OMPT --*- C++ -*-----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Common defines and typedefs for OMPT callback and tracing functionality.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OFFLOAD_INCLUDE_OMPTCOMMONDEFS_H
+#define OFFLOAD_INCLUDE_OMPTCOMMONDEFS_H
+
+#ifdef OMPT_SUPPORT
+
+#include "omp-tools.h"
+
+#pragma push_macro("DEBUG_PREFIX")
+#undef DEBUG_PREFIX
+#define DEBUG_PREFIX "OMPT"
+
+#define FUNCPTR_TO_PTR(x) ((void *)(uint64_t)x)
+
+#define FOREACH_OMPT_TARGET_CALLBACK(macro)                                    \
+  FOREACH_OMPT_DEVICE_EVENT(macro)                                             \
+  FOREACH_OMPT_NOEMI_EVENT(macro)                                              \
+  FOREACH_OMPT_EMI_EVENT(macro)
+
+// Common device tracing functions
+#define FOREACH_OMPT_DEVICE_TRACING_FN_COMMON(macro)                           \
+  macro(ompt_set_trace_ompt) macro(ompt_start_trace) macro(ompt_flush_trace)   \
+      macro(ompt_stop_trace) macro(ompt_advance_buffer_cursor)                 \
+          macro(ompt_get_record_type)
+
+// Supported device tracing entry points
+#define FOREACH_OMPT_DEVICE_TRACING_FN(macro)                                  \
+  FOREACH_OMPT_DEVICE_TRACING_FN_COMMON(macro)                                 \
+  macro(ompt_get_record_ompt) macro(ompt_get_device_time)                      \
+      macro(ompt_translate_time)
+
+// Device tracing functionalities, which are also e.g. coupled to mutexes
+#define FOREACH_OMPT_DEVICE_TRACING_FN_IMPLEMENTAIONS(macro)                   \
+  FOREACH_OMPT_DEVICE_TRACING_FN_COMMON(macro)                                 \
+  macro(ompt_set_timestamp) macro(ompt_set_granted_teams)
+
+#define OMPT_API_ROUTINE static
+
+#define OMPT_CALLBACK_AVAILABLE(fn) (llvm::omp::target::ompt::Initialized && fn)
+
+#define OMPT_IF_BUILT(stmt) stmt
+
+#define OMPT_IF_ENABLED(stmts)                                                 \
+  do {                                                                         \
+    if (llvm::omp::target::ompt::Initialized) {                                \
+      stmts                                                                    \
+    }                                                                          \
+  } while (0)
+
+#define OMPT_IF_TRACING_ENABLED(stmts)                                         \
+  do {                                                                         \
+    if (llvm::omp::target::ompt::TracingActive) {                              \
+      stmts                                                                    \
+    }                                                                          \
+  } while (0)
+
+#define OMPT_FRAME_FLAGS (ompt_frame_runtime | OMPT_FRAME_POSITION_DEFAULT)
+
+#if (__PPC64__ | __arm__)
+#define OMPT_GET_FRAME_ADDRESS(level) __builtin_frame_address(level)
+#define OMPT_FRAME_POSITION_DEFAULT ompt_frame_cfa
+#else
+#define OMPT_GET_FRAME_ADDRESS(level) __builtin_frame_address(level)
+#define OMPT_FRAME_POSITION_DEFAULT ompt_frame_framepointer
+#endif
+
+#define OMPT_PTR_UNKNOWN ((void *)0)
+
+#define performIfOmptInitialized(stmt)                                         \
+  do {                                                                         \
+    if (llvm::omp::target::ompt::Initialized) {                                \
+      stmt;                                                                    \
+    }                                                                          \
+  } while (0)
+
+#define performOmptCallback(CallbackName, ...)                                 \
+  do {                                                                         \
+    if (ompt_callback_##CallbackName##_fn)                                     \
+      ompt_callback_##CallbackName##_fn(__VA_ARGS__);                          \
+  } while (0)
+
+typedef ompt_set_result_t (*libomptarget_ompt_set_trace_ompt_t)(
+    int Device, unsigned int Enable, unsigned int EventTy);
+typedef int (*libomptarget_ompt_start_trace_t)(int,
+                                               ompt_callback_buffer_request_t,
+                                               ompt_callback_buffer_complete_t);
+typedef int (*libomptarget_ompt_flush_trace_t)(int);
+typedef int (*libomptarget_ompt_stop_trace_t)(int);
+typedef int (*libomptarget_ompt_advance_buffer_cursor_t)(
+    ompt_device_t *, ompt_buffer_t *, size_t, ompt_buffer_cursor_t,
+    ompt_buffer_cursor_t *);
+typedef ompt_get_record_ompt_t libomptarget_ompt_get_record_ompt_t;
+typedef ompt_device_time_t (*libomptarget_ompt_get_device_time_t)(
+    ompt_device_t *);
+typedef ompt_translate_time_t libomptarget_ompt_translate_time_t;
+typedef ompt_device_time_t (*libomptarget_ompt_get_device_time_t)(
+    ompt_device_t *);
+typedef ompt_record_t (*libomptarget_ompt_get_record_type_t)(
+    ompt_buffer_t *, ompt_buffer_cursor_t);
+typedef void (*libomptarget_ompt_set_timestamp_t)(uint64_t start, uint64_t end);
+typedef void (*libomptarget_ompt_set_granted_teams_t)(uint32_t);
+
+/// Function type def used for maintaining unique target region, target
+/// operations ids
+typedef uint64_t (*IdInterfaceTy)();
+
+#pragma pop_macro("DEBUG_PREFIX")
+
+#else
+#define performIfOmptInitialized(stmt)
+#define OMPT_IF_BUILT(stmt)
+#define OMPT_IF_ENABLED(stmts)
+#define OMPT_IF_TRACING_ENABLED(stmts)
+#endif // OMPT_SUPPORT
+
+#endif // OFFLOAD_INCLUDE_OMPTCOMMONDEFS_H
diff --git a/offload/include/OpenMP/OMPT/OmptEventInfoTy.h b/offload/include/OpenMP/OMPT/OmptEventInfoTy.h
new file mode 100644
index 0000000000000..7124b3a3ff501
--- /dev/null
+++ b/offload/include/OpenMP/OMPT/OmptEventInfoTy.h
@@ -0,0 +1,39 @@
+//===- OmptEventInfoTy.h - OMPT specific trace record data ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Data structure used to communicate OMPT specific profiler data from the
+// high-level libomptarget into the vendor-specific plugins
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OFFLOAD_INCLUDE_OPENMP_OMPT_OMPTEVENTINFOTY_H
+#define OFFLOAD_INCLUDE_OPENMP_OMPT_OMPTEVENTINFOTY_H
+
+#include "Shared/Debug.h"
+
+struct ompt_record_ompt_t;
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace ompt {
+
+/// Holds info needed to fill asynchronous trace records
+struct OmptEventInfoTy {
+  /// The granted number of teams at runtime
+  uint64_t NumTeams;
+  /// Pointer to the actual buffer storage location
+  ompt_record_ompt_t *TraceRecord;
+};
+
+} // namespace ompt
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+#endif // OFFLOAD_INCLUDE_OPENMP_OMPT_OMPTEVENTINFOTY_H
diff --git a/offload/libomptarget/CMakeLists.txt b/offload/libomptarget/CMakeLists.txt
index 8e6314b2a6eae..7909d55eff5c3 100644
--- a/offload/libomptarget/CMakeLists.txt
+++ b/offload/libomptarget/CMakeLists.txt
@@ -47,6 +47,22 @@ endforeach()
 
 target_compile_options(omptarget PRIVATE ${offload_compile_flags})
 target_link_options(omptarget PRIVATE ${offload_link_flags})
+if (OMPT_TARGET_DEFAULT AND LIBOMPTARGET_OMPT_SUPPORT)
+  # OmptProfiler.cpp provides the strong getProfilerToAttach() that routes
+  # plugin device events through OmptProfilerTy. It must be compiled directly
+  # into omptarget so its strong definition unconditionally overrides the weak
+  # fallback in GenericProfiler.cpp (which is embedded into each plugin archive
+  # via the PluginCommon OBJECT library and would otherwise win the link).
+  target_sources(omptarget PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}/../plugins-nextgen/common/OMPT/OmptProfiler.cpp
+  )
+  target_include_directories(omptarget PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}/../plugins-nextgen/common/OMPT
+    ${CMAKE_CURRENT_SOURCE_DIR}/../plugins-nextgen/common/include
+  )
+  # PluginOmpt provides OmptTracing.cpp (plugin-side GPU tracing).
+  target_link_libraries(omptarget PRIVATE PluginOmpt)
+endif()
 
 # libomptarget.so needs to be aware of where the plugins live as they
 # are now separated in the build directory.
diff --git a/offload/plugins-nextgen/common/CMakeLists.txt b/offload/plugins-nextgen/common/CMakeLists.txt
index 19ce102309a72..c3dcb0f53d218 100644
--- a/offload/plugins-nextgen/common/CMakeLists.txt
+++ b/offload/plugins-nextgen/common/CMakeLists.txt
@@ -37,6 +37,34 @@ endif()
 include(FindLibcCommonUtils)
 target_link_libraries(PluginCommon PRIVATE llvm-libc-common-utilities)
 
+if (OMPT_TARGET_DEFAULT AND LIBOMPTARGET_OMPT_SUPPORT)
+  # OmptTracing.cpp calls plugin-specific symbols (HSA profiling etc.) and
+  # is linked per-plugin. OmptProfiler.cpp is added directly to omptarget
+  # in offload/libomptarget/CMakeLists.txt to ensure its strong
+  # getProfilerToAttach() override is unconditionally included.
+  add_library(PluginOmpt STATIC OMPT/OmptTracing.cpp)
+  target_include_directories(PluginOmpt PUBLIC
+    OMPT
+    ${CMAKE_CURRENT_SOURCE_DIR}/include
+    ${CMAKE_CURRENT_BINARY_DIR}/include
+    ${LIBOMPTARGET_LLVM_INCLUDE_DIRS}
+    ${LIBOMPTARGET_BINARY_INCLUDE_DIR}
+    ${LIBOMPTARGET_INCLUDE_DIR}
+  )
+  target_compile_options(PluginOmpt PUBLIC ${offload_compile_flags} -fPIC)
+  target_link_options(PluginOmpt PUBLIC ${offload_link_flags})
+  target_compile_definitions(PluginOmpt PRIVATE
+    TARGET_NAME="Profiler"
+    DEBUG_PREFIX="OMPT"
+  )
+  add_dependencies(PluginOmpt PluginErrcodes)
+endif()
+
+if (TARGET flang_rt.runtime.static)
+  target_link_libraries(PluginCommon PRIVATE flang_rt.runtime.static)
+  target_compile_definitions(PluginCommon PRIVATE OFFLOAD_HAS_FLANG_RT)
+endif()
+
 # Define the TARGET_NAME and DEBUG_PREFIX.
 target_compile_definitions(PluginCommon PRIVATE
   TARGET_NAME=PluginInterface
diff --git a/offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h b/offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h
new file mode 100644
index 0000000000000..5744a784825da
--- /dev/null
+++ b/offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h
@@ -0,0 +1,133 @@
+//===- OmptDeviceTracing.h - Target independent OMPT callbacks --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Interface used by target-independent runtimes to coordinate registration and
+// invocation of OMPT tracing functionality.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_COMMON_OMPTDEVICETRACING_H
+#define OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_COMMON_OMPTDEVICETRACING_H
+
+#ifdef OMPT_SUPPORT
+
+#include "OpenMP/OMPT/OmptCommonDefs.h"
+
+#include "llvm/Support/DynamicLibrary.h"
+
+#include <map>
+#include <memory>
+
+#pragma push_macro("DEBUG_PREFIX")
+#undef DEBUG_PREFIX
+#define DEBUG_PREFIX "OMPT"
+
+extern void setOmptAsyncCopyProfile(bool Enable);
+extern void setGlobalOmptKernelProfile(void *Device, int Enable);
+extern uint64_t getSystemTimestampInNs();
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace ompt {
+
+// Declare OMPT device tracing function entry points
+#define declareOmptTracingFn(Name) extern libomptarget_##Name##_t Name##_fn;
+FOREACH_OMPT_DEVICE_TRACING_FN_IMPLEMENTAIONS(declareOmptTracingFn)
+#undef declareOmptTracingFn
+
+// Declare OMPT device tracing function mutexes
+#define declareOmptTracingFnMutex(Name) extern std::mutex Name##_mutex;
+FOREACH_OMPT_DEVICE_TRACING_FN_IMPLEMENTAIONS(declareOmptTracingFnMutex)
+#undef declareOmptTracingFnMutex
+
+extern std::mutex DeviceIdWritingMutex;
+
+/// Activate tracing on the given device
+void enableDeviceTracing(int DeviceId);
+
+/// Deactivate tracing on the given device
+void disableDeviceTracing(int DeviceId);
+
+/// Set 'start' and 'stop' in trace records
+void setOmptTimestamp(uint64_t StartTime, uint64_t EndTime);
+
+/// Set the linear function correlation between host and device clocks
+void setOmptHostToDeviceRate(double Slope, double Offset);
+
+/// Set / store the number of granted teams in trace records
+void setOmptGrantedNumTeams(uint64_t NumTeams);
+
+/// Lookup the given device pointer and return its RTL device ID
+int getDeviceId(ompt_device_t *Device);
+
+/// Map the given device pointer to the given DeviceId
+void setDeviceId(ompt_device_t *Device, int32_t DeviceId);
+
+/// Rempve the given device pointer from the current mapping
+void removeDeviceId(ompt_device_t *Device);
+
+/// Check whether the provided device is currently traced.
+bool isTracedDevice(int32_t DeviceId);
+
+/// Provide name based lookup for the device tracing functions
+extern ompt_interface_fn_t
+lookupDeviceTracingFn(const char *InterfaceFunctionName);
+
+/// Host to device linear clock correlation
+extern double HostToDeviceSlope;
+
+/// Host to device constant clock offset
+extern double HostToDeviceOffset;
+
+/// Mapping of device pointers to their corresponding RTL device ID
+extern std::map<ompt_device_t *, int32_t> Devices;
+
+/// Mapping of RTL device IDs to their currently enabled tracing event types.
+/// Note: Event type '0' (bit position) indicates if this device is traced.
+extern std::map<int32_t, uint64_t> TracedDevices;
+
+/// OMPT global tracing status. Indicates if at least one device is traced.
+extern bool TracingActive;
+
+/// Parent library pointer
+extern std::shared_ptr<llvm::sys::DynamicLibrary> ParentLibrary;
+
+/// Get the parent library by pointer. If it is not already set, it will set the
+/// parent library pointer.
+std::shared_ptr<llvm::sys::DynamicLibrary> getParentLibrary();
+
+/// Set the parent library by filename
+void setParentLibrary(const char *Filename);
+
+/// Search for FuncName inside the parent library and assign to FuncPtr.
+/// IMPORTANT: This function assumes that the *caller* holds the respective lock
+/// for FuncPtr.
+template <typename FT>
+void ensureFuncPtrLoaded(const std::string &FuncName, FT *FuncPtr) {
+  if (*FuncPtr == nullptr) {
+    if ((ParentLibrary == nullptr && getParentLibrary() == nullptr) ||
+        !ParentLibrary->isValid())
+      return;
+    void *SymbolPtr = ParentLibrary->getAddressOfSymbol(FuncName.c_str());
+    if (SymbolPtr == nullptr)
+      return;
+    *FuncPtr = reinterpret_cast<FT>(SymbolPtr);
+  }
+}
+
+} // namespace ompt
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+#pragma pop_macro("DEBUG_PREFIX")
+
+#endif // OMPT_SUPPORT
+
+#endif // OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_COMMON_OMPTDEVICETRACING_H
diff --git a/offload/plugins-nextgen/common/OMPT/OmptProfiler.cpp b/offload/plugins-nextgen/common/OMPT/OmptProfiler.cpp
new file mode 100644
index 0000000000000..4f9688a07f534
--- /dev/null
+++ b/offload/plugins-nextgen/common/OMPT/OmptProfiler.cpp
@@ -0,0 +1,164 @@
+//===- OmptProfiler.cpp - OMPT impl of GenericProfilerTy --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementation of OmptProfilerTy
+//
+//===----------------------------------------------------------------------===//
+
+#include "OmptProfiler.h"
+#include "OpenMP/OMPT/Interface.h"
+#include "PluginInterface.h"
+#include "Shared/Debug.h"
+
+#include <memory>
+
+using namespace llvm::omp::target;
+
+/// Strong override of getProfilerToAttach() that returns an OmptProfilerTy
+/// when OMPT is enabled. This overrides the weak default in
+/// GenericProfiler.cpp.
+std::unique_ptr<llvm::omp::target::plugin::GenericProfilerTy>
+getProfilerToAttach() {
+  return std::make_unique<llvm::omp::target::ompt::OmptProfilerTy>();
+}
+
+void ompt::OmptProfilerTy::handleInit(plugin::GenericDeviceTy *Device,
+                                      plugin::GenericPluginTy *Plugin) {
+  auto DeviceId = Device->getDeviceId();
+  auto DevicePtr = reinterpret_cast<ompt_device_t *>(Device);
+  ompt::setDeviceId(DevicePtr, Plugin->getUserId(DeviceId));
+
+  if (ompt::Initialized) {
+    bool ExpectedStatus = false;
+    if (OmptInitialized.compare_exchange_strong(ExpectedStatus, true))
+      performOmptCallback(device_initialize, Plugin->getUserId(DeviceId),
+                          /*type=*/Device->getComputeUnitKind().c_str(),
+                          /*device=*/DevicePtr,
+                          /*lookup=*/ompt::lookupDeviceTracingFn,
+                          /*documentation=*/nullptr);
+  }
+}
+
+void ompt::OmptProfilerTy::handleDeinit(
+    plugin::GenericDeviceTy *Device, target::plugin::GenericPluginTy *Plugin) {
+  auto DeviceId = Device->getDeviceId();
+
+  if (ompt::Initialized) {
+    bool ExpectedStatus = true;
+    if (OmptInitialized.compare_exchange_strong(ExpectedStatus, false))
+      performOmptCallback(device_finalize, Plugin->getUserId(DeviceId));
+  }
+  ompt::removeDeviceId(reinterpret_cast<ompt_device_t *>(Device));
+}
+
+void ompt::OmptProfilerTy::handleLoadBinary(plugin::GenericDeviceTy *Device,
+                                            plugin::GenericPluginTy *Plugin,
+                                            const StringRef InputTgtImage) {
+
+  if (!ompt::Initialized)
+    return;
+
+  auto DeviceId = Device->getDeviceId();
+  size_t Bytes = InputTgtImage.size();
+  performOmptCallback(
+      device_load, Plugin->getUserId(DeviceId),
+      /*FileName=*/nullptr, /*FileOffset=*/0, /*VmaInFile=*/nullptr,
+      /*ImgSize=*/Bytes,
+      /*HostAddr=*/const_cast<unsigned char *>(InputTgtImage.bytes_begin()),
+      /*DeviceAddr=*/nullptr, /* FIXME: ModuleId */ 0);
+}
+
+void ompt::OmptProfilerTy::handleDataAlloc(uint64_t StartNanos,
+                                           uint64_t EndNanos, void *HostPtr,
+                                           uint64_t Size, void *Data) {
+  ompt::setOmptTimestamp(StartNanos, EndNanos);
+}
+
+void ompt::OmptProfilerTy::handleDataDelete(uint64_t StartNanos,
+                                            uint64_t EndNanos, void *TgtPtr,
+                                            void *Data) {
+  ompt::setOmptTimestamp(StartNanos, EndNanos);
+}
+
+void ompt::OmptProfilerTy::handlePreKernelLaunch(
+    plugin::GenericDeviceTy *Device, uint32_t NumBlocks[3],
+    __tgt_async_info *AI) {
+  if (!ompt::isTracedDevice(
+          ompt::getDeviceId(reinterpret_cast<ompt_device_t *>(Device))))
+    return;
+
+  if (AI->ProfilerData == nullptr)
+    return;
+
+  auto ProfilerSpecificData =
+      reinterpret_cast<ompt::OmptEventInfoTy *>(AI->ProfilerData);
+  assert(ProfilerSpecificData && "Invalid ProfilerSpecificData");
+  // Set number of granted teams for OMPT
+  setOmptGrantedNumTeams(NumBlocks[0]);
+  ProfilerSpecificData->NumTeams = NumBlocks[0];
+}
+
+void ompt::OmptProfilerTy::handleKernelCompletion(uint64_t StartNanos,
+                                                  uint64_t EndNanos,
+                                                  void *Data) {
+
+  if (!isProfilingEnabled())
+    return;
+
+  /// Empty data means no tracing in OMPT
+  /// offload/include/OpenMP/OMPT/Interface.h line 492
+  if (!Data)
+    return;
+
+  ODBG(ODT_Tool) << "OMPT-Async: Time kernel for asynchronous execution: Start "
+                 << StartNanos << " End " << EndNanos;
+
+  auto OmptEventInfo = reinterpret_cast<ompt::OmptEventInfoTy *>(Data);
+  assert(OmptEventInfo && "Invalid OmptEventInfo");
+  assert(OmptEventInfo->TraceRecord && "Invalid TraceRecord");
+
+  ompt::RegionInterface.stopTargetSubmitTraceAsync(OmptEventInfo->TraceRecord,
+                                                   OmptEventInfo->NumTeams,
+                                                   StartNanos, EndNanos);
+
+  // Done processing, our responsibility to free the memory
+  freeProfilerDataEntry(OmptEventInfo);
+}
+
+void ompt::OmptProfilerTy::handleDataTransfer(uint64_t StartNanos,
+                                              uint64_t EndNanos, void *Data) {
+
+  if (!isProfilingEnabled())
+    return;
+
+  /// Empty data means no tracing in OMPT
+  /// offload/include/OpenMP/OMPT/Interface.h line 492
+  if (!Data)
+    return;
+
+  ODBG(ODT_Tool) << "OMPT-Async: Time data for asynchronous execution: Start "
+                 << StartNanos << " End " << EndNanos;
+
+  auto OmptEventInfo = reinterpret_cast<ompt::OmptEventInfoTy *>(Data);
+  assert(OmptEventInfo && "Invalid OmptEventInfo");
+  assert(OmptEventInfo->TraceRecord && "Invalid TraceRecord");
+
+  ompt::RegionInterface.stopTargetDataMovementTraceAsync(
+      OmptEventInfo->TraceRecord, StartNanos, EndNanos);
+
+  // Done processing, our responsibility to free the memory
+  freeProfilerDataEntry(OmptEventInfo);
+}
+
+bool ompt::OmptProfilerTy::isProfilingEnabled() { return ompt::TracingActive; }
+
+void ompt::OmptProfilerTy::setTimeConversionFactorsImpl(double Slope,
+                                                        double Offset) {
+  ODBG(ODT_Tool) << "Using Time Slope: " << Slope << " and Offset: " << Offset;
+  setOmptHostToDeviceRate(Slope, Offset);
+}
diff --git a/offload/plugins-nextgen/common/OMPT/OmptProfiler.h b/offload/plugins-nextgen/common/OMPT/OmptProfiler.h
new file mode 100644
index 0000000000000..9a25bda272d8a
--- /dev/null
+++ b/offload/plugins-nextgen/common/OMPT/OmptProfiler.h
@@ -0,0 +1,166 @@
+//===- OmptProfiler.h - OMPT specific impl of GenericProfilerTy -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// OMPT specific implementation of the GenericProfilerTy class.
+// This class uses the already existing implementation of OMPT to invoke
+// callbacks and perform tracing.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OFFLOAD_PLUGINS_NEXTGEN_COMMON_OMPT_OMPTPROFILERTY_H
+#define OFFLOAD_PLUGINS_NEXTGEN_COMMON_OMPT_OMPTPROFILERTY_H
+
+#include "GenericProfiler.h"
+
+#include "OmptDeviceTracing.h"
+#include "OpenMP/OMPT/Callback.h"
+#include "OpenMP/OMPT/OmptEventInfoTy.h"
+#include "Shared/Debug.h"
+#include "omp-tools.h"
+
+#include <functional>
+#include <tuple>
+
+#pragma push_macro("DEBUG_PREFIX")
+#undef DEBUG_PREFIX
+#define DEBUG_PREFIX "OMPT"
+
+extern uint64_t getSystemTimestampInNs();
+
+using namespace llvm::omp::target::debug;
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace plugin {
+struct GenericDeviceTy;
+struct GenericPluginTy;
+class GenericProfilerTy;
+
+} // namespace plugin
+
+namespace ompt {
+
+// From Callback.h / Callback.cpp
+extern bool Initialized;
+
+/**
+ * Implements an OMPT backend for the Profiler interface used in the plugins.
+ *
+ * Forwards / Implements the different generic hooks with OMPT semantics.
+ */
+class OmptProfilerTy : public plugin::GenericProfilerTy {
+public:
+  /** Public members **/
+  OmptProfilerTy() {
+
+    OmptInitialized.store(false);
+    // Bind the callbacks to this device's member functions
+#define bindOmptCallback(Name, Type, Code)                                     \
+  if (ompt::Initialized && ompt::lookupCallbackByCode) {                       \
+    ompt::lookupCallbackByCode((ompt_callbacks_t)(Code),                       \
+                               ((ompt_callback_t *)&(Name##_fn)));             \
+    ODBG(ODT_Tool) << "class bound " << #Name << "="                           \
+                   << ((void *)(uint64_t)Name##_fn);                           \
+  }
+
+    FOREACH_OMPT_DEVICE_EVENT(bindOmptCallback);
+#undef bindOmptCallback
+
+#define bindOmptTracingFunction(FunctionName)                                  \
+  if (ompt::Initialized && ompt::lookupDeviceTracingFn) {                      \
+    FunctionName##_fn = ompt::lookupDeviceTracingFn(#FunctionName);            \
+    ODBG(ODT_Tool) << "device tracing fn bound " << #FunctionName << "="       \
+                   << ((void *)(uint64_t)FunctionName##_fn);                   \
+  }
+
+    FOREACH_OMPT_DEVICE_TRACING_FN_COMMON(bindOmptTracingFunction);
+#undef bindOmptTracingFunction
+  }
+
+  bool isProfilingEnabled() override;
+
+  void handleInit(plugin::GenericDeviceTy *Device,
+                  plugin::GenericPluginTy *Plugin) override;
+
+  void handleDeinit(plugin::GenericDeviceTy *Device,
+                    plugin::GenericPluginTy *Plugin) override;
+
+  void handleLoadBinary(plugin::GenericDeviceTy *Device,
+                        plugin::GenericPluginTy *Plugin,
+                        const StringRef InputTgtImage) override;
+
+  void handleDataAlloc(uint64_t StartNanos, uint64_t EndNanos, void *HostPtr,
+                       uint64_t Size, void *Data) override;
+  void handleDataDelete(uint64_t StartNanos, uint64_t EndNanos, void *TgtPtr,
+                        void *Data) override;
+
+  void handlePreKernelLaunch(plugin::GenericDeviceTy *Device,
+                             uint32_t NumBlocks[3],
+                             __tgt_async_info *AI) override;
+
+  void handleKernelCompletion(uint64_t StartNanos, uint64_t EndNanos,
+                              void *Data) override;
+
+  void handleDataTransfer(uint64_t StartNanos, uint64_t EndNanos,
+                          void *Data) override;
+
+  void setTimeConversionFactorsImpl(double Slope, double Offset) override;
+
+  void *getProfilerSpecificData() override {
+    // TODO: This is ID is not used currently
+    uint64_t Id = OmptProfDataId.fetch_add(1);
+    {
+      std::scoped_lock Lock(ProfilerDataMutex);
+      ProfilerData[Id] = std::make_unique<OmptEventInfoTy>();
+      return ProfilerData[Id].get();
+    }
+  }
+
+  void freeProfilerDataEntry(OmptEventInfoTy *DataPtr) {
+    std::scoped_lock Lock(ProfilerDataMutex);
+
+    for (auto &Entry : ProfilerData)
+      if (Entry.second.get() == DataPtr) {
+        ProfilerData.erase(Entry.first);
+        break;
+      }
+  }
+
+private:
+  /// Holds a unique ID for each allocation of OmptEventInfoTy
+  std::atomic<uint64_t> OmptProfDataId{0};
+
+  /// Holds memory used to store OMPT specific data and pass it down from
+  /// libomptarget into the plugins.
+  std::map<uint64_t, std::unique_ptr<OmptEventInfoTy>> ProfilerData;
+
+  /// Lock to guard STL ProfilerData map
+  std::mutex ProfilerDataMutex;
+
+  /// OMPT callback functions
+#define defineOmptCallback(Name, Type, Code) Name##_t Name##_fn = nullptr;
+  FOREACH_OMPT_DEVICE_EVENT(defineOmptCallback)
+#undef defineOmptCallback
+
+  /// OMPT device tracing functions
+#define defineOmptTracingFunction(Name) ompt_interface_fn_t Name##_fn = nullptr;
+  FOREACH_OMPT_DEVICE_TRACING_FN_COMMON(defineOmptTracingFunction);
+#undef defineOmptTracingFunction
+
+  /// Internal representation for OMPT device (initialize & finalize)
+  std::atomic<bool> OmptInitialized;
+};
+} // namespace ompt
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+#pragma pop_macro("DEBUG_PREFIX")
+
+#endif
diff --git a/offload/plugins-nextgen/common/OMPT/OmptTracing.cpp b/offload/plugins-nextgen/common/OMPT/OmptTracing.cpp
new file mode 100644
index 0000000000000..7a8e312b3d476
--- /dev/null
+++ b/offload/plugins-nextgen/common/OMPT/OmptTracing.cpp
@@ -0,0 +1,318 @@
+//===-- OmptTracing.cpp - Target independent OpenMP target RTL --- C++ -*--===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementation of OMPT tracing interfaces for PluginInterface
+//
+//===----------------------------------------------------------------------===//
+
+#ifdef OMPT_SUPPORT
+
+#include "OmptDeviceTracing.h"
+#include "Shared/Debug.h"
+#include "omp-tools.h"
+
+#include "llvm/Support/DynamicLibrary.h"
+
+#include <atomic>
+#include <cassert>
+#include <cstdlib>
+#include <cstring>
+#include <memory>
+#include <mutex>
+
+#pragma push_macro("DEBUG_PREFIX")
+#undef DEBUG_PREFIX
+#define DEBUG_PREFIX "OMPT"
+
+// Define OMPT device tracing function entry points
+#define defineOmptTracingFn(Name)                                              \
+  libomptarget_##Name##_t llvm::omp::target::ompt::Name##_fn = nullptr;
+FOREACH_OMPT_DEVICE_TRACING_FN_IMPLEMENTAIONS(defineOmptTracingFn)
+#undef defineOmptTracingFn
+
+// Define OMPT device tracing function mutexes
+#define defineOmptTracingFnMutex(Name)                                         \
+  std::mutex llvm::omp::target::ompt::Name##_mutex;
+FOREACH_OMPT_DEVICE_TRACING_FN_IMPLEMENTAIONS(defineOmptTracingFnMutex)
+#undef defineOmptTracingFnMutex
+
+std::mutex llvm::omp::target::ompt::DeviceIdWritingMutex;
+
+using namespace llvm::omp::target::ompt;
+using namespace llvm::omp::target::debug;
+
+std::shared_ptr<llvm::sys::DynamicLibrary>
+    llvm::omp::target::ompt::ParentLibrary(nullptr);
+
+double llvm::omp::target::ompt::HostToDeviceSlope = .0;
+double llvm::omp::target::ompt::HostToDeviceOffset = .0;
+
+std::map<ompt_device_t *, int32_t> llvm::omp::target::ompt::Devices;
+
+std::shared_ptr<llvm::sys::DynamicLibrary>
+llvm::omp::target::ompt::getParentLibrary() {
+  static bool ParentLibraryAssigned = false;
+  if (!ParentLibraryAssigned) {
+    setParentLibrary("libomptarget.so");
+    ParentLibraryAssigned = true;
+  }
+  return ParentLibrary;
+}
+
+void llvm::omp::target::ompt::setParentLibrary(const char *Filename) {
+  if (ParentLibrary)
+    return;
+  std::string ErrorMsg;
+  ParentLibrary = std::make_shared<llvm::sys::DynamicLibrary>(
+      llvm::sys::DynamicLibrary::getPermanentLibrary(Filename, &ErrorMsg));
+  if ((ParentLibrary == nullptr) || (!ParentLibrary->isValid()))
+    REPORT() << "Failed to set parent library: " << ErrorMsg.c_str();
+}
+
+int llvm::omp::target::ompt::getDeviceId(ompt_device_t *Device) {
+  // Block other threads, which might trigger an erase (for the same device)
+  std::unique_lock<std::mutex> Lock(DeviceIdWritingMutex);
+  auto DeviceIterator = Devices.find(Device);
+  if (Device == nullptr || DeviceIterator == Devices.end()) {
+    REPORT() << "Failed to get ID for Device=" << Device;
+    return -1;
+  }
+  return DeviceIterator->second;
+}
+
+void llvm::omp::target::ompt::setDeviceId(ompt_device_t *Device,
+                                          int32_t DeviceId) {
+  assert(Device && "Mapping device ID to nullptr is not allowed");
+  if (Device == nullptr || DeviceId < 0) {
+    REPORT() << "Failed to set ID=%d for Device=" << DeviceId << Device;
+    return;
+  }
+  std::unique_lock<std::mutex> Lock(DeviceIdWritingMutex);
+  auto DeviceIterator = Devices.find(Device);
+  if (DeviceIterator != Devices.end()) {
+    auto CurrentDeviceId = DeviceIterator->second;
+    if (DeviceId == CurrentDeviceId) {
+      REPORT() << "Tried to duplicate OMPT Device= " << Device
+               << " ID=" << DeviceId;
+    } else {
+      REPORT() << "Tried to overwrite OMPT Device=" << Device
+               << " (ID=" << CurrentDeviceId << " with new ID=" << DeviceId;
+    }
+    return;
+  }
+  Devices.emplace(Device, DeviceId);
+}
+
+void llvm::omp::target::ompt::removeDeviceId(ompt_device_t *Device) {
+  int DeviceId = getDeviceId(Device);
+  if (DeviceId < 0) {
+    REPORT() << "Tried to remove Device= " << Device << " ID=" << DeviceId;
+    return;
+  }
+  std::unique_lock<std::mutex> Lock(DeviceIdWritingMutex);
+  Devices.erase(Device);
+  TracedDevices.erase(DeviceId);
+}
+
+OMPT_API_ROUTINE ompt_set_result_t ompt_set_trace_ompt(ompt_device_t *Device,
+                                                       unsigned int Enable,
+                                                       unsigned int EventTy) {
+  ODBG(ODT_Tool) << "Executing ompt_set_trace_ompt";
+
+  int DeviceId = getDeviceId(Device);
+  if (DeviceId < 0) {
+    REPORT() << "Failed to set trace events for Device=" << Device
+             << " (Unknown device) [Enable=" << Enable
+             << " EventTy=" << EventTy;
+    return ompt_set_never;
+  }
+
+  std::unique_lock<std::mutex> Lock(ompt_set_trace_ompt_mutex);
+  ensureFuncPtrLoaded<libomptarget_ompt_set_trace_ompt_t>(
+      "libomptarget_ompt_set_trace_ompt", &ompt_set_trace_ompt_fn);
+  assert(ompt_set_trace_ompt_fn && "libomptarget_ompt_set_trace_ompt loaded");
+  return ompt_set_trace_ompt_fn(DeviceId, Enable, EventTy);
+}
+
+OMPT_API_ROUTINE int
+ompt_start_trace(ompt_device_t *Device, ompt_callback_buffer_request_t Request,
+                 ompt_callback_buffer_complete_t Complete) {
+  ODBG(ODT_Tool) << "Executing ompt_start_trace";
+
+  int DeviceId = getDeviceId(Device);
+  if (DeviceId < 0) {
+    REPORT() << "Failed to start trace for Device=" << Device
+             << " (Unknown device";
+    // Indicate failure
+    return 0;
+  }
+
+  {
+    // Protect the function pointer
+    std::unique_lock<std::mutex> Lock(ompt_start_trace_mutex);
+
+    if (Request && Complete) {
+      llvm::omp::target::ompt::enableDeviceTracing(DeviceId);
+      // Enable asynchronous memory copy profiling
+      setOmptAsyncCopyProfile(/*Enable=*/true);
+      // Enable queue dispatch profiling
+      if (DeviceId >= 0)
+        setGlobalOmptKernelProfile(Device, /*Enable=*/1);
+      else
+        REPORT() << "May not enable kernel profiling for invalid device id="
+                 << DeviceId;
+    }
+
+    // Call libomptarget specific function
+    ensureFuncPtrLoaded<libomptarget_ompt_start_trace_t>(
+        "libomptarget_ompt_start_trace", &ompt_start_trace_fn);
+    assert(ompt_start_trace_fn && "libomptarget_ompt_start_trace loaded");
+  }
+  return ompt_start_trace_fn(DeviceId, Request, Complete);
+}
+
+OMPT_API_ROUTINE int ompt_flush_trace(ompt_device_t *Device) {
+  ODBG(ODT_Tool) << "Executing ompt_flush_trace";
+
+  std::unique_lock<std::mutex> Lock(ompt_flush_trace_mutex);
+  ensureFuncPtrLoaded<libomptarget_ompt_flush_trace_t>(
+      "libomptarget_ompt_flush_trace", &ompt_flush_trace_fn);
+  assert(ompt_flush_trace_fn && "libomptarget_ompt_flush_trace loaded");
+  return ompt_flush_trace_fn(getDeviceId(Device));
+}
+
+OMPT_API_ROUTINE int ompt_stop_trace(ompt_device_t *Device) {
+  ODBG(ODT_Tool) << "Executing ompt_stop_trace";
+
+  int DeviceId = getDeviceId(Device);
+  if (DeviceId < 0) {
+    REPORT() << "Failed to stop trace for Device=" << Device
+             << " (Unknown device)";
+    // Indicate failure
+    return 0;
+  }
+
+  {
+    // Protect the function pointer
+    std::unique_lock<std::mutex> Lock(ompt_stop_trace_mutex);
+    llvm::omp::target::ompt::disableDeviceTracing(DeviceId);
+    // Disable asynchronous memory copy profiling
+    setOmptAsyncCopyProfile(/*Enable=*/false);
+    // Disable queue dispatch profiling
+    if (DeviceId >= 0)
+      setGlobalOmptKernelProfile(Device, /*Enable=*/0);
+    else
+      REPORT() << "May not disable kernel profiling for invalid device id="
+               << DeviceId;
+    ensureFuncPtrLoaded<libomptarget_ompt_stop_trace_t>(
+        "libomptarget_ompt_stop_trace", &ompt_stop_trace_fn);
+    assert(ompt_stop_trace_fn && "libomptarget_ompt_stop_trace loaded");
+  }
+  return ompt_stop_trace_fn(DeviceId);
+}
+
+OMPT_API_ROUTINE ompt_record_ompt_t *
+ompt_get_record_ompt(ompt_buffer_t *Buffer, ompt_buffer_cursor_t CurrentPos) {
+  // TODO In debug mode, get the metadata associated with this buffer
+  // and assert that there are enough bytes for the current record
+
+  // Currently, no synchronization required since a disjoint set of
+  // trace records is handed over to a thread.
+
+  // Note that CurrentPos can be nullptr. In that case, we return
+  // nullptr. The tool has to handle that properly.
+  return (ompt_record_ompt_t *)CurrentPos;
+}
+
+OMPT_API_ROUTINE int ompt_advance_buffer_cursor(ompt_device_t *Device,
+                                                ompt_buffer_t *Buffer,
+                                                size_t Size,
+                                                ompt_buffer_cursor_t CurrentPos,
+                                                ompt_buffer_cursor_t *NextPos) {
+  // Note: The input parameter size is unused here. It refers to the
+  // bytes returned in the corresponding callback.
+  // Advance can be called concurrently, so synchronize setting the
+  // function pointer. The actual libomptarget function does not need
+  // to be synchronized since it must be working on logically disjoint
+  // buffers.
+  std::unique_lock<std::mutex> Lock(ompt_advance_buffer_cursor_mutex);
+  ensureFuncPtrLoaded<libomptarget_ompt_advance_buffer_cursor_t>(
+      "libomptarget_ompt_advance_buffer_cursor",
+      &ompt_advance_buffer_cursor_fn);
+  assert(ompt_advance_buffer_cursor_fn &&
+         "libomptarget_ompt_advance_buffer_cursor loaded");
+  return ompt_advance_buffer_cursor_fn(Device, Buffer, Size, CurrentPos,
+                                       NextPos);
+}
+
+OMPT_API_ROUTINE ompt_record_t
+ompt_get_record_type(ompt_buffer_t *Buffer, ompt_buffer_cursor_t CurrentPos) {
+  std::unique_lock<std::mutex> Lock(ompt_get_record_type_mutex);
+  ensureFuncPtrLoaded<libomptarget_ompt_get_record_type_t>(
+      "libomptarget_ompt_get_record_type", &ompt_get_record_type_fn);
+  assert(ompt_get_record_type_fn && "libomptarget_ompt_get_record_type loaded");
+  return ompt_get_record_type_fn(Buffer, CurrentPos);
+}
+
+OMPT_API_ROUTINE ompt_device_time_t
+ompt_get_device_time(ompt_device_t *Device) {
+  ODBG(ODT_Tool) << "Executing ompt_get_device_time";
+  return getSystemTimestampInNs();
+}
+
+OMPT_API_ROUTINE double ompt_translate_time(ompt_device_t *Device,
+                                            ompt_device_time_t DeviceTime) {
+  // Translate a device time to a meaningful timepoint in host time
+  // We do not need to account for clock-skew / drift. So simple linear
+  // translation using the host to device rate we obtained.
+  double TranslatedTime = DeviceTime * HostToDeviceSlope + HostToDeviceOffset;
+  ODBG(ODT_Tool) << "D2H translated time: " << TranslatedTime;
+
+  return TranslatedTime;
+}
+
+void llvm::omp::target::ompt::setOmptTimestamp(uint64_t StartTime,
+                                               uint64_t EndTime) {
+  std::unique_lock<std::mutex> Lock(ompt_set_timestamp_mutex);
+  ensureFuncPtrLoaded<libomptarget_ompt_set_timestamp_t>(
+      "libomptarget_ompt_set_timestamp", &ompt_set_timestamp_fn);
+  // No need to hold a lock
+  ompt_set_timestamp_fn(StartTime, EndTime);
+}
+
+void llvm::omp::target::ompt::setOmptHostToDeviceRate(double Slope,
+                                                      double Offset) {
+  HostToDeviceSlope = Slope;
+  HostToDeviceOffset = Offset;
+}
+
+void llvm::omp::target::ompt::setOmptGrantedNumTeams(uint64_t NumTeams) {
+  std::unique_lock<std::mutex> Lock(ompt_set_granted_teams_mutex);
+  ensureFuncPtrLoaded<libomptarget_ompt_set_granted_teams_t>(
+      "libomptarget_ompt_set_granted_teams", &ompt_set_granted_teams_fn);
+  // No need to hold a lock
+  ompt_set_granted_teams_fn(NumTeams);
+}
+
+ompt_interface_fn_t llvm::omp::target::ompt::lookupDeviceTracingFn(
+    const char *InterfaceFunctionName) {
+#define compareAgainst(AvailableFunction)                                      \
+  if (strcmp(InterfaceFunctionName, #AvailableFunction) == 0)                  \
+    return (ompt_interface_fn_t)AvailableFunction;
+
+  FOREACH_OMPT_DEVICE_TRACING_FN(compareAgainst);
+#undef compareAgainst
+
+  ODBG(ODT_Tool) << "Warning: Could not find requested function "
+                 << InterfaceFunctionName;
+  return (ompt_interface_fn_t) nullptr;
+}
+
+#pragma pop_macro("DEBUG_PREFIX")
+
+#endif // OMPT_SUPPORT

>From 323d92588cac6595fd8e53316c4669a9bf52d550 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 2 Apr 2026 07:40:55 -0500
Subject: [PATCH 05/11] [Offload][OMPT] Add tracing extensions to Interface.h

Extend the OMPT Interface class with trace record creation methods and
the async-aware TracerInterfaceRAII for device tracing support.

Key changes to Interface.h:
- Add includes for OmptEventInfoTy.h, APITypes.h, GenericProfiler.h
- Add extern TracingActive flag and isTracingEnabled() declaration
- Add trace start/stop methods for all target operations:
  startTargetDataAllocTrace, stopTargetDataAllocTrace,
  startTargetDataSubmitTrace, startTargetDataDeleteTrace,
  startTargetDataRetrieveTrace, stopTargetDataMovementTraceAsync,
  startTargetSubmitTrace, stopTargetSubmitTraceAsync, and
  corresponding methods for enter/exit/update/target regions
- Add getTraceGenerators<>() template overloads parallel to
  existing getCallbacks<>()
- Add private helpers: setTraceRecordCommon, setTraceRecordTargetDataOp,
  setTraceRecordTargetKernel, setTraceRecordTarget, announceTargetRegion
- Add TracerInterfaceRAII class: async-aware RAII that checks
  isTracingEnabled, calls begin trace generator, allocates
  OmptEventInfoTy via profiler, sets AsyncInfo->ProfilerData
- Rename CallbackPairTy to FunctionPairTy in InterfaceRAII
- Make InvokeInterfaceFunction return a value
- Replace OMPT_IF_BUILT with OMPT_IF_TRACING_ENABLED

Key changes to Callback.h:
- Replace inline macro/typedef definitions with include of
  OmptCommonDefs.h (shared definitions moved there in PR3)

Made-with: Cursor
Co-Authored-By: Claude <noreply at anthropic.com>
---
 offload/include/OpenMP/OMPT/Callback.h  |  26 +--
 offload/include/OpenMP/OMPT/Interface.h | 255 ++++++++++++++++++++++--
 2 files changed, 243 insertions(+), 38 deletions(-)

diff --git a/offload/include/OpenMP/OMPT/Callback.h b/offload/include/OpenMP/OMPT/Callback.h
index 9d545c643223f..7aff68ced0d79 100644
--- a/offload/include/OpenMP/OMPT/Callback.h
+++ b/offload/include/OpenMP/OMPT/Callback.h
@@ -16,34 +16,12 @@
 
 #ifdef OMPT_SUPPORT
 
-#include "omp-tools.h"
+#include "OmptCommonDefs.h"
 
 #pragma push_macro("DEBUG_PREFIX")
 #undef DEBUG_PREFIX
 #define DEBUG_PREFIX "OMPT"
 
-#define FOREACH_OMPT_TARGET_CALLBACK(macro)                                    \
-  FOREACH_OMPT_DEVICE_EVENT(macro)                                             \
-  FOREACH_OMPT_NOEMI_EVENT(macro)                                              \
-  FOREACH_OMPT_EMI_EVENT(macro)
-
-#define performIfOmptInitialized(stmt)                                         \
-  do {                                                                         \
-    if (llvm::omp::target::ompt::Initialized) {                                \
-      stmt;                                                                    \
-    }                                                                          \
-  } while (0)
-
-#define performOmptCallback(CallbackName, ...)                                 \
-  do {                                                                         \
-    if (ompt_callback_##CallbackName##_fn)                                     \
-      ompt_callback_##CallbackName##_fn(__VA_ARGS__);                          \
-  } while (0)
-
-/// Function type def used for maintaining unique target region, target
-/// operations ids
-typedef uint64_t (*IdInterfaceTy)();
-
 namespace llvm {
 namespace omp {
 namespace target {
@@ -98,8 +76,6 @@ extern bool Initialized;
 
 #pragma pop_macro("DEBUG_PREFIX")
 
-#else
-#define performIfOmptInitialized(stmt)
 #endif // OMPT_SUPPORT
 
 #endif // OFFLOAD_INCLUDE_OPENMP_OMPT_CALLBACK_H
diff --git a/offload/include/OpenMP/OMPT/Interface.h b/offload/include/OpenMP/OMPT/Interface.h
index 6961641769b76..54e7564c21ad8 100644
--- a/offload/include/OpenMP/OMPT/Interface.h
+++ b/offload/include/OpenMP/OMPT/Interface.h
@@ -16,14 +16,21 @@
 // Only provide functionality if target OMPT support is enabled
 #ifdef OMPT_SUPPORT
 #include "Callback.h"
+#include "OmptEventInfoTy.h"
+#include "Shared/APITypes.h"
+#include "Shared/Debug.h"
 #include "omp-tools.h"
 
+#include "GenericProfiler.h"
+
 #include "llvm/Support/ErrorHandling.h"
 
 #include <functional>
 #include <tuple>
 
-#define OMPT_IF_BUILT(stmt) stmt
+#pragma push_macro("DEBUG_PREFIX")
+#undef DEBUG_PREFIX
+#define DEBUG_PREFIX "OMPT"
 
 /// Callbacks for target regions require task_data representing the
 /// encountering task.
@@ -42,9 +49,17 @@ namespace ompt {
 static ompt_get_task_data_t ompt_get_task_data_fn;
 static ompt_get_target_task_data_t ompt_get_target_task_data_fn;
 
+/// OMPT global tracing status. Indicates if at least one device is traced.
+extern bool TracingActive;
+
+/// Check if this device traces the given event type
+extern bool isTracingEnabled(int DeviceId, unsigned int EventTy);
+
 /// Used to maintain execution state for this thread
 class Interface {
 public:
+  // Target data callbacks
+
   /// Top-level function for invoking callback before device data allocation
   void beginTargetDataAlloc(int64_t DeviceId, void *HstPtrBegin,
                             void **TgtPtrBegin, size_t Size, void *Code);
@@ -142,6 +157,85 @@ class Interface {
   /// Top-level function for invoking callback after target construct
   void endTarget(int64_t DeviceId, void *Code);
 
+  // Target data tracing
+
+  /// Top-level function for starting trace before device data allocation
+  void startTargetDataAllocTrace(int64_t DeviceId, void *HstPtrBegin,
+                                 void **TgtPtrBegin, size_t Size, void *Code);
+
+  /// Top-level function for stopping trace after device data allocation
+  ompt_record_ompt_t *stopTargetDataAllocTrace(int64_t DeviceId,
+                                               void *HstPtrBegin,
+                                               void **TgtPtrBegin, size_t Size,
+                                               void *Code);
+
+  /// Top-level function for starting trace before data submit
+  ompt_record_ompt_t *startTargetDataSubmitTrace(int64_t SrcDeviceId,
+                                                 void *SrcPtrBegin,
+                                                 int64_t DstDeviceId,
+                                                 void *DstPtrBegin, size_t Size,
+                                                 void *Code);
+
+  /// Top-level function for starting trace before device data deallocation
+  void startTargetDataDeleteTrace(int64_t DeviceId, void *TgtPtrBegin,
+                                  void *Code);
+
+  /// Top-level function for stopping trace after device data deallocation
+  ompt_record_ompt_t *stopTargetDataDeleteTrace(int64_t DeviceId,
+                                                void *TgtPtrBegin, void *Code);
+
+  /// Top-level function for starting trace before data retrieve
+  ompt_record_ompt_t *startTargetDataRetrieveTrace(int64_t SrcDeviceId,
+                                                   void *SrcPtrBegin,
+                                                   int64_t DstDeviceId,
+                                                   void *DstPtrBegin,
+                                                   size_t Size, void *Code);
+
+  ompt_record_ompt_t *
+  stopTargetDataMovementTraceAsync(ompt_record_ompt_t *DataPtr,
+                                   uint64_t NanosStart, uint64_t NanosEnd);
+
+  /// Top-level function for starting trace before kernel dispatch
+  ompt_record_ompt_t *startTargetSubmitTrace(int64_t DeviceId,
+                                             unsigned int NumTeams = 1);
+
+  ompt_record_ompt_t *stopTargetSubmitTraceAsync(ompt_record_ompt_t *DataPtr,
+                                                 unsigned int NumTeams,
+                                                 uint64_t NanosStart,
+                                                 uint64_t NanosStop);
+
+  // Target region tracing
+
+  /// Top-level function for starting trace before target enter data
+  /// construct
+  ompt_record_ompt_t *startTargetDataEnterTrace(int64_t DeviceId, void *Code);
+
+  /// Top-level function for stopping trace after target enter data
+  /// construct
+  ompt_record_ompt_t *stopTargetDataEnterTrace(int64_t DeviceId, void *Code);
+
+  /// Top-level function for starting trace before target exit data
+  /// construct
+  ompt_record_ompt_t *startTargetDataExitTrace(int64_t DeviceId, void *Code);
+
+  /// Top-level function for stopping trace after target exit data
+  /// construct
+  ompt_record_ompt_t *stopTargetDataExitTrace(int64_t DeviceId, void *Code);
+
+  /// Top-level function for starting trace before target update construct
+  ompt_record_ompt_t *startTargetUpdateTrace(int64_t DeviceId, void *Code);
+
+  /// Top-level function for stopping trace after target update construct
+  ompt_record_ompt_t *stopTargetUpdateTrace(int64_t DeviceId, void *Code);
+
+  // Target kernel tracing
+
+  /// Top-level function for starting trace before target construct
+  ompt_record_ompt_t *startTargetTrace(int64_t DeviceId, void *Code);
+
+  /// Top-level function for stopping trace after target construct
+  ompt_record_ompt_t *stopTargetTrace(int64_t DeviceId, void *Code);
+
   // Callback getter: Target data operations
   template <ompt_target_data_op_t OpType> auto getCallbacks() {
     if constexpr (OpType == ompt_target_data_alloc ||
@@ -217,6 +311,69 @@ class Interface {
     llvm_unreachable("Unhandled target operation!");
   }
 
+  // Callback getter: Target data operations
+  template <ompt_target_data_op_t OpType> auto getTraceGenerators() {
+    if constexpr (OpType == ompt_target_data_alloc ||
+                  OpType == ompt_target_data_alloc_async)
+      return std::make_pair(std::mem_fn(&Interface::startTargetDataAllocTrace),
+                            std::mem_fn(&Interface::stopTargetDataAllocTrace));
+
+    if constexpr (OpType == ompt_target_data_delete ||
+                  OpType == ompt_target_data_delete_async)
+      return std::make_pair(std::mem_fn(&Interface::startTargetDataDeleteTrace),
+                            std::mem_fn(&Interface::stopTargetDataDeleteTrace));
+
+    if constexpr (OpType == ompt_target_data_transfer_to_device ||
+                  OpType == ompt_target_data_transfer_to_device_async)
+      return std::make_pair(
+          std::mem_fn(&Interface::startTargetDataSubmitTrace),
+          std::mem_fn(&Interface::stopTargetDataMovementTraceAsync));
+
+    if constexpr (OpType == ompt_target_data_transfer_from_device ||
+                  OpType == ompt_target_data_transfer_from_device_async)
+      return std::make_pair(
+          std::mem_fn(&Interface::startTargetDataRetrieveTrace),
+          std::mem_fn(&Interface::stopTargetDataMovementTraceAsync));
+
+    llvm_unreachable("Unhandled target data operation type!");
+  }
+
+  // Callback getter: Target region operations
+  template <ompt_target_t OpType> auto getTraceGenerators() {
+    if constexpr (OpType == ompt_target_enter_data ||
+                  OpType == ompt_target_enter_data_nowait)
+      return std::make_pair(std::mem_fn(&Interface::startTargetDataEnterTrace),
+                            std::mem_fn(&Interface::stopTargetDataEnterTrace));
+
+    if constexpr (OpType == ompt_target_exit_data ||
+                  OpType == ompt_target_exit_data_nowait)
+      return std::make_pair(std::mem_fn(&Interface::startTargetDataExitTrace),
+                            std::mem_fn(&Interface::stopTargetDataExitTrace));
+
+    if constexpr (OpType == ompt_target_update ||
+                  OpType == ompt_target_update_nowait)
+      return std::make_pair(std::mem_fn(&Interface::startTargetUpdateTrace),
+                            std::mem_fn(&Interface::stopTargetUpdateTrace));
+
+    if constexpr (OpType == ompt_target || OpType == ompt_target_nowait)
+      return std::make_pair(std::mem_fn(&Interface::startTargetTrace),
+                            std::mem_fn(&Interface::stopTargetTrace));
+
+    llvm_unreachable("Unknown target region operation type!");
+  }
+
+  // Callback getter: Kernel launch operation
+  template <ompt_callbacks_t OpType> auto getTraceGenerators() {
+    // We use 'ompt_callbacks_t', because no other enum is currently available
+    // to model a kernel launch / target submit operation.
+    if constexpr (OpType == ompt_callback_target_submit)
+      return std::make_pair(
+          std::mem_fn(&Interface::startTargetSubmitTrace),
+          std::mem_fn(&Interface::stopTargetSubmitTraceAsync));
+
+    llvm_unreachable("Unhandled target operation!");
+  }
+
   /// Setters for target region and target operation correlation ids
   void setTargetDataValue(uint64_t DataValue) { TargetData.value = DataValue; }
   void setTargetDataPtr(void *DataPtr) { TargetData.ptr = DataPtr; }
@@ -240,6 +397,9 @@ class Interface {
   /// Target task data representing the target task region
   ompt_data_t *TargetTaskData = nullptr;
 
+  /// Used for marking begin of a data operation
+  void announceTargetRegion(const char *RegionName);
+
   /// Used for marking begin of a data operation
   void beginTargetDataOperation();
 
@@ -251,6 +411,23 @@ class Interface {
 
   /// Used for marking end of a target region
   void endTargetRegion();
+
+  // Called by all trace generation routines
+  void setTraceRecordCommon(ompt_record_ompt_t *DataPtr,
+                            ompt_callbacks_t CallbackType);
+  // Type specific helpers
+  void setTraceRecordTargetDataOp(ompt_record_target_data_op_t *Record,
+                                  ompt_target_data_op_t DataOpType,
+                                  void *SrcAddr, int64_t SrcDeviceNum,
+                                  void *DstAddr, int64_t DstDeviceNum,
+                                  size_t Bytes, void *CodePtr);
+
+  void setTraceRecordTargetKernel(ompt_record_target_kernel_t *Record,
+                                  unsigned int NumTeams);
+
+  void setTraceRecordTarget(ompt_record_target_t *Record, int64_t DeviceId,
+                            ompt_target_t TargetKind,
+                            ompt_scope_endpoint_t Endpoint, void *CodePtr);
 };
 
 /// Thread local state for target region and associated metadata
@@ -262,14 +439,14 @@ extern thread_local Interface RegionInterface;
 extern thread_local void *ReturnAddress;
 
 template <typename FuncTy, typename ArgsTy, size_t... IndexSeq>
-void InvokeInterfaceFunction(FuncTy Func, ArgsTy Args,
+auto InvokeInterfaceFunction(FuncTy Func, ArgsTy Args,
                              std::index_sequence<IndexSeq...>) {
-  std::invoke(Func, RegionInterface, std::get<IndexSeq>(Args)...);
+  return std::invoke(Func, RegionInterface, std::get<IndexSeq>(Args)...);
 }
 
-template <typename CallbackPairTy, typename... ArgsTy> class InterfaceRAII {
+template <typename FunctionPairTy, typename... ArgsTy> class InterfaceRAII {
 public:
-  InterfaceRAII(CallbackPairTy Callbacks, ArgsTy... Args)
+  InterfaceRAII(FunctionPairTy Callbacks, ArgsTy... Args)
       : Arguments(Args...), beginFunction(std::get<0>(Callbacks)),
         endFunction(std::get<1>(Callbacks)) {
     performIfOmptInitialized(begin());
@@ -290,14 +467,66 @@ template <typename CallbackPairTy, typename... ArgsTy> class InterfaceRAII {
   }
 
   std::tuple<ArgsTy...> Arguments;
-  typename CallbackPairTy::first_type beginFunction;
-  typename CallbackPairTy::second_type endFunction;
+  typename FunctionPairTy::first_type beginFunction;
+  typename FunctionPairTy::second_type endFunction;
 };
 
 // InterfaceRAII's class template argument deduction guide
-template <typename CallbackPairTy, typename... ArgsTy>
-InterfaceRAII(CallbackPairTy Callbacks, ArgsTy... Args)
-    -> InterfaceRAII<CallbackPairTy, ArgsTy...>;
+template <typename FunctionPairTy, typename... ArgsTy>
+InterfaceRAII(FunctionPairTy Callbacks, ArgsTy... Args)
+    -> InterfaceRAII<FunctionPairTy, ArgsTy...>;
+
+/// Similar to the original InterfaceRAII this class is used for tracing and
+/// extends the original with async capabilities. That is: It takes an
+/// additional AsyncInfo reference as argument to populate the relevant fields.
+/// The AsyncInfoTy propagates the info into the RTL / plugins.
+/// TracedDeviceId represents the trace record's device affinity. EventType is
+/// the callback type that needs to be enabled via ompt_set_trace_ompt.
+template <typename FunctionPairTy, typename AsyncInfoTy, typename... ArgsTy>
+class TracerInterfaceRAII {
+public:
+  TracerInterfaceRAII(FunctionPairTy Callbacks, AsyncInfoTy &AsyncInfo,
+                      plugin::GenericProfilerTy *Prof, int TracedDeviceId,
+                      ompt_callbacks_t EventType, ArgsTy... Args)
+      : Arguments(Args...), beginFunction(std::get<0>(Callbacks)) {
+    __tgt_async_info *AI = AsyncInfo;
+    if (isTracingEnabled(TracedDeviceId, EventType)) {
+      auto Record = begin();
+
+      // The Profiler can allocate specific data to be used to pass information
+      // from here to lower parts of the runtime system.
+      // NOTE: It is the responsibility of the programmer to ensure type
+      // compatibility and correct usage of the data. The profiler, however,
+      // OWNS the pointer and frees it at an appropriate time.
+      OmptEventInfoTy *ProfilerData =
+          reinterpret_cast<OmptEventInfoTy *>(Prof->getProfilerSpecificData());
+      ProfilerData->TraceRecord = Record;
+      ProfilerData->NumTeams = 0;
+
+      // Allows to pass down into the plugins via AsyncInfoTy
+      AI->ProfilerData = ProfilerData;
+    } else {
+      // Actively prevent further tracing of this event
+      AI->ProfilerData = nullptr;
+    }
+  }
+
+private:
+  auto begin() {
+    auto IndexSequence =
+        std::make_index_sequence<std::tuple_size_v<decltype(Arguments)>>{};
+    return InvokeInterfaceFunction(beginFunction, Arguments, IndexSequence);
+  }
+
+  std::tuple<ArgsTy...> Arguments;
+  typename FunctionPairTy::first_type beginFunction;
+  /// No end-function here, since the end is called asynchronously from the
+  /// plugins, once the operation has completed.
+};
+
+template <typename FunctionPairTy, typename... ArgsTy>
+TracerInterfaceRAII(FunctionPairTy Callbacks, ArgsTy... Args)
+    -> TracerInterfaceRAII<FunctionPairTy, ArgsTy...>;
 
 /// Used to set and reset the thread-local return address. The RAII is expected
 /// to be created at a runtime entry point when the return address should be
@@ -335,8 +564,8 @@ class ReturnAddressSetterRAII {
 // The getter returns the address stored in the thread local variable.
 #define OMPT_GET_RETURN_ADDRESS llvm::omp::target::ompt::ReturnAddress
 
-#else
-#define OMPT_IF_BUILT(stmt)
-#endif
+#pragma pop_macro("DEBUG_PREFIX")
+
+#endif // OMPT_SUPPORT
 
 #endif // OFFLOAD_INCLUDE_OPENMP_OMPT_INTERFACE_H

>From 97fa007a5e35dac47665ea0f19cc33b1b5fd3187 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 2 Apr 2026 07:44:36 -0500
Subject: [PATCH 06/11] [Offload][OMPT] Add OmptTracingBufferMgr trace buffer
 infrastructure

Add the trace buffer management subsystem for OMPT device tracing. This
manages allocation, population, and flushing of trace record buffers
between OpenMP worker threads and helper threads.

New files:
- OmptTracingBuffer.h: OmptTracingBufferMgr class with Buffer metadata,
  FlushInfo, TraceRecord types, thread-local per-device buffer pointers,
  helper thread management, and cursor-based record allocation
- OmptTracingBuffer.cpp: Full implementation of buffer lifecycle:
  assignCursor (lock-free fast path for existing buffers, locked path
  for new allocations), triggerFlushOnBufferFull, driveCompletion
  (helper thread main loop), invokeCallbacks, flushBuffer (dispatches
  buffer-completion callbacks for ranges of ready records),
  flushAllBuffers, helper thread start/shutdown/flush-and-shutdown
- OmptTracing.h: Libomptarget-side declarations for tracing state,
  buffer management callback registration, device tracing control,
  timestamp/clock correlation, and thread-local trace record fields.
  Implementation deferred to subsequent PR.

Changes to existing files:
- PluginManager.h: Add OmptTracingBufferMgr* TraceRecordManager member
  with getTraceRecordManager() accessor, include OmptTracingBuffer.h
- libomptarget/CMakeLists.txt: Add OmptTracingBuffer.cpp to omptarget

Key design points:
- Per-thread buffer assignment avoids contention on the common path
- Helper threads process buffers in flush-id order via condition variables
- LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL / FLUSH_ON_SHUTDOWN envars
  control automatic flushing behavior
- Buffer states: init -> ready -> released track record lifecycle
- Flush states: waiting -> processing prevent concurrent helper access

Made-with: Cursor
---
 offload/include/OpenMP/OMPT/OmptTracing.h     | 154 ++++
 .../include/OpenMP/OMPT/OmptTracingBuffer.h   | 412 ++++++++++
 offload/include/PluginManager.h               |  10 +-
 offload/libomptarget/CMakeLists.txt           |   1 +
 .../OpenMP/OMPT/OmptTracingBuffer.cpp         | 758 ++++++++++++++++++
 5 files changed, 1334 insertions(+), 1 deletion(-)
 create mode 100644 offload/include/OpenMP/OMPT/OmptTracing.h
 create mode 100644 offload/include/OpenMP/OMPT/OmptTracingBuffer.h
 create mode 100644 offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp

diff --git a/offload/include/OpenMP/OMPT/OmptTracing.h b/offload/include/OpenMP/OMPT/OmptTracing.h
new file mode 100644
index 0000000000000..2a892582923d5
--- /dev/null
+++ b/offload/include/OpenMP/OMPT/OmptTracing.h
@@ -0,0 +1,154 @@
+//===---- OmptTracing.h - Target independent OMPT callbacks --*- C++ -*----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Interface used by target-independent runtimes to coordinate registration and
+// invocation of OMPT tracing functionality.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OPENMP_LIBOMPTARGET_INCLUDE_OMPTTRACING_H
+#define OPENMP_LIBOMPTARGET_INCLUDE_OMPTTRACING_H
+
+#ifdef OMPT_SUPPORT
+
+#include <unordered_map>
+
+#include "OmptCommonDefs.h"
+#include "OmptTracingBuffer.h"
+
+#pragma push_macro("DEBUG_PREFIX")
+#undef DEBUG_PREFIX
+#define DEBUG_PREFIX "OMPT"
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace ompt {
+
+/// After a timestamp has been read, reset it.
+void resetTimestamp(uint64_t *T);
+
+/// A tool may register unique buffer-request and buffer-completion
+/// callback functions for a device. The following are utility functions to
+/// manage those functions.
+
+/// Given a device-id, return the corresponding buffer-request callback
+/// function.
+ompt_callback_buffer_request_t getBufferRequestFn(int DeviceId);
+
+/// Give a device-id, return the corresponding buffer-completion callback
+/// function.
+ompt_callback_buffer_complete_t getBufferCompleteFn(int DeviceId);
+
+/// Given a device-id, set the corresponding buffer-request and
+/// buffer-completion callback functions.
+void setBufferManagementFns(int DeviceId, ompt_callback_buffer_request_t ReqFn,
+                            ompt_callback_buffer_complete_t CmpltFn);
+
+/// Given a device-id, remove the corresponding buffer-request and
+/// buffer-completion callback functions.
+void removeBufferManagementFns(int DeviceId);
+
+/// Is device tracing stopped for all devices?
+bool isAllDeviceTracingStopped();
+
+/// Invoke callback function for buffer request events
+void ompt_callback_buffer_request(int DeviceId, ompt_buffer_t **BufferPtr,
+                                  size_t *Bytes);
+
+/// Invoke callback function for buffer complete events
+void ompt_callback_buffer_complete(int DeviceId, ompt_buffer_t *Buffer,
+                                   size_t Bytes,
+                                   ompt_buffer_cursor_t BeginCursor,
+                                   int BufferOwned);
+
+/// Set 'start' and 'stop' for the current trace record
+void setOmptTimestamp(uint64_t StartTime, uint64_t EndTime);
+
+/// Set the linear function correlation between host and device clocks
+void setOmptHostToDeviceRate(double Slope, double Offset);
+
+/// Set / store the number of granted teams
+void setOmptGrantedNumTeams(uint64_t NumTeams);
+
+/// Check if (1) tracing is globally active (2) the given device is actively
+/// traced and (3) the given event type is traced on the device
+bool isTracingEnabled(int DeviceId, unsigned int EventTy);
+
+/// Check if the given device is actively traced
+bool isTracedDevice(int DeviceId);
+
+/// Check if the given device is monitoring the provided tracing type
+bool isTracingTypeEnabled(int DeviceId, unsigned int EventTy);
+
+/// Check if the given device is monitoring the provided tracing type 'group'
+/// Where group means we will check for both: EMI and non-EMI event types
+bool isTracingTypeGroupEnabled(int DeviceId, unsigned int EventTy);
+
+/// Set whether the given tracing type should be monitored (or not) on the
+/// device
+void setTracingTypeEnabled(uint64_t &TracedEventTy, bool Enable,
+                           unsigned int EventTy);
+
+/// Set / reset the given tracing types (EventTy = 0 corresponds to 'all')
+ompt_set_result_t setTraceEventTy(int DeviceId, unsigned int Enable,
+                                  unsigned int EventTy);
+
+/// Return thread id
+uint64_t getThreadId();
+
+/// See TracedDevices in OmptDeviceTracing.h
+extern std::map<int32_t, uint64_t> TracedDevices;
+/// Activate tracing on the given device
+void enableDeviceTracing(int DeviceId);
+/// Deactivate tracing on the given device
+void disableDeviceTracing(int DeviceId);
+
+/// Mutexes to serialize invocation of device registration and checks
+extern std::mutex DeviceAccessMutex;
+
+/// Mutexes to serialize invocation of device-independent entry points
+extern std::mutex TraceAccessMutex;
+extern std::mutex TraceControlMutex;
+
+/// Ensure serialization of calls to std::hash
+extern std::mutex TraceHashThreadMutex;
+
+/// Protect map from device-id to the corresponding buffer-request and
+/// buffer-completion callback functions.
+extern std::mutex BufferManagementFnMutex;
+
+/// Map from device-id to the corresponding buffer-request and buffer-completion
+/// callback functions.
+extern std::unordered_map<int, std::pair<ompt_callback_buffer_request_t,
+                                         ompt_callback_buffer_complete_t>>
+    BufferManagementFns;
+
+/// Thread local variables used by the plugin to communicate OMPT information
+/// that are then used to populate trace records. This method assumes a
+/// synchronous implementation, otherwise it won't work.
+extern thread_local uint32_t TraceRecordNumGrantedTeams;
+extern thread_local uint64_t TraceRecordStartTime;
+extern thread_local uint64_t TraceRecordStopTime;
+
+/// Thread local thread-id.
+extern thread_local uint64_t ThreadId;
+
+/// OMPT global tracing status. Indicates if at least one device is traced.
+extern bool TracingActive;
+
+} // namespace ompt
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+#pragma pop_macro("DEBUG_PREFIX")
+
+#endif // OMPT_SUPPORT
+
+#endif // OPENMP_LIBOMPTARGET_INCLUDE_OMPTTRACING_H
diff --git a/offload/include/OpenMP/OMPT/OmptTracingBuffer.h b/offload/include/OpenMP/OMPT/OmptTracingBuffer.h
new file mode 100644
index 0000000000000..5c9f4bf33dae8
--- /dev/null
+++ b/offload/include/OpenMP/OMPT/OmptTracingBuffer.h
@@ -0,0 +1,412 @@
+//===- OmptTracingBuffer.h - Target independent OpenMP target RTL -- C++ *-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Interface to be used for generating and flushing OMPT device trace records.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OPENMP_LIBOMPTARGET_OMPTTRACINGBUFFER_H
+#define OPENMP_LIBOMPTARGET_OMPTTRACINGBUFFER_H
+
+#ifdef OMPT_SUPPORT
+
+#include <atomic>
+#include <cassert>
+#include <condition_variable>
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <set>
+#include <thread>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#include <omp-tools.h>
+
+#include "Shared/EnvironmentVar.h"
+
+// Maximum number of devices supported in device tracing. No device tracing
+// will be performed for any device-id larger than 1023.
+#define MAX_NUM_DEVICES 1024
+
+// TODO Start with 1 helper thread and add dynamically if required
+// Number of helper threads must not execeed 32 since the
+// thread-wait-tracker is 32 bits in length.
+#define OMPT_NUM_HELPER_THREADS 1
+
+/*
+ * Buffer manager for trace records generated by OpenMP master and
+ * worker threads. During device init, a tool may register a
+ * buffer-request and a buffer-completion callback. The buffer-request
+ * callback should be used to allocate new buffers as required. The
+ * buffer-complete callback should be used to return trace records to
+ * the tool.
+ *
+ * In addition to trace records, this class manages the helper threads
+ * for dispatching a range of trace records to the tool.
+ */
+class OmptTracingBufferMgr {
+public:
+  /*
+   * A trace record (TR) holds the trace data. Its type
+   * can be ompt or native. Currently, only ompt type is implemented.
+   */
+
+  /*
+   * A TR can be in the following states:
+   * TR_init: initial state
+   * TR_ready: An OpenMP thread marks a TR ready when it is done
+   * populating the TR
+   * TR_released: A helper thread marks a TR released after it has
+   * completed returning the TR to the tool
+   */
+  enum TRStatus { TR_init, TR_ready, TR_released };
+
+  /*
+   * Metadata capturing the state of a buffer of trace records. Once a
+   * buffer is allocated by an OpenMP worker thread, trace records are
+   * carved out from that buffer by that same OpenMP thread alone. Thus
+   * the allocated buffer is thread-specific from the allocation/population
+   * standpoint. But it may be manipulated by helper threads.
+   *
+   * Id, DeviceId, Start, and TotalBytes are not changed once set.
+   * RemainingBytes could be written multiple times but only by the same
+   * thread. But Cursor and IsFull may be read/written by an OpenMP worker
+   * thread and read by helper threads. Hence, accesses of
+   * this 2nd set of locations need to be atomic or synchronized.
+   */
+  struct Buffer {
+    uint64_t Id;                // Unique identifier of the buffer
+    int64_t DeviceId;           // Device for which this buffer is allocated
+    void *Start;                // Start of allocated space for trace records
+    size_t TotalBytes;          // Total number of bytes in the allocated space
+    size_t RemainingBytes;      // Total number of unused bytes
+                                // corresponding to Cursor
+    std::atomic<void *> Cursor; // Address of the last trace record carved out
+    std::atomic<bool> IsFull;   // true if no more trace records can be
+                                // accomodated, otherwise false
+    Buffer(uint64_t BufId, int64_t DevId, void *S, size_t Bytes, size_t Rem,
+           void *C, bool F)
+        : Id(BufId), DeviceId(DevId), Start(S), TotalBytes(Bytes),
+          RemainingBytes(Rem), Cursor(C), IsFull(F) {}
+    Buffer() = delete;
+    Buffer(const Buffer &) = delete;
+    Buffer &operator=(const Buffer &) = delete;
+  };
+  using BufPtr = std::shared_ptr<Buffer>;
+
+private:
+  /// Envar to control whether a buffer should be flushed when it gets full.
+  BoolEnvar OMPX_FlushOnBufferFull;
+
+  /// Envar to control whether all buffers should be flushed during shutdown.
+  BoolEnvar OMPX_FlushOnShutdown;
+
+  // Internal variable for tracking threads to wait for flush
+  uint32_t ThreadFlushTracker;
+
+  // Internal variable for tracking threads shutting down
+  uint32_t ThreadShutdownTracker;
+
+  using MapId2Buf = std::map<uint64_t, BufPtr>;
+
+  // Map from id to corresponding buffer. The ids are assigned in
+  // increasing order of creation.
+  MapId2Buf Id2BufferMap;
+
+  // Trace record. We currently support OMPT data type only. The state
+  // (TRStatus type) is maintained inline in the trace record. The
+  // tool is expected to access only the OMPT record.
+  struct TraceRecord {
+    ompt_record_ompt_t TR;
+    std::atomic<TRStatus> TRState;
+  };
+
+  // Thread-specific array of pointers to a buffer. The buffer pointed to
+  // is the last one allocated by this thread for a given device. The ith
+  // element points to the buffer for the ith device. At most MAX_NUM_DEVICES
+  // devices are supported.
+  static thread_local BufPtr ArrayOfBufPtr[MAX_NUM_DEVICES];
+
+  /*
+   * A buffer is flushed when it fills up or when the tool invokes
+   * flush_trace. So it's possible that the same buffer may be flushed
+   * more than once. When a buffer is flushed the first time, a unique
+   * id (flush-id) is generated and assigned to that buffer. Even if
+   * it is flushed again, the previously assigned id is maintained for
+   * that buffer. This id is loosely used to determine the order in
+   * which the buffers are processed and the corresponding trace
+   * records released to the tool.
+   */
+
+  struct FlushInfo {
+    uint64_t FlushId;
+    void *FlushCursor;
+    BufPtr FlushBuf;
+    FlushInfo() = default;
+    FlushInfo(uint64_t Id, void *CR, BufPtr Buf)
+        : FlushId{Id}, FlushCursor{CR}, FlushBuf{Buf} {}
+  };
+
+  /*
+   * A buffer may be in the following states:
+   * Flush_waiting: when a buffer is flushed, either because it is
+   * full or because the tool invokes ompt_flush_trace
+   * Flush_processing: when a helper thread claims the waiting buffer
+   * and is in the process of dispatching buffer-completion callbacks
+   * on an associated range of trace records. If all trace records are
+   * not released, the state may be reset to Flush_waiting after the
+   * buffer-completion callbacks return
+   */
+  enum BufferFlushStatus { Flush_waiting, Flush_processing };
+  struct FlushMd {
+    void *FlushCursor;
+    BufPtr FlushBuf;
+    BufferFlushStatus FlushStatus;
+    FlushMd(void *CR, BufPtr Buf, BufferFlushStatus Status)
+        : FlushCursor{CR}, FlushBuf{Buf}, FlushStatus{Status} {}
+    FlushMd() = delete;
+  };
+
+  using MapId2Md = std::map<uint64_t, FlushMd>;
+
+  /*
+   * A map from a flush-id to metadata containing the current
+   * cursor. the corresponding buffer, and its flushed status. If a
+   * buffer is flushed multiple times, the cursor is updated to the
+   * furthest one
+   */
+  MapId2Md Id2FlushMdMap;
+
+  using UMapBufPtr2Id = std::unordered_map<BufPtr, uint64_t>;
+
+  // A hash map from a buffer address to the corresponding flush-id
+  UMapBufPtr2Id FlushBufPtr2IdMap;
+
+  using USetCursor = std::unordered_set<void *>;
+
+  USetCursor LastCursors;
+
+  using UMapThd2Id = std::unordered_map<std::thread::id, uint32_t>;
+
+  // A hash map from a helper thread id to an integer
+  UMapThd2Id HelperThreadIdMap;
+
+  // Mutex to protect Id2BufferMap and Cursor2BufMdMap
+  std::mutex BufferMgrMutex;
+
+  // Mutex to protect FlushBufPtr2IdMap and Id2FlushMdMap
+  std::mutex FlushMutex;
+
+  // Mutex to protect metadata tracking last cursors of buffer-completion
+  // callbacks
+  std::mutex LastCursorMutex;
+
+  // Condition variable used by helper thread to signal that flush is requested
+  std::condition_variable FlushCv;
+
+  // Condition variable used while waiting for flushing to complete
+  std::condition_variable ThreadFlushCv;
+
+  // Condition variable used while waiting for threads to shutdown
+  std::condition_variable ThreadShutdownCv;
+
+  // TODO Separate out the helper thread into its own class
+  std::vector<std::thread> CompletionThreads;
+
+  /// Called when a buffer \p Buf may be flushed with \p Cursor as the
+  /// last allocated trace record in the buffer.
+  /// triggerFlushOnBufferFull should be called without holding any lock.
+  void triggerFlushOnBufferFull(void *Cursor, BufPtr Buf);
+
+  // Called to dispatch buffer-completion callbacks for the trace records in
+  // this buffer
+  void flushBuffer(FlushInfo);
+
+  // Dispatch a buffer-completion callback with a range of trace records
+  void dispatchCallback(int64_t DeviceId, void *Buffer, void *FirstCursor,
+                        void *LastCursor);
+
+  // Add a last cursor
+  void addLastCursor(void *Cursor) {
+    std::unique_lock<std::mutex> Lock(LastCursorMutex);
+    LastCursors.emplace(Cursor);
+  }
+
+  // Remove a last cursor
+  void removeLastCursor(void *Cursor) {
+    std::unique_lock<std::mutex> Lock(LastCursorMutex);
+    assert(LastCursors.find(Cursor) != LastCursors.end());
+    LastCursors.erase(Cursor);
+  }
+
+  // Given a trace record pointer, initialize its metadata
+  void initTraceRecordMetaData(void *Rec);
+
+  // Given a device-id, get/set a pointer to the last allocated buffer metadata.
+  BufPtr getDeviceSpecificBuffer(int64_t DevId);
+  void setDeviceSpecificBuffer(int64_t DevId, BufPtr Buf);
+
+  // Reserve a candidate buffer for flushing, preventing other helper threads
+  // from accessing it
+  FlushInfo findAndReserveFlushedBuf(uint64_t FlushId);
+
+  // Unreserve a buffer so that other helper threads can process it
+  void unreserveFlushedBuf(const FlushInfo &);
+
+  // All done with this buffer, so the buffer and its metadata can be removed
+  void destroyFlushedBuf(const FlushInfo &);
+
+  // Add a new buffer by an OpenMP thread so that a helper thread can process it
+  uint64_t addNewFlushEntry(BufPtr Buf, void *Cursor);
+
+  // Get the next trace record
+  void *getNextTR(void *TR);
+
+  // Given a buffer, return the latest cursor
+  void *getBufferCursor(BufPtr);
+
+  // Is no more space remaining for trace records in this buffer?
+  bool isBufferFull(const FlushInfo &);
+
+  // Have all trace records in this buffer been returned to the tool?
+  bool isBufferOwned(const FlushInfo &);
+
+  // Dispatch a buffer-completion callback and indicate that the buffer can be
+  // deallocated
+  void dispatchBufferOwnedCallback(const FlushInfo &);
+
+  // Main entry point for a helper thread
+  void driveCompletion();
+
+  // Examine the flushed buffers and dispatch buffer-completion callbacks
+  void invokeCallbacks();
+
+  // The caller does not hold a lock while calling this method
+  void waitForFlushCompletion();
+
+  // Given a thread number, set the corresponding bit in the flush
+  // tracker. The caller must hold the flush lock.
+  void setThreadFlush(uint32_t ThreadNum) {
+    ThreadFlushTracker |= (1 << ThreadNum);
+  }
+
+  // Reset this thread's flush bit. The caller must hold the flush lock
+  void resetThisThreadFlush() {
+    std::thread::id ID = std::this_thread::get_id();
+    assert(HelperThreadIdMap.find(ID) != HelperThreadIdMap.end());
+    ThreadFlushTracker &= ~(1 << HelperThreadIdMap[ID]);
+  }
+
+  // Given a thread number, set the corresponding bit in the shutdown
+  // tracker. The caller must hold the flush lock.
+  void setThreadShutdown(uint32_t ThreadNum) {
+    ThreadShutdownTracker |= (1 << ThreadNum);
+  }
+
+  // Reset this thread's shutdown bit. The caller must hold the flush
+  // lock
+  void resetThisThreadShutdown() {
+    std::thread::id ID = std::this_thread::get_id();
+    assert(HelperThreadIdMap.find(ID) != HelperThreadIdMap.end());
+    ThreadShutdownTracker &= ~(1 << HelperThreadIdMap[ID]);
+  }
+
+  // Return true if this thread's flush bit is set. The caller must
+  // hold the flush lock
+  bool isThisThreadFlushWaitedUpon() {
+    std::thread::id ID = std::this_thread::get_id();
+    assert(HelperThreadIdMap.find(ID) != HelperThreadIdMap.end());
+    return (ThreadFlushTracker & (1 << HelperThreadIdMap[ID])) != 0;
+  }
+
+  // Return true if this thread's shutdown bit is set. The caller must
+  // hold the flush lock
+  bool isThisThreadShutdownWaitedUpon() {
+    std::thread::id ID = std::this_thread::get_id();
+    assert(HelperThreadIdMap.find(ID) != HelperThreadIdMap.end());
+    return (ThreadShutdownTracker & (1 << HelperThreadIdMap[ID])) != 0;
+  }
+
+  // The caller must not hold the flush lock
+  bool amIHelperThread() {
+    std::unique_lock<std::mutex> flush_lock(FlushMutex);
+    if (HelperThreadIdMap.find(std::this_thread::get_id()) !=
+        HelperThreadIdMap.end())
+      return true;
+    return false;
+  }
+
+  // The caller must not hold the flush lock
+  bool areHelperThreadsAvailable();
+
+  // The caller must hold the appropriate lock
+  void init();
+
+  // The caller must hold the flush lock
+  void createHelperThreads();
+
+  // The caller must hold the flush lock
+  void destroyHelperThreads();
+
+public:
+  OmptTracingBufferMgr()
+      : OMPX_FlushOnBufferFull("LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL", true),
+        OMPX_FlushOnShutdown("LIBOMPTARGET_OMPT_FLUSH_ON_SHUTDOWN", true) {
+    // no need to hold locks for init() since object is getting constructed
+    // here.
+    init();
+  }
+
+  OmptTracingBufferMgr(const OmptTracingBufferMgr &) = delete;
+  OmptTracingBufferMgr &operator=(const OmptTracingBufferMgr &) = delete;
+
+  // The caller must not hold the flush lock
+  void startHelperThreads();
+
+  // The caller must not hold the flush lock. The helper threads are shut down
+  // without flushing any outstanding trace records.
+  void shutdownHelperThreads();
+
+  // The caller must not hold the flush lock. The helper threads are shut down
+  // after flushing all outstanding trace records for all devices.
+  void flushAndShutdownHelperThreads();
+
+  // Assign a cursor for a new trace record. This will assign a trace record
+  // for the provided device-id, allocating a new buffer if required.
+  void *assignCursor(ompt_callbacks_t Type, int64_t DeviceId);
+
+  // Get the size of a trace record
+  size_t getTRSize() { return sizeof(TraceRecord); }
+
+  // Get the status of a trace record. This function does not acquire
+  // a lock. If locking is required, the caller must hold a lock.
+  TRStatus getTRStatus(void *Rec);
+
+  // Set the status of a trace record. This function does not acquire
+  // a lock. If locking is required, the caller must hold a lock.
+  void setTRStatus(void *Rec, TRStatus);
+
+  // Is this a last cursor of a buffer completion callback?
+  bool isLastCursor(void *Cursor) {
+    std::unique_lock<std::mutex> Lock(LastCursorMutex);
+    return LastCursors.find(Cursor) != LastCursors.end();
+  }
+
+  // Called for flushing outstanding buffers for the provided device-id.
+  int flushAllBuffers(int DeviceId);
+};
+
+#else
+class OmptTracingBufferMgr {};
+#endif // OMPT_SUPPORT
+
+#endif // OPENMP_LIBOMPTARGET_OMPTTRACINGBUFFER_H
diff --git a/offload/include/PluginManager.h b/offload/include/PluginManager.h
index 6c6fdebe76dff..972afd962cbc1 100644
--- a/offload/include/PluginManager.h
+++ b/offload/include/PluginManager.h
@@ -13,6 +13,7 @@
 #ifndef OMPTARGET_PLUGIN_MANAGER_H
 #define OMPTARGET_PLUGIN_MANAGER_H
 
+#include "OpenMP/OMPT/OmptTracingBuffer.h"
 #include "PluginInterface.h"
 
 #include "DeviceImage.h"
@@ -48,7 +49,7 @@ struct PluginManager {
   /// Exclusive accessor type for the device container.
   using ExclusiveDevicesAccessorTy = Accessor<DeviceContainerTy>;
 
-  PluginManager() {}
+  PluginManager() : TraceRecordManager(nullptr) {}
 
   void init();
 
@@ -150,6 +151,11 @@ struct PluginManager {
     return count;
   }
 
+  auto getTraceRecordManager() const {
+    assert(TraceRecordManager && "Trace record manager not initialized");
+    return TraceRecordManager;
+  }
+
 private:
   bool RTLsLoaded = false;
   llvm::SmallVector<__tgt_bin_desc *> DelayedBinDesc;
@@ -176,6 +182,8 @@ struct PluginManager {
   /// Devices associated with plugins, accesses to the container are exclusive.
   ProtectedObj<DeviceContainerTy> Devices;
 
+  OmptTracingBufferMgr *TraceRecordManager;
+
   /// References to upgraded legacy offloading entries.
   std::list<llvm::SmallVector<llvm::offloading::EntryTy, 0>> LegacyEntries;
   std::list<llvm::SmallVector<__tgt_device_image, 0>> LegacyImages;
diff --git a/offload/libomptarget/CMakeLists.txt b/offload/libomptarget/CMakeLists.txt
index 7909d55eff5c3..3301c01ac90a3 100644
--- a/offload/libomptarget/CMakeLists.txt
+++ b/offload/libomptarget/CMakeLists.txt
@@ -17,6 +17,7 @@ add_library(omptarget SHARED
   OpenMP/Mapping.cpp
   OpenMP/InteropAPI.cpp
   OpenMP/OMPT/Callback.cpp
+  OpenMP/OMPT/OmptTracingBuffer.cpp
 
   KernelLanguage/API.cpp
 )
diff --git a/offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp b/offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp
new file mode 100644
index 0000000000000..5ac1974cea03b
--- /dev/null
+++ b/offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp
@@ -0,0 +1,758 @@
+//=== OmptTracingBuffer.cpp - Target independent OpenMP target RTL -- C++ -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementation of OMPT device trace record generation and flushing.
+//
+//===----------------------------------------------------------------------===//
+
+#ifdef OMPT_SUPPORT
+
+#include "OpenMP/OMPT/OmptTracingBuffer.h"
+#include "OpenMP/OMPT/OmptTracing.h"
+#include "Shared/Debug.h"
+
+#include <atomic>
+#include <cassert>
+#include <cstdlib>
+#include <limits>
+
+using namespace llvm::omp::target::debug;
+
+// When set to true, helper threads terminate their work
+static bool DoneTracing{false};
+
+// Unique buffer id in creation order
+static std::atomic<uint64_t> BufId{0};
+
+// Unique id in buffer flush order
+static std::atomic<uint64_t> FlushId{0};
+
+thread_local OmptTracingBufferMgr::BufPtr
+    OmptTracingBufferMgr::ArrayOfBufPtr[MAX_NUM_DEVICES];
+
+static uint64_t get_and_inc_buf_id() { return BufId++; }
+
+static uint64_t get_and_inc_flush_id() { return FlushId++; }
+static uint64_t get_flush_id() { return FlushId; }
+
+/*
+ * Used by OpenMP threads for assigning space for a trace record. If
+ * there is no space in the last buffer allocated by this thread, the
+ * last buffer is marked full and scheduled for flushing. Otherwise,
+ * space is assigned for a trace record and the new cursor returned.
+ * Since the memory allocated by a thread is used by that thread alone
+ * for creating trace records, a lock need not be held. In the less
+ * common branch when memory is allocated, a lock needs to be acquired
+ * for updating shared metadata. The common path of allocating a trace
+ * record from an existing buffer proceeds without locking.
+ */
+void *OmptTracingBufferMgr::assignCursor(ompt_callbacks_t Type,
+                                         int64_t DeviceId) {
+  // The caller should handle nullptr by not tracing for this event.
+  if (DeviceId < 0 || DeviceId > MAX_NUM_DEVICES - 1)
+    return nullptr;
+
+  size_t RecSize = getTRSize();
+
+  // If the buffer fills up, it will be scheduled for flushing with the
+  // following cursor.
+  void *ToBeFlushedCursor = nullptr;
+  BufPtr ToBeFlushedBuf = nullptr;
+
+  // Thread local buffer pointer should be non-null once an allocation
+  // has been done by this thread.
+  BufPtr DeviceBuf = getDeviceSpecificBuffer(DeviceId);
+  if (DeviceBuf != nullptr) {
+    assert(DeviceBuf->DeviceId == DeviceId && "Unexpected device id in buffer");
+    void *OldCursor = DeviceBuf->Cursor.load(std::memory_order_acquire);
+    // Try to assign a trace record from the last allocated buffer
+    if (RecSize <= DeviceBuf->RemainingBytes) {
+      assert((char *)DeviceBuf->Start + DeviceBuf->TotalBytes -
+                 DeviceBuf->RemainingBytes ==
+             (char *)OldCursor + RecSize);
+      DeviceBuf->RemainingBytes -= RecSize;
+
+      // Note the trace record status must be initialized before setting
+      // the cursor, ensuring that a helper thread always sees an initialized
+      // trace record status.
+      void *NewCursor = (char *)OldCursor + RecSize;
+      initTraceRecordMetaData(NewCursor);
+      DeviceBuf->Cursor.store(NewCursor, std::memory_order_release);
+
+      ODBG(ODT_Tool) << "Thread " << llvm::omp::target::ompt::getThreadId()
+                     << ": Assigned " << RecSize << " bytes at " << NewCursor
+                     << " in existing buffer " << DeviceBuf->Start
+                     << " for device " << DeviceId;
+      return NewCursor;
+    } else {
+      ToBeFlushedCursor = OldCursor;
+      ToBeFlushedBuf = DeviceBuf;
+
+      // Mark that no space is present for any more trace records.
+      // The following is atomic but there is no logical order between when
+      // it is set here and when it is checked by a helper thread. That works
+      // because the helper thread uses this info to decide whether a buffer
+      // can be scheduled for removal. In the worst case, the buffer will be
+      // removed late.
+      DeviceBuf->IsFull.store(true, std::memory_order_release);
+    }
+  }
+  void *NewBuffer = nullptr;
+  size_t TotalBytes = 0;
+  // TODO Move the buffer allocation to a helper thread
+  llvm::omp::target::ompt::ompt_callback_buffer_request(DeviceId, &NewBuffer,
+                                                        &TotalBytes);
+
+  // The caller should handle nullptr by not tracing for this event.
+  if (NewBuffer == nullptr || TotalBytes < RecSize)
+    return nullptr;
+
+  uint64_t NewBufId = get_and_inc_buf_id();
+  auto new_buf = std::make_shared<Buffer>(
+      NewBufId, DeviceId, /*Start=*/NewBuffer, TotalBytes,
+      /*RemainingBytes=*/TotalBytes - RecSize,
+      /*Cursor=*/NewBuffer,
+      /*IsFull=*/false);
+
+  // Initialize trace record status before publishing it to helper threads.
+  initTraceRecordMetaData(new_buf->Cursor.load(std::memory_order_acquire));
+  setDeviceSpecificBuffer(DeviceId, new_buf);
+
+  // Make this trace record visible to helper threads by adding to shared
+  // metadata.
+  std::unique_lock<std::mutex> lck(BufferMgrMutex);
+  assert(Id2BufferMap.find(NewBufId) == Id2BufferMap.end());
+  Id2BufferMap[NewBufId] = new_buf;
+  lck.unlock();
+
+  // Schedule the full buffer for flushing till the corresponding cursor.
+  if (OMPX_FlushOnBufferFull && ToBeFlushedCursor)
+    triggerFlushOnBufferFull(ToBeFlushedCursor, ToBeFlushedBuf);
+
+  ODBG(ODT_Tool) << "Thread " << llvm::omp::target::ompt::getThreadId()
+                 << ": Assigned " << RecSize << " bytes at " << NewBuffer
+                 << " in new buffer with id " << NewBufId << " for device "
+                 << DeviceId;
+  return NewBuffer;
+}
+
+/*
+ * Called by an OpenMP thread when a buffer fills up and should be
+ * flushed. This function assigns a new FlushId to the buffer, adds
+ * to the flush-related metadata and wakes up a helper thread to
+ * dispatch a buffer-completion callback. This function should be
+ * called without holding any lock.
+ * Note lock order: buf_lock -> flush_lock
+ */
+void OmptTracingBufferMgr::triggerFlushOnBufferFull(void *cursor, BufPtr Buf) {
+  std::unique_lock<std::mutex> buf_lock(BufferMgrMutex);
+
+  // Between calling this function and this check, a flush-all may have
+  // delivered this buffer to the tool and deleted it. So the buffer
+  // may not exist.
+  if (Id2BufferMap.find(Buf->Id) == Id2BufferMap.end())
+    return;
+
+  // Cannot assert that the state of the cursor is ready since a
+  // different thread may be in the process of populating it. If it
+  // remains in init state when the range of trace records is
+  // determined for dispatching the buffer-completion callback, it
+  // will not be included.
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  uint64_t flush_id;
+  auto flush_itr = FlushBufPtr2IdMap.find(Buf);
+  if (flush_itr == FlushBufPtr2IdMap.end()) {
+    // This buffer has not been flushed yet
+    addNewFlushEntry(Buf, cursor);
+  } else {
+    // This buffer has been flushed before
+    flush_id = flush_itr->second;
+    auto flush_md_itr = Id2FlushMdMap.find(flush_id);
+    assert(flush_md_itr != Id2FlushMdMap.end());
+    flush_md_itr->second.FlushCursor = cursor; // update the cursor
+    // Do not update the flush status since it may be under processing
+    // by another thread
+    ODBG(ODT_Tool) << "Updated id " << flush_id << " cursor " << cursor
+                   << " buf " << flush_md_itr->second.FlushBuf->Start;
+  }
+  flush_lock.unlock();
+  buf_lock.unlock();
+
+  // Wake up a helper thread to invoke the buffer-completion callback
+  FlushCv.notify_one();
+}
+
+// This is the driver routine for the completion thread
+void OmptTracingBufferMgr::driveCompletion() {
+  while (true) {
+    bool should_signal_workers = false;
+    std::unique_lock<std::mutex> flush_lock(FlushMutex);
+    if (DoneTracing) {
+      // An upper layer serializes flush_trace and stop_trace. In
+      // addition, before DoneTracing is set, a flush is performed as
+      // part of stop_trace. So assert that no flush is in progress.
+      assert(ThreadFlushTracker == 0);
+      break;
+    }
+    FlushCv.wait(flush_lock, [this] {
+      return DoneTracing ||
+             (!Id2FlushMdMap.empty() &&
+              llvm::omp::target::ompt::TracingActive) ||
+             isThisThreadFlushWaitedUpon();
+    });
+    if (isThisThreadFlushWaitedUpon()) {
+      resetThisThreadFlush();
+      if (ThreadFlushTracker == 0)
+        should_signal_workers = true;
+    }
+    flush_lock.unlock();
+
+    invokeCallbacks();
+
+    if (should_signal_workers)
+      ThreadFlushCv.notify_all();
+
+    // There is a scenario where a buffer was processed but not full
+    // or owned, so it was put back in waiting state. So this thread
+    // would not wait but keep on looping without having any actual
+    // work until new trace records are added and this thread
+    // signaled. Hence, this thread yields.
+    std::this_thread::yield();
+  }
+  bool is_last_helper = false;
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  assert(DoneTracing && "Helper thread exiting but not yet done");
+  assert(isThisThreadShutdownWaitedUpon() &&
+         "Helper thread exiting but not waited upon");
+  resetThisThreadShutdown();
+  if (ThreadShutdownTracker == 0)
+    is_last_helper = true;
+  flush_lock.unlock();
+  if (is_last_helper)
+    ThreadShutdownCv.notify_all();
+
+  // Note that some trace records may have been written but not
+  // delivered to the tool. If flush/stop APIs are not called by the
+  // tool, those trace records may never be delivered to the tool and
+  // the corresponding buffers not reclaimed. TODO Explore whether
+  // this cleanup must be done.
+}
+
+/*
+ * Called by a buffer-completion helper thread. This function examines
+ * the flushed buffers in flush order and dispatches
+ * callbacks. Lock holding is minimized by reserving a buffer,
+ * processing it, and then unreserving it if there are more trace
+ * records to flush later. If all trace records are flushed, a
+ * callback is dispatched informing the tool that the buffer can be
+ * deallocated. If the buffer can be deallocated, all metadata is
+ * destroyed.
+ * Note that this function must be called without holding any locks.
+ */
+void OmptTracingBufferMgr::invokeCallbacks() {
+  ODBG(ODT_Tool) << "Looking for callbacks to invoke";
+  auto max_id = std::numeric_limits<uint64_t>::max();
+  auto curr_id = max_id;
+  auto end_id = get_flush_id();
+  ODBG(ODT_Tool) << "End id is " << end_id;
+  while (true) {
+    // Set the status of the flushed buffer to in-processing so that
+    // another helper thread does not process it concurrently. An
+    // OpenMP worker thread may, however, populate a trace record in a
+    // reserved buffer concurrently.
+    FlushInfo flush_info = findAndReserveFlushedBuf(curr_id);
+
+    // no entry found, nothing to process
+    if (curr_id == max_id && flush_info.FlushCursor == nullptr)
+      return;
+
+    if (flush_info.FlushCursor != nullptr) {
+      // increment curr_id to get the candidate for the next iteration
+      curr_id = flush_info.FlushId + 1;
+    } else {
+      assert(curr_id != max_id && "Cannot increment max id");
+      ++curr_id;
+    }
+
+    ODBG(ODT_Tool) << "Next id will be " << curr_id;
+
+    if (flush_info.FlushCursor == nullptr) {
+      // This buffer must have been processed already
+      if (curr_id < end_id)
+        continue;
+      else
+        return; // nothing else to process
+    }
+
+    ODBG(ODT_Tool) << "Buf " << flush_info.FlushBuf->Start << " Cursor "
+                   << flush_info.FlushCursor << " Id " << flush_info.FlushId
+                   << " will be flushed";
+
+    // Examine the status of the trace records and dispatch
+    // buffer-completion callbacks as appropriate.
+    flushBuffer(flush_info);
+
+    // TODO optimize to set buffer-owned in the same pass above.
+    // Currently, this is the only way a buffer is deallocated
+    if (isBufferFull(flush_info)) {
+      // All trace records have been delivered to the tool
+      if (isBufferOwned(flush_info)) {
+        // erase element from buffer and flush maps
+        destroyFlushedBuf(flush_info);
+
+        // dispatch callback with a null range and have the tool
+        // deallocate the buffer
+        dispatchBufferOwnedCallback(flush_info);
+      } else {
+        unreserveFlushedBuf(flush_info);
+      }
+    } else {
+      unreserveFlushedBuf(flush_info);
+    }
+    if (curr_id >= end_id)
+      return;
+  }
+}
+
+/*
+ * This function is called on a buffer that is already reserved by
+ * this thread. Buffer-completion callbacks are dispatched for every
+ * range of trace records that are ready.
+ * This routine must be called without holding locks
+ */
+void OmptTracingBufferMgr::flushBuffer(FlushInfo flush_info) {
+  assert(flush_info.FlushBuf && "Cannot flush an empty buffer");
+  assert(flush_info.FlushCursor && "Cannot flush upto a null cursor");
+
+  void *curr_tr = flush_info.FlushBuf->Start;
+  void *last_tr = flush_info.FlushCursor;
+  // Compute a range [first_cursor,last_cursor] to flush
+  void *first_cursor = nullptr;
+  void *last_cursor = nullptr;
+  while (curr_tr <= last_tr) {
+    TRStatus tr_status = getTRStatus(curr_tr);
+    if (tr_status == TR_init || tr_status == TR_released) {
+      if (first_cursor == nullptr) {
+        // This TR won't be part of a range
+        assert(last_cursor == nullptr &&
+               "Begin/last cursors mutually inconsistent");
+      } else {
+        // End the current interval
+        dispatchCallback(flush_info.FlushBuf->DeviceId,
+                         flush_info.FlushBuf->Start, first_cursor, last_cursor);
+        first_cursor = last_cursor = nullptr;
+      }
+    } else {
+      assert(tr_status == TR_ready && "Unknown trace record status");
+      setTRStatus(curr_tr, TR_released);
+      if (first_cursor == nullptr)
+        first_cursor = curr_tr;
+      last_cursor = curr_tr;
+    }
+    curr_tr = getNextTR(curr_tr);
+  }
+  if (first_cursor != nullptr) {
+    assert(last_cursor != nullptr);
+    dispatchCallback(flush_info.FlushBuf->DeviceId, flush_info.FlushBuf->Start,
+                     first_cursor, last_cursor);
+  }
+}
+
+// Given a range of trace records, dispatch a buffer-completion callback
+void OmptTracingBufferMgr::dispatchCallback(int64_t DeviceId, void *Buffer,
+                                            void *FirstCursor,
+                                            void *LastCursor) {
+  assert(FirstCursor != nullptr && LastCursor != nullptr &&
+         "Callback with nullptr");
+  addLastCursor(LastCursor);
+
+  // This is best effort.
+  // There is a small window when the buffer-completion callback may
+  // be invoked even after tracing has been disabled.
+  // Note that we don't want to hold a lock when dispatching the callback.
+  if (llvm::omp::target::ompt::isTracedDevice(DeviceId)) {
+    ODBG(ODT_Tool) << "Dispatch callback w/ range (inclusive) to be flushed: "
+                   << FirstCursor << " -> " << LastCursor;
+    llvm::omp::target::ompt::ompt_callback_buffer_complete(
+        DeviceId, Buffer,
+        /* bytes returned in this callback */
+        (char *)getNextTR(LastCursor) - (char *)FirstCursor,
+        (ompt_buffer_cursor_t)FirstCursor, false /* buffer_owned */);
+  }
+
+  removeLastCursor(LastCursor);
+}
+
+// Dispatch a buffer-completion callback with buffer_owned set so that
+// the tool can deallocate the buffer
+void OmptTracingBufferMgr::dispatchBufferOwnedCallback(
+    const FlushInfo &flush_info) {
+  // This is best effort.
+  // There is a small window when the buffer-completion callback may
+  // be invoked even after tracing has been disabled.
+  // Note that we don't want to hold a lock when dispatching the callback.
+  if (llvm::omp::target::ompt::isTracedDevice(flush_info.FlushBuf->DeviceId)) {
+    ODBG(ODT_Tool) << "Dispatch callback with buffer "
+                   << flush_info.FlushBuf->Start << " owned";
+    llvm::omp::target::ompt::ompt_callback_buffer_complete(
+        flush_info.FlushBuf->DeviceId, flush_info.FlushBuf->Start, 0,
+        (ompt_buffer_cursor_t)0, true /* buffer owned */);
+  }
+}
+
+void OmptTracingBufferMgr::initTraceRecordMetaData(void *Rec) {
+  setTRStatus(Rec, TR_init);
+}
+
+OmptTracingBufferMgr::BufPtr
+OmptTracingBufferMgr::getDeviceSpecificBuffer(int64_t DeviceId) {
+  if (DeviceId < 0 || DeviceId > MAX_NUM_DEVICES - 1) {
+    REPORT() << "getDeviceSpecificBuffer: Device id " << DeviceId
+             << " invalid or exceeds supported max: " << MAX_NUM_DEVICES - 1;
+    return nullptr;
+  }
+  return ArrayOfBufPtr[DeviceId];
+}
+
+void OmptTracingBufferMgr::setDeviceSpecificBuffer(int64_t DeviceId,
+                                                   BufPtr Buf) {
+  if (DeviceId < 0 || DeviceId > MAX_NUM_DEVICES - 1) {
+    REPORT() << "setDeviceSpecificBuffer: Device id " << DeviceId
+             << " invalid or exceeds supported max: " << MAX_NUM_DEVICES - 1;
+    return;
+  }
+  ArrayOfBufPtr[DeviceId] = Buf;
+}
+
+void OmptTracingBufferMgr::setTRStatus(void *Rec, TRStatus Status) {
+  TraceRecord *TR = static_cast<TraceRecord *>(Rec);
+  TR->TRState.store(Status, std::memory_order_release);
+}
+
+OmptTracingBufferMgr::TRStatus OmptTracingBufferMgr::getTRStatus(void *Rec) {
+  return static_cast<TraceRecord *>(Rec)->TRState.load(
+      std::memory_order_acquire);
+}
+
+void *OmptTracingBufferMgr::getNextTR(void *TR) {
+  size_t RecSize = getTRSize();
+  // warning: no overflow check done
+  return (char *)TR + RecSize;
+}
+
+bool OmptTracingBufferMgr::isBufferFull(const FlushInfo &flush_info) {
+  std::unique_lock<std::mutex> buf_lock(BufferMgrMutex);
+  return flush_info.FlushBuf->IsFull;
+}
+
+void *OmptTracingBufferMgr::getBufferCursor(BufPtr buf) {
+  return buf->Cursor.load(std::memory_order_acquire);
+}
+
+/*
+ * Traverse all the trace records of a buffer and return true if all
+ * of them have been released to the tool, otherwise return false
+ */
+bool OmptTracingBufferMgr::isBufferOwned(const FlushInfo &flush_info) {
+  assert(isBufferFull(flush_info) && "Compute buffer-owned when it is full");
+  void *curr_tr = flush_info.FlushBuf->Start;
+  // Since the buffer is full, the cursor must be the last valid
+  // TR. Note that this may be more up-to-date than the cursor in the
+  // flush_info. Use the last valid TR to avoid dropping trace records
+  void *last_tr = getBufferCursor(flush_info.FlushBuf);
+  while (curr_tr <= last_tr) {
+    if (getTRStatus(curr_tr) != TR_released)
+      return false;
+    curr_tr = getNextTR(curr_tr);
+  }
+  return true;
+}
+
+/*
+ * A buffer must be reserved by a thread before it can be processed
+ * and callbacks dispatched for that buffer. Reservation is done by
+ * setting the status to in-processing.
+ *
+ * If a buffer is found in the flush metadata for the given id and it
+ * is not in in-processing mode, reserve it by setting its mode to
+ * in-processing and return the corresponding flush metadata. If the
+ * given id is set to max, return the first waiting buffer in the
+ * list of buffers to be flushed.
+ */
+OmptTracingBufferMgr::FlushInfo
+OmptTracingBufferMgr::findAndReserveFlushedBuf(uint64_t FlushId) {
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  MapId2Md::iterator flush_itr;
+  if (FlushId == std::numeric_limits<uint64_t>::max()) {
+    // Reserve the first waiting buffer and return it
+    if (Id2FlushMdMap.empty())
+      return FlushInfo();
+    for (flush_itr = Id2FlushMdMap.begin(); flush_itr != Id2FlushMdMap.end();
+         ++flush_itr) {
+      // Reserve only if waiting
+      if (flush_itr->second.FlushStatus == Flush_waiting)
+        break;
+    }
+    if (flush_itr == Id2FlushMdMap.end())
+      return FlushInfo();
+  } else {
+    flush_itr = Id2FlushMdMap.find(FlushId);
+    if (flush_itr == Id2FlushMdMap.end() ||
+        flush_itr->second.FlushStatus == Flush_processing)
+      return FlushInfo();
+  }
+  assert(flush_itr->second.FlushStatus == Flush_waiting);
+  flush_itr->second.FlushStatus = Flush_processing;
+  // Update the metadata cursor since more trace records may have been
+  // generated.
+  flush_itr->second.FlushCursor =
+      flush_itr->second.FlushBuf->Cursor.load(std::memory_order_acquire);
+
+  FlushInfo flush_info(flush_itr->first, flush_itr->second.FlushCursor,
+                       flush_itr->second.FlushBuf);
+  ODBG(ODT_Tool) << "Reserved buffer: flush_id:" << flush_itr->first
+                 << ", cursor:" << flush_itr->second.FlushCursor
+                 << ", buf:" << flush_itr->second.FlushBuf->Start;
+  return flush_info;
+}
+
+/*
+ * Given a buffer, verify that it is in processing state and set its
+ * status to waiting, removing the reservation. The same thread that
+ * reserved it should be unreserving it but currently there is no such
+ * check.
+ */
+void OmptTracingBufferMgr::unreserveFlushedBuf(const FlushInfo &flush_info) {
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  auto itr = Id2FlushMdMap.find(flush_info.FlushId);
+  assert(itr != Id2FlushMdMap.end() &&
+         itr->second.FlushStatus == Flush_processing);
+  itr->second.FlushStatus = Flush_waiting;
+  ODBG(ODT_Tool) << "Unreserved buffer: flush_id:" << flush_info.FlushId
+                 << ", cursor:" << flush_info.FlushCursor
+                 << ", buf:" << flush_info.FlushBuf->Start;
+}
+
+/*
+ * This function must be called after all of the trace records in the
+ * buffer have been released to the tool. The buffer is removed from
+ * all metadata maps.
+ * Note lock order: buf_lock -> flush_lock
+ */
+void OmptTracingBufferMgr::destroyFlushedBuf(const FlushInfo &flush_info) {
+  ODBG(ODT_Tool) << "Destroying buffer: flush_id:" << flush_info.FlushId
+                 << ", cursor:" << flush_info.FlushCursor
+                 << ", buf:" << flush_info.FlushBuf->Start;
+
+  BufPtr buf = flush_info.FlushBuf;
+
+  std::unique_lock<std::mutex> buf_lock(BufferMgrMutex);
+  Id2BufferMap.erase(buf->Id);
+
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  auto flush_itr = Id2FlushMdMap.find(flush_info.FlushId);
+  assert(flush_itr != Id2FlushMdMap.end());
+  assert(flush_itr->second.FlushBuf == buf);
+  Id2FlushMdMap.erase(flush_itr);
+  FlushBufPtr2IdMap.erase(buf);
+}
+
+/*
+ * Generate a new flush id and add the buffer to the flush metadata
+ * maps. This function must be called while holding the flush lock.
+ */
+uint64_t OmptTracingBufferMgr::addNewFlushEntry(BufPtr Buf, void *Cursor) {
+  assert(FlushBufPtr2IdMap.find(Buf) == FlushBufPtr2IdMap.end());
+  uint64_t FlushId = get_and_inc_flush_id();
+  FlushBufPtr2IdMap.emplace(Buf, FlushId);
+  assert(Id2FlushMdMap.find(FlushId) == Id2FlushMdMap.end());
+  Id2FlushMdMap.emplace(FlushId, FlushMd(Cursor, Buf, Flush_waiting));
+
+  ODBG(ODT_Tool) << "Added new flush id " << FlushId << " cursor " << Cursor
+                 << " buf " << Buf->Start;
+
+  return FlushId;
+}
+
+/*
+ * Called by ompt_flush_trace and ompt_stop_trace. Traverse the
+ * existing buffers in creation order and flush all the ready TRs
+ */
+int OmptTracingBufferMgr::flushAllBuffers(int DeviceId) {
+  ODBG(ODT_Tool) << "Flushing buffers for device " << DeviceId << " :: START";
+  // Overloading MAX_NUM_DEVICES to mean all devices.
+  if (DeviceId < 0 || DeviceId > MAX_NUM_DEVICES)
+    return 0; // failed to flush
+
+  if (!areHelperThreadsAvailable())
+    return 0; // failed to flush
+
+  // If flush is called from a helper thread, just bail out
+  if (amIHelperThread())
+    return 0; // failed to flush
+
+  // To avoid holding the mutex for too long, get the ids of the first
+  // and the last TRs under lock, and then go through that range,
+  // holding the mutex for an individual TR
+  std::unique_lock<std::mutex> buf_lock(BufferMgrMutex);
+  if (Id2BufferMap.empty())
+    return 1; // no trace records to flush
+  uint64_t curr_buf_id = Id2BufferMap.begin()->first;
+  uint64_t last_buf_id = Id2BufferMap.rbegin()->first;
+  buf_lock.unlock();
+
+  while (curr_buf_id <= last_buf_id) {
+    std::unique_lock<std::mutex> buf_lock(BufferMgrMutex);
+    // Another thread may have deleted this buffer by now
+    auto buf_itr = Id2BufferMap.find(curr_buf_id);
+    if (buf_itr == Id2BufferMap.end()) {
+      ++curr_buf_id;
+      continue;
+    }
+    BufPtr curr_buf = buf_itr->second;
+
+    // If the device-id does not match, skip it. A device-id of MAX_NUM_DEVICES
+    // indicates flushing for all devices.
+    if (DeviceId != MAX_NUM_DEVICES && curr_buf->DeviceId != DeviceId) {
+      ++curr_buf_id;
+      continue;
+    }
+
+    // If this buffer is in the flush-map, skip it. It is either in
+    // process by another thread or will be processed
+    std::unique_lock<std::mutex> flush_lock(FlushMutex);
+    auto flush_itr = FlushBufPtr2IdMap.find(curr_buf);
+    if (flush_itr != FlushBufPtr2IdMap.end()) {
+      ++curr_buf_id;
+      continue;
+    }
+    // This buffer has not been flushed yet
+    void *CurrBufCursor = getBufferCursor(curr_buf);
+    uint64_t flush_id = addNewFlushEntry(curr_buf, CurrBufCursor);
+    (void)flush_id; // Silence warning.
+    ODBG(ODT_Tool) << "flushAllBuffers: Added new id " << flush_id << " cursor "
+                   << CurrBufCursor << " buf " << curr_buf->Start;
+
+    flush_lock.unlock();
+    buf_lock.unlock();
+
+    ++curr_buf_id;
+  }
+
+  ODBG(ODT_Tool) << "Flushing buffers for device " << DeviceId << " :: WAIT";
+
+  // This is best effort. It is possible that some trace records are
+  // not flushed when the wait is done.
+  waitForFlushCompletion();
+
+  ODBG(ODT_Tool) << "Flushing buffers for device " << DeviceId << " :: STOP";
+
+  return 1; // success
+}
+
+void OmptTracingBufferMgr::waitForFlushCompletion() {
+  {
+    std::unique_lock<std::mutex> flush_lock(FlushMutex);
+    // Setting the flush bit for a given helper thread indicates that the worker
+    // thread is ready for the helper thread to do some work.
+    for (uint32_t i = 0; i < OMPT_NUM_HELPER_THREADS; ++i)
+      setThreadFlush(i);
+  }
+
+  // Wake up all helper threads to invoke buffer-completion callbacks.
+  FlushCv.notify_all();
+
+  // Now wait for all helper threads  to complete flushing.
+  {
+    std::unique_lock<std::mutex> flush_lock(FlushMutex);
+    ThreadFlushCv.wait(flush_lock, [this] { return ThreadFlushTracker == 0; });
+  }
+}
+
+void OmptTracingBufferMgr::init() {
+  for (int i = 0; i < MAX_NUM_DEVICES; ++i)
+    ArrayOfBufPtr[i] = nullptr;
+  ThreadFlushTracker = 0;
+  ThreadShutdownTracker = 0;
+  DoneTracing = false; // TODO make it a class member
+}
+
+void OmptTracingBufferMgr::startHelperThreads() {
+  // All helper threads are stopped while holding FlushMutex. So if
+  // any helper thread is present, just return. This takes care of
+  // repeated calls to start-trace.
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  if (!HelperThreadIdMap.empty()) {
+    assert(!DoneTracing && "Helper threads exist but tracing is done");
+    return;
+  }
+  init();
+  createHelperThreads();
+}
+
+bool OmptTracingBufferMgr::areHelperThreadsAvailable() {
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  if (DoneTracing // If another thread called stop, assume there are no threads
+      || HelperThreadIdMap.empty() // Threads were never started
+  ) {
+    // Don't assert on HelperThreadIdMap since shutdown by another
+    // thread may be in progress
+    return false;
+  }
+  return true;
+}
+
+void OmptTracingBufferMgr::shutdownHelperThreads() {
+  if (!areHelperThreadsAvailable())
+    return;
+
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  // If I am destroying the threads, then at least one thread must be present
+  assert(!CompletionThreads.empty());
+  assert(!HelperThreadIdMap.empty());
+  assert(ThreadShutdownTracker == 0);
+
+  // Set the done flag which helper threads will look at
+  DoneTracing = true;
+  // Wait to make sure all helper threads exit
+  for (uint32_t i = 0; i < OMPT_NUM_HELPER_THREADS; ++i)
+    setThreadShutdown(i);
+  // Signal indicating that DoneTracing is set
+  FlushCv.notify_all();
+  ThreadShutdownCv.wait(flush_lock,
+                        [this] { return ThreadShutdownTracker == 0; });
+
+  // Now destroy all the helper threads
+  destroyHelperThreads();
+}
+
+void OmptTracingBufferMgr::flushAndShutdownHelperThreads() {
+  std::unique_lock<std::mutex> Lock(llvm::omp::target::ompt::TraceControlMutex);
+  // Flush buffers for all devices.
+  if (OMPX_FlushOnShutdown)
+    flushAllBuffers(MAX_NUM_DEVICES);
+  else
+    waitForFlushCompletion(); // Dont initiate but wait for outstanding flushes.
+  shutdownHelperThreads();
+}
+
+void OmptTracingBufferMgr::createHelperThreads() {
+  for (uint32_t i = 0; i < OMPT_NUM_HELPER_THREADS; ++i) {
+    CompletionThreads.emplace_back(
+        std::thread(&OmptTracingBufferMgr::driveCompletion, this));
+    HelperThreadIdMap[CompletionThreads.back().get_id()] = i;
+  }
+}
+
+void OmptTracingBufferMgr::destroyHelperThreads() {
+  for (auto &thd : CompletionThreads)
+    thd.join();
+  CompletionThreads.clear();
+  HelperThreadIdMap.clear();
+}
+#endif

>From 9335c95349c578e38171252f2766ea551cf197fb Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 2 Apr 2026 07:52:40 -0500
Subject: [PATCH 07/11] [Offload][OMPT] Add tracing orchestration and
 libomptarget integration

Wire the OMPT device tracing subsystem into libomptarget, completing the tracing pipeline from record production through buffer management to tool delivery. Include the synchronous fallback and initialization fixes in the integration that requires them.

Co-Authored-By: Claude <noreply at anthropic.com>
---
 offload/libomptarget/CMakeLists.txt           |   1 +
 offload/libomptarget/OpenMP/OMPT/Callback.cpp |  12 +
 .../libomptarget/OpenMP/OMPT/OmptTracing.cpp  | 891 ++++++++++++++++++
 offload/libomptarget/PluginManager.cpp        |  22 +
 offload/libomptarget/device.cpp               |  58 +-
 offload/libomptarget/exports                  |   8 +
 offload/libomptarget/interface.cpp            |  44 +-
 offload/libomptarget/omptarget.cpp            |   6 +-
 8 files changed, 1014 insertions(+), 28 deletions(-)
 create mode 100644 offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp

diff --git a/offload/libomptarget/CMakeLists.txt b/offload/libomptarget/CMakeLists.txt
index 3301c01ac90a3..7f2ff7329a536 100644
--- a/offload/libomptarget/CMakeLists.txt
+++ b/offload/libomptarget/CMakeLists.txt
@@ -17,6 +17,7 @@ add_library(omptarget SHARED
   OpenMP/Mapping.cpp
   OpenMP/InteropAPI.cpp
   OpenMP/OMPT/Callback.cpp
+  OpenMP/OMPT/OmptTracing.cpp
   OpenMP/OMPT/OmptTracingBuffer.cpp
 
   KernelLanguage/API.cpp
diff --git a/offload/libomptarget/OpenMP/OMPT/Callback.cpp b/offload/libomptarget/OpenMP/OMPT/Callback.cpp
index 1e03f1455d1b2..071276b3903ca 100644
--- a/offload/libomptarget/OpenMP/OMPT/Callback.cpp
+++ b/offload/libomptarget/OpenMP/OMPT/Callback.cpp
@@ -12,9 +12,13 @@
 
 #ifdef OMPT_SUPPORT
 
+#include <atomic>
+#include <cassert>
 #include <cstdlib>
 #include <cstring>
 #include <memory>
+#include <mutex>
+#include <thread>
 
 #include "Shared/Debug.h"
 
@@ -23,7 +27,9 @@
 #include "OpenMP/OMPT/Interface.h"
 
 #include "llvm/Support/DynamicLibrary.h"
+#include "llvm/Support/ErrorHandling.h"
 
+#pragma push_macro("DEBUG_PREFIX")
 #undef DEBUG_PREFIX
 #define DEBUG_PREFIX "OMPT"
 
@@ -441,6 +447,11 @@ void Interface::endTarget(int64_t DeviceId, void *Code) {
   endTargetRegion();
 }
 
+void Interface::announceTargetRegion(const char *RegionName) {
+  ODBG(ODT_Tool) << "in Interface::target_region_" << RegionName
+                 << " target_id=" << TargetData.value;
+}
+
 void Interface::beginTargetDataOperation() {
   ODBG(ODT_Tool) << "in ompt_target_region_begin (TargetRegionId = "
                  << TargetData.value << ")";
@@ -562,4 +573,5 @@ void llvm::omp::target::ompt::connectLibrary() {
   ODBG(ODT_Tool) << "Exiting connectLibrary";
 }
 
+#pragma pop_macro("DEBUG_PREFIX")
 #endif // OMPT_SUPPORT
diff --git a/offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp b/offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp
new file mode 100644
index 0000000000000..77c771786f29b
--- /dev/null
+++ b/offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp
@@ -0,0 +1,891 @@
+//===-- OmptTracing.cpp - Target independent OpenMP target RTL --- C++ -*--===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementation of OMPT tracing interfaces for target independent layer
+//
+//===----------------------------------------------------------------------===//
+
+#ifdef OMPT_SUPPORT
+
+#include "OpenMP/OMPT/OmptTracing.h"
+#include "OpenMP/OMPT/Callback.h"
+#include "OpenMP/OMPT/Interface.h"
+#include "OpenMP/OMPT/OmptTracingBuffer.h"
+#include "PluginManager.h"
+#include "Shared/Debug.h"
+#include "omp-tools.h"
+
+#include "llvm/Support/DynamicLibrary.h"
+
+#include <atomic>
+#include <cassert>
+#include <cstdlib>
+#include <cstring>
+#include <memory>
+#include <mutex>
+#include <thread>
+
+#pragma push_macro("DEBUG_PREFIX")
+#undef DEBUG_PREFIX
+#define DEBUG_PREFIX "OMPT"
+
+using namespace llvm::omp::target::ompt;
+
+std::mutex llvm::omp::target::ompt::DeviceAccessMutex;
+std::mutex llvm::omp::target::ompt::TraceAccessMutex;
+std::mutex llvm::omp::target::ompt::TraceControlMutex;
+std::mutex llvm::omp::target::ompt::TraceHashThreadMutex;
+std::mutex llvm::omp::target::ompt::BufferManagementFnMutex;
+
+std::unordered_map<int /*DeviceId*/, std::pair<ompt_callback_buffer_request_t,
+                                               ompt_callback_buffer_complete_t>>
+    llvm::omp::target::ompt::BufferManagementFns;
+
+thread_local uint32_t llvm::omp::target::ompt::TraceRecordNumGrantedTeams = 0;
+thread_local uint64_t llvm::omp::target::ompt::TraceRecordStartTime = 0;
+thread_local uint64_t llvm::omp::target::ompt::TraceRecordStopTime = 0;
+thread_local uint64_t llvm::omp::target::ompt::ThreadId =
+    std::numeric_limits<uint64_t>::max();
+
+std::map<int32_t, uint64_t> llvm::omp::target::ompt::TracedDevices;
+
+bool llvm::omp::target::ompt::TracingActive = false;
+
+void llvm::omp::target::ompt::resetTimestamp(uint64_t *T) { *T = 0; }
+
+ompt_callback_buffer_request_t
+llvm::omp::target::ompt::getBufferRequestFn(int DeviceId) {
+  std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
+  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
+  if (BufferMgrItr == BufferManagementFns.end()) {
+    return nullptr;
+  }
+  return BufferMgrItr->second.first;
+}
+
+ompt_callback_buffer_complete_t
+llvm::omp::target::ompt::getBufferCompleteFn(int DeviceId) {
+  std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
+  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
+  if (BufferMgrItr == BufferManagementFns.end()) {
+    return nullptr;
+  }
+  return BufferMgrItr->second.second;
+}
+
+void llvm::omp::target::ompt::setBufferManagementFns(
+    int DeviceId, ompt_callback_buffer_request_t ReqFn,
+    ompt_callback_buffer_complete_t CmpltFn) {
+  std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
+  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
+  if (BufferMgrItr != BufferManagementFns.end()) {
+    REPORT()
+        << "Buffer request and complete functions already exist for device  "
+        << DeviceId << "ignoring ...";
+    return;
+  }
+  BufferManagementFns[DeviceId] = std::make_pair(ReqFn, CmpltFn);
+}
+
+void llvm::omp::target::ompt::removeBufferManagementFns(int DeviceId) {
+  std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
+  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
+  if (BufferMgrItr == BufferManagementFns.end()) {
+    REPORT() << "Buffer request and complete functions don't exist for device  "
+             << DeviceId << "ignoring ...";
+    return;
+  }
+  BufferManagementFns.erase(BufferMgrItr);
+}
+
+bool llvm::omp::target::ompt::isAllDeviceTracingStopped() {
+  std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
+  return BufferManagementFns.empty();
+}
+
+void llvm::omp::target::ompt::ompt_callback_buffer_request(
+    int DeviceId, ompt_buffer_t **BufferPtr, size_t *Bytes) {
+  if (auto Fn = getBufferRequestFn(DeviceId))
+    Fn(DeviceId, BufferPtr, Bytes);
+}
+
+void llvm::omp::target::ompt::ompt_callback_buffer_complete(
+    int DeviceId, ompt_buffer_t *Buffer, size_t Bytes,
+    ompt_buffer_cursor_t BeginCursor, int BufferOwned) {
+  if (auto Fn = getBufferCompleteFn(DeviceId))
+    Fn(DeviceId, Buffer, Bytes, BeginCursor, BufferOwned);
+}
+
+inline void setDeviceTracing(uint64_t &TracingTypes) {
+  // Set bit 0 to indicate generally enabled device tracing.
+  TracingTypes |= 1UL;
+}
+
+inline void resetDeviceTracing(uint64_t &TracingTypes) {
+  // Reset bit 0 to indicate generally disabled device tracing.
+  TracingTypes &= ~(1UL);
+}
+
+inline bool checkDeviceTracingState(const uint64_t &TracingTypes) {
+  // Return state of bit 0 to indicate if device is actively traced.
+  return TracingTypes & 1UL;
+}
+
+void llvm::omp::target::ompt::enableDeviceTracing(int DeviceId) {
+  std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
+  auto Device = TracedDevices.find(DeviceId);
+  if (Device == TracedDevices.end()) {
+    uint64_t TracingTypes{0};
+    setDeviceTracing(TracingTypes);
+    TracedDevices.emplace(DeviceId, TracingTypes);
+  } else
+    setDeviceTracing(Device->second);
+  // In any case: at least one device is traced
+  TracingActive = true;
+}
+
+void llvm::omp::target::ompt::disableDeviceTracing(int DeviceId) {
+  std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
+  auto Device = TracedDevices.find(DeviceId);
+  if (Device == TracedDevices.end()) {
+    uint64_t TracingTypes{0};
+    resetDeviceTracing(TracingTypes);
+    TracedDevices.emplace(DeviceId, TracingTypes);
+  } else
+    resetDeviceTracing(Device->second);
+
+  // Check for actively traced devices
+  for (auto &Dev : TracedDevices)
+    if (checkDeviceTracingState(Dev.second))
+      return;
+
+  // If no device is currently traced: set global tracing flag to false
+  TracingActive = false;
+}
+
+bool llvm::omp::target::ompt::isTracingEnabled(int DeviceId,
+                                               unsigned int EventTy) {
+  return TracingActive && isTracedDevice(DeviceId) &&
+         isTracingTypeGroupEnabled(DeviceId, EventTy);
+}
+
+bool llvm::omp::target::ompt::isTracedDevice(int DeviceId) {
+  std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
+  auto Device = TracedDevices.find(DeviceId);
+  if (Device != TracedDevices.end())
+    return checkDeviceTracingState(Device->second);
+
+  return false;
+}
+
+bool llvm::omp::target::ompt::isTracingTypeEnabled(int DeviceId,
+                                                   unsigned int EventTy) {
+  std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
+  // Make sure we do not shift more than std::numeric_limits<uint64_t>::digits
+  assert(EventTy < 64 && "Shift limit exceeded: EventTy must be less than 64");
+  auto Device = TracedDevices.find(DeviceId);
+  if (Device != TracedDevices.end() && EventTy < 64)
+    return (Device->second & (1UL << EventTy));
+  return false;
+}
+
+bool llvm::omp::target::ompt::isTracingTypeGroupEnabled(int DeviceId,
+                                                        unsigned int EventTy) {
+  std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
+  // Make sure we do not shift more than std::numeric_limits<uint64_t>::digits
+  assert(EventTy < 64 && "Shift limit exceeded: EventTy must be less than 64");
+  auto Device = TracedDevices.find(DeviceId);
+  if (Device != TracedDevices.end() && EventTy < 64) {
+    auto TracedEvents = Device->second;
+    switch (EventTy) {
+    case ompt_callbacks_t::ompt_callback_target:
+    case ompt_callbacks_t::ompt_callback_target_emi:
+      return ((TracedEvents & (1UL << ompt_callback_target))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_emi)));
+    case ompt_callbacks_t::ompt_callback_target_data_op:
+    case ompt_callbacks_t::ompt_callback_target_data_op_emi:
+      return ((TracedEvents & (1UL << ompt_callback_target_data_op))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_data_op_emi)));
+    case ompt_callbacks_t::ompt_callback_target_submit:
+    case ompt_callbacks_t::ompt_callback_target_submit_emi:
+      return ((TracedEvents & (1UL << ompt_callback_target_submit))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_submit_emi)));
+    // Special case: EventTy == 0 -> Check all EventTy
+    case 0:
+      return ((TracedEvents & (1UL << ompt_callback_target))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_emi))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_data_op))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_data_op_emi))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_submit))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_submit_emi)));
+    }
+  }
+  return false;
+}
+
+void llvm::omp::target::ompt::setTracingTypeEnabled(uint64_t &TracedEventTy,
+                                                    bool Enable,
+                                                    unsigned int EventTy) {
+  // Make sure we do not shift more than std::numeric_limits<uint64_t>::digits
+  assert(EventTy < 64 && "Shift limit exceeded: EventTy must be less than 64");
+  if (EventTy < 64) {
+    if (Enable)
+      TracedEventTy |= (1UL << EventTy);
+    else
+      TracedEventTy &= ~(1UL << EventTy);
+  }
+}
+
+ompt_set_result_t
+llvm::omp::target::ompt::setTraceEventTy(int DeviceId, unsigned int Enable,
+                                         unsigned int EventTy) {
+  if (DeviceId < 0) {
+    REPORT() << "Failed to set trace event type for DeviceId=" << DeviceId;
+    return ompt_set_never;
+  }
+
+  ODBG(debug::ODT_Tool) << "Executing setTraceEventTy: DeviceId=" << DeviceId
+                        << " Enable=" << Enable << " EventTy=" << EventTy;
+
+  std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
+  if (TracedDevices.find(DeviceId) == TracedDevices.end())
+    TracedDevices.emplace(DeviceId, 0UL);
+
+  auto &TracedEventTy = TracedDevices[DeviceId];
+  bool Enabled = Enable > 0;
+  if (EventTy == 0) {
+    // Set / reset all supported types
+    setTracingTypeEnabled(TracedEventTy, Enabled,
+                          ompt_callbacks_t::ompt_callback_target);
+    setTracingTypeEnabled(TracedEventTy, Enabled,
+                          ompt_callbacks_t::ompt_callback_target_data_op);
+    setTracingTypeEnabled(TracedEventTy, Enabled,
+                          ompt_callbacks_t::ompt_callback_target_submit);
+    setTracingTypeEnabled(TracedEventTy, Enabled,
+                          ompt_callbacks_t::ompt_callback_target_emi);
+    setTracingTypeEnabled(TracedEventTy, Enabled,
+                          ompt_callbacks_t::ompt_callback_target_data_op_emi);
+    setTracingTypeEnabled(TracedEventTy, Enabled,
+                          ompt_callbacks_t::ompt_callback_target_submit_emi);
+
+    if (Enabled) {
+      // Event subset is enabled
+      return ompt_set_sometimes;
+    } else {
+      // All events are disabled
+      return ompt_set_always;
+    }
+  }
+
+  switch (EventTy) {
+  case ompt_callbacks_t::ompt_callback_target:
+  case ompt_callbacks_t::ompt_callback_target_data_op:
+  case ompt_callbacks_t::ompt_callback_target_submit:
+  case ompt_callbacks_t::ompt_callback_target_emi:
+  case ompt_callbacks_t::ompt_callback_target_data_op_emi:
+  case ompt_callbacks_t::ompt_callback_target_submit_emi: {
+    setTracingTypeEnabled(TracedEventTy, Enabled, EventTy);
+    return ompt_set_always;
+  }
+  default: {
+    if (Enabled) {
+      // Unimplemented
+      return ompt_set_never;
+    } else {
+      // Always disabled anyways
+      return ompt_set_always;
+    }
+  }
+  }
+}
+
+uint64_t llvm::omp::target::ompt::getThreadId() {
+  // Grab the value from thread local storage, if valid.
+  if (ThreadId != std::numeric_limits<uint64_t>::max())
+    return ThreadId;
+  // Otherwise set it, protecting the hash with a lock.
+  std::unique_lock<std::mutex> Lock(TraceHashThreadMutex);
+  ThreadId = std::hash<std::thread::id>()(std::this_thread::get_id());
+  return ThreadId;
+}
+
+void Interface::setTraceRecordCommon(ompt_record_ompt_t *DataPtr,
+                                     ompt_callbacks_t CallbackType) {
+  DataPtr->type = CallbackType;
+
+  if (CallbackType == ompt_callback_target)
+    DataPtr->time = 0; // Currently, no consumer, so no need to set it
+  else {
+    DataPtr->time = TraceRecordStartTime;
+    resetTimestamp(&TraceRecordStartTime);
+  }
+
+  DataPtr->thread_id = getThreadId();
+  DataPtr->target_id = TargetData.value;
+}
+
+void Interface::setTraceRecordTargetDataOp(ompt_record_target_data_op_t *Record,
+                                           ompt_target_data_op_t DataOpType,
+                                           void *SrcAddr, int64_t SrcDeviceNum,
+                                           void *DstAddr, int64_t DstDeviceNum,
+                                           size_t Bytes, void *CodePtr) {
+  Record->host_op_id = HostOpId;
+  Record->optype = DataOpType;
+  Record->src_addr = SrcAddr;
+  Record->src_device_num = SrcDeviceNum;
+  Record->dest_addr = DstAddr;
+  Record->dest_device_num = DstDeviceNum;
+  Record->bytes = Bytes;
+
+  Record->end_time = TraceRecordStopTime;
+  resetTimestamp(&TraceRecordStopTime);
+
+  Record->codeptr_ra = CodePtr;
+}
+
+void Interface::setTraceRecordTargetKernel(ompt_record_target_kernel_t *Record,
+                                           unsigned int NumTeams) {
+  Record->host_op_id = HostOpId;
+  Record->requested_num_teams = NumTeams;
+  Record->granted_num_teams = TraceRecordNumGrantedTeams;
+
+  Record->end_time = TraceRecordStopTime;
+  resetTimestamp(&TraceRecordStopTime);
+}
+
+void Interface::setTraceRecordTarget(ompt_record_target_t *Record,
+                                     int64_t DeviceId, ompt_target_t TargetKind,
+                                     ompt_scope_endpoint_t Endpoint,
+                                     void *CodePtr) {
+  Record->kind = TargetKind;
+  Record->endpoint = Endpoint;
+  Record->device_num = DeviceId;
+  assert(TaskData);
+  Record->task_id = TaskData->value;
+  Record->target_id = TargetData.value;
+  Record->codeptr_ra = CodePtr;
+}
+
+void Interface::startTargetDataAllocTrace(int64_t DeviceId, void *HstPtrBegin,
+                                          void **TgtPtrBegin, size_t Size,
+                                          void *Code) {}
+
+ompt_record_ompt_t *Interface::stopTargetDataAllocTrace(int64_t DeviceId,
+                                                        void *HstPtrBegin,
+                                                        void **TgtPtrBegin,
+                                                        size_t Size,
+                                                        void *Code) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target_data_op))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr = (ompt_record_ompt_t *)TRM->assignCursor(
+      ompt_callback_target_data_op, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target_data_op);
+  setTraceRecordTargetDataOp(&DataPtr->record.target_data_op,
+                             ompt_target_data_alloc, HstPtrBegin,
+                             /*SrcDeviceNum=*/omp_get_initial_device(),
+                             *TgtPtrBegin, DeviceId, Size, Code);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Generated trace record: " << DataPtr
+                        << " (ompt_target_data_alloc)";
+  return DataPtr;
+}
+
+void Interface::startTargetDataDeleteTrace(int64_t DeviceId, void *TgtPtrBegin,
+                                           void *Code) {}
+
+ompt_record_ompt_t *Interface::stopTargetDataDeleteTrace(int64_t DeviceId,
+                                                         void *TgtPtrBegin,
+                                                         void *Code) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target_data_op))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr = (ompt_record_ompt_t *)TRM->assignCursor(
+      ompt_callback_target_data_op, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target_data_op);
+  setTraceRecordTargetDataOp(&DataPtr->record.target_data_op,
+                             ompt_target_data_delete, TgtPtrBegin, DeviceId,
+                             /*DstAddr=*/nullptr,
+                             /*DstDeviceNum=*/-1, /*Bytes=*/0, Code);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Generated trace record: " << DataPtr
+                        << " (ompt_target_data_delete)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *
+Interface::startTargetDataSubmitTrace(int64_t SrcDeviceId, void *SrcPtrBegin,
+                                      int64_t DstDeviceId, void *DstPtrBegin,
+                                      size_t Size, void *Code) {
+  if (!isTracingEnabled(DstDeviceId, ompt_callback_target_data_op))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr = (ompt_record_ompt_t *)TRM->assignCursor(
+      ompt_callback_target_data_op, DstDeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target_data_op);
+  DataPtr->time = 0; // Set to sanity value and let "stop" function fix it
+
+  // Set some of the data-op specific fields here
+  setTraceRecordTargetDataOp(&DataPtr->record.target_data_op,
+                             ompt_target_data_transfer_to_device, SrcPtrBegin,
+                             SrcDeviceId, DstPtrBegin, DstDeviceId, Size, Code);
+
+  ODBG(debug::ODT_Tool) << "OMPT-Async: Returning data trace record buf ptr "
+                        << DataPtr;
+  return DataPtr;
+}
+
+ompt_record_ompt_t *
+Interface::startTargetDataRetrieveTrace(int64_t SrcDeviceId, void *SrcPtrBegin,
+                                        int64_t DstDeviceId, void *DstPtrBegin,
+                                        size_t Size, void *Code) {
+  if (!isTracingEnabled(SrcDeviceId, ompt_callback_target_data_op))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr = (ompt_record_ompt_t *)TRM->assignCursor(
+      ompt_callback_target_data_op, SrcDeviceId);
+
+  if (!DataPtr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target_data_op);
+  DataPtr->time = 0; // Set to sanity value and let "stop" function fix it
+
+  // Set some of the data-op specific fields here
+  setTraceRecordTargetDataOp(&DataPtr->record.target_data_op,
+                             ompt_target_data_transfer_from_device, SrcPtrBegin,
+                             SrcDeviceId, DstPtrBegin, DstDeviceId, Size, Code);
+
+  ODBG(debug::ODT_Tool) << "OMPT-Async: Returning data trace record buf ptr "
+                        << DataPtr;
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::stopTargetDataMovementTraceAsync(
+    ompt_record_ompt_t *DataPtr, uint64_t NanosStart, uint64_t NanosEnd) {
+  // Finalize the data that comes from the plugin.
+  DataPtr->time = NanosStart;
+  auto Record = static_cast<ompt_record_target_data_op_t *>(
+      &DataPtr->record.target_data_op);
+  Record->end_time = NanosEnd;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "OMPT-Async: Completed target_data trace record "
+                        << DataPtr;
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::startTargetSubmitTrace(int64_t DeviceId,
+                                                      unsigned int NumTeams) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target_submit))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr = (ompt_record_ompt_t *)TRM->assignCursor(
+      ompt_callback_target_submit, DeviceId);
+
+  // Set all known entries and leave remaining to the stop function
+  setTraceRecordCommon(DataPtr, ompt_callback_target_submit);
+  DataPtr->time = 0; // Set to sanity value and let "stop" function fix it
+  // Kernel specific things
+  DataPtr->record.target_kernel.requested_num_teams = NumTeams;
+  DataPtr->record.target_kernel.host_op_id = getHostOpId();
+
+  // May be null if event is not traced
+  ODBG(debug::ODT_Tool) << "OMPT-Async: Returning kernel trace record buf ptr "
+                        << DataPtr;
+  return DataPtr;
+}
+
+ompt_record_ompt_t *
+Interface::stopTargetSubmitTraceAsync(ompt_record_ompt_t *DataPtr,
+                                      unsigned int NumTeams,
+                                      uint64_t NanosStart, uint64_t NanosStop) {
+  // Common fields
+  DataPtr->time = NanosStart;
+  // Submit specific
+  DataPtr->record.target_kernel.end_time = NanosStop;
+  DataPtr->record.target_kernel.granted_num_teams = NumTeams;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  // Ready Record
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "OMPT-Async: Completed trace record buf ptr "
+                        << DataPtr;
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::startTargetDataEnterTrace(int64_t DeviceId,
+                                                         void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId,
+                       ompt_target_enter_data, ompt_scope_begin, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Returning trace record buf ptr " << DataPtr
+                        << " (ompt_target_enter_data)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::stopTargetDataEnterTrace(int64_t DeviceId,
+                                                        void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId,
+                       ompt_target_enter_data, ompt_scope_end, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Generated trace record " << DataPtr
+                        << " (ompt_target_enter_data)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::startTargetDataExitTrace(int64_t DeviceId,
+                                                        void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId, ompt_target_exit_data,
+                       ompt_scope_begin, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Returning trace record buf ptr " << DataPtr
+                        << " (ompt_target_exit_data)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::stopTargetDataExitTrace(int64_t DeviceId,
+                                                       void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId, ompt_target_exit_data,
+                       ompt_scope_end, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Generated trace record " << DataPtr
+                        << " (ompt_target_exit_data)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::startTargetUpdateTrace(int64_t DeviceId,
+                                                      void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId, ompt_target_update,
+                       ompt_scope_begin, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Returning trace record buf ptr " << DataPtr
+                        << " (ompt_target_update)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::stopTargetUpdateTrace(int64_t DeviceId,
+                                                     void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId, ompt_target_update,
+                       ompt_scope_end, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Generated trace record " << DataPtr
+                        << " (ompt_target_update)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::startTargetTrace(int64_t DeviceId,
+                                                void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId, ompt_target,
+                       ompt_scope_begin, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Returning trace record buf ptr " << DataPtr
+                        << " (ompt_target)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::stopTargetTrace(int64_t DeviceId,
+                                               void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId, ompt_target,
+                       ompt_scope_end, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+
+  ODBG(debug::ODT_Tool) << "Generated trace record " << DataPtr
+                        << " (ompt_target)";
+  return DataPtr;
+}
+
+extern "C" {
+// Device-independent entry point for ompt_set_trace_ompt
+ompt_set_result_t libomptarget_ompt_set_trace_ompt(int DeviceId,
+                                                   unsigned int Enable,
+                                                   unsigned int EventTy) {
+  std::unique_lock<std::mutex> Lock(TraceAccessMutex);
+  return llvm::omp::target::ompt::setTraceEventTy(DeviceId, Enable, EventTy);
+}
+
+// Device-independent entry point for ompt_start_trace
+int libomptarget_ompt_start_trace(int DeviceId,
+                                  ompt_callback_buffer_request_t Request,
+                                  ompt_callback_buffer_complete_t Complete) {
+  if (!PM) {
+    REPORT() << "Failed to start trace for DeviceId=" << DeviceId
+             << " (invalid plugin manager)";
+    // Indicate failure
+    return 0;
+  }
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  std::unique_lock<std::mutex> Lock(TraceControlMutex);
+  if (Request && Complete) {
+    // Set buffer related functions
+    llvm::omp::target::ompt::setBufferManagementFns(DeviceId, Request,
+                                                    Complete);
+    llvm::omp::target::ompt::enableDeviceTracing(DeviceId);
+    TRM->startHelperThreads();
+    // Success
+    return 1;
+  }
+  // Failure
+  return 0;
+}
+
+// Device-independent entry point for ompt_flush_trace
+int libomptarget_ompt_flush_trace(int DeviceId) {
+  if (!PM) {
+    REPORT() << "Failed to flush trace for DeviceId=" << DeviceId
+             << " (invalid plugin manager)";
+    // Indicate failure
+    return 0;
+  }
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  std::unique_lock<std::mutex> Lock(TraceControlMutex);
+  return TRM->flushAllBuffers(DeviceId);
+}
+
+// Device independent entry point for ompt_stop_trace
+int libomptarget_ompt_stop_trace(int DeviceId) {
+  if (!PM) {
+    REPORT() << "Failed to stop trace for DeviceId=" << DeviceId
+             << " (invalid plugin manager)";
+    // Indicate failure
+    return 0;
+  }
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  std::unique_lock<std::mutex> Lock(TraceControlMutex);
+
+  // Schedule flushing of trace records for this device
+  int Status = TRM->flushAllBuffers(DeviceId);
+
+  // De-register this device so that no more traces are collected
+  // or delivered for this device until an ompt_start_trace is
+  // invoked for this device.
+  removeBufferManagementFns(DeviceId);
+
+  // If no device is being traced, shut down the helper threads. A
+  // subsequent ompt_start_trace will start up the helper threads.
+  if (isAllDeviceTracingStopped()) {
+    // TODO shutdown should perhaps return a status
+    TRM->shutdownHelperThreads();
+    llvm::omp::target::ompt::disableDeviceTracing(DeviceId);
+  }
+  return Status;
+}
+
+// Device independent entry point for ompt_advance_buffer_cursor
+// Note: The input parameter size is unused here. It refers to the
+// bytes returned in the corresponding callback.
+int libomptarget_ompt_advance_buffer_cursor(ompt_device_t *Device,
+                                            ompt_buffer_t *Buffer, size_t Size,
+                                            ompt_buffer_cursor_t CurrentPos,
+                                            ompt_buffer_cursor_t *NextPos) {
+  if (!PM) {
+    REPORT() << "Failed to advance buffer cursor for Device=" << Device
+             << " (invalid plugin manager)";
+    // Indicate failure
+    return false;
+  }
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  char *TraceRecord = (char *)CurrentPos;
+  // Don't assert if CurrentPos is null, just indicate end of buffer
+  if (TraceRecord == nullptr || TRM->isLastCursor(TraceRecord)) {
+    *NextPos = 0;
+    return false;
+  }
+  // TODO In debug mode, assert that the metadata points to the
+  // input parameter buffer
+
+  size_t TRSize = TRM->getTRSize();
+  *NextPos = (ompt_buffer_cursor_t)(TraceRecord + TRSize);
+  ODBG(llvm::omp::target::debug::ODT_Tool)
+      << "Advanced buffer pointer by " << TRSize << " bytes to "
+      << TraceRecord + TRSize;
+  return true;
+}
+
+// This function is invoked before the kernel launch. So, when the trace record
+// is populated after kernel completion, TraceRecordNumGrantedTeams is already
+// updated.
+void libomptarget_ompt_set_granted_teams(uint32_t NumTeams) {
+  TraceRecordNumGrantedTeams = NumTeams;
+}
+
+// Assume a synchronous implementation and set thread local variables to track
+// timestamps. The thread local variables can then be used to populate trace
+// records.
+void libomptarget_ompt_set_timestamp(uint64_t Start, uint64_t Stop) {
+  TraceRecordStartTime = Start;
+  TraceRecordStopTime = Stop;
+}
+
+// Device-independent entry point to query for the trace format used.
+// Currently, only OMPT format is supported.
+ompt_record_t
+libomptarget_ompt_get_record_type(ompt_buffer_t *Buffer,
+                                  ompt_buffer_cursor_t CurrentPos) {
+  // TODO: When different OMPT trace buffer formats supported, this needs to be
+  // fixed.
+  return ompt_record_t::ompt_record_ompt;
+}
+} // extern "C"
+
+#pragma pop_macro("DEBUG_PREFIX")
+
+#endif // OMPT_SUPPORT
diff --git a/offload/libomptarget/PluginManager.cpp b/offload/libomptarget/PluginManager.cpp
index 41b653a60adfd..5ef8200c7006b 100644
--- a/offload/libomptarget/PluginManager.cpp
+++ b/offload/libomptarget/PluginManager.cpp
@@ -12,6 +12,9 @@
 
 #include "PluginManager.h"
 #include "OffloadPolicy.h"
+#include "OpenMP/OMPT/Callback.h"
+#include "OpenMP/OMPT/OmptCommonDefs.h"
+#include "OpenMP/OMPT/OmptTracing.h"
 #include "Shared/Debug.h"
 #include "Shared/Profile.h"
 #include "device.h"
@@ -47,6 +50,12 @@ void PluginManager::init() {
   } while (false);
 #include "Shared/Targets.def"
 
+#ifdef OMPT_SUPPORT
+  assert(TraceRecordManager == nullptr &&
+         "Expected trace record manager to be null");
+  TraceRecordManager = new OmptTracingBufferMgr();
+#endif
+
   ODBG(ODT_Init) << "RTLs loaded!";
 }
 
@@ -54,6 +63,16 @@ void PluginManager::deinit() {
   TIMESCOPE();
   ODBG(ODT_Deinit) << "Unloading RTLs...";
 
+#ifdef OMPT_SUPPORT
+  // When offloading is disabled, the TRM is not initialized.
+  if (!OffloadPolicy::isOffloadDisabled()) {
+    assert(TraceRecordManager != nullptr &&
+           "Trace record manager should have been non-null");
+    delete TraceRecordManager;
+    TraceRecordManager = nullptr;
+  }
+#endif
+
   for (auto &Plugin : Plugins) {
     if (!Plugin->is_initialized())
       continue;
@@ -323,6 +342,9 @@ int target(ident_t *Loc, DeviceTy &Device, void *HostPtr,
 void PluginManager::unregisterLib(__tgt_bin_desc *Desc) {
   ODBG(ODT_Deinit) << "Unloading target library!";
 
+  OMPT_IF_TRACING_ENABLED(
+      PM->getTraceRecordManager()->flushAndShutdownHelperThreads(););
+
   Desc = upgradeLegacyEntries(Desc);
 
   PM->RTLsMtx.lock();
diff --git a/offload/libomptarget/device.cpp b/offload/libomptarget/device.cpp
index 18339e9afe975..770b74ca35597 100644
--- a/offload/libomptarget/device.cpp
+++ b/offload/libomptarget/device.cpp
@@ -15,6 +15,8 @@
 #include "OpenMP/Mapping.h"
 #include "OpenMP/OMPT/Callback.h"
 #include "OpenMP/OMPT/Interface.h"
+#include "OpenMP/OMPT/OmptCommonDefs.h"
+#include "OpenMP/OMPT/OmptTracing.h"
 #include "PluginManager.h"
 #include "Shared/APITypes.h"
 #include "Shared/Debug.h"
@@ -34,7 +36,8 @@
 #include <thread>
 
 #ifdef OMPT_SUPPORT
-using namespace llvm::omp::target::ompt;
+using namespace llvm::omp::target;
+using namespace ompt;
 #endif
 
 using namespace llvm::omp::target::plugin;
@@ -249,10 +252,15 @@ DeviceTy::loadBinary(__tgt_device_image *Img) {
 void *DeviceTy::allocData(int64_t Size, void *HstPtr, int32_t Kind) {
   /// RAII to establish tool anchors before and after data allocation
   void *TargetPtr = nullptr;
-  OMPT_IF_BUILT(InterfaceRAII TargetDataAllocRAII(
-                    RegionInterface.getCallbacks<ompt_target_data_alloc>(),
-                    DeviceID, HstPtr, &TargetPtr, Size,
-                    /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
+  OMPT_IF_BUILT(
+      InterfaceRAII TargetDataAllocRAII(
+          RegionInterface.getCallbacks<ompt_target_data_alloc>(), DeviceID,
+          HstPtr, &TargetPtr, Size,
+          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+      InterfaceRAII TargetDataAllocTraceRAII(
+          RegionInterface.getTraceGenerators<ompt_target_data_alloc>(),
+          RTLDeviceID, HstPtr, &TargetPtr, Size,
+          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
 
   TargetPtr = RTL->data_alloc(RTLDeviceID, Size, HstPtr, Kind);
   return TargetPtr;
@@ -260,11 +268,15 @@ void *DeviceTy::allocData(int64_t Size, void *HstPtr, int32_t Kind) {
 
 int32_t DeviceTy::deleteData(void *TgtAllocBegin, int32_t Kind) {
   /// RAII to establish tool anchors before and after data deletion
-  OMPT_IF_BUILT(InterfaceRAII TargetDataDeleteRAII(
-                    RegionInterface.getCallbacks<ompt_target_data_delete>(),
-                    DeviceID, TgtAllocBegin,
-                    /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
-
+  OMPT_IF_BUILT(
+      InterfaceRAII TargetDataDeleteRAII(
+          RegionInterface.getCallbacks<ompt_target_data_delete>(), DeviceID,
+          TgtAllocBegin,
+          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+      InterfaceRAII TargetDataDeleteTraceRAII(
+          RegionInterface.getTraceGenerators<ompt_target_data_delete>(),
+          DeviceID, TgtAllocBegin,
+          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
   return RTL->data_delete(RTLDeviceID, TgtAllocBegin, Kind);
 }
 
@@ -281,6 +293,13 @@ int32_t DeviceTy::submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size,
       InterfaceRAII TargetDataSubmitRAII(
           RegionInterface.getCallbacks<ompt_target_data_transfer_to_device>(),
           omp_initial_device, HstPtrBegin, DeviceID, TgtPtrBegin, Size,
+          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+      TracerInterfaceRAII TargetDataSubmitTraceRAII(
+          RegionInterface
+              .getTraceGenerators<ompt_target_data_transfer_to_device>(),
+          AsyncInfo, RTL->getProfiler(), /*TracedDeviceId=*/DeviceID,
+          /*EventType=*/ompt_callback_target_data_op, omp_initial_device,
+          HstPtrBegin, DeviceID, TgtPtrBegin, Size,
           /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
 
   return RTL->data_submit_async(RTLDeviceID, TgtPtrBegin, HstPtrBegin, Size,
@@ -301,6 +320,13 @@ int32_t DeviceTy::retrieveData(void *HstPtrBegin, void *TgtPtrBegin,
       InterfaceRAII TargetDataRetrieveRAII(
           RegionInterface.getCallbacks<ompt_target_data_transfer_from_device>(),
           DeviceID, TgtPtrBegin, omp_initial_device, HstPtrBegin, Size,
+          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+      TracerInterfaceRAII TargetDataRetrieveTraceRAII(
+          RegionInterface
+              .getTraceGenerators<ompt_target_data_transfer_from_device>(),
+          AsyncInfo, RTL->getProfiler(), /*TracedDeviceId=*/DeviceID,
+          /*EventType=*/ompt_callback_target_data_op, DeviceID, TgtPtrBegin,
+          omp_initial_device, HstPtrBegin, Size,
           /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
 
   return RTL->data_retrieve_async(RTLDeviceID, HstPtrBegin, TgtPtrBegin, Size,
@@ -320,11 +346,15 @@ int32_t DeviceTy::dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr,
       InterfaceRAII TargetDataExchangeRAII(
           RegionInterface.getCallbacks<ompt_target_data_transfer_from_device>(),
           RTLDeviceID, SrcPtr, DstDev.RTLDeviceID, DstPtr, Size,
+          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+      TracerInterfaceRAII TargetDataExchangeTraceRAII(
+          RegionInterface
+              .getTraceGenerators<ompt_target_data_transfer_from_device>(),
+          AsyncInfo, RTL->getProfiler(), /*TracedDeviceId=*/RTLDeviceID,
+          /*EventType=*/ompt_callback_target_data_op, RTLDeviceID, SrcPtr,
+          DstDev.RTLDeviceID, DstPtr, Size,
           /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
-  if (!AsyncInfo) {
-    return RTL->data_exchange(RTLDeviceID, SrcPtr, DstDev.RTLDeviceID, DstPtr,
-                              Size);
-  }
+
   return RTL->data_exchange_async(RTLDeviceID, SrcPtr, DstDev.RTLDeviceID,
                                   DstPtr, Size, AsyncInfo);
 }
diff --git a/offload/libomptarget/exports b/offload/libomptarget/exports
index 1831c43cc5f29..6e404ed059096 100644
--- a/offload/libomptarget/exports
+++ b/offload/libomptarget/exports
@@ -67,6 +67,14 @@ VERS1.0 {
     llvm_omp_target_unlock_mem;
     __tgt_set_info_flag;
     __tgt_print_device_info;
+    libomptarget_ompt_set_trace_ompt;
+    libomptarget_ompt_start_trace;
+    libomptarget_ompt_flush_trace;
+    libomptarget_ompt_stop_trace;
+    libomptarget_ompt_set_granted_teams;
+    libomptarget_ompt_set_timestamp;
+    libomptarget_ompt_advance_buffer_cursor;
+    libomptarget_ompt_get_record_type;
     omp_get_interop_ptr;
     omp_get_interop_str;
     omp_get_interop_int;
diff --git a/offload/libomptarget/interface.cpp b/offload/libomptarget/interface.cpp
index 5d7d948711b99..1e04d10dab8e4 100644
--- a/offload/libomptarget/interface.cpp
+++ b/offload/libomptarget/interface.cpp
@@ -14,6 +14,7 @@
 #include "OpenMP/OMPT/Interface.h"
 #include "OffloadPolicy.h"
 #include "OpenMP/OMPT/Callback.h"
+#include "OpenMP/OMPT/OmptCommonDefs.h"
 #include "OpenMP/omp.h"
 #include "PluginManager.h"
 #include "omptarget.h"
@@ -153,19 +154,30 @@ targetData(ident_t *Loc, int64_t DeviceId, int32_t ArgNum, void **ArgsBase,
   AsyncInfoTy &AsyncInfo = TargetAsyncInfo;
 
   /// RAII to establish tool anchors before and after data begin / end / update
-  OMPT_IF_BUILT(assert((TargetDataFunction == targetDataBegin ||
-                        TargetDataFunction == targetDataEnd ||
-                        TargetDataFunction == targetDataUpdate) &&
-                       "Encountered unexpected TargetDataFunction during "
-                       "execution of targetData");
-                auto CallbackFunctions =
-                    (TargetDataFunction == targetDataBegin)
-                        ? RegionInterface.getCallbacks<ompt_target_enter_data>()
-                    : (TargetDataFunction == targetDataEnd)
-                        ? RegionInterface.getCallbacks<ompt_target_exit_data>()
-                        : RegionInterface.getCallbacks<ompt_target_update>();
-                InterfaceRAII TargetDataRAII(CallbackFunctions, DeviceId,
-                                             OMPT_GET_RETURN_ADDRESS);)
+  OMPT_IF_BUILT(
+      assert((TargetDataFunction == targetDataBegin ||
+              TargetDataFunction == targetDataEnd ||
+              TargetDataFunction == targetDataUpdate) &&
+             "Encountered unexpected TargetDataFunction during "
+             "execution of targetData");
+      auto CallbackFunctions =
+          (TargetDataFunction == targetDataBegin)
+              ? RegionInterface.getCallbacks<ompt_target_enter_data>()
+          : (TargetDataFunction == targetDataEnd)
+              ? RegionInterface.getCallbacks<ompt_target_exit_data>()
+              : RegionInterface.getCallbacks<ompt_target_update>();
+
+      auto TraceGenerators =
+          (TargetDataFunction == targetDataBegin)
+              ? RegionInterface.getTraceGenerators<ompt_target_enter_data>()
+          : (TargetDataFunction == targetDataEnd)
+              ? RegionInterface.getTraceGenerators<ompt_target_exit_data>()
+              : RegionInterface.getTraceGenerators<ompt_target_update>();
+
+      InterfaceRAII TargetDataRAII(CallbackFunctions, DeviceId,
+                                   /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+      InterfaceRAII TargetDataTraceRAII(TraceGenerators, DeviceId,
+                                        /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
 
   int Rc = OFFLOAD_SUCCESS;
 
@@ -433,6 +445,9 @@ static inline int targetKernel(ident_t *Loc, int64_t DeviceId, int32_t NumTeams,
   /// RAII to establish tool anchors before and after target region
   OMPT_IF_BUILT(InterfaceRAII TargetRAII(
                     RegionInterface.getCallbacks<ompt_target>(), DeviceId,
+                    /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+                InterfaceRAII TargetTraceRAII(
+                    RegionInterface.getTraceGenerators<ompt_target>(), DeviceId,
                     /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
 
   int Rc = OFFLOAD_SUCCESS;
@@ -542,6 +557,9 @@ EXTERN int __tgt_target_kernel_replay(
   /// RAII to establish tool anchors before and after target region
   OMPT_IF_BUILT(InterfaceRAII TargetRAII(
                     RegionInterface.getCallbacks<ompt_target>(), DeviceId,
+                    /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+                InterfaceRAII TargetTraceRAII(
+                    RegionInterface.getTraceGenerators<ompt_target>(), DeviceId,
                     /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
 
   AsyncInfoTy AsyncInfo(*DeviceOrErr);
diff --git a/offload/libomptarget/omptarget.cpp b/offload/libomptarget/omptarget.cpp
index 24169e68a29c7..187e7b5351d4b 100644
--- a/offload/libomptarget/omptarget.cpp
+++ b/offload/libomptarget/omptarget.cpp
@@ -2363,8 +2363,12 @@ int target(ident_t *Loc, DeviceTy &Device, void *HostPtr,
     // No need to guard this with OMPT_IF_BUILT
     InterfaceRAII TargetSubmitRAII(
         RegionInterface.getCallbacks<ompt_callback_target_submit>(), NumTeams);
-#endif
 
+    TracerInterfaceRAII TargetTraceRAII(
+        RegionInterface.getTraceGenerators<ompt_callback_target_submit>(),
+        AsyncInfo, Device.RTL->getProfiler(), /*TracedDeviceId=*/DeviceId,
+        /*EventType=*/ompt_callback_target_submit, DeviceId, NumTeams);
+#endif
     Ret = Device.launchKernel(TgtEntryPtr, TgtArgs.data(), TgtOffsets.data(),
                               KernelArgs, nullptr, AsyncInfo);
   }

>From 48e9253d97ed8ab98f4290bed0d5068c5311b574 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Mon, 13 Apr 2026 06:26:35 -0500
Subject: [PATCH 08/11] [Offload][OMPT] Fix global destructor warnings in OMPT
 tracing globals

Replace five globals with non-trivial destructors with function-local
statics, following the LLVM canonical pattern (see Signals.cpp). This
resolves -Werror=global-constructors failures that were triggered by
the destructor of std::map, std::unordered_map, std::shared_ptr, and
a thread_local array of shared_ptr.

Affected globals:
- ParentLibrary (shared_ptr) in plugin OmptTracing.cpp -> file-local
  getParentLibraryStorage() helper
- Devices (map) in plugin OmptTracing.cpp -> getDevices()
- BufferManagementFns (unordered_map) in libomptarget OmptTracing.cpp
  -> file-local getBufferMgmt() helper
- TracedDevices (map) in libomptarget OmptTracing.cpp -> getTracedDevices()
- ArrayOfBufPtr (thread_local shared_ptr array) in OmptTracingBuffer.cpp
  -> getArrayOfBufPtr()

extern declarations are replaced with getter function declarations in
the corresponding headers (OmptDeviceTracing.h, OmptTracing.h,
OmptTracingBuffer.h). All use-sites are updated accordingly.

Made-with: Cursor
---
 offload/include/OpenMP/OMPT/OmptTracing.h     | 11 +--
 .../include/OpenMP/OMPT/OmptTracingBuffer.h   |  5 +-
 .../libomptarget/OpenMP/OMPT/OmptTracing.cpp  | 72 ++++++++++---------
 .../OpenMP/OMPT/OmptTracingBuffer.cpp         | 12 ++--
 .../common/OMPT/OmptDeviceTracing.h           | 20 +++---
 .../common/OMPT/OmptTracing.cpp               | 41 ++++++-----
 6 files changed, 85 insertions(+), 76 deletions(-)

diff --git a/offload/include/OpenMP/OMPT/OmptTracing.h b/offload/include/OpenMP/OMPT/OmptTracing.h
index 2a892582923d5..4c8c049a9ae48 100644
--- a/offload/include/OpenMP/OMPT/OmptTracing.h
+++ b/offload/include/OpenMP/OMPT/OmptTracing.h
@@ -102,8 +102,9 @@ ompt_set_result_t setTraceEventTy(int DeviceId, unsigned int Enable,
 /// Return thread id
 uint64_t getThreadId();
 
-/// See TracedDevices in OmptDeviceTracing.h
-extern std::map<int32_t, uint64_t> TracedDevices;
+/// Access the map of RTL device IDs to their currently enabled tracing event
+/// types. See also OmptDeviceTracing.h.
+std::map<int32_t, uint64_t> &getTracedDevices();
 /// Activate tracing on the given device
 void enableDeviceTracing(int DeviceId);
 /// Deactivate tracing on the given device
@@ -123,12 +124,6 @@ extern std::mutex TraceHashThreadMutex;
 /// buffer-completion callback functions.
 extern std::mutex BufferManagementFnMutex;
 
-/// Map from device-id to the corresponding buffer-request and buffer-completion
-/// callback functions.
-extern std::unordered_map<int, std::pair<ompt_callback_buffer_request_t,
-                                         ompt_callback_buffer_complete_t>>
-    BufferManagementFns;
-
 /// Thread local variables used by the plugin to communicate OMPT information
 /// that are then used to populate trace records. This method assumes a
 /// synchronous implementation, otherwise it won't work.
diff --git a/offload/include/OpenMP/OMPT/OmptTracingBuffer.h b/offload/include/OpenMP/OMPT/OmptTracingBuffer.h
index 5c9f4bf33dae8..a9b7f8d569fcb 100644
--- a/offload/include/OpenMP/OMPT/OmptTracingBuffer.h
+++ b/offload/include/OpenMP/OMPT/OmptTracingBuffer.h
@@ -133,8 +133,9 @@ class OmptTracingBufferMgr {
   // Thread-specific array of pointers to a buffer. The buffer pointed to
   // is the last one allocated by this thread for a given device. The ith
   // element points to the buffer for the ith device. At most MAX_NUM_DEVICES
-  // devices are supported.
-  static thread_local BufPtr ArrayOfBufPtr[MAX_NUM_DEVICES];
+  // devices are supported. Stored as a function-local thread_local to avoid
+  // a global destructor.
+  static BufPtr *getArrayOfBufPtr();
 
   /*
    * A buffer is flushed when it fills up or when the tool invokes
diff --git a/offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp b/offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp
index 77c771786f29b..ad4bb58ff1701 100644
--- a/offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp
+++ b/offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp
@@ -42,17 +42,25 @@ std::mutex llvm::omp::target::ompt::TraceControlMutex;
 std::mutex llvm::omp::target::ompt::TraceHashThreadMutex;
 std::mutex llvm::omp::target::ompt::BufferManagementFnMutex;
 
-std::unordered_map<int /*DeviceId*/, std::pair<ompt_callback_buffer_request_t,
-                                               ompt_callback_buffer_complete_t>>
-    llvm::omp::target::ompt::BufferManagementFns;
-
 thread_local uint32_t llvm::omp::target::ompt::TraceRecordNumGrantedTeams = 0;
 thread_local uint64_t llvm::omp::target::ompt::TraceRecordStartTime = 0;
 thread_local uint64_t llvm::omp::target::ompt::TraceRecordStopTime = 0;
 thread_local uint64_t llvm::omp::target::ompt::ThreadId =
     std::numeric_limits<uint64_t>::max();
 
-std::map<int32_t, uint64_t> llvm::omp::target::ompt::TracedDevices;
+// File-local helper returning a reference to the buffer-management map stored
+// as a function-local static, avoiding a global destructor.
+static auto &getBufferMgmt() {
+  static std::unordered_map<int, std::pair<ompt_callback_buffer_request_t,
+                                           ompt_callback_buffer_complete_t>>
+      M;
+  return M;
+}
+
+std::map<int32_t, uint64_t> &llvm::omp::target::ompt::getTracedDevices() {
+  static std::map<int32_t, uint64_t> TracedDevices;
+  return TracedDevices;
+}
 
 bool llvm::omp::target::ompt::TracingActive = false;
 
@@ -61,8 +69,8 @@ void llvm::omp::target::ompt::resetTimestamp(uint64_t *T) { *T = 0; }
 ompt_callback_buffer_request_t
 llvm::omp::target::ompt::getBufferRequestFn(int DeviceId) {
   std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
-  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
-  if (BufferMgrItr == BufferManagementFns.end()) {
+  auto BufferMgrItr = getBufferMgmt().find(DeviceId);
+  if (BufferMgrItr == getBufferMgmt().end()) {
     return nullptr;
   }
   return BufferMgrItr->second.first;
@@ -71,8 +79,8 @@ llvm::omp::target::ompt::getBufferRequestFn(int DeviceId) {
 ompt_callback_buffer_complete_t
 llvm::omp::target::ompt::getBufferCompleteFn(int DeviceId) {
   std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
-  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
-  if (BufferMgrItr == BufferManagementFns.end()) {
+  auto BufferMgrItr = getBufferMgmt().find(DeviceId);
+  if (BufferMgrItr == getBufferMgmt().end()) {
     return nullptr;
   }
   return BufferMgrItr->second.second;
@@ -82,30 +90,30 @@ void llvm::omp::target::ompt::setBufferManagementFns(
     int DeviceId, ompt_callback_buffer_request_t ReqFn,
     ompt_callback_buffer_complete_t CmpltFn) {
   std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
-  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
-  if (BufferMgrItr != BufferManagementFns.end()) {
+  auto BufferMgrItr = getBufferMgmt().find(DeviceId);
+  if (BufferMgrItr != getBufferMgmt().end()) {
     REPORT()
         << "Buffer request and complete functions already exist for device  "
         << DeviceId << "ignoring ...";
     return;
   }
-  BufferManagementFns[DeviceId] = std::make_pair(ReqFn, CmpltFn);
+  getBufferMgmt()[DeviceId] = std::make_pair(ReqFn, CmpltFn);
 }
 
 void llvm::omp::target::ompt::removeBufferManagementFns(int DeviceId) {
   std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
-  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
-  if (BufferMgrItr == BufferManagementFns.end()) {
+  auto BufferMgrItr = getBufferMgmt().find(DeviceId);
+  if (BufferMgrItr == getBufferMgmt().end()) {
     REPORT() << "Buffer request and complete functions don't exist for device  "
              << DeviceId << "ignoring ...";
     return;
   }
-  BufferManagementFns.erase(BufferMgrItr);
+  getBufferMgmt().erase(BufferMgrItr);
 }
 
 bool llvm::omp::target::ompt::isAllDeviceTracingStopped() {
   std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
-  return BufferManagementFns.empty();
+  return getBufferMgmt().empty();
 }
 
 void llvm::omp::target::ompt::ompt_callback_buffer_request(
@@ -138,11 +146,11 @@ inline bool checkDeviceTracingState(const uint64_t &TracingTypes) {
 
 void llvm::omp::target::ompt::enableDeviceTracing(int DeviceId) {
   std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
-  auto Device = TracedDevices.find(DeviceId);
-  if (Device == TracedDevices.end()) {
+  auto Device = getTracedDevices().find(DeviceId);
+  if (Device == getTracedDevices().end()) {
     uint64_t TracingTypes{0};
     setDeviceTracing(TracingTypes);
-    TracedDevices.emplace(DeviceId, TracingTypes);
+    getTracedDevices().emplace(DeviceId, TracingTypes);
   } else
     setDeviceTracing(Device->second);
   // In any case: at least one device is traced
@@ -151,16 +159,16 @@ void llvm::omp::target::ompt::enableDeviceTracing(int DeviceId) {
 
 void llvm::omp::target::ompt::disableDeviceTracing(int DeviceId) {
   std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
-  auto Device = TracedDevices.find(DeviceId);
-  if (Device == TracedDevices.end()) {
+  auto Device = getTracedDevices().find(DeviceId);
+  if (Device == getTracedDevices().end()) {
     uint64_t TracingTypes{0};
     resetDeviceTracing(TracingTypes);
-    TracedDevices.emplace(DeviceId, TracingTypes);
+    getTracedDevices().emplace(DeviceId, TracingTypes);
   } else
     resetDeviceTracing(Device->second);
 
   // Check for actively traced devices
-  for (auto &Dev : TracedDevices)
+  for (auto &Dev : getTracedDevices())
     if (checkDeviceTracingState(Dev.second))
       return;
 
@@ -176,8 +184,8 @@ bool llvm::omp::target::ompt::isTracingEnabled(int DeviceId,
 
 bool llvm::omp::target::ompt::isTracedDevice(int DeviceId) {
   std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
-  auto Device = TracedDevices.find(DeviceId);
-  if (Device != TracedDevices.end())
+  auto Device = getTracedDevices().find(DeviceId);
+  if (Device != getTracedDevices().end())
     return checkDeviceTracingState(Device->second);
 
   return false;
@@ -188,8 +196,8 @@ bool llvm::omp::target::ompt::isTracingTypeEnabled(int DeviceId,
   std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
   // Make sure we do not shift more than std::numeric_limits<uint64_t>::digits
   assert(EventTy < 64 && "Shift limit exceeded: EventTy must be less than 64");
-  auto Device = TracedDevices.find(DeviceId);
-  if (Device != TracedDevices.end() && EventTy < 64)
+  auto Device = getTracedDevices().find(DeviceId);
+  if (Device != getTracedDevices().end() && EventTy < 64)
     return (Device->second & (1UL << EventTy));
   return false;
 }
@@ -199,8 +207,8 @@ bool llvm::omp::target::ompt::isTracingTypeGroupEnabled(int DeviceId,
   std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
   // Make sure we do not shift more than std::numeric_limits<uint64_t>::digits
   assert(EventTy < 64 && "Shift limit exceeded: EventTy must be less than 64");
-  auto Device = TracedDevices.find(DeviceId);
-  if (Device != TracedDevices.end() && EventTy < 64) {
+  auto Device = getTracedDevices().find(DeviceId);
+  if (Device != getTracedDevices().end() && EventTy < 64) {
     auto TracedEvents = Device->second;
     switch (EventTy) {
     case ompt_callbacks_t::ompt_callback_target:
@@ -253,10 +261,10 @@ llvm::omp::target::ompt::setTraceEventTy(int DeviceId, unsigned int Enable,
                         << " Enable=" << Enable << " EventTy=" << EventTy;
 
   std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
-  if (TracedDevices.find(DeviceId) == TracedDevices.end())
-    TracedDevices.emplace(DeviceId, 0UL);
+  if (getTracedDevices().find(DeviceId) == getTracedDevices().end())
+    getTracedDevices().emplace(DeviceId, 0UL);
 
-  auto &TracedEventTy = TracedDevices[DeviceId];
+  auto &TracedEventTy = getTracedDevices()[DeviceId];
   bool Enabled = Enable > 0;
   if (EventTy == 0) {
     // Set / reset all supported types
diff --git a/offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp b/offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp
index 5ac1974cea03b..92609e6e690be 100644
--- a/offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp
+++ b/offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp
@@ -32,8 +32,10 @@ static std::atomic<uint64_t> BufId{0};
 // Unique id in buffer flush order
 static std::atomic<uint64_t> FlushId{0};
 
-thread_local OmptTracingBufferMgr::BufPtr
-    OmptTracingBufferMgr::ArrayOfBufPtr[MAX_NUM_DEVICES];
+OmptTracingBufferMgr::BufPtr *OmptTracingBufferMgr::getArrayOfBufPtr() {
+  thread_local BufPtr Array[MAX_NUM_DEVICES]{};
+  return Array;
+}
 
 static uint64_t get_and_inc_buf_id() { return BufId++; }
 
@@ -416,7 +418,7 @@ OmptTracingBufferMgr::getDeviceSpecificBuffer(int64_t DeviceId) {
              << " invalid or exceeds supported max: " << MAX_NUM_DEVICES - 1;
     return nullptr;
   }
-  return ArrayOfBufPtr[DeviceId];
+  return getArrayOfBufPtr()[DeviceId];
 }
 
 void OmptTracingBufferMgr::setDeviceSpecificBuffer(int64_t DeviceId,
@@ -426,7 +428,7 @@ void OmptTracingBufferMgr::setDeviceSpecificBuffer(int64_t DeviceId,
              << " invalid or exceeds supported max: " << MAX_NUM_DEVICES - 1;
     return;
   }
-  ArrayOfBufPtr[DeviceId] = Buf;
+  getArrayOfBufPtr()[DeviceId] = Buf;
 }
 
 void OmptTracingBufferMgr::setTRStatus(void *Rec, TRStatus Status) {
@@ -676,7 +678,7 @@ void OmptTracingBufferMgr::waitForFlushCompletion() {
 
 void OmptTracingBufferMgr::init() {
   for (int i = 0; i < MAX_NUM_DEVICES; ++i)
-    ArrayOfBufPtr[i] = nullptr;
+    getArrayOfBufPtr()[i] = nullptr;
   ThreadFlushTracker = 0;
   ThreadShutdownTracker = 0;
   DoneTracing = false; // TODO make it a class member
diff --git a/offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h b/offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h
index 5744a784825da..ba9e1dcd6ccc3 100644
--- a/offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h
+++ b/offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h
@@ -85,19 +85,17 @@ extern double HostToDeviceSlope;
 /// Host to device constant clock offset
 extern double HostToDeviceOffset;
 
-/// Mapping of device pointers to their corresponding RTL device ID
-extern std::map<ompt_device_t *, int32_t> Devices;
+/// Access the map of device pointers to their corresponding RTL device ID
+std::map<ompt_device_t *, int32_t> &getDevices();
 
-/// Mapping of RTL device IDs to their currently enabled tracing event types.
-/// Note: Event type '0' (bit position) indicates if this device is traced.
-extern std::map<int32_t, uint64_t> TracedDevices;
+/// Access the map of RTL device IDs to their currently enabled tracing event
+/// types. Note: Event type '0' (bit position) indicates if this device is
+/// traced.
+std::map<int32_t, uint64_t> &getTracedDevices();
 
 /// OMPT global tracing status. Indicates if at least one device is traced.
 extern bool TracingActive;
 
-/// Parent library pointer
-extern std::shared_ptr<llvm::sys::DynamicLibrary> ParentLibrary;
-
 /// Get the parent library by pointer. If it is not already set, it will set the
 /// parent library pointer.
 std::shared_ptr<llvm::sys::DynamicLibrary> getParentLibrary();
@@ -111,10 +109,10 @@ void setParentLibrary(const char *Filename);
 template <typename FT>
 void ensureFuncPtrLoaded(const std::string &FuncName, FT *FuncPtr) {
   if (*FuncPtr == nullptr) {
-    if ((ParentLibrary == nullptr && getParentLibrary() == nullptr) ||
-        !ParentLibrary->isValid())
+    auto Lib = getParentLibrary();
+    if (!Lib || !Lib->isValid())
       return;
-    void *SymbolPtr = ParentLibrary->getAddressOfSymbol(FuncName.c_str());
+    void *SymbolPtr = Lib->getAddressOfSymbol(FuncName.c_str());
     if (SymbolPtr == nullptr)
       return;
     *FuncPtr = reinterpret_cast<FT>(SymbolPtr);
diff --git a/offload/plugins-nextgen/common/OMPT/OmptTracing.cpp b/offload/plugins-nextgen/common/OMPT/OmptTracing.cpp
index 7a8e312b3d476..637452e535695 100644
--- a/offload/plugins-nextgen/common/OMPT/OmptTracing.cpp
+++ b/offload/plugins-nextgen/common/OMPT/OmptTracing.cpp
@@ -46,39 +46,44 @@ std::mutex llvm::omp::target::ompt::DeviceIdWritingMutex;
 using namespace llvm::omp::target::ompt;
 using namespace llvm::omp::target::debug;
 
-std::shared_ptr<llvm::sys::DynamicLibrary>
-    llvm::omp::target::ompt::ParentLibrary(nullptr);
-
 double llvm::omp::target::ompt::HostToDeviceSlope = .0;
 double llvm::omp::target::ompt::HostToDeviceOffset = .0;
 
-std::map<ompt_device_t *, int32_t> llvm::omp::target::ompt::Devices;
+// File-local helper to hold the parent library shared_ptr as a function-local
+// static, avoiding a global destructor.
+static std::shared_ptr<llvm::sys::DynamicLibrary> &getParentLibraryStorage() {
+  static std::shared_ptr<llvm::sys::DynamicLibrary> Lib(nullptr);
+  return Lib;
+}
 
 std::shared_ptr<llvm::sys::DynamicLibrary>
 llvm::omp::target::ompt::getParentLibrary() {
-  static bool ParentLibraryAssigned = false;
-  if (!ParentLibraryAssigned) {
+  if (!getParentLibraryStorage()) {
     setParentLibrary("libomptarget.so");
-    ParentLibraryAssigned = true;
   }
-  return ParentLibrary;
+  return getParentLibraryStorage();
 }
 
 void llvm::omp::target::ompt::setParentLibrary(const char *Filename) {
-  if (ParentLibrary)
+  if (getParentLibraryStorage())
     return;
   std::string ErrorMsg;
-  ParentLibrary = std::make_shared<llvm::sys::DynamicLibrary>(
+  getParentLibraryStorage() = std::make_shared<llvm::sys::DynamicLibrary>(
       llvm::sys::DynamicLibrary::getPermanentLibrary(Filename, &ErrorMsg));
-  if ((ParentLibrary == nullptr) || (!ParentLibrary->isValid()))
+  if (!getParentLibraryStorage() || !getParentLibraryStorage()->isValid())
     REPORT() << "Failed to set parent library: " << ErrorMsg.c_str();
 }
 
+std::map<ompt_device_t *, int32_t> &llvm::omp::target::ompt::getDevices() {
+  static std::map<ompt_device_t *, int32_t> Devices;
+  return Devices;
+}
+
 int llvm::omp::target::ompt::getDeviceId(ompt_device_t *Device) {
   // Block other threads, which might trigger an erase (for the same device)
   std::unique_lock<std::mutex> Lock(DeviceIdWritingMutex);
-  auto DeviceIterator = Devices.find(Device);
-  if (Device == nullptr || DeviceIterator == Devices.end()) {
+  auto DeviceIterator = getDevices().find(Device);
+  if (Device == nullptr || DeviceIterator == getDevices().end()) {
     REPORT() << "Failed to get ID for Device=" << Device;
     return -1;
   }
@@ -93,8 +98,8 @@ void llvm::omp::target::ompt::setDeviceId(ompt_device_t *Device,
     return;
   }
   std::unique_lock<std::mutex> Lock(DeviceIdWritingMutex);
-  auto DeviceIterator = Devices.find(Device);
-  if (DeviceIterator != Devices.end()) {
+  auto DeviceIterator = getDevices().find(Device);
+  if (DeviceIterator != getDevices().end()) {
     auto CurrentDeviceId = DeviceIterator->second;
     if (DeviceId == CurrentDeviceId) {
       REPORT() << "Tried to duplicate OMPT Device= " << Device
@@ -105,7 +110,7 @@ void llvm::omp::target::ompt::setDeviceId(ompt_device_t *Device,
     }
     return;
   }
-  Devices.emplace(Device, DeviceId);
+  getDevices().emplace(Device, DeviceId);
 }
 
 void llvm::omp::target::ompt::removeDeviceId(ompt_device_t *Device) {
@@ -115,8 +120,8 @@ void llvm::omp::target::ompt::removeDeviceId(ompt_device_t *Device) {
     return;
   }
   std::unique_lock<std::mutex> Lock(DeviceIdWritingMutex);
-  Devices.erase(Device);
-  TracedDevices.erase(DeviceId);
+  getDevices().erase(Device);
+  getTracedDevices().erase(DeviceId);
 }
 
 OMPT_API_ROUTINE ompt_set_result_t ompt_set_trace_ompt(ompt_device_t *Device,

>From 482ae352db6d3fb5d2c65c6ab268b8198900f0a9 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 21 May 2026 08:43:39 -0500
Subject: [PATCH 09/11] [OpenMP] Add OMPT device tracing tests

---
 .../callbacks.h                               | 340 ++++++++++++++++++
 .../veccopy_ompt_selective_tracing_dataop.cpp |  66 ++++
 .../callbacks.h                               | 340 ++++++++++++++++++
 .../veccopy_ompt_selective_tracing_kernel.cpp |  70 ++++
 .../callbacks.h                               | 332 +++++++++++++++++
 .../veccopy-ompt-target-data-tracing-emi.cpp  | 146 ++++++++
 .../callbacks.h                               | 333 +++++++++++++++++
 .../veccopy-ompt-target-emi-tracing-dag.cpp   |  81 +++++
 .../callbacks.h                               | 332 +++++++++++++++++
 .../veccopy-ompt-target-emi-tracing.cpp       |  78 ++++
 .../callbacks.h                               | 332 +++++++++++++++++
 .../veccopy-ompt-target-emi-tracing.cpp       |  67 ++++
 .../callbacks.h                               | 332 +++++++++++++++++
 .../veccopy-ompt-target-emi-tracing.cpp       |  66 ++++
 .../veccopy-ompt-target-tracing/callbacks.h   | 292 +++++++++++++++
 .../veccopy-ompt-target-tracing.cpp           |  78 ++++
 16 files changed, 3285 insertions(+)
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/veccopy_ompt_selective_tracing_dataop.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/veccopy_ompt_selective_tracing_kernel.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/veccopy-ompt-target-data-tracing-emi.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/veccopy-ompt-target-emi-tracing-dag.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/veccopy-ompt-target-emi-tracing.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/veccopy-ompt-target-emi-tracing.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/veccopy-ompt-target-emi-tracing.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing/veccopy-ompt-target-tracing.cpp

diff --git a/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/callbacks.h
new file mode 100644
index 0000000000000..5cd611990d32b
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/callbacks.h
@@ -0,0 +1,340 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+  // Only enable device tracing for dataop events explicitly
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d filename:%s host_adddr:%p device_addr:%p"
+         " bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+  assert(DeviceMapPtr != nullptr && "DeviceMapPtr must be valid");
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d optype=%d target_task_data=%p "
+         "(0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) src=%p "
+         "src_device_num=%d dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, optype, target_task_data, target_task_data_value,
+         target_data, target_data->value, host_op_id, *host_op_id, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d req_num_teams=%d target_data=%p "
+         "(0x%lx) host_op_id=%p (0x%lx)\n",
+         endpoint, requested_num_teams, target_data, target_data->value,
+         host_op_id, *host_op_id);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+#if EMI
+  register_ompt_callback(ompt_callback_target_data_op_emi);
+  register_ompt_callback(ompt_callback_target_emi);
+  register_ompt_callback(ompt_callback_target_submit_emi);
+#else
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+#endif
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/veccopy_ompt_selective_tracing_dataop.cpp b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/veccopy_ompt_selective_tracing_dataop.cpp
new file mode 100644
index 0000000000000..c07bff4230688
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/veccopy_ompt_selective_tracing_dataop.cpp
@@ -0,0 +1,66 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
+/// CHECK: rec={{0x[0-9a-fA-F]+}} type=9
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=10
+/// CHECK: rec={{0x[0-9a-fA-F]+}} type=9
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
+
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
+/// CHECK: rec={{0x[0-9a-fA-F]+}} type=9
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=10
+/// CHECK: rec={{0x[0-9a-fA-F]+}} type=9
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
diff --git a/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/callbacks.h
new file mode 100644
index 0000000000000..9655d718c7c48
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/callbacks.h
@@ -0,0 +1,340 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+  // Only enable device tracing for kernel submit events
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d filename:%s host_adddr:%p device_addr:%p"
+         " bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+  assert(DeviceMapPtr != nullptr && "DeviceMapPtr must be valid");
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d optype=%d target_task_data=%p "
+         "(0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) src=%p "
+         "src_device_num=%d dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, optype, target_task_data, target_task_data_value,
+         target_data, target_data->value, host_op_id, *host_op_id, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d req_num_teams=%d target_data=%p "
+         "(0x%lx) host_op_id=%p (0x%lx)\n",
+         endpoint, requested_num_teams, target_data, target_data->value,
+         host_op_id, *host_op_id);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+#if EMI
+  register_ompt_callback(ompt_callback_target_data_op_emi);
+  register_ompt_callback(ompt_callback_target_emi);
+  register_ompt_callback(ompt_callback_target_submit_emi);
+#else
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+#endif
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/veccopy_ompt_selective_tracing_kernel.cpp b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/veccopy_ompt_selective_tracing_kernel.cpp
new file mode 100644
index 0000000000000..285f1f1011268
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/veccopy_ompt_selective_tracing_kernel.cpp
@@ -0,0 +1,70 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// XFAIL: *
+// clang-format on
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  for (auto Device : *DeviceMapPtr)
+    flush_trace(Device);
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=9
+/// CHECK: rec={{0x[0-9a-fA-F]+}} type=10
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=9
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
+
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=9
+/// CHECK: rec={{0x[0-9a-fA-F]+}} type=10
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=9
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/callbacks.h
new file mode 100644
index 0000000000000..d05f05a4a4501
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/callbacks.h
@@ -0,0 +1,332 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+#define EMI 1
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d filename:%s host_adddr:%p device_addr:%p"
+         " bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d optype=%d target_task_data=%p "
+         "(0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) src=%p "
+         "src_device_num=%d dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, optype, target_task_data, target_task_data_value,
+         target_data, target_data->value, host_op_id, *host_op_id, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d req_num_teams=%d target_data=%p "
+         "(0x%lx) host_op_id=%p (0x%lx)\n",
+         endpoint, requested_num_teams, target_data, target_data->value,
+         host_op_id, *host_op_id);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+#if EMI
+  register_ompt_callback(ompt_callback_target_data_op_emi);
+  register_ompt_callback(ompt_callback_target_emi);
+  register_ompt_callback(ompt_callback_target_submit_emi);
+#else
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+#endif
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/veccopy-ompt-target-data-tracing-emi.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/veccopy-ompt-target-data-tracing-emi.cpp
new file mode 100644
index 0000000000000..35a38f4eba739
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/veccopy-ompt-target-data-tracing-emi.cpp
@@ -0,0 +1,146 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+#define N 100000
+
+#pragma omp declare target
+int c[N];
+#pragma omp end declare target
+
+int main() {
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+  for (i = 0; i < N; i++)
+    c[i] = 0;
+
+#pragma omp target enter data map(to : a)
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+#pragma omp target exit data map(from : a)
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+#pragma omp target parallel for map(alloc : c)
+  {
+    for (int j = 0; j < N; j++)
+      c[j] = 2 * j + 1;
+  }
+#pragma omp target update from(c) nowait
+#pragma omp barrier
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+  int rc = 0;
+  for (i = 0; i < N; i++) {
+    if (a[i] != i) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+  }
+
+  for (i = 0; i < N; i++) {
+    if (c[i] != 2 * i + 1) {
+      rc++;
+      printf("Wrong value: c[%d]=%d\n", i, c[i]);
+    }
+  }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+// > OMPT device callback related checks below. <
+
+/// CHECK-NOT: Callback Target EMI:
+/// CHECK-NOT: device_num=-1
+
+/// CHECK: Callback Init:
+/// CHECK: Callback Load:
+
+/// CHECK-DAG: Callback Target EMI: kind=2 endpoint=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=2
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=2
+/// CHECK-DAG: Callback Target EMI: kind=2 endpoint=2
+
+/// CHECK-DAG: Callback Target EMI: kind=1 endpoint=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=2
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=2
+/// CHECK-DAG: Callback Submit EMI: endpoint=1 req_num_teams=1
+/// CHECK-DAG: Callback Submit EMI: endpoint=2 req_num_teams=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=3
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=3
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=4
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=4
+/// CHECK-DAG: Callback Target EMI: kind=1 endpoint=2
+/// CHECK-DAG: Callback Target EMI: kind=3 endpoint=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=3
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=3
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=4
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=4
+/// CHECK-DAG: Callback Target EMI: kind=3 endpoint=2
+
+/// CHECK-DAG: Callback Target EMI: kind=1 endpoint=1
+/// CHECK-DAG: Callback Submit EMI: endpoint=1 req_num_teams=1
+/// CHECK-DAG: Callback Submit EMI: endpoint=2 req_num_teams=1
+/// CHECK-DAG: Callback Target EMI: kind=1 endpoint=2
+/// CHECK-DAG: Callback Target EMI: kind=4 endpoint=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=3
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=3
+/// CHECK-DAG: Callback Target EMI: kind=4 endpoint=2
+
+/// CHECK-DAG: Success
+/// CHECK-DAG: Callback Fini:
+
+// > OMPT device tracing related checks below. <
+
+
+
+// Note: Split checks for record address and content. That way we do not imply
+//       any order. Records 01-06 and 12-17 occur interleaved and belong to the
+//       first target region. 07-11 occur interleaved with 18-22 and belong to
+//       the second target region.
+
+// Note: These addresses will only occur once. They are only captured to
+//       indicate their existence.
+
+/// CHECK-DAG: type=8 (Target task)
+/// CHECK-DAG: type=9 (Target data op)
+
+// Note: ADDRX_11 may not trigger a final callback.
+
+// Note: ADDRX_11 may not be deallocated.
+
+/// CHECK-NOT: host_op_id=0x0
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/callbacks.h
new file mode 100644
index 0000000000000..f5724fa7a6852
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/callbacks.h
@@ -0,0 +1,333 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// Enable / disable OMPT's 'External Monitoring Interface'
+#define EMI 1
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d filename:%s host_adddr:%p device_addr:%p"
+         " bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d optype=%d target_task_data=%p "
+         "(0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) src=%p "
+         "src_device_num=%d dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, optype, target_task_data, target_task_data_value,
+         target_data, target_data->value, host_op_id, *host_op_id, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d req_num_teams=%d target_data=%p "
+         "(0x%lx) host_op_id=%p (0x%lx)\n",
+         endpoint, requested_num_teams, target_data, target_data->value,
+         host_op_id, *host_op_id);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+#if EMI
+  register_ompt_callback(ompt_callback_target_data_op_emi);
+  register_ompt_callback(ompt_callback_target_emi);
+  register_ompt_callback(ompt_callback_target_submit_emi);
+#else
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+#endif
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/veccopy-ompt-target-emi-tracing-dag.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/veccopy-ompt-target-emi-tracing-dag.cpp
new file mode 100644
index 0000000000000..0fdae19017ab2
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/veccopy-ompt-target-emi-tracing-dag.cpp
@@ -0,0 +1,81 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+/// CHECK-NOT: host_op_id=0x0
+
+
+
+// Note: Split checks for record address and content. That way we do not imply
+//       any order. Records 01-06 and 12-17 occur interleaved and belong to the
+//       first target region. 07-11 occur interleaved with 18-22 and belong to
+//       the second target region.
+
+/// CHECK-DAG: type=8 (Target task)
+/// CHECK-DAG: type=9 (Target data op)
+
+// Note: These addresses will only occur once. They are only captured to
+//       indicate their existence.
+
+
+// Note: ADDRX_11 may not trigger a final callback.
+
+// Note: ADDRX_11 may not be deallocated.
+
+/// CHECK-DAG: Success
+
+/// CHECK-NOT: host_op_id=0x0
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/callbacks.h
new file mode 100644
index 0000000000000..d05f05a4a4501
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/callbacks.h
@@ -0,0 +1,332 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+#define EMI 1
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d filename:%s host_adddr:%p device_addr:%p"
+         " bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d optype=%d target_task_data=%p "
+         "(0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) src=%p "
+         "src_device_num=%d dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, optype, target_task_data, target_task_data_value,
+         target_data, target_data->value, host_op_id, *host_op_id, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d req_num_teams=%d target_data=%p "
+         "(0x%lx) host_op_id=%p (0x%lx)\n",
+         endpoint, requested_num_teams, target_data, target_data->value,
+         host_op_id, *host_op_id);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+#if EMI
+  register_ompt_callback(ompt_callback_target_data_op_emi);
+  register_ompt_callback(ompt_callback_target_emi);
+  register_ompt_callback(ompt_callback_target_submit_emi);
+#else
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+#endif
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/veccopy-ompt-target-emi-tracing.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/veccopy-ompt-target-emi-tracing.cpp
new file mode 100644
index 0000000000000..f9dfb47ffff43
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/veccopy-ompt-target-emi-tracing.cpp
@@ -0,0 +1,78 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-generic && env LIBOMPTARGET_OMPT_FLUSH_ON_SHUTDOWN=false LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL=false %libomptarget-run-generic | %fcheck-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+/*
+ * This test is run with LIBOMPTARGET_OMPT_FLUSH_ON_SHUTDOWN=false and
+ * LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL=false while doing explicit flushing
+ * using ompt_flush_trace. The intention is to check whether trace records are
+ * properly flushed when the program/tool uses the ompt_flush_trace API. There
+ * should be 23 trace records returned to the tool.
+ */
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+/// CHECK-NOT: host_op_id=0x0
+
+/// CHECK: rec=
+
+/// CHECK-DAG: Success
+
+/// The user calls flush before printing success, so
+/// no more records should be returned here.
+
+/// CHECK-NOT: host_op_id=0x0
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/callbacks.h
new file mode 100644
index 0000000000000..d05f05a4a4501
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/callbacks.h
@@ -0,0 +1,332 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+#define EMI 1
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d filename:%s host_adddr:%p device_addr:%p"
+         " bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d optype=%d target_task_data=%p "
+         "(0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) src=%p "
+         "src_device_num=%d dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, optype, target_task_data, target_task_data_value,
+         target_data, target_data->value, host_op_id, *host_op_id, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d req_num_teams=%d target_data=%p "
+         "(0x%lx) host_op_id=%p (0x%lx)\n",
+         endpoint, requested_num_teams, target_data, target_data->value,
+         host_op_id, *host_op_id);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+#if EMI
+  register_ompt_callback(ompt_callback_target_data_op_emi);
+  register_ompt_callback(ompt_callback_target_emi);
+  register_ompt_callback(ompt_callback_target_submit_emi);
+#else
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+#endif
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/veccopy-ompt-target-emi-tracing.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/veccopy-ompt-target-emi-tracing.cpp
new file mode 100644
index 0000000000000..abf2426d2eee9
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/veccopy-ompt-target-emi-tracing.cpp
@@ -0,0 +1,67 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-generic && env LIBOMPTARGET_OMPT_FLUSH_ON_SHUTDOWN=false %libomptarget-run-generic | %fcheck-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+/*
+ * This test is run with LIBOMPTARGET_OMPT_FLUSH_ON_SHUTDOWN=false and
+ * ompt_flush_trace is not invoked by the user/tool. The intention is to check
+ * whether trace records are properly flushed as a buffer fills up. Currently,
+ * the buffer size in callbacks.h implies that every buffer holds 2 trace
+ * records. With this assumption, there should be 22 trace records. The last
+ * trace record is not flushed because the last buffer is not yet full.
+ */
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+/// CHECK-NOT: host_op_id=0x0
+
+/// CHECK: rec=
+/// CHECK-NOT: host_op_id=0x0
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/callbacks.h
new file mode 100644
index 0000000000000..d05f05a4a4501
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/callbacks.h
@@ -0,0 +1,332 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+#define EMI 1
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d filename:%s host_adddr:%p device_addr:%p"
+         " bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d optype=%d target_task_data=%p "
+         "(0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) src=%p "
+         "src_device_num=%d dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, optype, target_task_data, target_task_data_value,
+         target_data, target_data->value, host_op_id, *host_op_id, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d req_num_teams=%d target_data=%p "
+         "(0x%lx) host_op_id=%p (0x%lx)\n",
+         endpoint, requested_num_teams, target_data, target_data->value,
+         host_op_id, *host_op_id);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+#if EMI
+  register_ompt_callback(ompt_callback_target_data_op_emi);
+  register_ompt_callback(ompt_callback_target_emi);
+  register_ompt_callback(ompt_callback_target_submit_emi);
+#else
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+#endif
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/veccopy-ompt-target-emi-tracing.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/veccopy-ompt-target-emi-tracing.cpp
new file mode 100644
index 0000000000000..7f4e99b83e4e3
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/veccopy-ompt-target-emi-tracing.cpp
@@ -0,0 +1,66 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-generic && env LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL=false %libomptarget-run-generic | %fcheck-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+/*
+ * This test is run with LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL=false and
+ * ompt_flush_trace is not invoked by the user/tool. The intention is to check
+ * whether trace records are properly flushed on shutdown. 23 trace records
+ * should be flushed during shutdown.
+ */
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+/// CHECK-NOT: host_op_id=0x0
+
+/// CHECK: rec=
+
+/// CHECK-NOT: host_op_id=0x0
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-tracing/callbacks.h
new file mode 100644
index 0000000000000..cfb5fd4147ebf
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing/callbacks.h
@@ -0,0 +1,292 @@
+// Copyright © Advanced Micro Devices, Inc., or its affiliates.
+//
+// SPDX-License-Identifier:  MIT
+
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// Enable / disable OMPT's 'External Monitoring Interface'
+#define EMI 0
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE (4 * 1024)
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+static ompt_get_record_type_t ompt_get_record_type_fn = 0;
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+// Note: This callback must handle a null begin cursor. Currently,
+// ompt_get_record_ompt, print_record_ompt, and
+// ompt_advance_buffer_cursor handle a null cursor.
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+
+    if (ompt_get_record_type_fn(buffer, current) != ompt_record_ompt) {
+      printf("WARNING: received non-ompt type buffer object\n");
+    }
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n", device_num,
+         type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  ompt_get_record_type_fn =
+      (ompt_get_record_type_t)lookup("ompt_get_record_type");
+  if (!ompt_get_record_type_fn) {
+    printf("WARNING: No function ompt_get_record_type found in device "
+           "callbacks\n");
+  }
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Load: device_num:%d filename:%s host_adddr:%p device_addr:%p "
+         "bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: target_id=%lu host_op_id=%lu optype=%d src=%p "
+         "src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         target_id, host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: target_id=%lu kind=%d endpoint=%d device_num=%d "
+         "code=%p\n",
+         target_id, kind, endpoint, device_num, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing/veccopy-ompt-target-tracing.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-tracing/veccopy-ompt-target-tracing.cpp
new file mode 100644
index 0000000000000..d393bc63b2e14
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing/veccopy-ompt-target-tracing.cpp
@@ -0,0 +1,78 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+// Copyright © Advanced Micro Devices, Inc., or its affiliates.
+//
+// SPDX-License-Identifier:  MIT
+
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+// > OMPT device tracing related checks below. <
+
+// Note: This test will allocate one buffer, big enough to hold all trace
+//       records, hence there will be only one allocation.
+
+
+// Note: Split checks for record address and content. That way we do not imply
+//       any order. Records may / will occur interleaved.
+
+// Note: These addresses will only occur once. They are only captured to
+//       indicate their existence.
+
+/// CHECK-DAG: type=8 (Target task)
+/// CHECK-DAG: type=9 (Target data op)
+
+// Note: ADDRX_01 may not trigger a final callback.
+// Note: ADDRX_01 may not be deallocated.
+
+/// CHECK-NOT: host_op_id=0x0

>From 89b5cfcf0b49e5b1358e52b236ff94925f57a749 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 21 May 2026 08:44:24 -0500
Subject: [PATCH 10/11] [Openmp] Add OMPT device tracing support tests

These tests concern functionality used by parts of the OMPT device
tracing implementation.
---
 .../veccopy-ompt-ctor-1/Inputs/callbacks.h    | 322 +++++++++++++++++
 .../Inputs/veccopy-ctor-1.cpp                 |  35 ++
 .../tracing/veccopy-ompt-ctor-1/veccopy.cpp   |  51 +++
 .../callbacks.h                               | 286 ++++++++++++++++
 .../veccopy-ompt-target-default-device.cpp    |  87 +++++
 .../veccopy-ompt-target-devices/callbacks.h   | 324 ++++++++++++++++++
 .../veccopy-ompt-target-devices.cpp           |  87 +++++
 .../callbacks.h                               | 136 ++++++++
 .../veccopy-ompt-target.c                     |  77 +++++
 .../callbacks.h                               | 131 +++++++
 .../veccopy-ompt-target.c                     |  77 +++++
 11 files changed, 1613 insertions(+)
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/veccopy-ctor-1.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-ctor-1/veccopy.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-default-device/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-default-device/veccopy-ompt-target-default-device.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-devices/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-devices/veccopy-ompt-target-devices.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-translate-time/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-translate-time/veccopy-ompt-target.c
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/veccopy-ompt-target.c

diff --git a/offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/callbacks.h
new file mode 100644
index 0000000000000..78bd4fe39b87b
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/callbacks.h
@@ -0,0 +1,322 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+#define EMI 1
+
+// Tool related code below
+#include <omp-tools.h>
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  printf("rec=%p type=%2d time=%lu thread_id=%lu target_id=%lu\n", rec,
+         rec->type, rec->time, rec->thread_id, rec->target_id);
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("\tRecord Target task: target_id=0x%lx kind=%d endpoint=%d "
+           "device=%d task_id=%lu codeptr=%p\n",
+           target_rec.target_id, target_rec.kind, target_rec.endpoint,
+           target_rec.device_num, target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("\t  Record Target data op: target_id=0x%lx host_op_id=0x%lx "
+           "optype=%d src_addr=%p src_device=%d "
+           "dest_addr=%p dest_device=%d bytes=%lu end_time=%lu duration=%lu ns "
+           "codeptr=%p\n",
+           rec->target_id, target_data_op_rec.host_op_id,
+           target_data_op_rec.optype, target_data_op_rec.src_addr,
+           target_data_op_rec.src_device_num, target_data_op_rec.dest_addr,
+           target_data_op_rec.dest_device_num, target_data_op_rec.bytes,
+           target_data_op_rec.end_time, target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("\t  Record Target kernel:  target_id=0x%lx host_op_id=0x%lx "
+           "requested_num_teams=%u granted_num_teams=%u "
+           "end_time=%lu duration=%lu ns\n",
+           rec->target_id, target_kernel_rec.host_op_id,
+           target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n", device_num,
+         type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Load: device_num:%d filename:%s host_adddr:%p device_addr:%p "
+         "bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d target_task_data=%p (0x%lx) "
+         "target_data=%p (0x%lx) host_op_id=%p (0x%lx) optype=%d src=%p "
+         "src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, target_task_data, target_task_data_value, target_data,
+         target_data->value, host_op_id, *host_op_id, optype, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d target_data=%p (0x%lx) "
+         "host_op_id=%p (0x%lx) req_num_teams=%d\n",
+         endpoint, target_data, target_data->value, host_op_id, *host_op_id,
+         requested_num_teams);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  ompt_set_callback(ompt_callback_device_initialize,
+                    (ompt_callback_t)&on_ompt_callback_device_initialize);
+  ompt_set_callback(ompt_callback_device_load,
+                    (ompt_callback_t)&on_ompt_callback_device_load);
+#if EMI
+  ompt_set_callback(ompt_callback_target_submit_emi,
+                    (ompt_callback_t)&on_ompt_callback_target_submit_emi);
+  ompt_set_callback(ompt_callback_target_data_op_emi,
+                    (ompt_callback_t)&on_ompt_callback_target_data_op_emi);
+  ompt_set_callback(ompt_callback_target_emi,
+                    (ompt_callback_t)&on_ompt_callback_target_emi);
+#else
+  ompt_set_callback(ompt_callback_target_submit,
+                    (ompt_callback_t)&on_ompt_callback_target_submit);
+  ompt_set_callback(ompt_callback_target_data_op,
+                    (ompt_callback_t)&on_ompt_callback_target_data_op);
+  ompt_set_callback(ompt_callback_target,
+                    (ompt_callback_t)&on_ompt_callback_target);
+#endif
+  return 1; //success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/veccopy-ctor-1.cpp b/offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/veccopy-ctor-1.cpp
new file mode 100644
index 0000000000000..137a60735f345
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/veccopy-ctor-1.cpp
@@ -0,0 +1,35 @@
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int status;
+
+__attribute__((constructor(101))) void veccopy_init() {
+  int N = 10;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  if (a[5] != b[5])
+    status = 1;
+  else
+    status = 0;
+}
diff --git a/offload/test/ompt/tracing/veccopy-ompt-ctor-1/veccopy.cpp b/offload/test/ompt/tracing/veccopy-ompt-ctor-1/veccopy.cpp
new file mode 100644
index 0000000000000..76af6a9b6c33f
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-ctor-1/veccopy.cpp
@@ -0,0 +1,51 @@
+// clang-format off
+// RUN: %clangxx-generic %s %S/Inputs/veccopy-ctor-1.cpp -o %t -Xoffload-linker -lompdevice && %libomptarget-run-generic | %fcheck-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+/*
+ * Test whether OMPT support works for OpenMP directives
+ * in a global constructor without using a shared library.
+ * This currently succeeds.
+ */
+
+#include <omp.h>
+#include <stdio.h>
+
+extern int status;
+
+int main() {
+  int N = 10;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong varlue: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc && !status)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+/// CHECK: Record Target task
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-default-device/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-default-device/callbacks.h
new file mode 100644
index 0000000000000..5df9985ced0c3
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-default-device/callbacks.h
@@ -0,0 +1,286 @@
+#include <cassert>
+#include <memory>
+#include <unordered_set>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// Enable / disable OMPT's 'External Monitoring Interface'
+#define EMI 0
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+static ompt_get_record_type_t ompt_get_record_type_fn = 0;
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request_default_device(
+    int device_num, ompt_buffer_t **buffer, size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback for default "
+         "device %d\n",
+         *bytes, *buffer, device_num);
+}
+
+// Note: This callback must handle a null begin cursor. Currently,
+// ompt_get_record_ompt, print_record_ompt, and
+// ompt_advance_buffer_cursor handle a null cursor.
+static void on_ompt_callback_buffer_complete_default_device(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback for default device: \
+  device_num=%d, buffer=%p, num_bytes=%lu, begin=%p, buffer_owned=%d\n",
+         device_num, buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+
+    if (ompt_get_record_type_fn(buffer, current) != ompt_record_ompt) {
+      printf("WARNING: received non-ompt type buffer object\n");
+    }
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+  ompt_set_trace_ompt(Device, /*enable=*/1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, /*enable=*/1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, /*enable=*/1, ompt_callback_target_submit);
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+  if (device_num != omp_get_default_device())
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device,
+                          &on_ompt_callback_buffer_request_default_device,
+                          &on_ompt_callback_buffer_complete_default_device);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n", device_num,
+         type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  ompt_get_record_type_fn =
+      (ompt_get_record_type_t)lookup("ompt_get_record_type");
+  if (!ompt_get_record_type_fn) {
+    printf("WARNING: No function ompt_get_record_type found in device "
+           "callbacks\n");
+  }
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Load: device_num:%d filename:%s host_adddr:%p device_addr:%p "
+         "bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: target_id=%lu host_op_id=%lu optype=%d src=%p "
+         "src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         target_id, host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: target_id=%lu kind=%d endpoint=%d device_num=%d "
+         "code=%p\n",
+         target_id, kind, endpoint, device_num, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-default-device/veccopy-ompt-target-default-device.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-default-device/veccopy-ompt-target-default-device.cpp
new file mode 100644
index 0000000000000..6446cda7e7fbb
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-default-device/veccopy-ompt-target-default-device.cpp
@@ -0,0 +1,87 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+#include <cassert>
+#include <omp.h>
+#include <stdio.h>
+
+// This test starts device tracing on the default device only (see
+// start_trace in callbacks.h). However, if more devices are
+// available, it calls flush and stop on the other devices as
+// well. The intention is to check correct runtime behavior if a tool
+// invokes flush or stop on a device that was not started. The runtime
+// should just return without doing anything.
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+  // Warm up
+#pragma omp target
+  {
+  }
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+  for (int dev = 0; dev < omp_get_num_devices(); ++dev) {
+#pragma omp target teams distribute parallel for device(dev)
+    {
+      for (int j = 0; j < N; j++)
+        a[j] = b[j];
+    }
+  }
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+// > OMPT device tracing related checks below. <
+
+
+
+// Note: Split checks for record address and content. That way we do not imply
+//       any order. Records may / will occur interleaved.
+
+// Note: These addresses will only occur once. They are only captured to
+//       indicate their existence.
+
+/// CHECK-DAG: type=8 (Target task)
+/// CHECK-DAG: type=9 (Target data op)
+
+// Note: ADDRX_07 may not trigger a final callback.
+
+// Note: ADDRX_07 may not be deallocated.
+
+/// CHECK-NOT: host_op_id=0x0
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-devices/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-devices/callbacks.h
new file mode 100644
index 0000000000000..0625b1a7b1522
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-devices/callbacks.h
@@ -0,0 +1,324 @@
+#include <cassert>
+#include <memory>
+#include <unordered_set>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// Enable / disable OMPT's 'External Monitoring Interface'
+#define EMI 0
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+static ompt_get_record_type_t ompt_get_record_type_fn = 0;
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request_default_device(
+    int device_num, ompt_buffer_t **buffer, size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback for default "
+         "device %d\n",
+         *bytes, *buffer, device_num);
+}
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback for device %d\n",
+         *bytes, *buffer, device_num);
+}
+
+// Note: This callback must handle a null begin cursor. Currently,
+// ompt_get_record_ompt, print_record_ompt, and
+// ompt_advance_buffer_cursor handle a null cursor.
+static void on_ompt_callback_buffer_complete_default_device(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback for default device: \
+  device_num=%d, buffer=%p, num_bytes=%lu, begin=%p, buffer_owned=%d\n",
+         device_num, buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+
+    if (ompt_get_record_type_fn(buffer, current) != ompt_record_ompt) {
+      printf("WARNING: received non-ompt type buffer object\n");
+    }
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+// Note: This callback must handle a null begin cursor. Currently,
+// ompt_get_record_ompt, print_record_ompt, and
+// ompt_advance_buffer_cursor handle a null cursor.
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: \
+  device_num=%d, buffer=%p, num_bytes=%lu, begin=%p, buffer_owned=%d\n",
+         device_num, buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+
+    if (ompt_get_record_type_fn(buffer, current) != ompt_record_ompt) {
+      printf("WARNING: received non-ompt type buffer object\n");
+    }
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+  ompt_set_trace_ompt(Device, /*enable=*/1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, /*enable=*/1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, /*enable=*/1, ompt_callback_target_submit);
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  if (device_num == omp_get_default_device())
+    return ompt_start_trace(Device,
+                            &on_ompt_callback_buffer_request_default_device,
+                            &on_ompt_callback_buffer_complete_default_device);
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n", device_num,
+         type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  ompt_get_record_type_fn =
+      (ompt_get_record_type_t)lookup("ompt_get_record_type");
+  if (!ompt_get_record_type_fn) {
+    printf("WARNING: No function ompt_get_record_type found in device "
+           "callbacks\n");
+  }
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Load: device_num:%d filename:%s host_adddr:%p device_addr:%p "
+         "bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: target_id=%lu host_op_id=%lu optype=%d src=%p "
+         "src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         target_id, host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: target_id=%lu kind=%d endpoint=%d device_num=%d "
+         "code=%p\n",
+         target_id, kind, endpoint, device_num, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-devices/veccopy-ompt-target-devices.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-devices/veccopy-ompt-target-devices.cpp
new file mode 100644
index 0000000000000..0c6c0b001cff9
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-devices/veccopy-ompt-target-devices.cpp
@@ -0,0 +1,87 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+#include <cassert>
+#include <omp.h>
+#include <stdio.h>
+
+// This test starts device tracing on all available devices (see
+// start_trace in callbacks.h). It subsequently calls flush and stop
+// on all devices like a tool would normally do.
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+  // Warm up
+#pragma omp target
+  {
+  }
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+  for (int dev = 0; dev < omp_get_num_devices(); ++dev) {
+#pragma omp target teams distribute parallel for device(dev)
+    {
+      for (int j = 0; j < N; j++)
+        a[j] = b[j];
+    }
+  }
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+// > OMPT device tracing related checks below. <
+
+// Note: Since this test may (theoretically) run on an arbitrary number of
+//       devices, we may only check for the minimal set of records.
+
+
+
+// Note: Split checks for record address and content. That way we do not imply
+//       any order. Records may / will occur interleaved.
+
+// Note: These addresses will only occur once. They are only captured to
+//       indicate their existence.
+
+/// CHECK-DAG: type=8 (Target task)
+/// CHECK-DAG: type=9 (Target data op)
+
+// Note: ADDRX_07 may not trigger a final callback.
+
+// Note: ADDRX_07 may not be deallocated.
+
+/// CHECK-NOT: host_op_id=0x0
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-translate-time/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-translate-time/callbacks.h
new file mode 100644
index 0000000000000..0bb080d863a9b
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-translate-time/callbacks.h
@@ -0,0 +1,136 @@
+#include <assert.h>
+#include <math.h>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_translate_time_t ompt_translate_time_fn = 0;
+static ompt_get_device_time_t ompt_get_device_time_fn = 0;
+
+// OMPT callbacks
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+
+  ompt_device_time_t device_time = 0;
+
+  ompt_translate_time_fn = (ompt_translate_time_t)lookup("ompt_translate_time");
+  ompt_get_device_time_fn =
+      (ompt_get_device_time_t)lookup("ompt_get_device_time");
+
+  if (ompt_get_device_time_fn) {
+    uint64_t time = ompt_get_device_time_fn(NULL);
+    printf("The device time can be queried: %lu\n", time);
+    device_time = time;
+  } else {
+    printf("Could not determine the device time\n");
+  }
+
+  if (ompt_translate_time_fn) {
+    assert(device_time > 0 && "Obtain device time was successfull");
+    printf("Translate time available\n");
+    double omp_wtime = omp_get_wtime();
+    device_time = ompt_get_device_time_fn(NULL);
+    double omp_device_wtime = ompt_translate_time_fn(NULL, device_time);
+    printf("OMPT: Host time: %f, Device time: %f\n", omp_wtime,
+           omp_device_wtime);
+  } else {
+    printf("Could not translate time.\n");
+  }
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d module_id:%lu filename:%s host_adddr:%p "
+         "device_addr:%p bytes:%lu\n",
+         device_num, module_id, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  assert(codeptr_ra != 0);
+  // Both src and dest must not be null
+  assert(src_addr != 0 || dest_addr != 0);
+  printf("  Callback DataOp: target_id=%lu host_op_id=%lu optype=%d src=%p "
+         "src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         target_id, host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  assert(codeptr_ra != 0);
+  printf("Callback Target: target_id=%lu kind=%d endpoint=%d device_num=%d "
+         "code=%p\n",
+         target_id, kind, endpoint, device_num, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+
+  return 1; //success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-translate-time/veccopy-ompt-target.c b/offload/test/ompt/tracing/veccopy-ompt-target-translate-time/veccopy-ompt-target.c
new file mode 100644
index 0000000000000..97936f987afc2
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-translate-time/veccopy-ompt-target.c
@@ -0,0 +1,77 @@
+// clang-format off
+// RUN: %libomptarget-compile-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+/// CHECK: Callback Init:
+/// CHECK: The device time can be queried: [[X:[0-9]+]]
+/// CHECK: Translate time available
+/// CHECK: Callback Load:
+/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2
+
+/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2
+/// CHECK: Callback Fini:
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/callbacks.h
new file mode 100644
index 0000000000000..66495a7f45f6b
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/callbacks.h
@@ -0,0 +1,131 @@
+#include <assert.h>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_get_record_type_t ompt_get_record_type_fn = 0;
+static ompt_get_device_time_t ompt_get_device_time_fn = 0;
+
+// OMPT callbacks
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+
+  ompt_get_record_type_fn =
+      (ompt_get_record_type_t)lookup("ompt_get_record_type");
+  ompt_get_device_time_fn =
+      (ompt_get_device_time_t)lookup("ompt_get_device_time");
+
+  if (ompt_get_record_type_fn) {
+    ompt_buffer_cursor_t buf;
+    // FIXME: For now, we pass a NULL buffer
+    ompt_record_t rec_type = ompt_get_record_type_fn(NULL, buf);
+    printf("Record Type: %s\n",
+           ((rec_type == ompt_record_ompt) ? "OMPT" : "Unknown"));
+  } else {
+    printf("Could not determine Record Type");
+  }
+
+  if (ompt_get_device_time_fn) {
+    uint64_t time = ompt_get_device_time_fn(NULL);
+    printf("The device time can be queried: %lu\n", time);
+  } else {
+    printf("Could not determine the device time \n");
+  }
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d module_id:%lu filename:%s host_adddr:%p "
+         "device_addr:%p bytes:%lu\n",
+         device_num, module_id, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  assert(codeptr_ra != 0);
+  // Both src and dest must not be null
+  assert(src_addr != 0 || dest_addr != 0);
+  printf("  Callback DataOp: target_id=%lu host_op_id=%lu optype=%d src=%p "
+         "src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         target_id, host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  assert(codeptr_ra != 0);
+  printf("Callback Target: target_id=%lu kind=%d endpoint=%d device_num=%d "
+         "code=%p\n",
+         target_id, kind, endpoint, device_num, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+
+  return 1; //success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/veccopy-ompt-target.c b/offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/veccopy-ompt-target.c
new file mode 100644
index 0000000000000..e636072826bb0
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/veccopy-ompt-target.c
@@ -0,0 +1,77 @@
+// clang-format off
+// RUN: %libomptarget-compile-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+/// CHECK: Callback Init:
+/// CHECK: Record Type: OMPT
+/// CHECK: The device time can be queried: [[X:[0-9]+]]
+/// CHECK: Callback Load:
+/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2
+
+/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2
+/// CHECK: Callback Fini:

>From 4477dd2993eefe69cc143e7722307c916150aab3 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 21 May 2026 09:11:16 -0500
Subject: [PATCH 11/11] [OMPT] Add test for synchronous path w/ OMPT

---
 .../large_mapping_sync_tracing.cpp            | 81 +++++++++++++++++++
 1 file changed, 81 insertions(+)
 create mode 100644 offload/test/ompt/tracing/large-mapping-sync-tracing/large_mapping_sync_tracing.cpp

diff --git a/offload/test/ompt/tracing/large-mapping-sync-tracing/large_mapping_sync_tracing.cpp b/offload/test/ompt/tracing/large-mapping-sync-tracing/large_mapping_sync_tracing.cpp
new file mode 100644
index 0000000000000..44939536d30f3
--- /dev/null
+++ b/offload/test/ompt/tracing/large-mapping-sync-tracing/large_mapping_sync_tracing.cpp
@@ -0,0 +1,81 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-generic && env LIBOMPTARGET_AMDGPU_MAX_ASYNC_COPY_BYTES=1 LIBOMPTARGET_OMPT_FLUSH_ON_SHUTDOWN=false LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL=false %libomptarget-run-generic | %fcheck-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+
+// Exercises OMPT device tracing for AMDGPU's synchronous large-copy path.
+// The transfer size is intentionally above the plugin's default async-copy
+// threshold, while the RUN line also lowers the threshold to keep the test
+// independent of future default changes.
+
+#include <omp.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "../veccopy-ompt-target-data-tracing-emi/callbacks.h"
+
+#define COPY_BYTES (50 * 1024 * 1024)
+
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  if (omp_get_num_devices() == 0) {
+    printf("Success\n");
+    return 0;
+  }
+
+  const size_t N = COPY_BYTES / sizeof(int);
+  int *Input = (int *)malloc(COPY_BYTES);
+  int *Output = (int *)malloc(COPY_BYTES);
+
+  if (!Input || !Output) {
+    free(Input);
+    free(Output);
+    printf("Failure\n");
+    return 1;
+  }
+
+  for (size_t I = 0; I < N; ++I) {
+    Input[I] = (int)(I & 0x7fff);
+    Output[I] = 0;
+  }
+
+#pragma omp target teams distribute parallel for map(to : Input[0 : N])        \
+    map(from : Output[0 : N])
+  for (size_t I = 0; I < N; ++I)
+    Output[I] = Input[I] + 1;
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+  int Failed = 0;
+  for (size_t I = 0; I < N; ++I) {
+    if (Output[I] != Input[I] + 1) {
+      Failed = 1;
+      break;
+    }
+  }
+
+  free(Input);
+  free(Output);
+
+  printf("%s\n", Failed ? "Failure" : "Success");
+  return Failed;
+}
+
+// clang-format off
+
+/// CHECK: Callback Init:
+/// CHECK: Callback Load:
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=2 {{.*}} bytes=52428800
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=2 {{.*}} bytes=52428800
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=3 {{.*}} bytes=52428800
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=3 {{.*}} bytes=52428800
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=4
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=4
+
+/// CHECK-DAG: type=9 (Target data op) {{.*}} optype=1 {{.*}} bytes=52428800
+/// CHECK: Success


From openmp-commits at lists.llvm.org  Wed Aug  5 06:29:38 2026
From: openmp-commits at lists.llvm.org (Jannick Kremer via Openmp-commits)
Date: Wed, 05 Aug 2026 06:29:38 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [Clang][OpenMP] Add
 `#pragma omp flatten` loop directive + `depth` clause (PR #206977)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206977@github.com>
Message-ID: <6a733ac2.8e75e0cb.1bed60.0ac3@mx.google.com>


https://github.com/DeinAlptraum commented:

I'm the maintainer for the libclang python bindings (cindex.py). That part LGTM

https://github.com/llvm/llvm-project/pull/206977

From openmp-commits at lists.llvm.org  Wed Aug  5 06:58:02 2026
From: openmp-commits at lists.llvm.org (Amit Tiwari via Openmp-commits)
Date: Wed, 05 Aug 2026 06:58:02 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [Clang][OpenMP] Add
 `#pragma omp flatten` loop directive + `depth` clause (PR #206977)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206977@github.com>
Message-ID: <6a73416a.492e23e4.45e2c.09b0@mx.google.com>


loopacino wrote:

> I'm the maintainer for the libclang python bindings (cindex.py). That part LGTM

Sure, thanks.

https://github.com/llvm/llvm-project/pull/206977

From openmp-commits at lists.llvm.org  Wed Aug  5 07:55:15 2026
From: openmp-commits at lists.llvm.org (Michael Klemm via Openmp-commits)
Date: Wed, 05 Aug 2026 07:55:15 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [openmp][WebAssembly] add
 support for wasm64 (PR #181669)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/181669@github.com>
Message-ID: <6a734ed3.dbb26164.10167b.20c0@mx.google.com>


https://github.com/mjklemm approved this pull request.

LGTM.

https://github.com/llvm/llvm-project/pull/181669

From openmp-commits at lists.llvm.org  Wed Aug  5 09:01:09 2026
From: openmp-commits at lists.llvm.org (Sam Clegg via Openmp-commits)
Date: Wed, 05 Aug 2026 09:01:09 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [openmp][WebAssembly] Add
 support for wasm64 (PR #181669)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/181669@github.com>
Message-ID: <6a735e45.f06f4994.2c262d.0b59@mx.google.com>


https://github.com/sbc100 edited https://github.com/llvm/llvm-project/pull/181669

From openmp-commits at lists.llvm.org  Wed Aug  5 09:01:30 2026
From: openmp-commits at lists.llvm.org (Sam Clegg via Openmp-commits)
Date: Wed, 05 Aug 2026 09:01:30 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [openmp][WebAssembly] Add
 support for wasm64 (PR #181669)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/181669@github.com>
Message-ID: <6a735e5a.eb165e5c.46348.15f8@mx.google.com>


https://github.com/sbc100 edited https://github.com/llvm/llvm-project/pull/181669

From openmp-commits at lists.llvm.org  Wed Aug  5 09:01:35 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Wed, 05 Aug 2026 09:01:35 -0700 (PDT)
Subject: [Openmp-commits] [openmp] 019fc9e - [openmp][WebAssembly] Add
	support for wasm64 (#181669)
Message-ID: <6a735e5f.d76d81af.2931b1.135c@mx.google.com>


Author: arsnyder16
Date: 2026-08-05T09:01:27-07:00
New Revision: 019fc9efa2dc8f36d9b4794caeca0738a06db667

URL: https://github.com/llvm/llvm-project/commit/019fc9efa2dc8f36d9b4794caeca0738a06db667
DIFF: https://github.com/llvm/llvm-project/commit/019fc9efa2dc8f36d9b4794caeca0738a06db667.diff

LOG: [openmp][WebAssembly] Add support for wasm64 (#181669)

Building on the existing ability to build wasm32, this change mostly
redefines `M_ARCH_WASM` to `M_ARCH_WASM32` and then introduces
`M_ARCH_WASM64` based on `__wasm64__` define

Added: 
    clang/test/OpenMP/wasm_codegen.c

Modified: 
    llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
    openmp/CMakeLists.txt
    openmp/cmake/modules/LibompUtils.cmake
    openmp/runtime/CMakeLists.txt
    openmp/runtime/cmake/LibompGetArchitecture.cmake
    openmp/runtime/src/kmp_gsupport.cpp
    openmp/runtime/src/kmp_os.h
    openmp/runtime/src/kmp_platform.h
    openmp/runtime/src/kmp_runtime.cpp
    openmp/runtime/src/z_Linux_asm.S
    openmp/runtime/src/z_Linux_util.cpp

Removed: 
    


################################################################################
diff  --git a/clang/test/OpenMP/wasm_codegen.c b/clang/test/OpenMP/wasm_codegen.c
new file mode 100644
index 0000000000000..0d40cd68fb710
--- /dev/null
+++ b/clang/test/OpenMP/wasm_codegen.c
@@ -0,0 +1,58 @@
+// REQUIRES: webassembly-registered-target
+// RUN: %clang_cc1 -triple wasm32-unknown-emscripten -fopenmp -fnoopenmp-use-tls -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,WASM32
+// RUN: %clang_cc1 -triple wasm64-unknown-emscripten -fopenmp -fnoopenmp-use-tls -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK,WASM64
+// RUN: %clang_cc1 -triple wasm32-unknown-emscripten -fopenmp -fnoopenmp-use-tls -emit-obj -o %t.wasm32.o %s
+// RUN: %clang_cc1 -triple wasm64-unknown-emscripten -fopenmp -fnoopenmp-use-tls -emit-obj -o %t.wasm64.o %s
+
+// Exercise representative host OpenMP lowering for both WebAssembly pointer
+// models. WebAssembly does not support common linkage, so compiler-generated
+// locks and caches must have internal linkage.
+// CHECK-DAG: @.gomp_critical_user_.var = internal global [8 x i32] zeroinitializer
+// CHECK-DAG: @.gomp_critical_user_named.var = internal global [8 x i32] zeroinitializer
+// CHECK-DAG: @.gomp_critical_user_.reduction.var = internal global [8 x i32] zeroinitializer
+// CHECK-DAG: @threadprivate_var.cache. = internal global ptr null
+
+// The dependency record and size_t parameters must follow the target pointer
+// width rather than assuming wasm32.
+// WASM32-DAG: %struct.kmp_depend_info = type { i32, i32, i8 }
+// WASM64-DAG: %struct.kmp_depend_info = type { i64, i64, i8 }
+// WASM32-DAG: call ptr @__kmpc_omp_task_alloc({{.*}}i32 {{[0-9]+}}, i32 {{[0-9]+}}, ptr
+// WASM64-DAG: call ptr @__kmpc_omp_task_alloc({{.*}}i64 {{[0-9]+}}, i64 {{[0-9]+}}, ptr
+// WASM32-DAG: call ptr @__kmpc_threadprivate_cached({{.*}}i32 4, ptr @threadprivate_var.cache.)
+// WASM64-DAG: call ptr @__kmpc_threadprivate_cached({{.*}}i64 4, ptr @threadprivate_var.cache.)
+// WASM32-DAG: call i32 @__kmpc_reduce_nowait({{.*}}i32 4, ptr
+// WASM64-DAG: call i32 @__kmpc_reduce_nowait({{.*}}i64 8, ptr
+
+// CHECK-DAG: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(
+// CHECK-DAG: call void @__kmpc_for_static_init_4(
+// CHECK-DAG: call i32 @__kmpc_omp_task_with_deps(
+// CHECK-DAG: call i32 @__kmpc_omp_taskwait(
+// CHECK-DAG: call void @__kmpc_critical(
+
+void critical_regions(void) {
+#pragma omp critical
+  {}
+
+#pragma omp critical(named)
+  {}
+}
+
+int threadprivate_var;
+#pragma omp threadprivate(threadprivate_var)
+
+int openmp_constructs(int *values, int count) {
+  int sum = 0;
+#pragma omp parallel for reduction(+ : sum)
+  for (int i = 0; i < count; ++i)
+    sum += values[i];
+
+#pragma omp task shared(values) depend(inout : values[0 : count])
+  values[0] = sum;
+
+#pragma omp taskwait
+
+#pragma omp critical(named)
+  threadprivate_var += sum;
+
+  return sum;
+}

diff  --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index c85cfe15d058c..5a363d0ac3dbd 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -10257,7 +10257,7 @@ GlobalVariable *OpenMPIRBuilder::getOrCreateInternalVariable(
                                : M.getTargetTriple().isAMDGPU()
                                    ? 0
                                    : DL.getDefaultGlobalsAddressSpace();
-    auto Linkage = this->M.getTargetTriple().getArch() == Triple::wasm32
+    auto Linkage = this->M.getTargetTriple().isWasm()
                        ? GlobalValue::InternalLinkage
                        : GlobalValue::CommonLinkage;
     auto *GV = new GlobalVariable(M, Ty, /*IsConstant=*/false, Linkage,

diff  --git a/openmp/CMakeLists.txt b/openmp/CMakeLists.txt
index 53ae187c8824f..f9a89859be7af 100644
--- a/openmp/CMakeLists.txt
+++ b/openmp/CMakeLists.txt
@@ -134,11 +134,12 @@ set(OPENMP_TEST_Fortran_FLAGS "${CMAKE_Fortran_FLAGS}" CACHE STRING
   "Additional compiler flags to use for testing Fortran programs (e.g. additional module search paths via -fintrinsic-modules-path )")
 
 set(ENABLE_LIBOMPTARGET ON)
-# Currently libomptarget cannot be compiled on Windows or MacOS X.
+# Currently libomptarget cannot be compiled on Windows, MacOS X, or WebAssembly.
 # Since the device plugins are only supported on Linux anyway,
 # there is no point in trying to compile libomptarget on other OSes.
 # 32-bit systems are not supported either.
-if (APPLE OR WIN32 OR WASM OR NOT "cxx_std_17" IN_LIST CMAKE_CXX_COMPILE_FEATURES
+if (APPLE OR WIN32 OR "${CMAKE_SYSTEM_NAME}" MATCHES "^(Emscripten|WASI)$" OR
+    NOT "cxx_std_17" IN_LIST CMAKE_CXX_COMPILE_FEATURES
     OR NOT CMAKE_SIZEOF_VOID_P EQUAL 8 OR "${CMAKE_SYSTEM_NAME}" MATCHES "AIX")
   set(ENABLE_LIBOMPTARGET OFF)
 endif()

diff  --git a/openmp/cmake/modules/LibompUtils.cmake b/openmp/cmake/modules/LibompUtils.cmake
index 796487e2f9218..9dd80c28dcaf0 100644
--- a/openmp/cmake/modules/LibompUtils.cmake
+++ b/openmp/cmake/modules/LibompUtils.cmake
@@ -125,6 +125,10 @@ function(libomp_get_legal_arch return_arch_string)
     set(${return_arch_string} "SPARC" PARENT_SCOPE)
   elseif(SPARCV9)
     set(${return_arch_string} "SPARCV9" PARENT_SCOPE)
+  elseif(WASM32)
+    set(${return_arch_string} "WASM32" PARENT_SCOPE)
+  elseif(WASM64)
+    set(${return_arch_string} "WASM64" PARENT_SCOPE)
   else()
     set(${return_arch_string} "${LIBOMP_ARCH}" PARENT_SCOPE)
     libomp_warning_say("libomp_get_legal_arch(): Warning: Unknown architecture: Using ${LIBOMP_ARCH}")

diff  --git a/openmp/runtime/CMakeLists.txt b/openmp/runtime/CMakeLists.txt
index c0f5b5a4b1596..0256bfaf4a892 100644
--- a/openmp/runtime/CMakeLists.txt
+++ b/openmp/runtime/CMakeLists.txt
@@ -60,8 +60,10 @@ elseif(LIBOMP_NATIVE_ARCH MATCHES "sparcv9")
   set(LIBOMP_ARCH sparcv9)
 elseif(LIBOMP_NATIVE_ARCH MATCHES "sparc")
   set(LIBOMP_ARCH sparc)
-elseif(LIBOMP_NATIVE_ARCH MATCHES "wasm")
+elseif(LIBOMP_NATIVE_ARCH MATCHES "wasm32")
   set(LIBOMP_ARCH wasm32)
+elseif(LIBOMP_NATIVE_ARCH MATCHES "wasm64")
+  set(LIBOMP_ARCH wasm64)
 else()
   # last ditch effort
   libomp_get_architecture(LIBOMP_ARCH)
@@ -86,7 +88,7 @@ if(LIBOMP_ENABLE_ARM64X)
   include(arm64x)
 endif()
 
-libomp_check_variable(LIBOMP_ARCH 32e x86_64 32 i386 arm ppc ppc64 ppc64le aarch64 aarch64_32 aarch64_a64fx arm64ec mic mips mips64 riscv64 loongarch64 ve s390x sparc sparcv9 wasm32)
+libomp_check_variable(LIBOMP_ARCH 32e x86_64 32 i386 arm ppc ppc64 ppc64le aarch64 aarch64_32 aarch64_a64fx arm64ec mic mips mips64 riscv64 loongarch64 ve s390x sparc sparcv9 wasm32 wasm64)
 
 set(LIBOMP_LIB_TYPE normal CACHE STRING
   "Performance,Profiling,Stubs library (normal/profile/stubs)")
@@ -181,7 +183,8 @@ set(RISCV64 FALSE)
 set(LOONGARCH64 FALSE)
 set(VE FALSE)
 set(S390X FALSE)
-set(WASM FALSE)
+set(WASM32 FALSE)
+set(WASM64 FALSE)
 set(PPC FALSE)
 set(SPARC FALSE)
 set(SPARCV9 FALSE)
@@ -221,8 +224,10 @@ elseif("${LIBOMP_ARCH}" STREQUAL "ve") # VE architecture
     set(VE TRUE)
 elseif("${LIBOMP_ARCH}" STREQUAL "s390x") # S390x (Z) architecture
     set(S390X TRUE)
-elseif("${LIBOMP_ARCH}" STREQUAL "wasm32") # WebAssembly architecture
-    set(WASM TRUE)
+elseif("${LIBOMP_ARCH}" STREQUAL "wasm32") # WebAssembly 32 bit architecture
+    set(WASM32 TRUE)
+elseif("${LIBOMP_ARCH}" STREQUAL "wasm64") # WebAssembly 64 bit architecture
+    set(WASM64 TRUE)
 elseif("${LIBOMP_ARCH}" STREQUAL "sparc") # SPARC architecture
     set(SPARC TRUE)
 elseif("${LIBOMP_ARCH}" STREQUAL "sparcv9") # SPARC V9 architecture
@@ -336,7 +341,7 @@ set(LIBOMP_USE_CANCEL_THREADS "${LIBOMP_HAVE_PTHREAD_CANCEL}" CACHE BOOL
 set(LIBOMP_ENABLE_SHARED TRUE CACHE BOOL
   "Shared library instead of static library?")
 
-if(WASM)
+if(WASM32 OR WASM64)
   libomp_warning_say("The WebAssembly build currently only supports static libraries; forcing LIBOMP_ENABLE_SHARED to false")
   set(LIBOMP_ENABLE_SHARED FALSE)
 endif()

diff  --git a/openmp/runtime/cmake/LibompGetArchitecture.cmake b/openmp/runtime/cmake/LibompGetArchitecture.cmake
index 56377a397bb40..77d3e3f90d47e 100644
--- a/openmp/runtime/cmake/LibompGetArchitecture.cmake
+++ b/openmp/runtime/cmake/LibompGetArchitecture.cmake
@@ -62,6 +62,8 @@ function(libomp_get_architecture return_arch)
       #error ARCHITECTURE=s390x
     #elif defined(__wasm32__)
       #error ARCHITECTURE=wasm32
+    #elif defined(__wasm64__)
+      #error ARCHITECTURE=wasm64
     #elif defined(__sparcv9)
       #error ARCHITECTURE=sparcv9
     #elif defined(__sparc)

diff  --git a/openmp/runtime/src/kmp_gsupport.cpp b/openmp/runtime/src/kmp_gsupport.cpp
index 0d04045f7b165..be95e0f3de6e2 100644
--- a/openmp/runtime/src/kmp_gsupport.cpp
+++ b/openmp/runtime/src/kmp_gsupport.cpp
@@ -357,7 +357,7 @@ void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_ORDERED_END)(void) {
 // They come in two flavors: 64-bit unsigned, and either 32-bit signed
 // (IA-32 architecture) or 64-bit signed (Intel(R) 64).
 
-#if KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_MIPS || KMP_ARCH_WASM ||          \
+#if KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_MIPS || KMP_ARCH_WASM32 ||        \
     KMP_ARCH_PPC || KMP_ARCH_AARCH64_32 || KMP_ARCH_SPARC32
 #define KMP_DISPATCH_INIT __kmp_aux_dispatch_init_4
 #define KMP_DISPATCH_FINI_CHUNK __kmp_aux_dispatch_fini_chunk_4

diff  --git a/openmp/runtime/src/kmp_os.h b/openmp/runtime/src/kmp_os.h
index c5da19dc2407b..be8c121e18736 100644
--- a/openmp/runtime/src/kmp_os.h
+++ b/openmp/runtime/src/kmp_os.h
@@ -178,18 +178,19 @@ typedef unsigned long long kmp_uint64;
 #define KMP_UINT64_SPEC "llu"
 #endif /* KMP_OS_UNIX */
 
-#if KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_MIPS || KMP_ARCH_WASM ||          \
+#if KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_MIPS || KMP_ARCH_WASM32 ||        \
     KMP_ARCH_PPC || KMP_ARCH_AARCH64_32 || KMP_ARCH_SPARC32
 #define KMP_SIZE_T_SPEC KMP_UINT32_SPEC
 #elif KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 ||                 \
     KMP_ARCH_MIPS64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 ||             \
-    KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_SPARC64 || KMP_ARCH_ARM64EC
+    KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_SPARC64 || KMP_ARCH_WASM64 ||    \
+    KMP_ARCH_ARM64EC
 #define KMP_SIZE_T_SPEC KMP_UINT64_SPEC
 #else
 #error "Can't determine size_t printf format specifier."
 #endif
 
-#if KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_WASM || KMP_ARCH_PPC
+#if KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_WASM32 || KMP_ARCH_PPC
 #define KMP_SIZE_T_MAX (0xFFFFFFFF)
 #else
 #define KMP_SIZE_T_MAX (0xFFFFFFFFFFFFFFFF)
@@ -218,7 +219,8 @@ typedef kmp_uint32 kmp_uint;
 #define KMP_INT_MIN ((kmp_int32)0x80000000)
 
 // stdarg handling
-#if (KMP_ARCH_ARM || KMP_ARCH_X86_64 || KMP_ARCH_AARCH64 || KMP_ARCH_WASM) &&  \
+#if (KMP_ARCH_ARM || KMP_ARCH_X86_64 || KMP_ARCH_AARCH64 || KMP_ARCH_WASM32 || \
+     KMP_ARCH_WASM64) &&                                                       \
     (KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_OPENBSD || KMP_OS_DRAGONFLY ||  \
      KMP_OS_LINUX || KMP_OS_WASI)
 typedef va_list *kmp_va_list;
@@ -1155,7 +1157,7 @@ extern kmp_real64 __kmp_xchg_real64(volatile kmp_real64 *p, kmp_real64 v);
   KMP_COMPARE_AND_STORE_REL64((volatile kmp_int64 *)(volatile void *)&(a),     \
                               (kmp_int64)(b), (kmp_int64)(c))
 
-#if KMP_ARCH_X86 || KMP_ARCH_MIPS || KMP_ARCH_WASM || KMP_ARCH_PPC
+#if KMP_ARCH_X86 || KMP_ARCH_MIPS || KMP_ARCH_WASM32 || KMP_ARCH_PPC
 // What about ARM?
 #define TCR_PTR(a) ((void *)TCR_4(a))
 #define TCW_PTR(a, b) TCW_4((a), (b))

diff  --git a/openmp/runtime/src/kmp_platform.h b/openmp/runtime/src/kmp_platform.h
index 7d88c02e9cf37..3cafd90225614 100644
--- a/openmp/runtime/src/kmp_platform.h
+++ b/openmp/runtime/src/kmp_platform.h
@@ -249,7 +249,11 @@
 #endif
 
 #if defined(__wasm32__)
-#define KMP_ARCH_WASM 1
+#define KMP_ARCH_WASM32 1
+#endif
+
+#if defined(__wasm64__)
+#define KMP_ARCH_WASM64 1
 #endif
 
 #define KMP_ARCH_PPC64                                                         \
@@ -283,7 +287,7 @@
 
 /* Specify 32 bit architectures here */
 #define KMP_32_BIT_ARCH                                                        \
-  (KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_MIPS || KMP_ARCH_WASM ||           \
+  (KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_MIPS || KMP_ARCH_WASM32 ||         \
    KMP_ARCH_PPC || KMP_ARCH_AARCH64_32 || KMP_ARCH_SPARC32)
 
 // Platforms which support Intel(R) Many Integrated Core Architecture
@@ -294,8 +298,9 @@
 #if (1 != KMP_ARCH_X86 + KMP_ARCH_X86_64 + KMP_ARCH_ARM + KMP_ARCH_PPC64 +     \
               KMP_ARCH_AARCH64 + KMP_ARCH_MIPS + KMP_ARCH_MIPS64 +             \
               KMP_ARCH_RISCV64 + KMP_ARCH_LOONGARCH64 + KMP_ARCH_VE +          \
-              KMP_ARCH_S390X + KMP_ARCH_WASM + KMP_ARCH_PPC +                  \
-              KMP_ARCH_AARCH64_32 + KMP_ARCH_SPARC + KMP_ARCH_ARM64EC)
+              KMP_ARCH_S390X + KMP_ARCH_WASM32 + KMP_ARCH_WASM64 +             \
+              KMP_ARCH_PPC + KMP_ARCH_AARCH64_32 + KMP_ARCH_SPARC +            \
+              KMP_ARCH_ARM64EC)
 #error Unknown or unsupported architecture
 #endif
 

diff  --git a/openmp/runtime/src/kmp_runtime.cpp b/openmp/runtime/src/kmp_runtime.cpp
index c402645af9ad6..234a211bcfba3 100644
--- a/openmp/runtime/src/kmp_runtime.cpp
+++ b/openmp/runtime/src/kmp_runtime.cpp
@@ -8924,7 +8924,8 @@ __kmp_determine_reduction_method(
 
 #if KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 ||                   \
     KMP_ARCH_MIPS64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 ||             \
-    KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_WASM || KMP_ARCH_ARM64EC
+    KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_WASM32 || KMP_ARCH_WASM64 ||     \
+    KMP_ARCH_ARM64EC
 
 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD ||     \
     KMP_OS_OPENBSD || KMP_OS_WINDOWS || KMP_OS_DARWIN || KMP_OS_HAIKU ||       \
@@ -8956,7 +8957,7 @@ __kmp_determine_reduction_method(
        // KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_WASI || KMP_OS_AIX
 
 #elif KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_AARCH || KMP_ARCH_MIPS ||       \
-    KMP_ARCH_WASM || KMP_ARCH_PPC || KMP_ARCH_AARCH64_32 || KMP_ARCH_SPARC
+    KMP_ARCH_PPC || KMP_ARCH_AARCH64_32 || KMP_ARCH_SPARC
 
 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD ||     \
     KMP_OS_OPENBSD || KMP_OS_WINDOWS || KMP_OS_HAIKU || KMP_OS_HURD ||         \

diff  --git a/openmp/runtime/src/z_Linux_asm.S b/openmp/runtime/src/z_Linux_asm.S
index 867cb3024921d..68b755a089412 100644
--- a/openmp/runtime/src/z_Linux_asm.S
+++ b/openmp/runtime/src/z_Linux_asm.S
@@ -2512,7 +2512,7 @@ KMP_PREFIX_UNDERSCORE(__kmp_unnamed_critical_addr):
 #if KMP_OS_LINUX
 # if KMP_ARCH_ARM || KMP_ARCH_AARCH64
 .section .note.GNU-stack,"",%progbits
-# elif !KMP_ARCH_WASM
+# elif !KMP_ARCH_WASM32 && !KMP_ARCH_WASM64 
 .section .note.GNU-stack,"", at progbits
 # endif
 #endif

diff  --git a/openmp/runtime/src/z_Linux_util.cpp b/openmp/runtime/src/z_Linux_util.cpp
index f42fc68488c91..bd72a3d46a97f 100644
--- a/openmp/runtime/src/z_Linux_util.cpp
+++ b/openmp/runtime/src/z_Linux_util.cpp
@@ -321,7 +321,8 @@ int __kmp_futex_determine_capable() {
 
 #endif // KMP_USE_FUTEX
 
-#if (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_WASM) && (!KMP_ASM_INTRINS)
+#if (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_WASM32 || KMP_ARCH_WASM64) && \
+    (!KMP_ASM_INTRINS)
 /* Only 32-bit "add-exchange" instruction on IA-32 architecture causes us to
    use compare_and_store for these routines */
 
@@ -381,7 +382,7 @@ kmp_uint32 __kmp_test_then_and32(volatile kmp_uint32 *p, kmp_uint32 d) {
   return old_value;
 }
 
-#if KMP_ARCH_X86 || KMP_ARCH_WASM
+#if KMP_ARCH_X86 || KMP_ARCH_WASM32
 kmp_int8 __kmp_test_then_add8(volatile kmp_int8 *p, kmp_int8 d) {
   kmp_int8 old_value, new_value;
 


        

From openmp-commits at lists.llvm.org  Wed Aug  5 09:01:37 2026
From: openmp-commits at lists.llvm.org (Sam Clegg via Openmp-commits)
Date: Wed, 05 Aug 2026 09:01:37 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [openmp][WebAssembly] Add
 support for wasm64 (PR #181669)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/181669@github.com>
Message-ID: <6a735e61.f06f4994.2c262d.168a@mx.google.com>


https://github.com/sbc100 closed https://github.com/llvm/llvm-project/pull/181669

From openmp-commits at lists.llvm.org  Thu Aug  6 06:45:45 2026
From: openmp-commits at lists.llvm.org (Michael Kruse via Openmp-commits)
Date: Thu, 06 Aug 2026 06:45:45 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [mlir] [openmp] [Clang][OpenMP]
 Canonicalize Intra-tiles in Loop Tiling (PR #191114)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/191114@github.com>
Message-ID: <6a749009.e12b82d4.480d.11c6@mx.google.com>


https://github.com/Meinersbur commented:

I get a crash with this code:
```
$ cat tile2d.c
void f(int n) {
#pragma omp tile sizes(3,5)
#pragma omp tile sizes(2)
  for (int i = 0; i < n; ++i)
    body(i);
}
$ bin/clang -fopenmp tile2d.c
clang-24: /home/meinersbur/src/llvm/work/_src/clang/lib/Sema/SemaOpenMP.cpp:14766: clang::SemaOpenMP::checkTransformableLoopNest(clang::OpenMPDirectiveKind, clang::Stmt*, int, llvm::SmallVectorImpl<clang::OMPLoopBasedDirective::HelperExprs>&, clang::Stmt*&, llvm::SmallVectorImpl<llvm::SmallVector<clang::Stmt*> >&)::<lambda(unsigned int, clang::Stmt*)>: Assertion `isa<CXXForRangeStmt>(CurStmt) && "Expected canonical for or range-based for loops."' failed.
PLEASE submit a bug report to https://github.com/llvm/llvm-project/issues/ and include the crash backtrace and dumped files.
Stack dump:
0.      Program arguments: /home/meinersbur/src/llvm/work/release/bin/clang-24 -cc1 -triple x86_64-unknown-linux-gnu -emit-obj -dumpdir a- -disable-free -clear-ast-before-backend -main-file-name tile2d.c -mrelocation-model pic -pic-level 2 -pic-is-pie -mframe-pointer=all -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -fdebug-compilation-dir=/home/meinersbur/src/llvm/work/release -fcoverage-compilation-dir=/home/meinersbur/src/llvm/work/release -resource-dir /home/meinersbur/src/llvm/work/release/lib/clang/24 -internal-isystem /home/meinersbur/src/llvm/work/release/lib/clang/24/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/13/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -ferror-limit 19 -fmessage-length=250 -fopenmp -fgnuc-version=4.2.1 -fskip-odr-check-in-gmf -faddrsig -fdwarf2-cfi-asm -o /tmp/meinersbur/tile2d-f57fad.o -x c tile2d.c
1.      tile2d.c:8:1: current parser token '}'
2.      tile2d.c:3:15: parsing function body 'f'
3.      tile2d.c:3:15: in compound statement ('{}')
 #0 0x00007f01c7fa9051 llvm::sys::PrintStackTrace(llvm::raw_ostream&, int) (/home/meinersbur/src/llvm/work/release/bin/../lib/libLLVMSupport.so.24.0git+0x2ec051)
 #1 0x00007f01c7fa595c llvm::sys::RunSignalHandlers() (/home/meinersbur/src/llvm/work/release/bin/../lib/libLLVMSupport.so.24.0git+0x2e895c)
 #2 0x00007f01c7fa5abe SignalHandler(int, siginfo_t*, void*) Signals.cpp:0:0
 #3 0x00007f01c7fa5b7c SignalHandlerTerminate(int, siginfo_t*, void*) Signals.cpp:0:0
 #4 0x00007f01c7855330 (/lib/x86_64-linux-gnu/libc.so.6+0x45330)
 #5 0x00007f01c78aec0c __pthread_kill_implementation ./nptl/pthread_kill.c:44:76
 #6 0x00007f01c78aec0c __pthread_kill_internal ./nptl/pthread_kill.c:78:10
 #7 0x00007f01c78aec0c pthread_kill ./nptl/pthread_kill.c:89:10
 #8 0x00007f01c785527e raise ./signal/../sysdeps/posix/raise.c:27:6
 #9 0x00007f01c78388ff abort ./stdlib/abort.c:81:7
#10 0x00007f01c783881b _nl_load_domain ./intl/loadmsgcat.c:1177:9
#11 0x00007f01c784b517 (/lib/x86_64-linux-gnu/libc.so.6+0x3b517)
#12 0x00007f01c41af307 bool llvm::function_ref<bool (unsigned int, clang::Stmt*)>::callback_fn<clang::SemaOpenMP::checkTransformableLoopNest(llvm::omp::Directive, clang::Stmt*, int, llvm::SmallVectorImpl<clang::OMPLoopBasedDirective::HelperExprs>&, clang::Stmt*&, llvm::SmallVectorImpl<llvm::SmallVector<clang::Stmt*, 6u>>&)::'lambda'(unsigned int, clang::Stmt*)>(long, unsigned int, clang::Stmt*) SemaOpenMP.cpp:0:0
#13 0x00007f01c67490f1 clang::OMPLoopBasedDirective::doForAllLoops(clang::Stmt*, bool, unsigned int, llvm::function_ref<bool (unsigned int, clang::Stmt*)>, llvm::function_ref<void (clang::OMPLoopTransformationDirective*)>) (/home/meinersbur/src/llvm/work/release/bin/../lib/../lib/libclangAST.so.24.0git+0xfe50f1)
#14 0x00007f01c4138499 clang::SemaOpenMP::checkTransformableLoopNest(llvm::omp::Directive, clang::Stmt*, int, llvm::SmallVectorImpl<clang::OMPLoopBasedDirective::HelperExprs>&, clang::Stmt*&, llvm::SmallVectorImpl<llvm::SmallVector<clang::Stmt*, 6u>>&) (/home/meinersbur/src/llvm/work/release/bin/../lib/../lib/libclangSema.so.24.0git+0xc6e499)
#15 0x00007f01c421f5ef clang::SemaOpenMP::ActOnOpenMPTileDirective(llvm::ArrayRef<clang::OMPClause*>, clang::Stmt*, clang::SourceLocation, clang::SourceLocation) (/home/meinersbur/src/llvm/work/release/bin/../lib/../lib/libclangSema.so.24.0git+0xd555ef)
....
```

> One case remains unsupported: collapsing through stacked #pragma omp tile directives, where the inner tile's floor is body-derived from the outer reinterpreted tile. Since the fix for this may span in the shared collapse working, I've kept it out of this PR and marked it XFAIL ([openmp/runtime/test/transform/tile/parallel-wsloop-collapse-stacked-tile-deep.cpp](https://github.com/llvm/llvm-project/pull/url)) with a note in the central comments. Happy to follow up in a separate change–lmk if you prefer it here.

I'd implement this by sinking the Predicate of OMPInvariantPredicateBoundAttr further: If checkOpenMPIterationSpace encounters an OMPInvariantPredicateBoundAttr. then remember the body it to emit into its own OMPInvariantPredicateBoundAttr.

https://github.com/llvm/llvm-project/pull/191114

From openmp-commits at lists.llvm.org  Thu Aug  6 06:45:45 2026
From: openmp-commits at lists.llvm.org (Michael Kruse via Openmp-commits)
Date: Thu, 06 Aug 2026 06:45:45 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [mlir] [openmp] [Clang][OpenMP]
 Canonicalize Intra-tiles in Loop Tiling (PR #191114)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/191114@github.com>
Message-ID: <6a749009.83d2da6f.361d7c.c8c0@mx.google.com>


================
@@ -1,3 +1,4 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
----------------
Meinersbur wrote:

This just a regeneration of the test in main; the original test passes even with this PR

https://github.com/llvm/llvm-project/pull/191114

From openmp-commits at lists.llvm.org  Thu Aug  6 06:45:45 2026
From: openmp-commits at lists.llvm.org (Michael Kruse via Openmp-commits)
Date: Thu, 06 Aug 2026 06:45:45 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [mlir] [openmp] [Clang][OpenMP]
 Canonicalize Intra-tiles in Loop Tiling (PR #191114)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/191114@github.com>
Message-ID: <6a749009.e2619d42.1728c.aba3@mx.google.com>


https://github.com/Meinersbur edited https://github.com/llvm/llvm-project/pull/191114

From openmp-commits at lists.llvm.org  Thu Aug  6 06:45:46 2026
From: openmp-commits at lists.llvm.org (Michael Kruse via Openmp-commits)
Date: Thu, 06 Aug 2026 06:45:46 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [mlir] [openmp] [Clang][OpenMP]
 Canonicalize Intra-tiles in Loop Tiling (PR #191114)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/191114@github.com>
Message-ID: <6a74900a.b400e095.39a928.d49b@mx.google.com>


================
@@ -0,0 +1,521 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
+// Regression tests for standalone `#pragma omp tile` lowering (no consuming
+// directive). The intra-tile loop is emitted in its min-bounded form
+//   for (.tile.iv = .floor.iv; .tile.iv < min(.floor.iv + tilesize, N); ++.tile.iv)
+// so the upper bound is materialized as a `min()` (cond.true/cond.false/phi over
+// the two `.tile_*.iv`/`.floor_*.iv` counters). There is no `.tile.cnt`
+// rectangular counter and no body-guard predicate here; those only appear when a
+// directive (e.g. `collapse`) consumes the tile -- see
+// tile_collapse_reinterpret_codegen.cpp.
+//
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fopenmp -emit-llvm -o - %s | FileCheck %s --check-prefix=IR
+// expected-no-diagnostics
+
+extern "C" void body(int);
+
+// Trip count 6, tile 4 → the last tile is partial; the min() upper bound stops
+// the tile loop at the trip count (no separate body guard).
+// IR-LABEL: define dso_local void @_Z18remainder_6_tile_4v(
+// IR-SAME: ) #[[ATTR0:[0-9]+]] {
+// IR-NEXT:  [[ENTRY:.*:]]
+// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    store i32 0, ptr [[I]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND:.*]]
+// IR:       [[FOR_COND]]:
+// IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 6
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END9:.*]]
+// IR:       [[FOR_BODY]]:
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND1:.*]]
+// IR:       [[FOR_COND1]]:
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 6, [[ADD]]
+// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 4
+// IR-NEXT:    br label %[[COND_END]]
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ 6, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
+// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
+// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY5]]:
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
+// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void @body(i32 noundef [[TMP6]])
+// IR-NEXT:    br label %[[FOR_INC:.*]]
+// IR:       [[FOR_INC]]:
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP7]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP2:![0-9]+]]
+// IR:       [[FOR_END]]:
+// IR-NEXT:    br label %[[FOR_INC7:.*]]
+// IR:       [[FOR_INC7]]:
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP8]], 4
+// IR-NEXT:    store i32 [[ADD8]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP4:![0-9]+]]
+// IR:       [[FOR_END9]]:
+// IR-NEXT:    ret void
+//
+void remainder_6_tile_4(void) {
+  // Rectangular tile counter and constant tile-size bound.
+#pragma omp tile sizes(4)
----------------
Meinersbur wrote:

None of the tests in this file need intra-tile loops being canonical.

https://github.com/llvm/llvm-project/pull/191114

From openmp-commits at lists.llvm.org  Thu Aug  6 06:45:46 2026
From: openmp-commits at lists.llvm.org (Michael Kruse via Openmp-commits)
Date: Thu, 06 Aug 2026 06:45:46 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [mlir] [openmp] [Clang][OpenMP]
 Canonicalize Intra-tiles in Loop Tiling (PR #191114)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/191114@github.com>
Message-ID: <6a74900a.d786cb60.bce86.aeeb@mx.google.com>


================
@@ -0,0 +1,14 @@
+// RUN: %clang_cc1 -verify -triple x86_64-unknown-linux-gnu -fclang-abi-compat=latest -std=c++17 -fopenmp -O2 -vectorize-loops -vectorize-slp -emit-llvm %s -o - | FileCheck %s
+// REQUIRES: x86-registered-target
+// expected-no-diagnostics
+
+// Standalone tile stays min-bounded (no body predicate) and vectorizes.
+
+// CHECK-LABEL: define {{.*}} @vec_tile(
+// CHECK: fadd <4 x float>
+// CHECK-NOT: omp.body.next
+extern "C" void vec_tile(float *a, float *b, int n) {
+#pragma omp tile sizes(64)
----------------
Meinersbur wrote:

This test is independent of the intra-tile loop being canonical. In needs to be affected e.g. by a `collapse(2)` or another tile to make a difference.

https://github.com/llvm/llvm-project/pull/191114

From openmp-commits at lists.llvm.org  Thu Aug  6 06:45:46 2026
From: openmp-commits at lists.llvm.org (Michael Kruse via Openmp-commits)
Date: Thu, 06 Aug 2026 06:45:46 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [mlir] [openmp] [Clang][OpenMP]
 Canonicalize Intra-tiles in Loop Tiling (PR #191114)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/191114@github.com>
Message-ID: <6a74900a.146286d8.24572.78bf@mx.google.com>


================
@@ -15257,6 +15367,13 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
         ForStmt(Context, InitStmt.get(), CondExpr.get(), nullptr,
                 IncrStmt.get(), Inner, LoopHelper.Init->getBeginLoc(),
                 LoopHelper.Init->getBeginLoc(), LoopHelper.Inc->getEndLoc());
+
+    // Attach the droppable reinterpretation attribute to the intra-tile loop.
+    auto *Hint = OMPInvariantPredicateBoundAttr::CreateImplicit(
----------------
Meinersbur wrote:

In a previous version of this patch you only added the predicated if it could not shown that the itertion is a multiple of a tile size. Could we still do that here?

I had AI point out to me that the optimizer will not always be able to optimize-away this predicate, due to the constant loop bound being outside the outlined body that contains the predicate.

https://github.com/llvm/llvm-project/pull/191114

From openmp-commits at lists.llvm.org  Thu Aug  6 06:45:46 2026
From: openmp-commits at lists.llvm.org (Michael Kruse via Openmp-commits)
Date: Thu, 06 Aug 2026 06:45:46 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [mlir] [openmp] [Clang][OpenMP]
 Canonicalize Intra-tiles in Loop Tiling (PR #191114)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/191114@github.com>
Message-ID: <6a74900a.fa459026.33c12c.154a@mx.google.com>


================
@@ -9668,8 +9693,18 @@ static bool checkOpenMPIterationSpace(
     return HasErrors;
 
   // Build the loop's iteration space representation.
-  ResultIterSpaces[CurrentNestedLoopCount].PreCond = ISC.buildPreCond(
-      DSA.getCurScope(), For ? For->getCond() : CXXFor->getCond(), Captures);
+  ResultIterSpaces[CurrentNestedLoopCount].PreCond =
+      ISC.buildPreCond(DSA.getCurScope(), EffectiveCond, Captures);
+  // Reinterpreted intra-tile: precondition is trivially true (.floor.iv
+  // unreadable here); overshoot handled by Predicate.
----------------
Meinersbur wrote:

Should this be in the else branch of `if (TileRectCond)`?

https://github.com/llvm/llvm-project/pull/191114

From openmp-commits at lists.llvm.org  Thu Aug  6 06:45:48 2026
From: openmp-commits at lists.llvm.org (Michael Kruse via Openmp-commits)
Date: Thu, 06 Aug 2026 06:45:48 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [mlir] [openmp] [Clang][OpenMP]
 Canonicalize Intra-tiles in Loop Tiling (PR #191114)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/191114@github.com>
Message-ID: <6a74900c.e912d209.2eee3f.2839@mx.google.com>


================
@@ -0,0 +1,32 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+
+// XFAIL: *
+// collapse(3) through stacked tiles is not supported.
----------------
Meinersbur wrote:

If unsupported, it should report a compiler error, not silently miscompile.

https://github.com/llvm/llvm-project/pull/191114

From openmp-commits at lists.llvm.org  Thu Aug  6 06:45:48 2026
From: openmp-commits at lists.llvm.org (Michael Kruse via Openmp-commits)
Date: Thu, 06 Aug 2026 06:45:48 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [mlir] [openmp] [Clang][OpenMP]
 Canonicalize Intra-tiles in Loop Tiling (PR #191114)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/191114@github.com>
Message-ID: <6a74900c.f2123fa7.685e3.1306@mx.google.com>


================
@@ -10,12 +10,57 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/AST/ASTContext.h"
 #include "clang/AST/StmtOpenMP.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Attr.h"
+#include "clang/AST/Stmt.h"
 
 using namespace clang;
 using namespace llvm::omp;
 
+/// Returns true if the attributed statement carries the internal `omp tile`
+/// intra-tile reinterpretation hint (see OMPInvariantPredicateBoundAttr).
+static bool hasOMPInvariantPredicateBound(const AttributedStmt *AS) {
----------------
Meinersbur wrote:

Consider using `getSpecificAttr<OMPInvariantPredicateBoundAttr>(AS->getAttrs())` instead of these helpers.

https://github.com/llvm/llvm-project/pull/191114

From openmp-commits at lists.llvm.org  Thu Aug  6 06:45:48 2026
From: openmp-commits at lists.llvm.org (Michael Kruse via Openmp-commits)
Date: Thu, 06 Aug 2026 06:45:48 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [mlir] [openmp] [Clang][OpenMP]
 Canonicalize Intra-tiles in Loop Tiling (PR #191114)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/191114@github.com>
Message-ID: <6a74900c.91e9324c.af086.0a56@mx.google.com>


================
@@ -0,0 +1,108 @@
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fclang-abi-compat=latest -std=c++20 -fopenmp -ast-print %s | FileCheck %s --check-prefix=PRINT
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fclang-abi-compat=latest -std=c++20 -fopenmp -emit-llvm %s -o - | FileCheck %s --check-prefix=IR
+// expected-no-diagnostics
+
+// Codegen tests for how a loop-associated directive (e.g. 'omp for' with a
+// 'collapse' clause) consumes a '#pragma omp tile' intra-tile loop.
+//
+// These tests pin the key IR invariants of the below design:
+// The intra-tile loop is stored in its min-bounded form
+//   for (.tile.iv = .floor.iv; .tile.iv < min(.floor.iv + tilesize, N); ++.tile.iv)
+// and carries an internal OMPInvariantPredicateBoundAttr hint. When a directive
----------------
Meinersbur wrote:

Consider testing the presence of OMPInvariantPredicateBoundAttr with `-ast-dump`

https://github.com/llvm/llvm-project/pull/191114

From openmp-commits at lists.llvm.org  Thu Aug  6 06:45:48 2026
From: openmp-commits at lists.llvm.org (Michael Kruse via Openmp-commits)
Date: Thu, 06 Aug 2026 06:45:48 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [mlir] [openmp] [Clang][OpenMP]
 Canonicalize Intra-tiles in Loop Tiling (PR #191114)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/191114@github.com>
Message-ID: <6a74900c.6b60e315.2fd0be.c590@mx.google.com>


================
@@ -0,0 +1,51 @@
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fclang-abi-compat=latest -std=c++20 -fopenmp -ast-print %s | FileCheck %s
+// expected-no-diagnostics
+
+// -ast-print round-trip tests: a `#pragma omp tile` directive and its associated
+// loop must print back as written. This checks parsing/printing only; for the
+// generated IR (min-bounded lowering) see tile_rect_codegen_ir.cpp.
+
+extern "C" void body(...) {}
+
+// CHECK-LABEL: void rect_tile_1d(
+void rect_tile_1d() {
+  // Partial last tile (trip count 4, tile 5); -ast-print prints the original loop.
+  // CHECK: #pragma omp tile sizes(5)
+  // CHECK-NEXT: for (int i = 7; i < 17; i += 3)
+  #pragma omp tile sizes(5)
----------------
Meinersbur wrote:

None of the tests in this file need the intra-tile loops to be canonical.

https://github.com/llvm/llvm-project/pull/191114

From openmp-commits at lists.llvm.org  Thu Aug  6 08:40:55 2026
From: openmp-commits at lists.llvm.org (=?UTF-8?Q?Michael_Halkenh=C3=A4user?= via Openmp-commits)
Date: Thu, 06 Aug 2026 08:40:55 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMP][OMPT] Error when
 registering EMI and non-EMI callbacks (PR #213697)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213697@github.com>
Message-ID: <6a74ab07.a1accbb4.2dd632.1ce1@mx.google.com>


https://github.com/mhalk edited https://github.com/llvm/llvm-project/pull/213697

From openmp-commits at lists.llvm.org  Thu Aug  6 08:40:55 2026
From: openmp-commits at lists.llvm.org (=?UTF-8?Q?Michael_Halkenh=C3=A4user?= via Openmp-commits)
Date: Thu, 06 Aug 2026 08:40:55 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMP][OMPT] Error when
 registering EMI and non-EMI callbacks (PR #213697)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213697@github.com>
Message-ID: <6a74ab07.6c8668f2.39fcc1.72a8@mx.google.com>


https://github.com/mhalk approved this pull request.

Looks good to me!

https://github.com/llvm/llvm-project/pull/213697

From openmp-commits at lists.llvm.org  Thu Aug  6 08:40:55 2026
From: openmp-commits at lists.llvm.org (=?UTF-8?Q?Michael_Halkenh=C3=A4user?= via Openmp-commits)
Date: Thu, 06 Aug 2026 08:40:55 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMP][OMPT] Error when
 registering EMI and non-EMI callbacks (PR #213697)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213697@github.com>
Message-ID: <6a74ab07.92d2873c.36630e.85a5@mx.google.com>


================
@@ -597,10 +597,35 @@ OMPT_API_ROUTINE ompt_set_result_t ompt_set_callback(ompt_callbacks_t which,
     else                                                                       \
       return ompt_set_always;
 
-    FOREACH_OMPT_EVENT(ompt_event_macro)
+    FOREACH_OMPT_HOST_EVENT(ompt_event_macro)
+    FOREACH_OMPT_DEVICE_EVENT(ompt_event_macro)
 
 #undef ompt_event_macro
 
+    // OpenMP v5.2, p. 503, l. 18-20:
+    // Restrictions to the ompt_callback_target_emi and ompt_callback_target
+    // callbacks are as follows:
+    //   These callbacks must not be registered at the same time.
+    //
+    // Similar restrictions apply to target_data_op, target_submit,
+    // and target_map. Hence, handle registration of them separately to ensure
+    // only one can be registered at the same time. */
----------------
mhalk wrote:

Stray ` */`?

https://github.com/llvm/llvm-project/pull/213697

From openmp-commits at lists.llvm.org  Thu Aug  6 09:11:14 2026
From: openmp-commits at lists.llvm.org (CHANDRA GHALE via Openmp-commits)
Date: Thu, 06 Aug 2026 09:11:14 -0700 (PDT)
Subject: [Openmp-commits] [flang] [llvm] [mlir] [openmp] [Flang][OpenMP]
 Lower scan directive and inscan reduction modifier (PR #206747)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206747@github.com>
Message-ID: <6a74b222.510f576d.25a50.7692@mx.google.com>


chandraghale wrote:

ping @chichunchen 

https://github.com/llvm/llvm-project/pull/206747

From openmp-commits at lists.llvm.org  Fri Aug  7 01:28:48 2026
From: openmp-commits at lists.llvm.org (=?UTF-8?Q?Jan_Andr=C3=A9_Reuter?= via Openmp-commits)
Date: Fri, 07 Aug 2026 01:28:48 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [Offload][OMPT][NFCI] Call
 ompt_libomp_connect instead of dlopen libomp (PR #214556)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/214556@github.com>
Message-ID: <6a759740.d2cb4adc.dcd01.6645@mx.google.com>


https://github.com/Thyre approved this pull request.

I did some testing with examples of prior issues I've encountered with the initialization sequence and those still look fine. The general changes look good to me, and I like getting rid of the `dlopen` approach done previously.

As mentioned in the PR description, `libomptarget.so.[...]` already links against `libomp.so`, so the symbols should be found AFAICT.

Please wait for further feedback before merging though.

https://github.com/llvm/llvm-project/pull/214556

From openmp-commits at lists.llvm.org  Fri Aug  7 01:30:20 2026
From: openmp-commits at lists.llvm.org (=?UTF-8?Q?Jan_Andr=C3=A9_Reuter?= via Openmp-commits)
Date: Fri, 07 Aug 2026 01:30:20 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMP][OMPT] Error when
 registering EMI and non-EMI callbacks (PR #213697)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213697@github.com>
Message-ID: <6a75979c.871e6cae.3779bd.c1bc@mx.google.com>


================
@@ -597,10 +597,35 @@ OMPT_API_ROUTINE ompt_set_result_t ompt_set_callback(ompt_callbacks_t which,
     else                                                                       \
       return ompt_set_always;
 
-    FOREACH_OMPT_EVENT(ompt_event_macro)
+    FOREACH_OMPT_HOST_EVENT(ompt_event_macro)
+    FOREACH_OMPT_DEVICE_EVENT(ompt_event_macro)
 
 #undef ompt_event_macro
 
+    // OpenMP v5.2, p. 503, l. 18-20:
+    // Restrictions to the ompt_callback_target_emi and ompt_callback_target
+    // callbacks are as follows:
+    //   These callbacks must not be registered at the same time.
+    //
+    // Similar restrictions apply to target_data_op, target_submit,
+    // and target_map. Hence, handle registration of them separately to ensure
+    // only one can be registered at the same time. */
----------------
Thyre wrote:

Looks like I missed that while converting the comment style. Thanks for spotting!
This will be resolved with the next (force) push, doing another round of testing before that to make sure I haven't missed anything.

https://github.com/llvm/llvm-project/pull/213697

From openmp-commits at lists.llvm.org  Fri Aug  7 02:40:15 2026
From: openmp-commits at lists.llvm.org (=?UTF-8?Q?Jan_Andr=C3=A9_Reuter?= via Openmp-commits)
Date: Fri, 07 Aug 2026 02:40:15 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMP][OMPT] Error when
 registering EMI and non-EMI callbacks (PR #213697)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213697@github.com>
Message-ID: <6a75a7ff.2915925c.19f4ed.fb98@mx.google.com>


https://github.com/Thyre updated https://github.com/llvm/llvm-project/pull/213697

>From 35af9f995a2054c7fcd1eb9991e2b2e261528281 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Andr=C3=A9=20Reuter?= <jan at zyten.de>
Date: Mon, 3 Aug 2026 17:29:16 +0200
Subject: [PATCH] [OpenMP][OMPT] Error when registering EMI and non-EMI
 callbacks
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The OpenMP specification includes a restriction for registering both EMI and
non-EMI callbacks (e.g. `ompt_callback_target`, OpenMP v5.2, p. 503, l. 18-20):

  These callbacks must not be registered at the same time.

However, this was not respected in the runtime. A tool was able to register
both, receiving `ompt_set_always` for each `ompt_set_callback` call.
When an event associated with the EMI or non-EMI callbacks was dispatched
and both callbacks were registered, only the EMI version was dispatched.

To align the runtime behavior with the OpenMP specification, return
`ompt_set_error` when a tool tries to register both the EMI and non-EMI variant
for a callback. With this, only the tool's first registration is used.

Update the offload `veccopy_disallow_both.c` test, which already tested the
prior, non-compliant, behavior.

Signed-off-by: Jan André Reuter <jan at zyten.de>
---
 offload/test/ompt/register_both.h          |  4 +++-
 offload/test/ompt/veccopy_disallow_both.c  | 19 ++++++++-------
 openmp/runtime/src/include/omp-tools.h.var | 13 +++++++++++
 openmp/runtime/src/ompt-general.cpp        | 27 +++++++++++++++++++++-
 4 files changed, 53 insertions(+), 10 deletions(-)

diff --git a/offload/test/ompt/register_both.h b/offload/test/ompt/register_both.h
index 77cb430aa1f97..a20b42bad2196 100644
--- a/offload/test/ompt/register_both.h
+++ b/offload/test/ompt/register_both.h
@@ -4,7 +4,9 @@
 #define register_ompt_callback_t(name, type)                                   \
   do {                                                                         \
     type f_##name = &on_##name;                                                \
-    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+    ompt_set_result_t result =                                                 \
+        ompt_set_callback(name, (ompt_callback_t)f_##name);                    \
+    if (result == ompt_set_never || result == ompt_set_error)                  \
       printf("0: Could not register callback '" #name "'\n");                  \
   } while (0)
 
diff --git a/offload/test/ompt/veccopy_disallow_both.c b/offload/test/ompt/veccopy_disallow_both.c
index bfc67c5f4d274..2e24529042b37 100644
--- a/offload/test/ompt/veccopy_disallow_both.c
+++ b/offload/test/ompt/veccopy_disallow_both.c
@@ -7,8 +7,9 @@
 /*
  * Example OpenMP program that shows that both EMI and non-EMI
  * callbacks cannot be registered for the same type. In the
- * current implementation, the EMI callback overrides the non-EMI
- * callback.
+ * current implementation, the first registered callback takes
+ * precedence. Trying to register the other callback results in
+ * ompt_set_error for this callback.
  */
 
 #include <omp.h>
@@ -57,9 +58,11 @@ int main() {
 }
 
 // clang-format off
+/// CHECK: [[THREAD_ID:[0-9]+]]: Could not register callback 'ompt_callback_target_data_op'
+/// CHECK: [[THREAD_ID]]: Could not register callback 'ompt_callback_target_emi'
 /// CHECK: Callback Init:
 /// CHECK: Callback Load:
-/// CHECK: Callback Target EMI: kind=ompt_target endpoint=ompt_scope_begin
+/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=ompt_target endpoint=ompt_scope_begin
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_alloc
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_alloc
 /// CHECK-NOT: dest=(nil)
@@ -70,7 +73,7 @@ int main() {
 /// CHECK-NOT: dest=(nil)
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_transfer_to_device
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_transfer_to_device
-/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1
+/// CHECK: Callback Submit: target_id=[[TARGET_ID]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_transfer_from_device
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_transfer_from_device
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_transfer_from_device
@@ -79,8 +82,8 @@ int main() {
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_delete
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_delete
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_delete
-/// CHECK: Callback Target EMI: kind=ompt_target endpoint=ompt_scope_end
-/// CHECK: Callback Target EMI: kind=ompt_target endpoint=ompt_scope_begin
+/// CHECK: Callback Target: target_id=[[TARGET_ID]] kind=ompt_target endpoint=ompt_scope_end
+/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=ompt_target endpoint=ompt_scope_begin
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_alloc
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_alloc
 /// CHECK-NOT: dest=(nil)
@@ -91,7 +94,7 @@ int main() {
 /// CHECK-NOT: dest=(nil)
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_transfer_to_device
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_transfer_to_device
-/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0
+/// CHECK: Callback Submit: target_id=[[TARGET_ID]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_transfer_from_device
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_transfer_from_device
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_transfer_from_device
@@ -100,5 +103,5 @@ int main() {
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_delete
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_delete
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_delete
-/// CHECK: Callback Target EMI: kind=ompt_target endpoint=ompt_scope_end
+/// CHECK: Callback Target: target_id=[[TARGET_ID]] kind=ompt_target endpoint=ompt_scope_end
 /// CHECK: Callback Fini:
diff --git a/openmp/runtime/src/include/omp-tools.h.var b/openmp/runtime/src/include/omp-tools.h.var
index af9213a7290e7..46643dff5feab 100644
--- a/openmp/runtime/src/include/omp-tools.h.var
+++ b/openmp/runtime/src/include/omp-tools.h.var
@@ -185,6 +185,19 @@
     /* Optional Events */                                                                                                \
     macro (ompt_callback_target_map_emi,    ompt_callback_target_map_emi_t,    36) /* target map                      */
 
+#define FOREACH_MAPPED_OMPT_TARGET_EVENT(macro)                                                                          \
+    /* macro including correlated callback enum (e.g. target -- target_emi) for comparisons between both */              \
+    /*--- Mandatory Events ---*/                                                                                         \
+    macro (ompt_callback_target,             ompt_callback_target_emi,         ompt_callback_target_t,              8)   \
+    macro (ompt_callback_target_data_op,     ompt_callback_target_data_op_emi, ompt_callback_target_data_op_t,      9)   \
+    macro (ompt_callback_target_submit,      ompt_callback_target_submit_emi,  ompt_callback_target_submit_t,      10)   \
+    macro (ompt_callback_target_emi,         ompt_callback_target,             ompt_callback_target_emi_t,         33)   \
+    macro (ompt_callback_target_data_op_emi, ompt_callback_target_data_op,     ompt_callback_target_data_op_emi_t, 34)   \
+    macro (ompt_callback_target_submit_emi,  ompt_callback_target_submit,      ompt_callback_target_submit_emi_t,  35)   \
+    /* Optional Events */                                                                                                \
+    macro (ompt_callback_target_map,         ompt_callback_target_map_emi,     ompt_callback_target_map_t,         22)   \
+    macro (ompt_callback_target_map_emi,     ompt_callback_target_map,         ompt_callback_target_map_emi_t,     36)
+
 #define FOREACH_OMPT_50_TARGET_EVENT(macro)                                                                              \
     FOREACH_OMPT_DEVICE_EVENT(macro)                                                                                     \
     FOREACH_OMPT_NOEMI_EVENT(macro) 
diff --git a/openmp/runtime/src/ompt-general.cpp b/openmp/runtime/src/ompt-general.cpp
index 1682f7e74bd14..8f1ba4910815e 100644
--- a/openmp/runtime/src/ompt-general.cpp
+++ b/openmp/runtime/src/ompt-general.cpp
@@ -597,10 +597,35 @@ OMPT_API_ROUTINE ompt_set_result_t ompt_set_callback(ompt_callbacks_t which,
     else                                                                       \
       return ompt_set_always;
 
-    FOREACH_OMPT_EVENT(ompt_event_macro)
+    FOREACH_OMPT_HOST_EVENT(ompt_event_macro)
+    FOREACH_OMPT_DEVICE_EVENT(ompt_event_macro)
 
 #undef ompt_event_macro
 
+    // OpenMP v5.2, p. 503, l. 18-20:
+    // Restrictions to the ompt_callback_target_emi and ompt_callback_target
+    // callbacks are as follows:
+    //   These callbacks must not be registered at the same time.
+    //
+    // Similar restrictions apply to target_data_op, target_submit,
+    // and target_map. Hence, handle registration of them separately to ensure
+    // only one can be registered at the same time.
+#define ompt_target_event_macro(event_name, comparison_name, callback_type,    \
+                                event_id)                                      \
+  case event_name:                                                             \
+    if (ompt_callbacks.ompt_callback(comparison_name))                         \
+      return ompt_set_error;                                                   \
+    ompt_callbacks.ompt_callback(event_name) = (callback_type)callback;        \
+    ompt_enabled.event_name = (callback != 0);                                 \
+    if (callback)                                                              \
+      return ompt_event_implementation_status(event_name);                     \
+    else                                                                       \
+      return ompt_set_always;
+
+    FOREACH_MAPPED_OMPT_TARGET_EVENT(ompt_target_event_macro);
+
+#undef ompt_target_event_macro
+
   default:
     return ompt_set_error;
   }


From openmp-commits at lists.llvm.org  Fri Aug  7 02:41:51 2026
From: openmp-commits at lists.llvm.org (Jan Patrick Lehr via Openmp-commits)
Date: Fri, 07 Aug 2026 02:41:51 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OMPT] Upstream device tracing
	implementation (PR #200165)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/200165@github.com>
Message-ID: <6a75a85f.9edb5c9d.219adb.ebc5@mx.google.com>


https://github.com/jplehr updated https://github.com/llvm/llvm-project/pull/200165

>From 285be264d690413b95841194a05b32903583b0e4 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Tue, 4 Aug 2026 13:39:45 -0500
Subject: [PATCH 01/12] [OpenMP][OMPT] Test libomptarget connector lookup

Verify that the private target connector only exposes supported callbacks and does not retain the unwired frame-enter extension.

Co-Authored-By: Claude <noreply at anthropic.com>
---
 openmp/runtime/test/ompt/misc/target_lookup.c | 38 +++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 openmp/runtime/test/ompt/misc/target_lookup.c

diff --git a/openmp/runtime/test/ompt/misc/target_lookup.c b/openmp/runtime/test/ompt/misc/target_lookup.c
new file mode 100644
index 0000000000000..d062e759e1912
--- /dev/null
+++ b/openmp/runtime/test/ompt/misc/target_lookup.c
@@ -0,0 +1,38 @@
+// RUN: %libomp-compile-and-run | FileCheck %s
+// REQUIRES: ompt
+
+#include <omp-tools.h>
+#include <stdio.h>
+
+void ompt_libomp_connect(ompt_start_tool_result_t *result);
+
+static int tool_initialize(ompt_function_lookup_t lookup,
+                           int initial_device_num, ompt_data_t *tool_data) {
+  return 1;
+}
+
+static int target_initialize(ompt_function_lookup_t lookup,
+                             int initial_device_num, ompt_data_t *tool_data) {
+  printf("ompt_set_frame_enter: %s\n",
+         lookup("ompt_set_frame_enter") ? "present" : "absent");
+  return 1;
+}
+
+static void ompt_finalize(ompt_data_t *tool_data) {}
+
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t result = {&tool_initialize, &ompt_finalize,
+                                            0};
+  return &result;
+}
+
+int main() {
+  static ompt_start_tool_result_t result = {&target_initialize, &ompt_finalize,
+                                            0};
+  // This entry point initializes OMPT before invoking the target connector.
+  ompt_libomp_connect(&result);
+  return 0;
+}
+
+// CHECK: ompt_set_frame_enter: absent

>From b7a63c054e34442d519a3037c442da09e724fb11 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 2 Apr 2026 07:27:58 -0500
Subject: [PATCH 02/12] [Offload] Add GenericProfilerTy abstraction and
 APITypes extensions

Introduce GenericProfilerTy alongside the existing OMPT callback dispatch.
The weak profiler factory returns a no-op implementation, so the new hooks
are silent while the established callback path continues to handle OMPT
device events.

Co-Authored-By: Dhruva Chakrabarti <dhruva.chakrabarti at amd.com>
Co-Authored-By: Michael Halkenhauser <michaelgerald.halkenhauser at amd.com>
Co-Authored-By: Claude <noreply at anthropic.com>
---
 offload/include/Shared/APITypes.h             |   5 +
 offload/plugins-nextgen/common/CMakeLists.txt |   1 +
 .../common/include/GenericProfiler.h          | 183 ++++++++++++++++++
 .../common/include/PluginInterface.h          |  25 ++-
 .../common/src/GenericProfiler.cpp            |  36 ++++
 .../common/src/PluginInterface.cpp            |  26 ++-
 6 files changed, 273 insertions(+), 3 deletions(-)
 create mode 100644 offload/plugins-nextgen/common/include/GenericProfiler.h
 create mode 100644 offload/plugins-nextgen/common/src/GenericProfiler.cpp

diff --git a/offload/include/Shared/APITypes.h b/offload/include/Shared/APITypes.h
index 47d8c49bf7ef2..527aa8e617304 100644
--- a/offload/include/Shared/APITypes.h
+++ b/offload/include/Shared/APITypes.h
@@ -85,6 +85,11 @@ struct __tgt_async_info {
   /// ensure it is a valid location while the transfer to the device is
   /// happening.
   KernelLaunchEnvironmentTy KernelLaunchEnvironment;
+
+  /// Opaque handle for profiler-specific data (e.g., OMPT trace record info).
+  /// Owned by the profiler; the runtime threads this pointer through the plugin
+  /// layer to associate async operations with trace records.
+  void *ProfilerData = nullptr;
 };
 
 /// This struct contains all of the arguments to a target kernel region launch.
diff --git a/offload/plugins-nextgen/common/CMakeLists.txt b/offload/plugins-nextgen/common/CMakeLists.txt
index 6ad20796c3ca4..19ce102309a72 100644
--- a/offload/plugins-nextgen/common/CMakeLists.txt
+++ b/offload/plugins-nextgen/common/CMakeLists.txt
@@ -11,6 +11,7 @@ add_public_tablegen_target(PluginErrcodes)
 # don't want to export `PluginInterface` while `add_llvm_library` requires that.
 add_library(PluginCommon OBJECT
   src/PluginInterface.cpp
+  src/GenericProfiler.cpp
   src/GlobalHandler.cpp
   src/JIT.cpp
   src/RecordReplay.cpp
diff --git a/offload/plugins-nextgen/common/include/GenericProfiler.h b/offload/plugins-nextgen/common/include/GenericProfiler.h
new file mode 100644
index 0000000000000..5715a9f1fe349
--- /dev/null
+++ b/offload/plugins-nextgen/common/include/GenericProfiler.h
@@ -0,0 +1,183 @@
+//===- GenericProfiler.h - GenericProfiler interface for use in Plugins ---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// The GenericProfiler interface allows to implement profiler logic for various
+// backends, such as OMPT or other tracing mechanisms.
+// This enables the plugins to be agnostic of the actual high-level language
+// that is implemented.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OFFLOAD_PLUGINS_NEXTGEN_COMMON_INCLUDE_GENERICPROFILER_H
+#define OFFLOAD_PLUGINS_NEXTGEN_COMMON_INCLUDE_GENERICPROFILER_H
+
+#include "Shared/APITypes.h"
+
+#include <cstdint>
+#include <functional>
+#include <tuple>
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace plugin {
+
+struct GenericDeviceTy;
+struct GenericPluginTy;
+class GenericProfilerTy;
+
+template <typename FunT, typename... ArgsT, size_t... IdxSequence>
+void callViaIndexSeq(FunT F, GenericProfilerTy *P, uint64_t StartNanos,
+                     uint64_t EndNanos, std::tuple<ArgsT...> Args,
+                     std::index_sequence<IdxSequence...>) {
+  F(P, StartNanos, EndNanos, std::get<IdxSequence>(Args)...);
+}
+
+template <typename FunT, typename... ArgsT>
+void callViaUnpack(FunT F, GenericProfilerTy *P, uint64_t StartNanos,
+                   uint64_t EndNanos, std::tuple<ArgsT...> Tup) {
+  callViaIndexSeq(F, P, StartNanos, EndNanos, Tup,
+                  std::index_sequence_for<ArgsT...>{});
+}
+
+/// Abstraction layer to implement different profiler backends.
+///
+/// The plugins call into the GenericProfilerTy to handle the specific events
+/// with whatever specific backend was instantiated. For now, the supported
+/// backends are limited to an OMPT implementation.
+class GenericProfilerTy {
+public:
+  GenericProfilerTy() = default;
+  virtual ~GenericProfilerTy() = default;
+
+  /// Obtain a pointer to profiler-specific data, if any.
+  virtual void *getProfilerSpecificData() { return nullptr; }
+
+  virtual bool isProfilingEnabled() { return false; }
+
+  /// Set the factors which are used to interpolate the device clock compared to
+  /// the host clock. This follows a simple linear interpolation: Slope * <time>
+  /// + Offset.
+  void setTimeConversionFactors(double Slope, double Offset) {
+    HostToDeviceSlope = Slope;
+    HostToDeviceOffset = Offset;
+    setTimeConversionFactorsImpl(HostToDeviceSlope, HostToDeviceOffset);
+  }
+
+  /// Hook that is called when the plugin is initialized.
+  virtual void handleInit(GenericDeviceTy *Device, GenericPluginTy *Plugin) {}
+
+  /// Hook that is called when the plugin is de-initialized.
+  virtual void handleDeinit(GenericDeviceTy *Device, GenericPluginTy *Plugin) {}
+
+  /// Hook that is called when the device image is loaded.
+  virtual void handleLoadBinary(GenericDeviceTy *Device,
+                                GenericPluginTy *Plugin,
+                                const StringRef InputTgtImage) {}
+
+  /// Hook that is called when memory is allocated on the device.
+  virtual void handleDataAlloc(uint64_t StartNanos, uint64_t EndNanos,
+                               void *HostPtr, uint64_t Size, void *Data) {}
+
+  /// Hook that is called when memory is freed on the device.
+  virtual void handleDataDelete(uint64_t StartNanos, uint64_t EndNanos,
+                                void *TgtPtr, void *Data) {}
+
+  /// Hook that is called before launching a kernel.
+  virtual void handlePreKernelLaunch(GenericDeviceTy *Device,
+                                     uint32_t NumBlocks[3],
+                                     __tgt_async_info *AI) {}
+
+  /// Hook that is called when the kernel is finished to extract the specific
+  /// timing info for that kernel execution.
+  virtual void handleKernelCompletion(uint64_t StartNanos, uint64_t EndNanos,
+                                      void *Data) {}
+
+  /// Hook that is called when a data transfer happens to extract timing info
+  /// for that transfer.
+  virtual void handleDataTransfer(uint64_t StartNanos, uint64_t EndNanos,
+                                  void *Data) {}
+
+  /// Allow factors for time conversion between host and device.
+  virtual void setTimeConversionFactorsImpl(double Slope, double Offset) {}
+
+  /// RAII style timer that measures the elapsed time between construction and
+  /// destruction, then invokes a callback with the profiler, start/end times,
+  /// and any captured arguments.
+  template <typename FnT, typename... ArgsT> class ProfTimerTy {
+  public:
+    ProfTimerTy(FnT &&F, GenericProfilerTy *P, GenericDeviceTy *D, ArgsT... As)
+        : Fun(F), Prof(P), Dev(D), Args(As...) {
+      assert(Prof && "GenericProfilerTy is null");
+      assert(Dev && "GenericDeviceTy is null");
+      if (Prof)
+        StartTime = Prof->getDeviceTimeStamp(Dev);
+    }
+
+    ~ProfTimerTy() {
+      assert(Prof && "GenericProfilerTy is null");
+      assert(Dev && "GenericDeviceTy is null");
+      if (Prof) {
+        uint64_t EndTime = Prof->getDeviceTimeStamp(Dev);
+        callViaUnpack(Fun, Prof, StartTime, EndTime, Args);
+      }
+    }
+
+  private:
+    FnT Fun;
+    GenericProfilerTy *Prof;
+    GenericDeviceTy *Dev;
+    uint64_t StartTime = 0;
+    std::tuple<ArgsT...> Args;
+  };
+
+  template <typename FnT, typename... ArgsT>
+  [[maybe_unused]]
+  ProfTimerTy(FnT &&, GenericProfilerTy *, ArgsT...)
+      -> ProfTimerTy<FnT, ArgsT...>;
+
+  template <typename FnT, typename... ArgsT> friend class ProfTimerTy;
+
+  /// Returns an RAII style timer, which will handle data allocation timing.
+  [[nodiscard]] auto getScopedDataAllocTimer(GenericDeviceTy *Dev,
+                                             void *HostPtr, uint64_t Size,
+                                             void *ProfData = nullptr) {
+    return ProfTimerTy(
+        [](GenericProfilerTy *P, auto... args) {
+          assert(P && "P was null");
+          P->handleDataAlloc(args...);
+        },
+        this, Dev, HostPtr, Size, ProfData);
+  }
+
+  /// Returns an RAII style timer, which will handle data deletion timing.
+  [[nodiscard]] auto getScopedDataDeleteTimer(GenericDeviceTy *Dev,
+                                              void *TgtPtr,
+                                              void *ProfData = nullptr) {
+    return ProfTimerTy(
+        [](GenericProfilerTy *P, auto... args) {
+          assert(P && "P was null");
+          P->handleDataDelete(args...);
+        },
+        this, Dev, TgtPtr, ProfData);
+  }
+
+protected:
+  double HostToDeviceSlope = 1.0;
+  double HostToDeviceOffset = .0;
+
+private:
+  /// Vendor-specific implementation to obtain device time.
+  uint64_t getDeviceTimeStamp(GenericDeviceTy *D);
+};
+} // namespace plugin
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+#endif // OFFLOAD_PLUGINS_NEXTGEN_COMMON_INCLUDE_GENERICPROFILER_H
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index b3675e5a8700f..b6f19cd503137 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -29,6 +29,7 @@
 #include "Shared/Requirements.h"
 #include "Shared/Utils.h"
 
+#include "GenericProfiler.h"
 #include "GlobalHandler.h"
 #include "JIT.h"
 #include "MemoryManager.h"
@@ -57,6 +58,12 @@
 
 using namespace llvm::offload::debug;
 
+/// Factory function for creating a profiler. The default (weak) implementation
+/// returns a no-op GenericProfilerTy. When OMPT is enabled, a strong override
+/// returns an OmptProfilerTy instance.
+std::unique_ptr<llvm::omp::target::plugin::GenericProfilerTy>
+getProfilerToAttach();
+
 namespace llvm {
 namespace omp {
 namespace target {
@@ -1191,6 +1198,11 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
   uint32_t getDebugKind() const { return OMPX_DebugKind; }
   virtual uint64_t getClockFrequency() const { return CLOCKS_PER_SEC; }
 
+  /// Get a device-specific timestamp in nanoseconds, used by the profiler
+  /// for timing device operations. Subclasses should override this to provide
+  /// hardware-accurate timestamps (e.g., via HSA system info).
+  virtual uint64_t getDeviceTimeStamp() { return 0; }
+
   /// Get target compute unit kind (e.g., sm_80, or gfx908).
   virtual std::string getComputeUnitKind() const { return "unknown"; }
 
@@ -1469,7 +1481,8 @@ struct GenericPluginTy {
 
   /// Construct a plugin instance.
   GenericPluginTy(Triple::ArchType TA)
-      : GlobalHandler(nullptr), JIT(TA), RPCServer(nullptr) {}
+      : GlobalHandler(nullptr), JIT(TA), RPCServer(nullptr),
+        Profiler(getProfilerToAttach()) {}
 
   virtual ~GenericPluginTy() {}
 
@@ -1560,7 +1573,12 @@ struct GenericPluginTy {
   /// Tear down any target-specific doorbell resources.
   virtual Error deinitRPCDoorbell() { return Plugin::success(); }
 
-  /// Get a reference to the record and replay interface for the plugin.
+  /// Get a pointer to the profiler attached to this plugin.
+  GenericProfilerTy *getProfiler() {
+    assert(Profiler && "Profiler not initialized");
+    return Profiler.get();
+  }
+
   /// Initialize a device within the plugin.
   Error initDevice(int32_t DeviceId);
 
@@ -1842,6 +1860,9 @@ struct GenericPluginTy {
 
   /// The interface between the plugin and the GPU for host services.
   RPCServerTy *RPCServer;
+
+  /// The profiler backend attached to this plugin (e.g., OMPT).
+  std::unique_ptr<GenericProfilerTy> Profiler;
 };
 
 /// Auxiliary interface class for GenericDeviceResourceManagerTy. This class
diff --git a/offload/plugins-nextgen/common/src/GenericProfiler.cpp b/offload/plugins-nextgen/common/src/GenericProfiler.cpp
new file mode 100644
index 0000000000000..350a0cb5e378a
--- /dev/null
+++ b/offload/plugins-nextgen/common/src/GenericProfiler.cpp
@@ -0,0 +1,36 @@
+//===- GenericProfiler.cpp - GenericProfiler implementation ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//===----------------------------------------------------------------------===//
+
+#include "GenericProfiler.h"
+#include "PluginInterface.h"
+
+#include <cstdint>
+#include <memory>
+
+__attribute__((weak))
+std::unique_ptr<llvm::omp::target::plugin::GenericProfilerTy>
+getProfilerToAttach() {
+  return std::make_unique<llvm::omp::target::plugin::GenericProfilerTy>();
+}
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace plugin {
+
+uint64_t GenericProfilerTy::getDeviceTimeStamp(GenericDeviceTy *D) {
+  if (D)
+    return D->getDeviceTimeStamp();
+  return 0;
+}
+} // namespace plugin
+} // namespace target
+} // namespace omp
+} // namespace llvm
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index e539cbfc55324..d6f9d135a0091 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -26,6 +26,8 @@
 #include "omp-tools.h"
 #endif
 
+#include "GenericProfiler.h"
+
 #include "llvm/Bitcode/BitcodeReader.h"
 #include "llvm/Frontend/OpenMP/OMPConstants.h"
 #include "llvm/Support/Error.h"
@@ -47,7 +49,9 @@ using namespace llvm::offload::debug;
 AsyncInfoWrapperTy::AsyncInfoWrapperTy(GenericDeviceTy &Device,
                                        __tgt_async_info *AsyncInfoPtr)
     : Device(Device),
-      AsyncInfoPtr(AsyncInfoPtr ? AsyncInfoPtr : &LocalAsyncInfo) {}
+      AsyncInfoPtr(AsyncInfoPtr ? AsyncInfoPtr : &LocalAsyncInfo) {
+  LocalAsyncInfo.ProfilerData = nullptr;
+}
 
 Error AsyncInfoWrapperTy::synchronize() {
   assert(AsyncInfoPtr && "AsyncInfoWrapperTy already finalized");
@@ -168,9 +172,15 @@ GenericKernelTy::getKernelLaunchEnvironment(
        DPxPTR(&LocalKLE), DPxPTR(*AllocOrErr),
        sizeof(KernelLaunchEnvironmentTy));
 
+  // Temporarily suppress ProfilerData so the KLE upload is not traced as
+  // a user data operation.
+  __tgt_async_info *AI = AsyncInfoWrapper;
+  void *SavedProfilerData = AI->ProfilerData;
+  AI->ProfilerData = nullptr;
   auto Err = GenericDevice.dataSubmit(*AllocOrErr, &LocalKLE,
                                       sizeof(KernelLaunchEnvironmentTy),
                                       AsyncInfoWrapper);
+  AI->ProfilerData = SavedProfilerData;
   if (Err)
     return Err;
   return static_cast<KernelLaunchEnvironmentTy *>(*AllocOrErr);
@@ -331,6 +341,9 @@ Error GenericKernelTy::launch(GenericDeviceTy &GenericDevice, void **ArgPtrs,
     RRHandle = *RRHandleOrErr;
   }
 
+  GenericDevice.Plugin.getProfiler()->handlePreKernelLaunch(
+      &GenericDevice, EffectiveNumBlocks, AsyncInfoWrapper);
+
   if (auto Err = launchImpl(GenericDevice, EffectiveNumThreads,
                             EffectiveNumBlocks, DynBlockMemConf.NativeSize,
                             KernelArgs, LaunchParams, AsyncInfoWrapper))
@@ -567,6 +580,8 @@ Error GenericDeviceTy::init(GenericPluginTy &Plugin) {
   }
 #endif
 
+  Plugin.getProfiler()->handleInit(this, &Plugin);
+
   // Read and reinitialize the envars that depend on the device initialization.
   // Notice these two envars may change the stack size and heap size of the
   // device, so they need the device properly initialized.
@@ -668,6 +683,8 @@ Error GenericDeviceTy::deinit(GenericPluginTy &Plugin) {
   }
 #endif
 
+  Plugin.getProfiler()->handleDeinit(this, &Plugin);
+
   return deinitImpl();
 }
 Expected<DeviceImageTy *> GenericDeviceTy::loadBinary(GenericPluginTy &Plugin,
@@ -725,6 +742,8 @@ Expected<DeviceImageTy *> GenericDeviceTy::loadBinary(GenericPluginTy &Plugin,
   }
 #endif
 
+  Plugin.getProfiler()->handleLoadBinary(this, &Plugin, InputTgtImage);
+
   // Call any global constructors present on the device.
   if (auto Err = callGlobalConstructors(Plugin, *Image))
     return std::move(Err);
@@ -1006,6 +1025,9 @@ Error GenericDeviceTy::getDeviceMemorySize(uint64_t &DSize) {
 Expected<void *> GenericDeviceTy::dataAlloc(int64_t Size, void *HostPtr,
                                             TargetAllocTy Kind,
                                             size_t Alignment) {
+  auto ProfTimer =
+      Plugin.getProfiler()->getScopedDataAllocTimer(this, HostPtr, Size);
+
   void *Alloc = nullptr;
 
   // TODO Check alignment.
@@ -1073,6 +1095,8 @@ Expected<void *> GenericDeviceTy::dataAlloc(int64_t Size, void *HostPtr,
 }
 
 Error GenericDeviceTy::dataDelete(void *TgtPtr, TargetAllocTy Kind) {
+  auto ProfTimer = Plugin.getProfiler()->getScopedDataDeleteTimer(this, TgtPtr);
+
   // Free is a noop when recording or replaying.
   if (RecordReplay && RecordReplay->isRecordingOrReplaying())
     return RecordReplay->deallocate(TgtPtr);

>From 11e34b2f14dcd4f1553cb97848b814b67054f4b6 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 2 Apr 2026 07:36:31 -0500
Subject: [PATCH 03/12] [Offload][AMDGPU] Wire HSA profiling into
 GenericProfiler abstraction

Add device profiling infrastructure to the AMDGPU plugin so that the
GenericProfiler can receive nanosecond-accurate kernel execution and
data transfer timestamps from the HSA runtime.

Key changes:
- Add ProfilingInfoTy struct to transport HSA profiling data
- Add timeKernelInNsAsync/timeDataTransferInNsAsync callbacks that
  extract dispatch/copy times from HSA signals and call
  handleKernelCompletion/handleDataTransfer on the profiler
- Add getOrNullProfilerSpecificData helper to extract ProfilerData
  from AsyncInfoWrapperTy
- Add getDeviceTimeStamp() override using hsa_system_get_info
- Add getSystemTimestampInNs() for HSA system timestamp queries
- Add schedProfilerKernelTiming/schedProfilerDataTransferTiming to
  StreamSlotTy for scheduling profiler callbacks on stream slots
- Thread ProfilerSpecificData through pushKernelLaunch,
  pushMemoryCopyH2DAsync, pushMemoryCopyD2HAsync, pushMemoryCopyD2DAsync
- Extract ProfilerSpecificData in dataSubmitImpl, dataRetrieveImpl,
  dataExchangeImpl, and launchImpl
- Add stub getDeviceTimeStamp() to CUDA plugin

Made-with: Cursor
Co-Authored-By: Claude <noreply at anthropic.com>
---
 .../amdgpu/dynamic_hsa/hsa.cpp                |   1 +
 .../amdgpu/dynamic_hsa/hsa_ext_amd.h          |   8 +
 offload/plugins-nextgen/amdgpu/src/rtl.cpp    | 204 +++++++++++++++++-
 offload/plugins-nextgen/cuda/src/rtl.cpp      |   3 +
 4 files changed, 205 insertions(+), 11 deletions(-)

diff --git a/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa.cpp b/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa.cpp
index 5c7ec186b0ceb..99202574bfecc 100644
--- a/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa.cpp
+++ b/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa.cpp
@@ -71,6 +71,7 @@ DLWRAP(hsa_amd_signal_create, 5)
 DLWRAP(hsa_amd_signal_async_handler, 5)
 DLWRAP(hsa_amd_pointer_info, 5)
 DLWRAP(hsa_amd_profiling_get_dispatch_time, 3)
+DLWRAP(hsa_amd_profiling_get_async_copy_time, 2)
 DLWRAP(hsa_amd_profiling_set_profiler_enabled, 2)
 DLWRAP(hsa_code_object_reader_create_from_memory, 3)
 DLWRAP(hsa_code_object_reader_destroy, 1)
diff --git a/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa_ext_amd.h b/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa_ext_amd.h
index d26f9248e27ef..9fa740c664625 100644
--- a/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa_ext_amd.h
+++ b/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa_ext_amd.h
@@ -206,6 +206,14 @@ hsa_amd_profiling_get_dispatch_time(hsa_agent_t agent, hsa_signal_t signal,
 hsa_status_t hsa_amd_profiling_set_profiler_enabled(hsa_queue_t *queue,
                                                     int enable);
 
+typedef struct hsa_amd_profiling_async_copy_time_s {
+  uint64_t start;
+  uint64_t end;
+} hsa_amd_profiling_async_copy_time_t;
+
+hsa_status_t hsa_amd_profiling_get_async_copy_time(
+    hsa_signal_t signal, hsa_amd_profiling_async_copy_time_t *time);
+
 hsa_status_t hsa_amd_vmem_address_reserve(void **va, size_t size,
                                           uint64_t address, uint64_t flags);
 
diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index 29b5006f0d7b6..854e13839be86 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -30,6 +30,7 @@
 #include "Shared/Utils.h"
 #include "Utils/ELF.h"
 
+#include "GenericProfiler.h"
 #include "GlobalHandler.h"
 #include "OffloadAPI.h"
 #include "OpenMP/OMPT/Callback.h"
@@ -95,6 +96,88 @@ struct AMDGPUEventManagerTy;
 struct AMDGPUDeviceImageTy;
 struct AMDGPUMemoryManagerTy;
 struct AMDGPUMemoryPoolTy;
+struct AMDGPUSignalTy;
+
+/// Use to transport information to profiler timing functions.
+struct ProfilingInfoTy {
+  GenericPluginTy *Plugin;
+  hsa_agent_t Agent;
+  AMDGPUSignalTy *Signal;
+  double TicksToTime;
+  void *ProfilerSpecificData;
+};
+
+static ProfilingInfoTy *getProfilingInfo(void *Data);
+
+static std::pair<uint64_t, uint64_t>
+getKernelStartAndEndTime(const ProfilingInfoTy *Args);
+
+static std::pair<uint64_t, uint64_t>
+getCopyStartAndEndTime(const ProfilingInfoTy *Args);
+
+static Error timeKernelInNsAsync(void *Data);
+
+static Error timeDataTransferInNsAsync(void *Data) {
+  auto Args = getProfilingInfo(Data);
+  auto [Start, End] = getCopyStartAndEndTime(Args);
+  Args->Plugin->getProfiler()->handleDataTransfer(Start, End,
+                                                  Args->ProfilerSpecificData);
+  return Plugin::success();
+}
+
+static void *
+getOrNullProfilerSpecificData(AsyncInfoWrapperTy &AsyncInfoWrapper) {
+  __tgt_async_info *AI = AsyncInfoWrapper;
+  return AI ? AI->ProfilerData : nullptr;
+}
+
+} // namespace plugin
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+static double setTicksToTime() {
+  uint64_t TicksFrequency = 1;
+  double TicksToTime = 1.0;
+
+  hsa_status_t Status =
+      hsa_system_get_info(HSA_SYSTEM_INFO_TIMESTAMP_FREQUENCY, &TicksFrequency);
+  if (Status == HSA_STATUS_SUCCESS)
+    TicksToTime = (double)1e9 / (double)TicksFrequency;
+
+  return TicksToTime;
+}
+
+static double TicksToTime = 1.0;
+
+static void setHSATicksToTimeConstant() { TicksToTime = setTicksToTime(); }
+
+/// Get the current HSA-based system timestamp in nanoseconds.
+/// Called by OmptTracing.cpp (from PluginOmpt) for device time queries.
+uint64_t getSystemTimestampInNs() {
+  uint64_t TimeStamp = 0;
+  hsa_status_t Status =
+      hsa_system_get_info(HSA_SYSTEM_INFO_TIMESTAMP, &TimeStamp);
+  if (Status != HSA_STATUS_SUCCESS)
+    return 0;
+  return TimeStamp;
+}
+
+/// Enable or disable HSA async copy profiling for OMPT device tracing.
+/// Called by OmptTracing.cpp when a tool activates/deactivates device tracing.
+/// Full HSA profiling integration (enabling per-copy timing signals) will be
+/// wired up in a follow-on commit.
+void setOmptAsyncCopyProfile(bool Enable) {}
+
+/// Enable or disable HSA queue kernel profiling for OMPT device tracing.
+/// Called by OmptTracing.cpp when a tool activates/deactivates device tracing.
+/// Full HSA queue profiling integration will be wired up in a follow-on commit.
+void setGlobalOmptKernelProfile(void *Device, int Enable) {}
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace plugin {
 
 namespace hsa_utils {
 
@@ -1066,6 +1149,7 @@ struct AMDGPUStreamTy {
       MemcpyArgsTy MemcpyArgs;
       ReleaseBufferArgsTy ReleaseBufferArgs;
       ReleaseSignalArgsTy ReleaseSignalArgs;
+      ProfilingInfoTy ProfilerArgs;
       void *CallbackArgs;
     };
 
@@ -1103,7 +1187,32 @@ struct AMDGPUStreamTy {
       return Plugin::success();
     }
 
-    /// Register a callback to be called on compleition
+    /// Schedule kernel timing measurement via the profiler on the slot.
+    Error schedProfilerKernelTiming(GenericDeviceTy *Device, hsa_agent_t Agent,
+                                    AMDGPUSignalTy *OutputSignal,
+                                    double TicksToTime,
+                                    void *ProfilerSpecificData) {
+      Callbacks.emplace_back(timeKernelInNsAsync);
+      ActionArgs.emplace_back().ProfilerArgs =
+          ProfilingInfoTy{&(Device->Plugin), Agent, OutputSignal, TicksToTime,
+                          ProfilerSpecificData};
+      return Plugin::success();
+    }
+
+    /// Schedule data transfer timing via the profiler on the slot.
+    Error schedProfilerDataTransferTiming(GenericDeviceTy *Device,
+                                          hsa_agent_t Agent,
+                                          AMDGPUSignalTy *OutputSignal,
+                                          double TicksToTime,
+                                          void *ProfilerSpecificData) {
+      Callbacks.emplace_back(timeDataTransferInNsAsync);
+      ActionArgs.emplace_back().ProfilerArgs =
+          ProfilingInfoTy{&(Device->Plugin), Agent, OutputSignal, TicksToTime,
+                          ProfilerSpecificData};
+      return Plugin::success();
+    }
+
+    /// Register a callback to be called on completion.
     Error schedCallback(AMDGPUStreamCallbackTy *Func, void *Data) {
       Callbacks.emplace_back(Func);
       ActionArgs.emplace_back().CallbackArgs = Data;
@@ -1128,6 +1237,12 @@ struct AMDGPUStreamTy {
         } else if (Callback == releaseSignalAction) {
           if (auto Err = releaseSignalAction(&ActionArg))
             return Err;
+        } else if (Callback == timeKernelInNsAsync) {
+          if (auto Err = timeKernelInNsAsync(&ActionArg))
+            return Err;
+        } else if (Callback == timeDataTransferInNsAsync) {
+          if (auto Err = timeDataTransferInNsAsync(&ActionArg))
+            return Err;
         } else if (Callback) {
           if (auto Err = Callback(ActionArg.CallbackArgs))
             return Err;
@@ -1399,7 +1514,8 @@ struct AMDGPUStreamTy {
   Error pushKernelLaunch(const AMDGPUKernelTy &Kernel, void *KernelArgs,
                          uint32_t NumThreads[3], uint32_t NumBlocks[3],
                          uint32_t GroupSize, uint64_t StackSize,
-                         AMDGPUMemoryManagerTy &MemoryManager) {
+                         AMDGPUMemoryManagerTy &MemoryManager,
+                         void *ProfilerSpecificData = nullptr) {
     if (Queue == nullptr)
       return Plugin::error(ErrorCode::INVALID_NULL_POINTER,
                            "target queue was nullptr");
@@ -1420,6 +1536,14 @@ struct AMDGPUStreamTy {
     if (auto Err = Slots[Curr].schedReleaseBuffer(KernelArgs, MemoryManager))
       return Err;
 
+#ifdef OMPT_SUPPORT
+    if (ProfilerSpecificData) {
+      if (auto Err = Slots[Curr].schedProfilerKernelTiming(
+              &Device, Agent, OutputSignal, TicksToTime, ProfilerSpecificData))
+        return Err;
+    }
+#endif
+
     // If we are running an RPC server we want to wake up the server thread
     // whenever there is a kernel running and let it sleep otherwise.
     if (Device.getRPCServer())
@@ -1482,7 +1606,8 @@ struct AMDGPUStreamTy {
   /// manager once the operation completes.
   Error pushMemoryCopyD2HAsync(void *Dst, const void *Src, void *Inter,
                                uint64_t CopySize,
-                               AMDGPUMemoryManagerTy &MemoryManager) {
+                               AMDGPUMemoryManagerTy &MemoryManager,
+                               void *ProfilerSpecificData = nullptr) {
     // Retrieve available signals for the operation's outputs.
     AMDGPUSignalTy *OutputSignals[2] = {};
     if (auto Err = SignalManager.getResources(/*Num=*/2, OutputSignals))
@@ -1545,6 +1670,7 @@ struct AMDGPUStreamTy {
   Error pushMemoryCopyH2DAsync(void *Dst, const void *Src, void *Inter,
                                uint64_t CopySize,
                                AMDGPUMemoryManagerTy &MemoryManager,
+                               void *ProfilerSpecificData = nullptr,
                                size_t NumTimes = 1) {
     // Retrieve available signals for the operation's outputs.
     AMDGPUSignalTy *OutputSignals[2] = {};
@@ -1620,7 +1746,8 @@ struct AMDGPUStreamTy {
 
   // AMDGPUDeviceTy is incomplete here, passing the underlying agent instead
   Error pushMemoryCopyD2DAsync(void *Dst, hsa_agent_t DstAgent, const void *Src,
-                               hsa_agent_t SrcAgent, uint64_t CopySize) {
+                               hsa_agent_t SrcAgent, uint64_t CopySize,
+                               void *ProfilerSpecificData = nullptr) {
     AMDGPUSignalTy *OutputSignal;
     if (auto Err = SignalManager.getResources(/*Num=*/1, &OutputSignal))
       return Err;
@@ -2286,6 +2413,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     if (auto Err = initMemoryPools())
       return Err;
 
+    setHSATicksToTimeConstant();
+
     char GPUName[64];
     if (auto Err = getDeviceAttr(HSA_AGENT_INFO_NAME, GPUName))
       return Err;
@@ -2630,6 +2759,9 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
   /// Returns the clock frequency for the given AMDGPU device.
   uint64_t getClockFrequency() const override { return ClockFrequency; }
 
+  /// Returns the current HSA system timestamp for profiling.
+  uint64_t getDeviceTimeStamp() override { return getSystemTimestampInNs(); }
+
   /// Returns the HSA system timestamp frequency. Zero means unavailable.
   uint64_t getSystemTimestampFrequency() const {
     return SystemTimestampFrequency;
@@ -2839,6 +2971,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     AMDGPUStreamTy *Stream = nullptr;
     void *PinnedPtr = nullptr;
 
+    auto ProfilerSpecificData = getOrNullProfilerSpecificData(AsyncInfoWrapper);
+
     // Use one-step asynchronous operation when host memory is already pinned.
     if (void *PinnedPtr =
             PinnedAllocs.getDeviceAccessiblePtrFromPinnedBuffer(HstPtr)) {
@@ -2889,7 +3023,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
       return Err;
 
     return Stream->pushMemoryCopyH2DAsync(TgtPtr, HstPtr, PinnedPtr, Size,
-                                          PinnedMemoryManager);
+                                          PinnedMemoryManager,
+                                          ProfilerSpecificData);
   }
 
   /// Retrieve data from the device (device to host transfer).
@@ -2898,6 +3033,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     AMDGPUStreamTy *Stream = nullptr;
     void *PinnedPtr = nullptr;
 
+    auto ProfilerSpecificData = getOrNullProfilerSpecificData(AsyncInfoWrapper);
+
     // Use one-step asynchronous operation when host memory is already pinned.
     if (void *PinnedPtr =
             PinnedAllocs.getDeviceAccessiblePtrFromPinnedBuffer(HstPtr)) {
@@ -2949,7 +3086,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
       return Err;
 
     return Stream->pushMemoryCopyD2HAsync(HstPtr, TgtPtr, PinnedPtr, Size,
-                                          PinnedMemoryManager);
+                                          PinnedMemoryManager,
+                                          ProfilerSpecificData);
   }
 
   Error dataMemcpyImpl(void *DstPtr, const void *SrcPtr, int64_t Size,
@@ -2985,6 +3123,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
                          AsyncInfoWrapperTy &AsyncInfoWrapper) override {
     AMDGPUDeviceTy &DstDevice = static_cast<AMDGPUDeviceTy &>(DstGenericDevice);
 
+    auto ProfilerSpecificData = getOrNullProfilerSpecificData(AsyncInfoWrapper);
+
     // For large transfers use synchronous behavior.
     if (Size >= OMPX_MaxAsyncCopyBytes) {
       if (AsyncInfoWrapper.hasQueue())
@@ -3013,7 +3153,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
       return Plugin::success();
 
     return Stream->pushMemoryCopyD2DAsync(DstPtr, DstDevice.getAgent(), SrcPtr,
-                                          getAgent(), (uint64_t)Size);
+                                          getAgent(), (uint64_t)Size,
+                                          ProfilerSpecificData);
   }
 
   /// Insert a data fence between previous data operations and the following
@@ -3089,9 +3230,9 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     if (auto Err = getStream(AsyncInfoWrapper, Stream))
       return Err;
 
-    return Stream->pushMemoryCopyH2DAsync(TgtPtr, PatternPtr, PinnedPtr,
-                                          PatternSize, PinnedMemoryManager,
-                                          Size / PatternSize);
+    return Stream->pushMemoryCopyH2DAsync(
+        TgtPtr, PatternPtr, PinnedPtr, PatternSize, PinnedMemoryManager,
+        /*ProfilerSpecificData=*/nullptr, Size / PatternSize);
   }
 
   /// Initialize the async info
@@ -4371,10 +4512,12 @@ Error AMDGPUKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
   // HSA requires the group segment size to include both static and dynamic.
   uint32_t TotalBlockMemSize = getStaticBlockMemSize() + DynBlockMemSize;
 
+  auto ProfilerSpecificData = getOrNullProfilerSpecificData(AsyncInfoWrapper);
+
   // Push the kernel launch into the stream.
   return Stream->pushKernelLaunch(*this, AllArgs, NumThreads, NumBlocks,
                                   TotalBlockMemSize, StackSize,
-                                  ArgsMemoryManager);
+                                  ArgsMemoryManager, ProfilerSpecificData);
 }
 
 Error AMDGPUKernelTy::printLaunchInfoDetails(GenericDeviceTy &GenericDevice,
@@ -4560,6 +4703,45 @@ void AMDGPUQueueTy::callbackError(hsa_status_t Status, hsa_queue_t *Source,
   FATAL_MESSAGE(1, "%s", toString(std::move(Err)).data());
 }
 
+/// Implementation of profiling helper functions.
+static ProfilingInfoTy *getProfilingInfo(void *Data) {
+  return reinterpret_cast<ProfilingInfoTy *>(Data);
+}
+
+static std::pair<uint64_t, uint64_t>
+getKernelStartAndEndTime(const ProfilingInfoTy *Args) {
+  hsa_amd_profiling_dispatch_time_t Time = {};
+  hsa_status_t Status = hsa_amd_profiling_get_dispatch_time(
+      Args->Agent, Args->Signal->get(), &Time);
+  if (Status != HSA_STATUS_SUCCESS)
+    return {0, 0};
+  return {static_cast<uint64_t>(Time.start * Args->TicksToTime),
+          static_cast<uint64_t>(Time.end * Args->TicksToTime)};
+}
+
+static std::pair<uint64_t, uint64_t>
+getCopyStartAndEndTime(const ProfilingInfoTy *Args) {
+  hsa_amd_profiling_async_copy_time_t Time = {};
+  hsa_status_t Status =
+      hsa_amd_profiling_get_async_copy_time(Args->Signal->get(), &Time);
+  if (Status != HSA_STATUS_SUCCESS)
+    return {0, 0};
+  return {static_cast<uint64_t>(Time.start * Args->TicksToTime),
+          static_cast<uint64_t>(Time.end * Args->TicksToTime)};
+}
+
+static Error timeKernelInNsAsync(void *Data) {
+  assert(Data && "Invalid data pointer");
+  auto ProfilerInfo = getProfilingInfo(Data);
+  assert(ProfilerInfo && "Invalid profiling info");
+  assert(ProfilerInfo->ProfilerSpecificData && "Invalid ProfilerSpecificData");
+
+  auto [StartTime, EndTime] = getKernelStartAndEndTime(ProfilerInfo);
+  ProfilerInfo->Plugin->getProfiler()->handleKernelCompletion(
+      StartTime, EndTime, ProfilerInfo->ProfilerSpecificData);
+  return Plugin::success();
+}
+
 } // namespace plugin
 } // namespace target
 } // namespace omp
diff --git a/offload/plugins-nextgen/cuda/src/rtl.cpp b/offload/plugins-nextgen/cuda/src/rtl.cpp
index 997dba577f3ce..4bb2ed1107049 100644
--- a/offload/plugins-nextgen/cuda/src/rtl.cpp
+++ b/offload/plugins-nextgen/cuda/src/rtl.cpp
@@ -1405,6 +1405,9 @@ struct CUDADeviceTy : public GenericDeviceTy {
   /// Returns the clock frequency for the given NVPTX device.
   uint64_t getClockFrequency() const override { return 1000000000; }
 
+  /// Device timestamp stub for CUDA - full profiling is a future extension.
+  uint64_t getDeviceTimeStamp() override { return 0; }
+
 private:
   using CUDAStreamManagerTy = GenericDeviceResourceManagerTy<CUDAStreamRef>;
   using CUDAEventManagerTy = GenericDeviceResourceManagerTy<CUDAEventRef>;

>From cd836285b816297db2a6e831fe994b39843f7df7 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 2 Apr 2026 07:40:01 -0500
Subject: [PATCH 04/12] [Offload][OMPT] Add OMPT profiler implementation

Add the OMPT-specific profiler types and plugin tracing implementation without compiling or activating them yet. The existing callback path remains authoritative until the tracing interface and orchestration dependencies are available.

Co-Authored-By: Claude <noreply at anthropic.com>
---
 offload/include/OpenMP/OMPT/OmptCommonDefs.h  | 127 +++++++
 offload/include/OpenMP/OMPT/OmptEventInfoTy.h |  39 +++
 offload/plugins-nextgen/common/CMakeLists.txt |   5 +
 .../common/OMPT/OmptDeviceTracing.h           | 134 ++++++++
 .../common/OMPT/OmptProfiler.cpp              | 164 +++++++++
 .../common/OMPT/OmptProfiler.h                | 166 +++++++++
 .../common/OMPT/OmptTracing.cpp               | 318 ++++++++++++++++++
 7 files changed, 953 insertions(+)
 create mode 100644 offload/include/OpenMP/OMPT/OmptCommonDefs.h
 create mode 100644 offload/include/OpenMP/OMPT/OmptEventInfoTy.h
 create mode 100644 offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h
 create mode 100644 offload/plugins-nextgen/common/OMPT/OmptProfiler.cpp
 create mode 100644 offload/plugins-nextgen/common/OMPT/OmptProfiler.h
 create mode 100644 offload/plugins-nextgen/common/OMPT/OmptTracing.cpp

diff --git a/offload/include/OpenMP/OMPT/OmptCommonDefs.h b/offload/include/OpenMP/OMPT/OmptCommonDefs.h
new file mode 100644
index 0000000000000..5391658e80262
--- /dev/null
+++ b/offload/include/OpenMP/OMPT/OmptCommonDefs.h
@@ -0,0 +1,127 @@
+//===------ OmptCommonDefs.h - Common definitions for OMPT --*- C++ -*-----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Common defines and typedefs for OMPT callback and tracing functionality.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OFFLOAD_INCLUDE_OMPTCOMMONDEFS_H
+#define OFFLOAD_INCLUDE_OMPTCOMMONDEFS_H
+
+#ifdef OMPT_SUPPORT
+
+#include "omp-tools.h"
+
+#pragma push_macro("DEBUG_PREFIX")
+#undef DEBUG_PREFIX
+#define DEBUG_PREFIX "OMPT"
+
+#define FUNCPTR_TO_PTR(x) ((void *)(uint64_t)x)
+
+#define FOREACH_OMPT_TARGET_CALLBACK(macro)                                    \
+  FOREACH_OMPT_DEVICE_EVENT(macro)                                             \
+  FOREACH_OMPT_NOEMI_EVENT(macro)                                              \
+  FOREACH_OMPT_EMI_EVENT(macro)
+
+// Common device tracing functions
+#define FOREACH_OMPT_DEVICE_TRACING_FN_COMMON(macro)                           \
+  macro(ompt_set_trace_ompt) macro(ompt_start_trace) macro(ompt_flush_trace)   \
+      macro(ompt_stop_trace) macro(ompt_advance_buffer_cursor)                 \
+          macro(ompt_get_record_type)
+
+// Supported device tracing entry points
+#define FOREACH_OMPT_DEVICE_TRACING_FN(macro)                                  \
+  FOREACH_OMPT_DEVICE_TRACING_FN_COMMON(macro)                                 \
+  macro(ompt_get_record_ompt) macro(ompt_get_device_time)                      \
+      macro(ompt_translate_time)
+
+// Device tracing functionalities, which are also e.g. coupled to mutexes
+#define FOREACH_OMPT_DEVICE_TRACING_FN_IMPLEMENTAIONS(macro)                   \
+  FOREACH_OMPT_DEVICE_TRACING_FN_COMMON(macro)                                 \
+  macro(ompt_set_timestamp) macro(ompt_set_granted_teams)
+
+#define OMPT_API_ROUTINE static
+
+#define OMPT_CALLBACK_AVAILABLE(fn) (llvm::omp::target::ompt::Initialized && fn)
+
+#define OMPT_IF_BUILT(stmt) stmt
+
+#define OMPT_IF_ENABLED(stmts)                                                 \
+  do {                                                                         \
+    if (llvm::omp::target::ompt::Initialized) {                                \
+      stmts                                                                    \
+    }                                                                          \
+  } while (0)
+
+#define OMPT_IF_TRACING_ENABLED(stmts)                                         \
+  do {                                                                         \
+    if (llvm::omp::target::ompt::TracingActive) {                              \
+      stmts                                                                    \
+    }                                                                          \
+  } while (0)
+
+#define OMPT_FRAME_FLAGS (ompt_frame_runtime | OMPT_FRAME_POSITION_DEFAULT)
+
+#if (__PPC64__ | __arm__)
+#define OMPT_GET_FRAME_ADDRESS(level) __builtin_frame_address(level)
+#define OMPT_FRAME_POSITION_DEFAULT ompt_frame_cfa
+#else
+#define OMPT_GET_FRAME_ADDRESS(level) __builtin_frame_address(level)
+#define OMPT_FRAME_POSITION_DEFAULT ompt_frame_framepointer
+#endif
+
+#define OMPT_PTR_UNKNOWN ((void *)0)
+
+#define performIfOmptInitialized(stmt)                                         \
+  do {                                                                         \
+    if (llvm::omp::target::ompt::Initialized) {                                \
+      stmt;                                                                    \
+    }                                                                          \
+  } while (0)
+
+#define performOmptCallback(CallbackName, ...)                                 \
+  do {                                                                         \
+    if (ompt_callback_##CallbackName##_fn)                                     \
+      ompt_callback_##CallbackName##_fn(__VA_ARGS__);                          \
+  } while (0)
+
+typedef ompt_set_result_t (*libomptarget_ompt_set_trace_ompt_t)(
+    int Device, unsigned int Enable, unsigned int EventTy);
+typedef int (*libomptarget_ompt_start_trace_t)(int,
+                                               ompt_callback_buffer_request_t,
+                                               ompt_callback_buffer_complete_t);
+typedef int (*libomptarget_ompt_flush_trace_t)(int);
+typedef int (*libomptarget_ompt_stop_trace_t)(int);
+typedef int (*libomptarget_ompt_advance_buffer_cursor_t)(
+    ompt_device_t *, ompt_buffer_t *, size_t, ompt_buffer_cursor_t,
+    ompt_buffer_cursor_t *);
+typedef ompt_get_record_ompt_t libomptarget_ompt_get_record_ompt_t;
+typedef ompt_device_time_t (*libomptarget_ompt_get_device_time_t)(
+    ompt_device_t *);
+typedef ompt_translate_time_t libomptarget_ompt_translate_time_t;
+typedef ompt_device_time_t (*libomptarget_ompt_get_device_time_t)(
+    ompt_device_t *);
+typedef ompt_record_t (*libomptarget_ompt_get_record_type_t)(
+    ompt_buffer_t *, ompt_buffer_cursor_t);
+typedef void (*libomptarget_ompt_set_timestamp_t)(uint64_t start, uint64_t end);
+typedef void (*libomptarget_ompt_set_granted_teams_t)(uint32_t);
+
+/// Function type def used for maintaining unique target region, target
+/// operations ids
+typedef uint64_t (*IdInterfaceTy)();
+
+#pragma pop_macro("DEBUG_PREFIX")
+
+#else
+#define performIfOmptInitialized(stmt)
+#define OMPT_IF_BUILT(stmt)
+#define OMPT_IF_ENABLED(stmts)
+#define OMPT_IF_TRACING_ENABLED(stmts)
+#endif // OMPT_SUPPORT
+
+#endif // OFFLOAD_INCLUDE_OMPTCOMMONDEFS_H
diff --git a/offload/include/OpenMP/OMPT/OmptEventInfoTy.h b/offload/include/OpenMP/OMPT/OmptEventInfoTy.h
new file mode 100644
index 0000000000000..7124b3a3ff501
--- /dev/null
+++ b/offload/include/OpenMP/OMPT/OmptEventInfoTy.h
@@ -0,0 +1,39 @@
+//===- OmptEventInfoTy.h - OMPT specific trace record data ------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Data structure used to communicate OMPT specific profiler data from the
+// high-level libomptarget into the vendor-specific plugins
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OFFLOAD_INCLUDE_OPENMP_OMPT_OMPTEVENTINFOTY_H
+#define OFFLOAD_INCLUDE_OPENMP_OMPT_OMPTEVENTINFOTY_H
+
+#include "Shared/Debug.h"
+
+struct ompt_record_ompt_t;
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace ompt {
+
+/// Holds info needed to fill asynchronous trace records
+struct OmptEventInfoTy {
+  /// The granted number of teams at runtime
+  uint64_t NumTeams;
+  /// Pointer to the actual buffer storage location
+  ompt_record_ompt_t *TraceRecord;
+};
+
+} // namespace ompt
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+#endif // OFFLOAD_INCLUDE_OPENMP_OMPT_OMPTEVENTINFOTY_H
diff --git a/offload/plugins-nextgen/common/CMakeLists.txt b/offload/plugins-nextgen/common/CMakeLists.txt
index 19ce102309a72..556b8ddf3311c 100644
--- a/offload/plugins-nextgen/common/CMakeLists.txt
+++ b/offload/plugins-nextgen/common/CMakeLists.txt
@@ -37,6 +37,11 @@ endif()
 include(FindLibcCommonUtils)
 target_link_libraries(PluginCommon PRIVATE llvm-libc-common-utilities)
 
+if (TARGET flang_rt.runtime.static)
+  target_link_libraries(PluginCommon PRIVATE flang_rt.runtime.static)
+  target_compile_definitions(PluginCommon PRIVATE OFFLOAD_HAS_FLANG_RT)
+endif()
+
 # Define the TARGET_NAME and DEBUG_PREFIX.
 target_compile_definitions(PluginCommon PRIVATE
   TARGET_NAME=PluginInterface
diff --git a/offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h b/offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h
new file mode 100644
index 0000000000000..4ffd1976e9730
--- /dev/null
+++ b/offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h
@@ -0,0 +1,134 @@
+//===- OmptDeviceTracing.h - Target independent OMPT callbacks --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Interface used by target-independent runtimes to coordinate registration and
+// invocation of OMPT tracing functionality.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_COMMON_OMPTDEVICETRACING_H
+#define OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_COMMON_OMPTDEVICETRACING_H
+
+#ifdef OMPT_SUPPORT
+
+#include "OpenMP/OMPT/OmptCommonDefs.h"
+
+#include "llvm/Support/DynamicLibrary.h"
+
+#include <map>
+#include <memory>
+#include <mutex>
+
+#pragma push_macro("DEBUG_PREFIX")
+#undef DEBUG_PREFIX
+#define DEBUG_PREFIX "OMPT"
+
+extern void setOmptAsyncCopyProfile(bool Enable);
+extern void setGlobalOmptKernelProfile(void *Device, int Enable);
+extern uint64_t getSystemTimestampInNs();
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace ompt {
+
+// Declare OMPT device tracing function entry points
+#define declareOmptTracingFn(Name) extern libomptarget_##Name##_t Name##_fn;
+FOREACH_OMPT_DEVICE_TRACING_FN_IMPLEMENTAIONS(declareOmptTracingFn)
+#undef declareOmptTracingFn
+
+// Declare OMPT device tracing function mutexes
+#define declareOmptTracingFnMutex(Name) extern std::mutex Name##_mutex;
+FOREACH_OMPT_DEVICE_TRACING_FN_IMPLEMENTAIONS(declareOmptTracingFnMutex)
+#undef declareOmptTracingFnMutex
+
+extern std::mutex DeviceIdWritingMutex;
+
+/// Activate tracing on the given device
+void enableDeviceTracing(int DeviceId);
+
+/// Deactivate tracing on the given device
+void disableDeviceTracing(int DeviceId);
+
+/// Set 'start' and 'stop' in trace records
+void setOmptTimestamp(uint64_t StartTime, uint64_t EndTime);
+
+/// Set the linear function correlation between host and device clocks
+void setOmptHostToDeviceRate(double Slope, double Offset);
+
+/// Set / store the number of granted teams in trace records
+void setOmptGrantedNumTeams(uint64_t NumTeams);
+
+/// Lookup the given device pointer and return its RTL device ID
+int getDeviceId(ompt_device_t *Device);
+
+/// Map the given device pointer to the given DeviceId
+void setDeviceId(ompt_device_t *Device, int32_t DeviceId);
+
+/// Rempve the given device pointer from the current mapping
+void removeDeviceId(ompt_device_t *Device);
+
+/// Check whether the provided device is currently traced.
+bool isTracedDevice(int32_t DeviceId);
+
+/// Provide name based lookup for the device tracing functions
+extern ompt_interface_fn_t
+lookupDeviceTracingFn(const char *InterfaceFunctionName);
+
+/// Host to device linear clock correlation
+extern double HostToDeviceSlope;
+
+/// Host to device constant clock offset
+extern double HostToDeviceOffset;
+
+/// Mapping of device pointers to their corresponding RTL device ID
+extern std::map<ompt_device_t *, int32_t> Devices;
+
+/// Mapping of RTL device IDs to their currently enabled tracing event types.
+/// Note: Event type '0' (bit position) indicates if this device is traced.
+extern std::map<int32_t, uint64_t> TracedDevices;
+
+/// OMPT global tracing status. Indicates if at least one device is traced.
+extern bool TracingActive;
+
+/// Parent library pointer
+extern std::shared_ptr<llvm::sys::DynamicLibrary> ParentLibrary;
+
+/// Get the parent library by pointer. If it is not already set, it will set the
+/// parent library pointer.
+std::shared_ptr<llvm::sys::DynamicLibrary> getParentLibrary();
+
+/// Set the parent library by filename
+void setParentLibrary(const char *Filename);
+
+/// Search for FuncName inside the parent library and assign to FuncPtr.
+/// IMPORTANT: This function assumes that the *caller* holds the respective lock
+/// for FuncPtr.
+template <typename FT>
+void ensureFuncPtrLoaded(const std::string &FuncName, FT *FuncPtr) {
+  if (*FuncPtr == nullptr) {
+    if ((ParentLibrary == nullptr && getParentLibrary() == nullptr) ||
+        !ParentLibrary->isValid())
+      return;
+    void *SymbolPtr = ParentLibrary->getAddressOfSymbol(FuncName.c_str());
+    if (SymbolPtr == nullptr)
+      return;
+    *FuncPtr = reinterpret_cast<FT>(SymbolPtr);
+  }
+}
+
+} // namespace ompt
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+#pragma pop_macro("DEBUG_PREFIX")
+
+#endif // OMPT_SUPPORT
+
+#endif // OPENMP_LIBOMPTARGET_PLUGINS_NEXTGEN_COMMON_OMPTDEVICETRACING_H
diff --git a/offload/plugins-nextgen/common/OMPT/OmptProfiler.cpp b/offload/plugins-nextgen/common/OMPT/OmptProfiler.cpp
new file mode 100644
index 0000000000000..4f9688a07f534
--- /dev/null
+++ b/offload/plugins-nextgen/common/OMPT/OmptProfiler.cpp
@@ -0,0 +1,164 @@
+//===- OmptProfiler.cpp - OMPT impl of GenericProfilerTy --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementation of OmptProfilerTy
+//
+//===----------------------------------------------------------------------===//
+
+#include "OmptProfiler.h"
+#include "OpenMP/OMPT/Interface.h"
+#include "PluginInterface.h"
+#include "Shared/Debug.h"
+
+#include <memory>
+
+using namespace llvm::omp::target;
+
+/// Strong override of getProfilerToAttach() that returns an OmptProfilerTy
+/// when OMPT is enabled. This overrides the weak default in
+/// GenericProfiler.cpp.
+std::unique_ptr<llvm::omp::target::plugin::GenericProfilerTy>
+getProfilerToAttach() {
+  return std::make_unique<llvm::omp::target::ompt::OmptProfilerTy>();
+}
+
+void ompt::OmptProfilerTy::handleInit(plugin::GenericDeviceTy *Device,
+                                      plugin::GenericPluginTy *Plugin) {
+  auto DeviceId = Device->getDeviceId();
+  auto DevicePtr = reinterpret_cast<ompt_device_t *>(Device);
+  ompt::setDeviceId(DevicePtr, Plugin->getUserId(DeviceId));
+
+  if (ompt::Initialized) {
+    bool ExpectedStatus = false;
+    if (OmptInitialized.compare_exchange_strong(ExpectedStatus, true))
+      performOmptCallback(device_initialize, Plugin->getUserId(DeviceId),
+                          /*type=*/Device->getComputeUnitKind().c_str(),
+                          /*device=*/DevicePtr,
+                          /*lookup=*/ompt::lookupDeviceTracingFn,
+                          /*documentation=*/nullptr);
+  }
+}
+
+void ompt::OmptProfilerTy::handleDeinit(
+    plugin::GenericDeviceTy *Device, target::plugin::GenericPluginTy *Plugin) {
+  auto DeviceId = Device->getDeviceId();
+
+  if (ompt::Initialized) {
+    bool ExpectedStatus = true;
+    if (OmptInitialized.compare_exchange_strong(ExpectedStatus, false))
+      performOmptCallback(device_finalize, Plugin->getUserId(DeviceId));
+  }
+  ompt::removeDeviceId(reinterpret_cast<ompt_device_t *>(Device));
+}
+
+void ompt::OmptProfilerTy::handleLoadBinary(plugin::GenericDeviceTy *Device,
+                                            plugin::GenericPluginTy *Plugin,
+                                            const StringRef InputTgtImage) {
+
+  if (!ompt::Initialized)
+    return;
+
+  auto DeviceId = Device->getDeviceId();
+  size_t Bytes = InputTgtImage.size();
+  performOmptCallback(
+      device_load, Plugin->getUserId(DeviceId),
+      /*FileName=*/nullptr, /*FileOffset=*/0, /*VmaInFile=*/nullptr,
+      /*ImgSize=*/Bytes,
+      /*HostAddr=*/const_cast<unsigned char *>(InputTgtImage.bytes_begin()),
+      /*DeviceAddr=*/nullptr, /* FIXME: ModuleId */ 0);
+}
+
+void ompt::OmptProfilerTy::handleDataAlloc(uint64_t StartNanos,
+                                           uint64_t EndNanos, void *HostPtr,
+                                           uint64_t Size, void *Data) {
+  ompt::setOmptTimestamp(StartNanos, EndNanos);
+}
+
+void ompt::OmptProfilerTy::handleDataDelete(uint64_t StartNanos,
+                                            uint64_t EndNanos, void *TgtPtr,
+                                            void *Data) {
+  ompt::setOmptTimestamp(StartNanos, EndNanos);
+}
+
+void ompt::OmptProfilerTy::handlePreKernelLaunch(
+    plugin::GenericDeviceTy *Device, uint32_t NumBlocks[3],
+    __tgt_async_info *AI) {
+  if (!ompt::isTracedDevice(
+          ompt::getDeviceId(reinterpret_cast<ompt_device_t *>(Device))))
+    return;
+
+  if (AI->ProfilerData == nullptr)
+    return;
+
+  auto ProfilerSpecificData =
+      reinterpret_cast<ompt::OmptEventInfoTy *>(AI->ProfilerData);
+  assert(ProfilerSpecificData && "Invalid ProfilerSpecificData");
+  // Set number of granted teams for OMPT
+  setOmptGrantedNumTeams(NumBlocks[0]);
+  ProfilerSpecificData->NumTeams = NumBlocks[0];
+}
+
+void ompt::OmptProfilerTy::handleKernelCompletion(uint64_t StartNanos,
+                                                  uint64_t EndNanos,
+                                                  void *Data) {
+
+  if (!isProfilingEnabled())
+    return;
+
+  /// Empty data means no tracing in OMPT
+  /// offload/include/OpenMP/OMPT/Interface.h line 492
+  if (!Data)
+    return;
+
+  ODBG(ODT_Tool) << "OMPT-Async: Time kernel for asynchronous execution: Start "
+                 << StartNanos << " End " << EndNanos;
+
+  auto OmptEventInfo = reinterpret_cast<ompt::OmptEventInfoTy *>(Data);
+  assert(OmptEventInfo && "Invalid OmptEventInfo");
+  assert(OmptEventInfo->TraceRecord && "Invalid TraceRecord");
+
+  ompt::RegionInterface.stopTargetSubmitTraceAsync(OmptEventInfo->TraceRecord,
+                                                   OmptEventInfo->NumTeams,
+                                                   StartNanos, EndNanos);
+
+  // Done processing, our responsibility to free the memory
+  freeProfilerDataEntry(OmptEventInfo);
+}
+
+void ompt::OmptProfilerTy::handleDataTransfer(uint64_t StartNanos,
+                                              uint64_t EndNanos, void *Data) {
+
+  if (!isProfilingEnabled())
+    return;
+
+  /// Empty data means no tracing in OMPT
+  /// offload/include/OpenMP/OMPT/Interface.h line 492
+  if (!Data)
+    return;
+
+  ODBG(ODT_Tool) << "OMPT-Async: Time data for asynchronous execution: Start "
+                 << StartNanos << " End " << EndNanos;
+
+  auto OmptEventInfo = reinterpret_cast<ompt::OmptEventInfoTy *>(Data);
+  assert(OmptEventInfo && "Invalid OmptEventInfo");
+  assert(OmptEventInfo->TraceRecord && "Invalid TraceRecord");
+
+  ompt::RegionInterface.stopTargetDataMovementTraceAsync(
+      OmptEventInfo->TraceRecord, StartNanos, EndNanos);
+
+  // Done processing, our responsibility to free the memory
+  freeProfilerDataEntry(OmptEventInfo);
+}
+
+bool ompt::OmptProfilerTy::isProfilingEnabled() { return ompt::TracingActive; }
+
+void ompt::OmptProfilerTy::setTimeConversionFactorsImpl(double Slope,
+                                                        double Offset) {
+  ODBG(ODT_Tool) << "Using Time Slope: " << Slope << " and Offset: " << Offset;
+  setOmptHostToDeviceRate(Slope, Offset);
+}
diff --git a/offload/plugins-nextgen/common/OMPT/OmptProfiler.h b/offload/plugins-nextgen/common/OMPT/OmptProfiler.h
new file mode 100644
index 0000000000000..9a25bda272d8a
--- /dev/null
+++ b/offload/plugins-nextgen/common/OMPT/OmptProfiler.h
@@ -0,0 +1,166 @@
+//===- OmptProfiler.h - OMPT specific impl of GenericProfilerTy -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// OMPT specific implementation of the GenericProfilerTy class.
+// This class uses the already existing implementation of OMPT to invoke
+// callbacks and perform tracing.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OFFLOAD_PLUGINS_NEXTGEN_COMMON_OMPT_OMPTPROFILERTY_H
+#define OFFLOAD_PLUGINS_NEXTGEN_COMMON_OMPT_OMPTPROFILERTY_H
+
+#include "GenericProfiler.h"
+
+#include "OmptDeviceTracing.h"
+#include "OpenMP/OMPT/Callback.h"
+#include "OpenMP/OMPT/OmptEventInfoTy.h"
+#include "Shared/Debug.h"
+#include "omp-tools.h"
+
+#include <functional>
+#include <tuple>
+
+#pragma push_macro("DEBUG_PREFIX")
+#undef DEBUG_PREFIX
+#define DEBUG_PREFIX "OMPT"
+
+extern uint64_t getSystemTimestampInNs();
+
+using namespace llvm::omp::target::debug;
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace plugin {
+struct GenericDeviceTy;
+struct GenericPluginTy;
+class GenericProfilerTy;
+
+} // namespace plugin
+
+namespace ompt {
+
+// From Callback.h / Callback.cpp
+extern bool Initialized;
+
+/**
+ * Implements an OMPT backend for the Profiler interface used in the plugins.
+ *
+ * Forwards / Implements the different generic hooks with OMPT semantics.
+ */
+class OmptProfilerTy : public plugin::GenericProfilerTy {
+public:
+  /** Public members **/
+  OmptProfilerTy() {
+
+    OmptInitialized.store(false);
+    // Bind the callbacks to this device's member functions
+#define bindOmptCallback(Name, Type, Code)                                     \
+  if (ompt::Initialized && ompt::lookupCallbackByCode) {                       \
+    ompt::lookupCallbackByCode((ompt_callbacks_t)(Code),                       \
+                               ((ompt_callback_t *)&(Name##_fn)));             \
+    ODBG(ODT_Tool) << "class bound " << #Name << "="                           \
+                   << ((void *)(uint64_t)Name##_fn);                           \
+  }
+
+    FOREACH_OMPT_DEVICE_EVENT(bindOmptCallback);
+#undef bindOmptCallback
+
+#define bindOmptTracingFunction(FunctionName)                                  \
+  if (ompt::Initialized && ompt::lookupDeviceTracingFn) {                      \
+    FunctionName##_fn = ompt::lookupDeviceTracingFn(#FunctionName);            \
+    ODBG(ODT_Tool) << "device tracing fn bound " << #FunctionName << "="       \
+                   << ((void *)(uint64_t)FunctionName##_fn);                   \
+  }
+
+    FOREACH_OMPT_DEVICE_TRACING_FN_COMMON(bindOmptTracingFunction);
+#undef bindOmptTracingFunction
+  }
+
+  bool isProfilingEnabled() override;
+
+  void handleInit(plugin::GenericDeviceTy *Device,
+                  plugin::GenericPluginTy *Plugin) override;
+
+  void handleDeinit(plugin::GenericDeviceTy *Device,
+                    plugin::GenericPluginTy *Plugin) override;
+
+  void handleLoadBinary(plugin::GenericDeviceTy *Device,
+                        plugin::GenericPluginTy *Plugin,
+                        const StringRef InputTgtImage) override;
+
+  void handleDataAlloc(uint64_t StartNanos, uint64_t EndNanos, void *HostPtr,
+                       uint64_t Size, void *Data) override;
+  void handleDataDelete(uint64_t StartNanos, uint64_t EndNanos, void *TgtPtr,
+                        void *Data) override;
+
+  void handlePreKernelLaunch(plugin::GenericDeviceTy *Device,
+                             uint32_t NumBlocks[3],
+                             __tgt_async_info *AI) override;
+
+  void handleKernelCompletion(uint64_t StartNanos, uint64_t EndNanos,
+                              void *Data) override;
+
+  void handleDataTransfer(uint64_t StartNanos, uint64_t EndNanos,
+                          void *Data) override;
+
+  void setTimeConversionFactorsImpl(double Slope, double Offset) override;
+
+  void *getProfilerSpecificData() override {
+    // TODO: This is ID is not used currently
+    uint64_t Id = OmptProfDataId.fetch_add(1);
+    {
+      std::scoped_lock Lock(ProfilerDataMutex);
+      ProfilerData[Id] = std::make_unique<OmptEventInfoTy>();
+      return ProfilerData[Id].get();
+    }
+  }
+
+  void freeProfilerDataEntry(OmptEventInfoTy *DataPtr) {
+    std::scoped_lock Lock(ProfilerDataMutex);
+
+    for (auto &Entry : ProfilerData)
+      if (Entry.second.get() == DataPtr) {
+        ProfilerData.erase(Entry.first);
+        break;
+      }
+  }
+
+private:
+  /// Holds a unique ID for each allocation of OmptEventInfoTy
+  std::atomic<uint64_t> OmptProfDataId{0};
+
+  /// Holds memory used to store OMPT specific data and pass it down from
+  /// libomptarget into the plugins.
+  std::map<uint64_t, std::unique_ptr<OmptEventInfoTy>> ProfilerData;
+
+  /// Lock to guard STL ProfilerData map
+  std::mutex ProfilerDataMutex;
+
+  /// OMPT callback functions
+#define defineOmptCallback(Name, Type, Code) Name##_t Name##_fn = nullptr;
+  FOREACH_OMPT_DEVICE_EVENT(defineOmptCallback)
+#undef defineOmptCallback
+
+  /// OMPT device tracing functions
+#define defineOmptTracingFunction(Name) ompt_interface_fn_t Name##_fn = nullptr;
+  FOREACH_OMPT_DEVICE_TRACING_FN_COMMON(defineOmptTracingFunction);
+#undef defineOmptTracingFunction
+
+  /// Internal representation for OMPT device (initialize & finalize)
+  std::atomic<bool> OmptInitialized;
+};
+} // namespace ompt
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+#pragma pop_macro("DEBUG_PREFIX")
+
+#endif
diff --git a/offload/plugins-nextgen/common/OMPT/OmptTracing.cpp b/offload/plugins-nextgen/common/OMPT/OmptTracing.cpp
new file mode 100644
index 0000000000000..7a8e312b3d476
--- /dev/null
+++ b/offload/plugins-nextgen/common/OMPT/OmptTracing.cpp
@@ -0,0 +1,318 @@
+//===-- OmptTracing.cpp - Target independent OpenMP target RTL --- C++ -*--===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementation of OMPT tracing interfaces for PluginInterface
+//
+//===----------------------------------------------------------------------===//
+
+#ifdef OMPT_SUPPORT
+
+#include "OmptDeviceTracing.h"
+#include "Shared/Debug.h"
+#include "omp-tools.h"
+
+#include "llvm/Support/DynamicLibrary.h"
+
+#include <atomic>
+#include <cassert>
+#include <cstdlib>
+#include <cstring>
+#include <memory>
+#include <mutex>
+
+#pragma push_macro("DEBUG_PREFIX")
+#undef DEBUG_PREFIX
+#define DEBUG_PREFIX "OMPT"
+
+// Define OMPT device tracing function entry points
+#define defineOmptTracingFn(Name)                                              \
+  libomptarget_##Name##_t llvm::omp::target::ompt::Name##_fn = nullptr;
+FOREACH_OMPT_DEVICE_TRACING_FN_IMPLEMENTAIONS(defineOmptTracingFn)
+#undef defineOmptTracingFn
+
+// Define OMPT device tracing function mutexes
+#define defineOmptTracingFnMutex(Name)                                         \
+  std::mutex llvm::omp::target::ompt::Name##_mutex;
+FOREACH_OMPT_DEVICE_TRACING_FN_IMPLEMENTAIONS(defineOmptTracingFnMutex)
+#undef defineOmptTracingFnMutex
+
+std::mutex llvm::omp::target::ompt::DeviceIdWritingMutex;
+
+using namespace llvm::omp::target::ompt;
+using namespace llvm::omp::target::debug;
+
+std::shared_ptr<llvm::sys::DynamicLibrary>
+    llvm::omp::target::ompt::ParentLibrary(nullptr);
+
+double llvm::omp::target::ompt::HostToDeviceSlope = .0;
+double llvm::omp::target::ompt::HostToDeviceOffset = .0;
+
+std::map<ompt_device_t *, int32_t> llvm::omp::target::ompt::Devices;
+
+std::shared_ptr<llvm::sys::DynamicLibrary>
+llvm::omp::target::ompt::getParentLibrary() {
+  static bool ParentLibraryAssigned = false;
+  if (!ParentLibraryAssigned) {
+    setParentLibrary("libomptarget.so");
+    ParentLibraryAssigned = true;
+  }
+  return ParentLibrary;
+}
+
+void llvm::omp::target::ompt::setParentLibrary(const char *Filename) {
+  if (ParentLibrary)
+    return;
+  std::string ErrorMsg;
+  ParentLibrary = std::make_shared<llvm::sys::DynamicLibrary>(
+      llvm::sys::DynamicLibrary::getPermanentLibrary(Filename, &ErrorMsg));
+  if ((ParentLibrary == nullptr) || (!ParentLibrary->isValid()))
+    REPORT() << "Failed to set parent library: " << ErrorMsg.c_str();
+}
+
+int llvm::omp::target::ompt::getDeviceId(ompt_device_t *Device) {
+  // Block other threads, which might trigger an erase (for the same device)
+  std::unique_lock<std::mutex> Lock(DeviceIdWritingMutex);
+  auto DeviceIterator = Devices.find(Device);
+  if (Device == nullptr || DeviceIterator == Devices.end()) {
+    REPORT() << "Failed to get ID for Device=" << Device;
+    return -1;
+  }
+  return DeviceIterator->second;
+}
+
+void llvm::omp::target::ompt::setDeviceId(ompt_device_t *Device,
+                                          int32_t DeviceId) {
+  assert(Device && "Mapping device ID to nullptr is not allowed");
+  if (Device == nullptr || DeviceId < 0) {
+    REPORT() << "Failed to set ID=%d for Device=" << DeviceId << Device;
+    return;
+  }
+  std::unique_lock<std::mutex> Lock(DeviceIdWritingMutex);
+  auto DeviceIterator = Devices.find(Device);
+  if (DeviceIterator != Devices.end()) {
+    auto CurrentDeviceId = DeviceIterator->second;
+    if (DeviceId == CurrentDeviceId) {
+      REPORT() << "Tried to duplicate OMPT Device= " << Device
+               << " ID=" << DeviceId;
+    } else {
+      REPORT() << "Tried to overwrite OMPT Device=" << Device
+               << " (ID=" << CurrentDeviceId << " with new ID=" << DeviceId;
+    }
+    return;
+  }
+  Devices.emplace(Device, DeviceId);
+}
+
+void llvm::omp::target::ompt::removeDeviceId(ompt_device_t *Device) {
+  int DeviceId = getDeviceId(Device);
+  if (DeviceId < 0) {
+    REPORT() << "Tried to remove Device= " << Device << " ID=" << DeviceId;
+    return;
+  }
+  std::unique_lock<std::mutex> Lock(DeviceIdWritingMutex);
+  Devices.erase(Device);
+  TracedDevices.erase(DeviceId);
+}
+
+OMPT_API_ROUTINE ompt_set_result_t ompt_set_trace_ompt(ompt_device_t *Device,
+                                                       unsigned int Enable,
+                                                       unsigned int EventTy) {
+  ODBG(ODT_Tool) << "Executing ompt_set_trace_ompt";
+
+  int DeviceId = getDeviceId(Device);
+  if (DeviceId < 0) {
+    REPORT() << "Failed to set trace events for Device=" << Device
+             << " (Unknown device) [Enable=" << Enable
+             << " EventTy=" << EventTy;
+    return ompt_set_never;
+  }
+
+  std::unique_lock<std::mutex> Lock(ompt_set_trace_ompt_mutex);
+  ensureFuncPtrLoaded<libomptarget_ompt_set_trace_ompt_t>(
+      "libomptarget_ompt_set_trace_ompt", &ompt_set_trace_ompt_fn);
+  assert(ompt_set_trace_ompt_fn && "libomptarget_ompt_set_trace_ompt loaded");
+  return ompt_set_trace_ompt_fn(DeviceId, Enable, EventTy);
+}
+
+OMPT_API_ROUTINE int
+ompt_start_trace(ompt_device_t *Device, ompt_callback_buffer_request_t Request,
+                 ompt_callback_buffer_complete_t Complete) {
+  ODBG(ODT_Tool) << "Executing ompt_start_trace";
+
+  int DeviceId = getDeviceId(Device);
+  if (DeviceId < 0) {
+    REPORT() << "Failed to start trace for Device=" << Device
+             << " (Unknown device";
+    // Indicate failure
+    return 0;
+  }
+
+  {
+    // Protect the function pointer
+    std::unique_lock<std::mutex> Lock(ompt_start_trace_mutex);
+
+    if (Request && Complete) {
+      llvm::omp::target::ompt::enableDeviceTracing(DeviceId);
+      // Enable asynchronous memory copy profiling
+      setOmptAsyncCopyProfile(/*Enable=*/true);
+      // Enable queue dispatch profiling
+      if (DeviceId >= 0)
+        setGlobalOmptKernelProfile(Device, /*Enable=*/1);
+      else
+        REPORT() << "May not enable kernel profiling for invalid device id="
+                 << DeviceId;
+    }
+
+    // Call libomptarget specific function
+    ensureFuncPtrLoaded<libomptarget_ompt_start_trace_t>(
+        "libomptarget_ompt_start_trace", &ompt_start_trace_fn);
+    assert(ompt_start_trace_fn && "libomptarget_ompt_start_trace loaded");
+  }
+  return ompt_start_trace_fn(DeviceId, Request, Complete);
+}
+
+OMPT_API_ROUTINE int ompt_flush_trace(ompt_device_t *Device) {
+  ODBG(ODT_Tool) << "Executing ompt_flush_trace";
+
+  std::unique_lock<std::mutex> Lock(ompt_flush_trace_mutex);
+  ensureFuncPtrLoaded<libomptarget_ompt_flush_trace_t>(
+      "libomptarget_ompt_flush_trace", &ompt_flush_trace_fn);
+  assert(ompt_flush_trace_fn && "libomptarget_ompt_flush_trace loaded");
+  return ompt_flush_trace_fn(getDeviceId(Device));
+}
+
+OMPT_API_ROUTINE int ompt_stop_trace(ompt_device_t *Device) {
+  ODBG(ODT_Tool) << "Executing ompt_stop_trace";
+
+  int DeviceId = getDeviceId(Device);
+  if (DeviceId < 0) {
+    REPORT() << "Failed to stop trace for Device=" << Device
+             << " (Unknown device)";
+    // Indicate failure
+    return 0;
+  }
+
+  {
+    // Protect the function pointer
+    std::unique_lock<std::mutex> Lock(ompt_stop_trace_mutex);
+    llvm::omp::target::ompt::disableDeviceTracing(DeviceId);
+    // Disable asynchronous memory copy profiling
+    setOmptAsyncCopyProfile(/*Enable=*/false);
+    // Disable queue dispatch profiling
+    if (DeviceId >= 0)
+      setGlobalOmptKernelProfile(Device, /*Enable=*/0);
+    else
+      REPORT() << "May not disable kernel profiling for invalid device id="
+               << DeviceId;
+    ensureFuncPtrLoaded<libomptarget_ompt_stop_trace_t>(
+        "libomptarget_ompt_stop_trace", &ompt_stop_trace_fn);
+    assert(ompt_stop_trace_fn && "libomptarget_ompt_stop_trace loaded");
+  }
+  return ompt_stop_trace_fn(DeviceId);
+}
+
+OMPT_API_ROUTINE ompt_record_ompt_t *
+ompt_get_record_ompt(ompt_buffer_t *Buffer, ompt_buffer_cursor_t CurrentPos) {
+  // TODO In debug mode, get the metadata associated with this buffer
+  // and assert that there are enough bytes for the current record
+
+  // Currently, no synchronization required since a disjoint set of
+  // trace records is handed over to a thread.
+
+  // Note that CurrentPos can be nullptr. In that case, we return
+  // nullptr. The tool has to handle that properly.
+  return (ompt_record_ompt_t *)CurrentPos;
+}
+
+OMPT_API_ROUTINE int ompt_advance_buffer_cursor(ompt_device_t *Device,
+                                                ompt_buffer_t *Buffer,
+                                                size_t Size,
+                                                ompt_buffer_cursor_t CurrentPos,
+                                                ompt_buffer_cursor_t *NextPos) {
+  // Note: The input parameter size is unused here. It refers to the
+  // bytes returned in the corresponding callback.
+  // Advance can be called concurrently, so synchronize setting the
+  // function pointer. The actual libomptarget function does not need
+  // to be synchronized since it must be working on logically disjoint
+  // buffers.
+  std::unique_lock<std::mutex> Lock(ompt_advance_buffer_cursor_mutex);
+  ensureFuncPtrLoaded<libomptarget_ompt_advance_buffer_cursor_t>(
+      "libomptarget_ompt_advance_buffer_cursor",
+      &ompt_advance_buffer_cursor_fn);
+  assert(ompt_advance_buffer_cursor_fn &&
+         "libomptarget_ompt_advance_buffer_cursor loaded");
+  return ompt_advance_buffer_cursor_fn(Device, Buffer, Size, CurrentPos,
+                                       NextPos);
+}
+
+OMPT_API_ROUTINE ompt_record_t
+ompt_get_record_type(ompt_buffer_t *Buffer, ompt_buffer_cursor_t CurrentPos) {
+  std::unique_lock<std::mutex> Lock(ompt_get_record_type_mutex);
+  ensureFuncPtrLoaded<libomptarget_ompt_get_record_type_t>(
+      "libomptarget_ompt_get_record_type", &ompt_get_record_type_fn);
+  assert(ompt_get_record_type_fn && "libomptarget_ompt_get_record_type loaded");
+  return ompt_get_record_type_fn(Buffer, CurrentPos);
+}
+
+OMPT_API_ROUTINE ompt_device_time_t
+ompt_get_device_time(ompt_device_t *Device) {
+  ODBG(ODT_Tool) << "Executing ompt_get_device_time";
+  return getSystemTimestampInNs();
+}
+
+OMPT_API_ROUTINE double ompt_translate_time(ompt_device_t *Device,
+                                            ompt_device_time_t DeviceTime) {
+  // Translate a device time to a meaningful timepoint in host time
+  // We do not need to account for clock-skew / drift. So simple linear
+  // translation using the host to device rate we obtained.
+  double TranslatedTime = DeviceTime * HostToDeviceSlope + HostToDeviceOffset;
+  ODBG(ODT_Tool) << "D2H translated time: " << TranslatedTime;
+
+  return TranslatedTime;
+}
+
+void llvm::omp::target::ompt::setOmptTimestamp(uint64_t StartTime,
+                                               uint64_t EndTime) {
+  std::unique_lock<std::mutex> Lock(ompt_set_timestamp_mutex);
+  ensureFuncPtrLoaded<libomptarget_ompt_set_timestamp_t>(
+      "libomptarget_ompt_set_timestamp", &ompt_set_timestamp_fn);
+  // No need to hold a lock
+  ompt_set_timestamp_fn(StartTime, EndTime);
+}
+
+void llvm::omp::target::ompt::setOmptHostToDeviceRate(double Slope,
+                                                      double Offset) {
+  HostToDeviceSlope = Slope;
+  HostToDeviceOffset = Offset;
+}
+
+void llvm::omp::target::ompt::setOmptGrantedNumTeams(uint64_t NumTeams) {
+  std::unique_lock<std::mutex> Lock(ompt_set_granted_teams_mutex);
+  ensureFuncPtrLoaded<libomptarget_ompt_set_granted_teams_t>(
+      "libomptarget_ompt_set_granted_teams", &ompt_set_granted_teams_fn);
+  // No need to hold a lock
+  ompt_set_granted_teams_fn(NumTeams);
+}
+
+ompt_interface_fn_t llvm::omp::target::ompt::lookupDeviceTracingFn(
+    const char *InterfaceFunctionName) {
+#define compareAgainst(AvailableFunction)                                      \
+  if (strcmp(InterfaceFunctionName, #AvailableFunction) == 0)                  \
+    return (ompt_interface_fn_t)AvailableFunction;
+
+  FOREACH_OMPT_DEVICE_TRACING_FN(compareAgainst);
+#undef compareAgainst
+
+  ODBG(ODT_Tool) << "Warning: Could not find requested function "
+                 << InterfaceFunctionName;
+  return (ompt_interface_fn_t) nullptr;
+}
+
+#pragma pop_macro("DEBUG_PREFIX")
+
+#endif // OMPT_SUPPORT

>From dc743088e53a1de84c6995d205ff3f081df1d764 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 2 Apr 2026 07:40:55 -0500
Subject: [PATCH 05/12] [Offload][OMPT] Add tracing extensions to Interface.h

Extend the OMPT Interface class with trace record creation methods and
the async-aware TracerInterfaceRAII for device tracing support.

Key changes to Interface.h:
- Add includes for OmptEventInfoTy.h, APITypes.h, GenericProfiler.h
- Add extern TracingActive flag and isTracingEnabled() declaration
- Add trace start/stop methods for all target operations:
  startTargetDataAllocTrace, stopTargetDataAllocTrace,
  startTargetDataSubmitTrace, startTargetDataDeleteTrace,
  startTargetDataRetrieveTrace, stopTargetDataMovementTraceAsync,
  startTargetSubmitTrace, stopTargetSubmitTraceAsync, and
  corresponding methods for enter/exit/update/target regions
- Add getTraceGenerators<>() template overloads parallel to
  existing getCallbacks<>()
- Add private helpers: setTraceRecordCommon, setTraceRecordTargetDataOp,
  setTraceRecordTargetKernel, setTraceRecordTarget, announceTargetRegion
- Add TracerInterfaceRAII class: async-aware RAII that checks
  isTracingEnabled, calls begin trace generator, allocates
  OmptEventInfoTy via profiler, sets AsyncInfo->ProfilerData
- Rename CallbackPairTy to FunctionPairTy in InterfaceRAII
- Make InvokeInterfaceFunction return a value
- Replace OMPT_IF_BUILT with OMPT_IF_TRACING_ENABLED

Key changes to Callback.h:
- Replace inline macro/typedef definitions with include of
  OmptCommonDefs.h (shared definitions moved there in PR3)

Made-with: Cursor
Co-Authored-By: Claude <noreply at anthropic.com>
---
 offload/include/OpenMP/OMPT/Callback.h  |  26 +--
 offload/include/OpenMP/OMPT/Interface.h | 255 ++++++++++++++++++++++--
 2 files changed, 243 insertions(+), 38 deletions(-)

diff --git a/offload/include/OpenMP/OMPT/Callback.h b/offload/include/OpenMP/OMPT/Callback.h
index 9d545c643223f..7aff68ced0d79 100644
--- a/offload/include/OpenMP/OMPT/Callback.h
+++ b/offload/include/OpenMP/OMPT/Callback.h
@@ -16,34 +16,12 @@
 
 #ifdef OMPT_SUPPORT
 
-#include "omp-tools.h"
+#include "OmptCommonDefs.h"
 
 #pragma push_macro("DEBUG_PREFIX")
 #undef DEBUG_PREFIX
 #define DEBUG_PREFIX "OMPT"
 
-#define FOREACH_OMPT_TARGET_CALLBACK(macro)                                    \
-  FOREACH_OMPT_DEVICE_EVENT(macro)                                             \
-  FOREACH_OMPT_NOEMI_EVENT(macro)                                              \
-  FOREACH_OMPT_EMI_EVENT(macro)
-
-#define performIfOmptInitialized(stmt)                                         \
-  do {                                                                         \
-    if (llvm::omp::target::ompt::Initialized) {                                \
-      stmt;                                                                    \
-    }                                                                          \
-  } while (0)
-
-#define performOmptCallback(CallbackName, ...)                                 \
-  do {                                                                         \
-    if (ompt_callback_##CallbackName##_fn)                                     \
-      ompt_callback_##CallbackName##_fn(__VA_ARGS__);                          \
-  } while (0)
-
-/// Function type def used for maintaining unique target region, target
-/// operations ids
-typedef uint64_t (*IdInterfaceTy)();
-
 namespace llvm {
 namespace omp {
 namespace target {
@@ -98,8 +76,6 @@ extern bool Initialized;
 
 #pragma pop_macro("DEBUG_PREFIX")
 
-#else
-#define performIfOmptInitialized(stmt)
 #endif // OMPT_SUPPORT
 
 #endif // OFFLOAD_INCLUDE_OPENMP_OMPT_CALLBACK_H
diff --git a/offload/include/OpenMP/OMPT/Interface.h b/offload/include/OpenMP/OMPT/Interface.h
index 6961641769b76..54e7564c21ad8 100644
--- a/offload/include/OpenMP/OMPT/Interface.h
+++ b/offload/include/OpenMP/OMPT/Interface.h
@@ -16,14 +16,21 @@
 // Only provide functionality if target OMPT support is enabled
 #ifdef OMPT_SUPPORT
 #include "Callback.h"
+#include "OmptEventInfoTy.h"
+#include "Shared/APITypes.h"
+#include "Shared/Debug.h"
 #include "omp-tools.h"
 
+#include "GenericProfiler.h"
+
 #include "llvm/Support/ErrorHandling.h"
 
 #include <functional>
 #include <tuple>
 
-#define OMPT_IF_BUILT(stmt) stmt
+#pragma push_macro("DEBUG_PREFIX")
+#undef DEBUG_PREFIX
+#define DEBUG_PREFIX "OMPT"
 
 /// Callbacks for target regions require task_data representing the
 /// encountering task.
@@ -42,9 +49,17 @@ namespace ompt {
 static ompt_get_task_data_t ompt_get_task_data_fn;
 static ompt_get_target_task_data_t ompt_get_target_task_data_fn;
 
+/// OMPT global tracing status. Indicates if at least one device is traced.
+extern bool TracingActive;
+
+/// Check if this device traces the given event type
+extern bool isTracingEnabled(int DeviceId, unsigned int EventTy);
+
 /// Used to maintain execution state for this thread
 class Interface {
 public:
+  // Target data callbacks
+
   /// Top-level function for invoking callback before device data allocation
   void beginTargetDataAlloc(int64_t DeviceId, void *HstPtrBegin,
                             void **TgtPtrBegin, size_t Size, void *Code);
@@ -142,6 +157,85 @@ class Interface {
   /// Top-level function for invoking callback after target construct
   void endTarget(int64_t DeviceId, void *Code);
 
+  // Target data tracing
+
+  /// Top-level function for starting trace before device data allocation
+  void startTargetDataAllocTrace(int64_t DeviceId, void *HstPtrBegin,
+                                 void **TgtPtrBegin, size_t Size, void *Code);
+
+  /// Top-level function for stopping trace after device data allocation
+  ompt_record_ompt_t *stopTargetDataAllocTrace(int64_t DeviceId,
+                                               void *HstPtrBegin,
+                                               void **TgtPtrBegin, size_t Size,
+                                               void *Code);
+
+  /// Top-level function for starting trace before data submit
+  ompt_record_ompt_t *startTargetDataSubmitTrace(int64_t SrcDeviceId,
+                                                 void *SrcPtrBegin,
+                                                 int64_t DstDeviceId,
+                                                 void *DstPtrBegin, size_t Size,
+                                                 void *Code);
+
+  /// Top-level function for starting trace before device data deallocation
+  void startTargetDataDeleteTrace(int64_t DeviceId, void *TgtPtrBegin,
+                                  void *Code);
+
+  /// Top-level function for stopping trace after device data deallocation
+  ompt_record_ompt_t *stopTargetDataDeleteTrace(int64_t DeviceId,
+                                                void *TgtPtrBegin, void *Code);
+
+  /// Top-level function for starting trace before data retrieve
+  ompt_record_ompt_t *startTargetDataRetrieveTrace(int64_t SrcDeviceId,
+                                                   void *SrcPtrBegin,
+                                                   int64_t DstDeviceId,
+                                                   void *DstPtrBegin,
+                                                   size_t Size, void *Code);
+
+  ompt_record_ompt_t *
+  stopTargetDataMovementTraceAsync(ompt_record_ompt_t *DataPtr,
+                                   uint64_t NanosStart, uint64_t NanosEnd);
+
+  /// Top-level function for starting trace before kernel dispatch
+  ompt_record_ompt_t *startTargetSubmitTrace(int64_t DeviceId,
+                                             unsigned int NumTeams = 1);
+
+  ompt_record_ompt_t *stopTargetSubmitTraceAsync(ompt_record_ompt_t *DataPtr,
+                                                 unsigned int NumTeams,
+                                                 uint64_t NanosStart,
+                                                 uint64_t NanosStop);
+
+  // Target region tracing
+
+  /// Top-level function for starting trace before target enter data
+  /// construct
+  ompt_record_ompt_t *startTargetDataEnterTrace(int64_t DeviceId, void *Code);
+
+  /// Top-level function for stopping trace after target enter data
+  /// construct
+  ompt_record_ompt_t *stopTargetDataEnterTrace(int64_t DeviceId, void *Code);
+
+  /// Top-level function for starting trace before target exit data
+  /// construct
+  ompt_record_ompt_t *startTargetDataExitTrace(int64_t DeviceId, void *Code);
+
+  /// Top-level function for stopping trace after target exit data
+  /// construct
+  ompt_record_ompt_t *stopTargetDataExitTrace(int64_t DeviceId, void *Code);
+
+  /// Top-level function for starting trace before target update construct
+  ompt_record_ompt_t *startTargetUpdateTrace(int64_t DeviceId, void *Code);
+
+  /// Top-level function for stopping trace after target update construct
+  ompt_record_ompt_t *stopTargetUpdateTrace(int64_t DeviceId, void *Code);
+
+  // Target kernel tracing
+
+  /// Top-level function for starting trace before target construct
+  ompt_record_ompt_t *startTargetTrace(int64_t DeviceId, void *Code);
+
+  /// Top-level function for stopping trace after target construct
+  ompt_record_ompt_t *stopTargetTrace(int64_t DeviceId, void *Code);
+
   // Callback getter: Target data operations
   template <ompt_target_data_op_t OpType> auto getCallbacks() {
     if constexpr (OpType == ompt_target_data_alloc ||
@@ -217,6 +311,69 @@ class Interface {
     llvm_unreachable("Unhandled target operation!");
   }
 
+  // Callback getter: Target data operations
+  template <ompt_target_data_op_t OpType> auto getTraceGenerators() {
+    if constexpr (OpType == ompt_target_data_alloc ||
+                  OpType == ompt_target_data_alloc_async)
+      return std::make_pair(std::mem_fn(&Interface::startTargetDataAllocTrace),
+                            std::mem_fn(&Interface::stopTargetDataAllocTrace));
+
+    if constexpr (OpType == ompt_target_data_delete ||
+                  OpType == ompt_target_data_delete_async)
+      return std::make_pair(std::mem_fn(&Interface::startTargetDataDeleteTrace),
+                            std::mem_fn(&Interface::stopTargetDataDeleteTrace));
+
+    if constexpr (OpType == ompt_target_data_transfer_to_device ||
+                  OpType == ompt_target_data_transfer_to_device_async)
+      return std::make_pair(
+          std::mem_fn(&Interface::startTargetDataSubmitTrace),
+          std::mem_fn(&Interface::stopTargetDataMovementTraceAsync));
+
+    if constexpr (OpType == ompt_target_data_transfer_from_device ||
+                  OpType == ompt_target_data_transfer_from_device_async)
+      return std::make_pair(
+          std::mem_fn(&Interface::startTargetDataRetrieveTrace),
+          std::mem_fn(&Interface::stopTargetDataMovementTraceAsync));
+
+    llvm_unreachable("Unhandled target data operation type!");
+  }
+
+  // Callback getter: Target region operations
+  template <ompt_target_t OpType> auto getTraceGenerators() {
+    if constexpr (OpType == ompt_target_enter_data ||
+                  OpType == ompt_target_enter_data_nowait)
+      return std::make_pair(std::mem_fn(&Interface::startTargetDataEnterTrace),
+                            std::mem_fn(&Interface::stopTargetDataEnterTrace));
+
+    if constexpr (OpType == ompt_target_exit_data ||
+                  OpType == ompt_target_exit_data_nowait)
+      return std::make_pair(std::mem_fn(&Interface::startTargetDataExitTrace),
+                            std::mem_fn(&Interface::stopTargetDataExitTrace));
+
+    if constexpr (OpType == ompt_target_update ||
+                  OpType == ompt_target_update_nowait)
+      return std::make_pair(std::mem_fn(&Interface::startTargetUpdateTrace),
+                            std::mem_fn(&Interface::stopTargetUpdateTrace));
+
+    if constexpr (OpType == ompt_target || OpType == ompt_target_nowait)
+      return std::make_pair(std::mem_fn(&Interface::startTargetTrace),
+                            std::mem_fn(&Interface::stopTargetTrace));
+
+    llvm_unreachable("Unknown target region operation type!");
+  }
+
+  // Callback getter: Kernel launch operation
+  template <ompt_callbacks_t OpType> auto getTraceGenerators() {
+    // We use 'ompt_callbacks_t', because no other enum is currently available
+    // to model a kernel launch / target submit operation.
+    if constexpr (OpType == ompt_callback_target_submit)
+      return std::make_pair(
+          std::mem_fn(&Interface::startTargetSubmitTrace),
+          std::mem_fn(&Interface::stopTargetSubmitTraceAsync));
+
+    llvm_unreachable("Unhandled target operation!");
+  }
+
   /// Setters for target region and target operation correlation ids
   void setTargetDataValue(uint64_t DataValue) { TargetData.value = DataValue; }
   void setTargetDataPtr(void *DataPtr) { TargetData.ptr = DataPtr; }
@@ -240,6 +397,9 @@ class Interface {
   /// Target task data representing the target task region
   ompt_data_t *TargetTaskData = nullptr;
 
+  /// Used for marking begin of a data operation
+  void announceTargetRegion(const char *RegionName);
+
   /// Used for marking begin of a data operation
   void beginTargetDataOperation();
 
@@ -251,6 +411,23 @@ class Interface {
 
   /// Used for marking end of a target region
   void endTargetRegion();
+
+  // Called by all trace generation routines
+  void setTraceRecordCommon(ompt_record_ompt_t *DataPtr,
+                            ompt_callbacks_t CallbackType);
+  // Type specific helpers
+  void setTraceRecordTargetDataOp(ompt_record_target_data_op_t *Record,
+                                  ompt_target_data_op_t DataOpType,
+                                  void *SrcAddr, int64_t SrcDeviceNum,
+                                  void *DstAddr, int64_t DstDeviceNum,
+                                  size_t Bytes, void *CodePtr);
+
+  void setTraceRecordTargetKernel(ompt_record_target_kernel_t *Record,
+                                  unsigned int NumTeams);
+
+  void setTraceRecordTarget(ompt_record_target_t *Record, int64_t DeviceId,
+                            ompt_target_t TargetKind,
+                            ompt_scope_endpoint_t Endpoint, void *CodePtr);
 };
 
 /// Thread local state for target region and associated metadata
@@ -262,14 +439,14 @@ extern thread_local Interface RegionInterface;
 extern thread_local void *ReturnAddress;
 
 template <typename FuncTy, typename ArgsTy, size_t... IndexSeq>
-void InvokeInterfaceFunction(FuncTy Func, ArgsTy Args,
+auto InvokeInterfaceFunction(FuncTy Func, ArgsTy Args,
                              std::index_sequence<IndexSeq...>) {
-  std::invoke(Func, RegionInterface, std::get<IndexSeq>(Args)...);
+  return std::invoke(Func, RegionInterface, std::get<IndexSeq>(Args)...);
 }
 
-template <typename CallbackPairTy, typename... ArgsTy> class InterfaceRAII {
+template <typename FunctionPairTy, typename... ArgsTy> class InterfaceRAII {
 public:
-  InterfaceRAII(CallbackPairTy Callbacks, ArgsTy... Args)
+  InterfaceRAII(FunctionPairTy Callbacks, ArgsTy... Args)
       : Arguments(Args...), beginFunction(std::get<0>(Callbacks)),
         endFunction(std::get<1>(Callbacks)) {
     performIfOmptInitialized(begin());
@@ -290,14 +467,66 @@ template <typename CallbackPairTy, typename... ArgsTy> class InterfaceRAII {
   }
 
   std::tuple<ArgsTy...> Arguments;
-  typename CallbackPairTy::first_type beginFunction;
-  typename CallbackPairTy::second_type endFunction;
+  typename FunctionPairTy::first_type beginFunction;
+  typename FunctionPairTy::second_type endFunction;
 };
 
 // InterfaceRAII's class template argument deduction guide
-template <typename CallbackPairTy, typename... ArgsTy>
-InterfaceRAII(CallbackPairTy Callbacks, ArgsTy... Args)
-    -> InterfaceRAII<CallbackPairTy, ArgsTy...>;
+template <typename FunctionPairTy, typename... ArgsTy>
+InterfaceRAII(FunctionPairTy Callbacks, ArgsTy... Args)
+    -> InterfaceRAII<FunctionPairTy, ArgsTy...>;
+
+/// Similar to the original InterfaceRAII this class is used for tracing and
+/// extends the original with async capabilities. That is: It takes an
+/// additional AsyncInfo reference as argument to populate the relevant fields.
+/// The AsyncInfoTy propagates the info into the RTL / plugins.
+/// TracedDeviceId represents the trace record's device affinity. EventType is
+/// the callback type that needs to be enabled via ompt_set_trace_ompt.
+template <typename FunctionPairTy, typename AsyncInfoTy, typename... ArgsTy>
+class TracerInterfaceRAII {
+public:
+  TracerInterfaceRAII(FunctionPairTy Callbacks, AsyncInfoTy &AsyncInfo,
+                      plugin::GenericProfilerTy *Prof, int TracedDeviceId,
+                      ompt_callbacks_t EventType, ArgsTy... Args)
+      : Arguments(Args...), beginFunction(std::get<0>(Callbacks)) {
+    __tgt_async_info *AI = AsyncInfo;
+    if (isTracingEnabled(TracedDeviceId, EventType)) {
+      auto Record = begin();
+
+      // The Profiler can allocate specific data to be used to pass information
+      // from here to lower parts of the runtime system.
+      // NOTE: It is the responsibility of the programmer to ensure type
+      // compatibility and correct usage of the data. The profiler, however,
+      // OWNS the pointer and frees it at an appropriate time.
+      OmptEventInfoTy *ProfilerData =
+          reinterpret_cast<OmptEventInfoTy *>(Prof->getProfilerSpecificData());
+      ProfilerData->TraceRecord = Record;
+      ProfilerData->NumTeams = 0;
+
+      // Allows to pass down into the plugins via AsyncInfoTy
+      AI->ProfilerData = ProfilerData;
+    } else {
+      // Actively prevent further tracing of this event
+      AI->ProfilerData = nullptr;
+    }
+  }
+
+private:
+  auto begin() {
+    auto IndexSequence =
+        std::make_index_sequence<std::tuple_size_v<decltype(Arguments)>>{};
+    return InvokeInterfaceFunction(beginFunction, Arguments, IndexSequence);
+  }
+
+  std::tuple<ArgsTy...> Arguments;
+  typename FunctionPairTy::first_type beginFunction;
+  /// No end-function here, since the end is called asynchronously from the
+  /// plugins, once the operation has completed.
+};
+
+template <typename FunctionPairTy, typename... ArgsTy>
+TracerInterfaceRAII(FunctionPairTy Callbacks, ArgsTy... Args)
+    -> TracerInterfaceRAII<FunctionPairTy, ArgsTy...>;
 
 /// Used to set and reset the thread-local return address. The RAII is expected
 /// to be created at a runtime entry point when the return address should be
@@ -335,8 +564,8 @@ class ReturnAddressSetterRAII {
 // The getter returns the address stored in the thread local variable.
 #define OMPT_GET_RETURN_ADDRESS llvm::omp::target::ompt::ReturnAddress
 
-#else
-#define OMPT_IF_BUILT(stmt)
-#endif
+#pragma pop_macro("DEBUG_PREFIX")
+
+#endif // OMPT_SUPPORT
 
 #endif // OFFLOAD_INCLUDE_OPENMP_OMPT_INTERFACE_H

>From 314fd5863cb621a6eb8d7908957aad3df1425237 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 2 Apr 2026 07:44:36 -0500
Subject: [PATCH 06/12] [Offload][OMPT] Add OmptTracingBufferMgr trace buffer
 infrastructure

Add the trace buffer management subsystem for OMPT device tracing. This
manages allocation, population, and flushing of trace record buffers
between OpenMP worker threads and helper threads.

New files:
- OmptTracingBuffer.h: OmptTracingBufferMgr class with Buffer metadata,
  FlushInfo, TraceRecord types, thread-local per-device buffer pointers,
  helper thread management, and cursor-based record allocation
- OmptTracingBuffer.cpp: Full implementation of buffer lifecycle:
  assignCursor (lock-free fast path for existing buffers, locked path
  for new allocations), triggerFlushOnBufferFull, driveCompletion
  (helper thread main loop), invokeCallbacks, flushBuffer (dispatches
  buffer-completion callbacks for ranges of ready records),
  flushAllBuffers, helper thread start/shutdown/flush-and-shutdown
- OmptTracing.h: Libomptarget-side declarations for tracing state,
  buffer management callback registration, device tracing control,
  timestamp/clock correlation, and thread-local trace record fields.
  Implementation deferred to subsequent PR.

Changes to existing files:
- PluginManager.h: Add OmptTracingBufferMgr* TraceRecordManager member
  with getTraceRecordManager() accessor, include OmptTracingBuffer.h
- libomptarget/CMakeLists.txt: Add OmptTracingBuffer.cpp to omptarget

Key design points:
- Per-thread buffer assignment avoids contention on the common path
- Helper threads process buffers in flush-id order via condition variables
- LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL / FLUSH_ON_SHUTDOWN envars
  control automatic flushing behavior
- Buffer states: init -> ready -> released track record lifecycle
- Flush states: waiting -> processing prevent concurrent helper access

Made-with: Cursor
---
 offload/include/OpenMP/OMPT/OmptTracing.h     | 154 ++++
 .../include/OpenMP/OMPT/OmptTracingBuffer.h   | 412 ++++++++++
 offload/include/PluginManager.h               |  10 +-
 offload/libomptarget/CMakeLists.txt           |   1 +
 .../OpenMP/OMPT/OmptTracingBuffer.cpp         | 758 ++++++++++++++++++
 5 files changed, 1334 insertions(+), 1 deletion(-)
 create mode 100644 offload/include/OpenMP/OMPT/OmptTracing.h
 create mode 100644 offload/include/OpenMP/OMPT/OmptTracingBuffer.h
 create mode 100644 offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp

diff --git a/offload/include/OpenMP/OMPT/OmptTracing.h b/offload/include/OpenMP/OMPT/OmptTracing.h
new file mode 100644
index 0000000000000..2a892582923d5
--- /dev/null
+++ b/offload/include/OpenMP/OMPT/OmptTracing.h
@@ -0,0 +1,154 @@
+//===---- OmptTracing.h - Target independent OMPT callbacks --*- C++ -*----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Interface used by target-independent runtimes to coordinate registration and
+// invocation of OMPT tracing functionality.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OPENMP_LIBOMPTARGET_INCLUDE_OMPTTRACING_H
+#define OPENMP_LIBOMPTARGET_INCLUDE_OMPTTRACING_H
+
+#ifdef OMPT_SUPPORT
+
+#include <unordered_map>
+
+#include "OmptCommonDefs.h"
+#include "OmptTracingBuffer.h"
+
+#pragma push_macro("DEBUG_PREFIX")
+#undef DEBUG_PREFIX
+#define DEBUG_PREFIX "OMPT"
+
+namespace llvm {
+namespace omp {
+namespace target {
+namespace ompt {
+
+/// After a timestamp has been read, reset it.
+void resetTimestamp(uint64_t *T);
+
+/// A tool may register unique buffer-request and buffer-completion
+/// callback functions for a device. The following are utility functions to
+/// manage those functions.
+
+/// Given a device-id, return the corresponding buffer-request callback
+/// function.
+ompt_callback_buffer_request_t getBufferRequestFn(int DeviceId);
+
+/// Give a device-id, return the corresponding buffer-completion callback
+/// function.
+ompt_callback_buffer_complete_t getBufferCompleteFn(int DeviceId);
+
+/// Given a device-id, set the corresponding buffer-request and
+/// buffer-completion callback functions.
+void setBufferManagementFns(int DeviceId, ompt_callback_buffer_request_t ReqFn,
+                            ompt_callback_buffer_complete_t CmpltFn);
+
+/// Given a device-id, remove the corresponding buffer-request and
+/// buffer-completion callback functions.
+void removeBufferManagementFns(int DeviceId);
+
+/// Is device tracing stopped for all devices?
+bool isAllDeviceTracingStopped();
+
+/// Invoke callback function for buffer request events
+void ompt_callback_buffer_request(int DeviceId, ompt_buffer_t **BufferPtr,
+                                  size_t *Bytes);
+
+/// Invoke callback function for buffer complete events
+void ompt_callback_buffer_complete(int DeviceId, ompt_buffer_t *Buffer,
+                                   size_t Bytes,
+                                   ompt_buffer_cursor_t BeginCursor,
+                                   int BufferOwned);
+
+/// Set 'start' and 'stop' for the current trace record
+void setOmptTimestamp(uint64_t StartTime, uint64_t EndTime);
+
+/// Set the linear function correlation between host and device clocks
+void setOmptHostToDeviceRate(double Slope, double Offset);
+
+/// Set / store the number of granted teams
+void setOmptGrantedNumTeams(uint64_t NumTeams);
+
+/// Check if (1) tracing is globally active (2) the given device is actively
+/// traced and (3) the given event type is traced on the device
+bool isTracingEnabled(int DeviceId, unsigned int EventTy);
+
+/// Check if the given device is actively traced
+bool isTracedDevice(int DeviceId);
+
+/// Check if the given device is monitoring the provided tracing type
+bool isTracingTypeEnabled(int DeviceId, unsigned int EventTy);
+
+/// Check if the given device is monitoring the provided tracing type 'group'
+/// Where group means we will check for both: EMI and non-EMI event types
+bool isTracingTypeGroupEnabled(int DeviceId, unsigned int EventTy);
+
+/// Set whether the given tracing type should be monitored (or not) on the
+/// device
+void setTracingTypeEnabled(uint64_t &TracedEventTy, bool Enable,
+                           unsigned int EventTy);
+
+/// Set / reset the given tracing types (EventTy = 0 corresponds to 'all')
+ompt_set_result_t setTraceEventTy(int DeviceId, unsigned int Enable,
+                                  unsigned int EventTy);
+
+/// Return thread id
+uint64_t getThreadId();
+
+/// See TracedDevices in OmptDeviceTracing.h
+extern std::map<int32_t, uint64_t> TracedDevices;
+/// Activate tracing on the given device
+void enableDeviceTracing(int DeviceId);
+/// Deactivate tracing on the given device
+void disableDeviceTracing(int DeviceId);
+
+/// Mutexes to serialize invocation of device registration and checks
+extern std::mutex DeviceAccessMutex;
+
+/// Mutexes to serialize invocation of device-independent entry points
+extern std::mutex TraceAccessMutex;
+extern std::mutex TraceControlMutex;
+
+/// Ensure serialization of calls to std::hash
+extern std::mutex TraceHashThreadMutex;
+
+/// Protect map from device-id to the corresponding buffer-request and
+/// buffer-completion callback functions.
+extern std::mutex BufferManagementFnMutex;
+
+/// Map from device-id to the corresponding buffer-request and buffer-completion
+/// callback functions.
+extern std::unordered_map<int, std::pair<ompt_callback_buffer_request_t,
+                                         ompt_callback_buffer_complete_t>>
+    BufferManagementFns;
+
+/// Thread local variables used by the plugin to communicate OMPT information
+/// that are then used to populate trace records. This method assumes a
+/// synchronous implementation, otherwise it won't work.
+extern thread_local uint32_t TraceRecordNumGrantedTeams;
+extern thread_local uint64_t TraceRecordStartTime;
+extern thread_local uint64_t TraceRecordStopTime;
+
+/// Thread local thread-id.
+extern thread_local uint64_t ThreadId;
+
+/// OMPT global tracing status. Indicates if at least one device is traced.
+extern bool TracingActive;
+
+} // namespace ompt
+} // namespace target
+} // namespace omp
+} // namespace llvm
+
+#pragma pop_macro("DEBUG_PREFIX")
+
+#endif // OMPT_SUPPORT
+
+#endif // OPENMP_LIBOMPTARGET_INCLUDE_OMPTTRACING_H
diff --git a/offload/include/OpenMP/OMPT/OmptTracingBuffer.h b/offload/include/OpenMP/OMPT/OmptTracingBuffer.h
new file mode 100644
index 0000000000000..5c9f4bf33dae8
--- /dev/null
+++ b/offload/include/OpenMP/OMPT/OmptTracingBuffer.h
@@ -0,0 +1,412 @@
+//===- OmptTracingBuffer.h - Target independent OpenMP target RTL -- C++ *-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Interface to be used for generating and flushing OMPT device trace records.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OPENMP_LIBOMPTARGET_OMPTTRACINGBUFFER_H
+#define OPENMP_LIBOMPTARGET_OMPTTRACINGBUFFER_H
+
+#ifdef OMPT_SUPPORT
+
+#include <atomic>
+#include <cassert>
+#include <condition_variable>
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <set>
+#include <thread>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+#include <omp-tools.h>
+
+#include "Shared/EnvironmentVar.h"
+
+// Maximum number of devices supported in device tracing. No device tracing
+// will be performed for any device-id larger than 1023.
+#define MAX_NUM_DEVICES 1024
+
+// TODO Start with 1 helper thread and add dynamically if required
+// Number of helper threads must not execeed 32 since the
+// thread-wait-tracker is 32 bits in length.
+#define OMPT_NUM_HELPER_THREADS 1
+
+/*
+ * Buffer manager for trace records generated by OpenMP master and
+ * worker threads. During device init, a tool may register a
+ * buffer-request and a buffer-completion callback. The buffer-request
+ * callback should be used to allocate new buffers as required. The
+ * buffer-complete callback should be used to return trace records to
+ * the tool.
+ *
+ * In addition to trace records, this class manages the helper threads
+ * for dispatching a range of trace records to the tool.
+ */
+class OmptTracingBufferMgr {
+public:
+  /*
+   * A trace record (TR) holds the trace data. Its type
+   * can be ompt or native. Currently, only ompt type is implemented.
+   */
+
+  /*
+   * A TR can be in the following states:
+   * TR_init: initial state
+   * TR_ready: An OpenMP thread marks a TR ready when it is done
+   * populating the TR
+   * TR_released: A helper thread marks a TR released after it has
+   * completed returning the TR to the tool
+   */
+  enum TRStatus { TR_init, TR_ready, TR_released };
+
+  /*
+   * Metadata capturing the state of a buffer of trace records. Once a
+   * buffer is allocated by an OpenMP worker thread, trace records are
+   * carved out from that buffer by that same OpenMP thread alone. Thus
+   * the allocated buffer is thread-specific from the allocation/population
+   * standpoint. But it may be manipulated by helper threads.
+   *
+   * Id, DeviceId, Start, and TotalBytes are not changed once set.
+   * RemainingBytes could be written multiple times but only by the same
+   * thread. But Cursor and IsFull may be read/written by an OpenMP worker
+   * thread and read by helper threads. Hence, accesses of
+   * this 2nd set of locations need to be atomic or synchronized.
+   */
+  struct Buffer {
+    uint64_t Id;                // Unique identifier of the buffer
+    int64_t DeviceId;           // Device for which this buffer is allocated
+    void *Start;                // Start of allocated space for trace records
+    size_t TotalBytes;          // Total number of bytes in the allocated space
+    size_t RemainingBytes;      // Total number of unused bytes
+                                // corresponding to Cursor
+    std::atomic<void *> Cursor; // Address of the last trace record carved out
+    std::atomic<bool> IsFull;   // true if no more trace records can be
+                                // accomodated, otherwise false
+    Buffer(uint64_t BufId, int64_t DevId, void *S, size_t Bytes, size_t Rem,
+           void *C, bool F)
+        : Id(BufId), DeviceId(DevId), Start(S), TotalBytes(Bytes),
+          RemainingBytes(Rem), Cursor(C), IsFull(F) {}
+    Buffer() = delete;
+    Buffer(const Buffer &) = delete;
+    Buffer &operator=(const Buffer &) = delete;
+  };
+  using BufPtr = std::shared_ptr<Buffer>;
+
+private:
+  /// Envar to control whether a buffer should be flushed when it gets full.
+  BoolEnvar OMPX_FlushOnBufferFull;
+
+  /// Envar to control whether all buffers should be flushed during shutdown.
+  BoolEnvar OMPX_FlushOnShutdown;
+
+  // Internal variable for tracking threads to wait for flush
+  uint32_t ThreadFlushTracker;
+
+  // Internal variable for tracking threads shutting down
+  uint32_t ThreadShutdownTracker;
+
+  using MapId2Buf = std::map<uint64_t, BufPtr>;
+
+  // Map from id to corresponding buffer. The ids are assigned in
+  // increasing order of creation.
+  MapId2Buf Id2BufferMap;
+
+  // Trace record. We currently support OMPT data type only. The state
+  // (TRStatus type) is maintained inline in the trace record. The
+  // tool is expected to access only the OMPT record.
+  struct TraceRecord {
+    ompt_record_ompt_t TR;
+    std::atomic<TRStatus> TRState;
+  };
+
+  // Thread-specific array of pointers to a buffer. The buffer pointed to
+  // is the last one allocated by this thread for a given device. The ith
+  // element points to the buffer for the ith device. At most MAX_NUM_DEVICES
+  // devices are supported.
+  static thread_local BufPtr ArrayOfBufPtr[MAX_NUM_DEVICES];
+
+  /*
+   * A buffer is flushed when it fills up or when the tool invokes
+   * flush_trace. So it's possible that the same buffer may be flushed
+   * more than once. When a buffer is flushed the first time, a unique
+   * id (flush-id) is generated and assigned to that buffer. Even if
+   * it is flushed again, the previously assigned id is maintained for
+   * that buffer. This id is loosely used to determine the order in
+   * which the buffers are processed and the corresponding trace
+   * records released to the tool.
+   */
+
+  struct FlushInfo {
+    uint64_t FlushId;
+    void *FlushCursor;
+    BufPtr FlushBuf;
+    FlushInfo() = default;
+    FlushInfo(uint64_t Id, void *CR, BufPtr Buf)
+        : FlushId{Id}, FlushCursor{CR}, FlushBuf{Buf} {}
+  };
+
+  /*
+   * A buffer may be in the following states:
+   * Flush_waiting: when a buffer is flushed, either because it is
+   * full or because the tool invokes ompt_flush_trace
+   * Flush_processing: when a helper thread claims the waiting buffer
+   * and is in the process of dispatching buffer-completion callbacks
+   * on an associated range of trace records. If all trace records are
+   * not released, the state may be reset to Flush_waiting after the
+   * buffer-completion callbacks return
+   */
+  enum BufferFlushStatus { Flush_waiting, Flush_processing };
+  struct FlushMd {
+    void *FlushCursor;
+    BufPtr FlushBuf;
+    BufferFlushStatus FlushStatus;
+    FlushMd(void *CR, BufPtr Buf, BufferFlushStatus Status)
+        : FlushCursor{CR}, FlushBuf{Buf}, FlushStatus{Status} {}
+    FlushMd() = delete;
+  };
+
+  using MapId2Md = std::map<uint64_t, FlushMd>;
+
+  /*
+   * A map from a flush-id to metadata containing the current
+   * cursor. the corresponding buffer, and its flushed status. If a
+   * buffer is flushed multiple times, the cursor is updated to the
+   * furthest one
+   */
+  MapId2Md Id2FlushMdMap;
+
+  using UMapBufPtr2Id = std::unordered_map<BufPtr, uint64_t>;
+
+  // A hash map from a buffer address to the corresponding flush-id
+  UMapBufPtr2Id FlushBufPtr2IdMap;
+
+  using USetCursor = std::unordered_set<void *>;
+
+  USetCursor LastCursors;
+
+  using UMapThd2Id = std::unordered_map<std::thread::id, uint32_t>;
+
+  // A hash map from a helper thread id to an integer
+  UMapThd2Id HelperThreadIdMap;
+
+  // Mutex to protect Id2BufferMap and Cursor2BufMdMap
+  std::mutex BufferMgrMutex;
+
+  // Mutex to protect FlushBufPtr2IdMap and Id2FlushMdMap
+  std::mutex FlushMutex;
+
+  // Mutex to protect metadata tracking last cursors of buffer-completion
+  // callbacks
+  std::mutex LastCursorMutex;
+
+  // Condition variable used by helper thread to signal that flush is requested
+  std::condition_variable FlushCv;
+
+  // Condition variable used while waiting for flushing to complete
+  std::condition_variable ThreadFlushCv;
+
+  // Condition variable used while waiting for threads to shutdown
+  std::condition_variable ThreadShutdownCv;
+
+  // TODO Separate out the helper thread into its own class
+  std::vector<std::thread> CompletionThreads;
+
+  /// Called when a buffer \p Buf may be flushed with \p Cursor as the
+  /// last allocated trace record in the buffer.
+  /// triggerFlushOnBufferFull should be called without holding any lock.
+  void triggerFlushOnBufferFull(void *Cursor, BufPtr Buf);
+
+  // Called to dispatch buffer-completion callbacks for the trace records in
+  // this buffer
+  void flushBuffer(FlushInfo);
+
+  // Dispatch a buffer-completion callback with a range of trace records
+  void dispatchCallback(int64_t DeviceId, void *Buffer, void *FirstCursor,
+                        void *LastCursor);
+
+  // Add a last cursor
+  void addLastCursor(void *Cursor) {
+    std::unique_lock<std::mutex> Lock(LastCursorMutex);
+    LastCursors.emplace(Cursor);
+  }
+
+  // Remove a last cursor
+  void removeLastCursor(void *Cursor) {
+    std::unique_lock<std::mutex> Lock(LastCursorMutex);
+    assert(LastCursors.find(Cursor) != LastCursors.end());
+    LastCursors.erase(Cursor);
+  }
+
+  // Given a trace record pointer, initialize its metadata
+  void initTraceRecordMetaData(void *Rec);
+
+  // Given a device-id, get/set a pointer to the last allocated buffer metadata.
+  BufPtr getDeviceSpecificBuffer(int64_t DevId);
+  void setDeviceSpecificBuffer(int64_t DevId, BufPtr Buf);
+
+  // Reserve a candidate buffer for flushing, preventing other helper threads
+  // from accessing it
+  FlushInfo findAndReserveFlushedBuf(uint64_t FlushId);
+
+  // Unreserve a buffer so that other helper threads can process it
+  void unreserveFlushedBuf(const FlushInfo &);
+
+  // All done with this buffer, so the buffer and its metadata can be removed
+  void destroyFlushedBuf(const FlushInfo &);
+
+  // Add a new buffer by an OpenMP thread so that a helper thread can process it
+  uint64_t addNewFlushEntry(BufPtr Buf, void *Cursor);
+
+  // Get the next trace record
+  void *getNextTR(void *TR);
+
+  // Given a buffer, return the latest cursor
+  void *getBufferCursor(BufPtr);
+
+  // Is no more space remaining for trace records in this buffer?
+  bool isBufferFull(const FlushInfo &);
+
+  // Have all trace records in this buffer been returned to the tool?
+  bool isBufferOwned(const FlushInfo &);
+
+  // Dispatch a buffer-completion callback and indicate that the buffer can be
+  // deallocated
+  void dispatchBufferOwnedCallback(const FlushInfo &);
+
+  // Main entry point for a helper thread
+  void driveCompletion();
+
+  // Examine the flushed buffers and dispatch buffer-completion callbacks
+  void invokeCallbacks();
+
+  // The caller does not hold a lock while calling this method
+  void waitForFlushCompletion();
+
+  // Given a thread number, set the corresponding bit in the flush
+  // tracker. The caller must hold the flush lock.
+  void setThreadFlush(uint32_t ThreadNum) {
+    ThreadFlushTracker |= (1 << ThreadNum);
+  }
+
+  // Reset this thread's flush bit. The caller must hold the flush lock
+  void resetThisThreadFlush() {
+    std::thread::id ID = std::this_thread::get_id();
+    assert(HelperThreadIdMap.find(ID) != HelperThreadIdMap.end());
+    ThreadFlushTracker &= ~(1 << HelperThreadIdMap[ID]);
+  }
+
+  // Given a thread number, set the corresponding bit in the shutdown
+  // tracker. The caller must hold the flush lock.
+  void setThreadShutdown(uint32_t ThreadNum) {
+    ThreadShutdownTracker |= (1 << ThreadNum);
+  }
+
+  // Reset this thread's shutdown bit. The caller must hold the flush
+  // lock
+  void resetThisThreadShutdown() {
+    std::thread::id ID = std::this_thread::get_id();
+    assert(HelperThreadIdMap.find(ID) != HelperThreadIdMap.end());
+    ThreadShutdownTracker &= ~(1 << HelperThreadIdMap[ID]);
+  }
+
+  // Return true if this thread's flush bit is set. The caller must
+  // hold the flush lock
+  bool isThisThreadFlushWaitedUpon() {
+    std::thread::id ID = std::this_thread::get_id();
+    assert(HelperThreadIdMap.find(ID) != HelperThreadIdMap.end());
+    return (ThreadFlushTracker & (1 << HelperThreadIdMap[ID])) != 0;
+  }
+
+  // Return true if this thread's shutdown bit is set. The caller must
+  // hold the flush lock
+  bool isThisThreadShutdownWaitedUpon() {
+    std::thread::id ID = std::this_thread::get_id();
+    assert(HelperThreadIdMap.find(ID) != HelperThreadIdMap.end());
+    return (ThreadShutdownTracker & (1 << HelperThreadIdMap[ID])) != 0;
+  }
+
+  // The caller must not hold the flush lock
+  bool amIHelperThread() {
+    std::unique_lock<std::mutex> flush_lock(FlushMutex);
+    if (HelperThreadIdMap.find(std::this_thread::get_id()) !=
+        HelperThreadIdMap.end())
+      return true;
+    return false;
+  }
+
+  // The caller must not hold the flush lock
+  bool areHelperThreadsAvailable();
+
+  // The caller must hold the appropriate lock
+  void init();
+
+  // The caller must hold the flush lock
+  void createHelperThreads();
+
+  // The caller must hold the flush lock
+  void destroyHelperThreads();
+
+public:
+  OmptTracingBufferMgr()
+      : OMPX_FlushOnBufferFull("LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL", true),
+        OMPX_FlushOnShutdown("LIBOMPTARGET_OMPT_FLUSH_ON_SHUTDOWN", true) {
+    // no need to hold locks for init() since object is getting constructed
+    // here.
+    init();
+  }
+
+  OmptTracingBufferMgr(const OmptTracingBufferMgr &) = delete;
+  OmptTracingBufferMgr &operator=(const OmptTracingBufferMgr &) = delete;
+
+  // The caller must not hold the flush lock
+  void startHelperThreads();
+
+  // The caller must not hold the flush lock. The helper threads are shut down
+  // without flushing any outstanding trace records.
+  void shutdownHelperThreads();
+
+  // The caller must not hold the flush lock. The helper threads are shut down
+  // after flushing all outstanding trace records for all devices.
+  void flushAndShutdownHelperThreads();
+
+  // Assign a cursor for a new trace record. This will assign a trace record
+  // for the provided device-id, allocating a new buffer if required.
+  void *assignCursor(ompt_callbacks_t Type, int64_t DeviceId);
+
+  // Get the size of a trace record
+  size_t getTRSize() { return sizeof(TraceRecord); }
+
+  // Get the status of a trace record. This function does not acquire
+  // a lock. If locking is required, the caller must hold a lock.
+  TRStatus getTRStatus(void *Rec);
+
+  // Set the status of a trace record. This function does not acquire
+  // a lock. If locking is required, the caller must hold a lock.
+  void setTRStatus(void *Rec, TRStatus);
+
+  // Is this a last cursor of a buffer completion callback?
+  bool isLastCursor(void *Cursor) {
+    std::unique_lock<std::mutex> Lock(LastCursorMutex);
+    return LastCursors.find(Cursor) != LastCursors.end();
+  }
+
+  // Called for flushing outstanding buffers for the provided device-id.
+  int flushAllBuffers(int DeviceId);
+};
+
+#else
+class OmptTracingBufferMgr {};
+#endif // OMPT_SUPPORT
+
+#endif // OPENMP_LIBOMPTARGET_OMPTTRACINGBUFFER_H
diff --git a/offload/include/PluginManager.h b/offload/include/PluginManager.h
index 6c6fdebe76dff..972afd962cbc1 100644
--- a/offload/include/PluginManager.h
+++ b/offload/include/PluginManager.h
@@ -13,6 +13,7 @@
 #ifndef OMPTARGET_PLUGIN_MANAGER_H
 #define OMPTARGET_PLUGIN_MANAGER_H
 
+#include "OpenMP/OMPT/OmptTracingBuffer.h"
 #include "PluginInterface.h"
 
 #include "DeviceImage.h"
@@ -48,7 +49,7 @@ struct PluginManager {
   /// Exclusive accessor type for the device container.
   using ExclusiveDevicesAccessorTy = Accessor<DeviceContainerTy>;
 
-  PluginManager() {}
+  PluginManager() : TraceRecordManager(nullptr) {}
 
   void init();
 
@@ -150,6 +151,11 @@ struct PluginManager {
     return count;
   }
 
+  auto getTraceRecordManager() const {
+    assert(TraceRecordManager && "Trace record manager not initialized");
+    return TraceRecordManager;
+  }
+
 private:
   bool RTLsLoaded = false;
   llvm::SmallVector<__tgt_bin_desc *> DelayedBinDesc;
@@ -176,6 +182,8 @@ struct PluginManager {
   /// Devices associated with plugins, accesses to the container are exclusive.
   ProtectedObj<DeviceContainerTy> Devices;
 
+  OmptTracingBufferMgr *TraceRecordManager;
+
   /// References to upgraded legacy offloading entries.
   std::list<llvm::SmallVector<llvm::offloading::EntryTy, 0>> LegacyEntries;
   std::list<llvm::SmallVector<__tgt_device_image, 0>> LegacyImages;
diff --git a/offload/libomptarget/CMakeLists.txt b/offload/libomptarget/CMakeLists.txt
index 8e6314b2a6eae..33b32d21e3c69 100644
--- a/offload/libomptarget/CMakeLists.txt
+++ b/offload/libomptarget/CMakeLists.txt
@@ -17,6 +17,7 @@ add_library(omptarget SHARED
   OpenMP/Mapping.cpp
   OpenMP/InteropAPI.cpp
   OpenMP/OMPT/Callback.cpp
+  OpenMP/OMPT/OmptTracingBuffer.cpp
 
   KernelLanguage/API.cpp
 )
diff --git a/offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp b/offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp
new file mode 100644
index 0000000000000..5ac1974cea03b
--- /dev/null
+++ b/offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp
@@ -0,0 +1,758 @@
+//=== OmptTracingBuffer.cpp - Target independent OpenMP target RTL -- C++ -===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementation of OMPT device trace record generation and flushing.
+//
+//===----------------------------------------------------------------------===//
+
+#ifdef OMPT_SUPPORT
+
+#include "OpenMP/OMPT/OmptTracingBuffer.h"
+#include "OpenMP/OMPT/OmptTracing.h"
+#include "Shared/Debug.h"
+
+#include <atomic>
+#include <cassert>
+#include <cstdlib>
+#include <limits>
+
+using namespace llvm::omp::target::debug;
+
+// When set to true, helper threads terminate their work
+static bool DoneTracing{false};
+
+// Unique buffer id in creation order
+static std::atomic<uint64_t> BufId{0};
+
+// Unique id in buffer flush order
+static std::atomic<uint64_t> FlushId{0};
+
+thread_local OmptTracingBufferMgr::BufPtr
+    OmptTracingBufferMgr::ArrayOfBufPtr[MAX_NUM_DEVICES];
+
+static uint64_t get_and_inc_buf_id() { return BufId++; }
+
+static uint64_t get_and_inc_flush_id() { return FlushId++; }
+static uint64_t get_flush_id() { return FlushId; }
+
+/*
+ * Used by OpenMP threads for assigning space for a trace record. If
+ * there is no space in the last buffer allocated by this thread, the
+ * last buffer is marked full and scheduled for flushing. Otherwise,
+ * space is assigned for a trace record and the new cursor returned.
+ * Since the memory allocated by a thread is used by that thread alone
+ * for creating trace records, a lock need not be held. In the less
+ * common branch when memory is allocated, a lock needs to be acquired
+ * for updating shared metadata. The common path of allocating a trace
+ * record from an existing buffer proceeds without locking.
+ */
+void *OmptTracingBufferMgr::assignCursor(ompt_callbacks_t Type,
+                                         int64_t DeviceId) {
+  // The caller should handle nullptr by not tracing for this event.
+  if (DeviceId < 0 || DeviceId > MAX_NUM_DEVICES - 1)
+    return nullptr;
+
+  size_t RecSize = getTRSize();
+
+  // If the buffer fills up, it will be scheduled for flushing with the
+  // following cursor.
+  void *ToBeFlushedCursor = nullptr;
+  BufPtr ToBeFlushedBuf = nullptr;
+
+  // Thread local buffer pointer should be non-null once an allocation
+  // has been done by this thread.
+  BufPtr DeviceBuf = getDeviceSpecificBuffer(DeviceId);
+  if (DeviceBuf != nullptr) {
+    assert(DeviceBuf->DeviceId == DeviceId && "Unexpected device id in buffer");
+    void *OldCursor = DeviceBuf->Cursor.load(std::memory_order_acquire);
+    // Try to assign a trace record from the last allocated buffer
+    if (RecSize <= DeviceBuf->RemainingBytes) {
+      assert((char *)DeviceBuf->Start + DeviceBuf->TotalBytes -
+                 DeviceBuf->RemainingBytes ==
+             (char *)OldCursor + RecSize);
+      DeviceBuf->RemainingBytes -= RecSize;
+
+      // Note the trace record status must be initialized before setting
+      // the cursor, ensuring that a helper thread always sees an initialized
+      // trace record status.
+      void *NewCursor = (char *)OldCursor + RecSize;
+      initTraceRecordMetaData(NewCursor);
+      DeviceBuf->Cursor.store(NewCursor, std::memory_order_release);
+
+      ODBG(ODT_Tool) << "Thread " << llvm::omp::target::ompt::getThreadId()
+                     << ": Assigned " << RecSize << " bytes at " << NewCursor
+                     << " in existing buffer " << DeviceBuf->Start
+                     << " for device " << DeviceId;
+      return NewCursor;
+    } else {
+      ToBeFlushedCursor = OldCursor;
+      ToBeFlushedBuf = DeviceBuf;
+
+      // Mark that no space is present for any more trace records.
+      // The following is atomic but there is no logical order between when
+      // it is set here and when it is checked by a helper thread. That works
+      // because the helper thread uses this info to decide whether a buffer
+      // can be scheduled for removal. In the worst case, the buffer will be
+      // removed late.
+      DeviceBuf->IsFull.store(true, std::memory_order_release);
+    }
+  }
+  void *NewBuffer = nullptr;
+  size_t TotalBytes = 0;
+  // TODO Move the buffer allocation to a helper thread
+  llvm::omp::target::ompt::ompt_callback_buffer_request(DeviceId, &NewBuffer,
+                                                        &TotalBytes);
+
+  // The caller should handle nullptr by not tracing for this event.
+  if (NewBuffer == nullptr || TotalBytes < RecSize)
+    return nullptr;
+
+  uint64_t NewBufId = get_and_inc_buf_id();
+  auto new_buf = std::make_shared<Buffer>(
+      NewBufId, DeviceId, /*Start=*/NewBuffer, TotalBytes,
+      /*RemainingBytes=*/TotalBytes - RecSize,
+      /*Cursor=*/NewBuffer,
+      /*IsFull=*/false);
+
+  // Initialize trace record status before publishing it to helper threads.
+  initTraceRecordMetaData(new_buf->Cursor.load(std::memory_order_acquire));
+  setDeviceSpecificBuffer(DeviceId, new_buf);
+
+  // Make this trace record visible to helper threads by adding to shared
+  // metadata.
+  std::unique_lock<std::mutex> lck(BufferMgrMutex);
+  assert(Id2BufferMap.find(NewBufId) == Id2BufferMap.end());
+  Id2BufferMap[NewBufId] = new_buf;
+  lck.unlock();
+
+  // Schedule the full buffer for flushing till the corresponding cursor.
+  if (OMPX_FlushOnBufferFull && ToBeFlushedCursor)
+    triggerFlushOnBufferFull(ToBeFlushedCursor, ToBeFlushedBuf);
+
+  ODBG(ODT_Tool) << "Thread " << llvm::omp::target::ompt::getThreadId()
+                 << ": Assigned " << RecSize << " bytes at " << NewBuffer
+                 << " in new buffer with id " << NewBufId << " for device "
+                 << DeviceId;
+  return NewBuffer;
+}
+
+/*
+ * Called by an OpenMP thread when a buffer fills up and should be
+ * flushed. This function assigns a new FlushId to the buffer, adds
+ * to the flush-related metadata and wakes up a helper thread to
+ * dispatch a buffer-completion callback. This function should be
+ * called without holding any lock.
+ * Note lock order: buf_lock -> flush_lock
+ */
+void OmptTracingBufferMgr::triggerFlushOnBufferFull(void *cursor, BufPtr Buf) {
+  std::unique_lock<std::mutex> buf_lock(BufferMgrMutex);
+
+  // Between calling this function and this check, a flush-all may have
+  // delivered this buffer to the tool and deleted it. So the buffer
+  // may not exist.
+  if (Id2BufferMap.find(Buf->Id) == Id2BufferMap.end())
+    return;
+
+  // Cannot assert that the state of the cursor is ready since a
+  // different thread may be in the process of populating it. If it
+  // remains in init state when the range of trace records is
+  // determined for dispatching the buffer-completion callback, it
+  // will not be included.
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  uint64_t flush_id;
+  auto flush_itr = FlushBufPtr2IdMap.find(Buf);
+  if (flush_itr == FlushBufPtr2IdMap.end()) {
+    // This buffer has not been flushed yet
+    addNewFlushEntry(Buf, cursor);
+  } else {
+    // This buffer has been flushed before
+    flush_id = flush_itr->second;
+    auto flush_md_itr = Id2FlushMdMap.find(flush_id);
+    assert(flush_md_itr != Id2FlushMdMap.end());
+    flush_md_itr->second.FlushCursor = cursor; // update the cursor
+    // Do not update the flush status since it may be under processing
+    // by another thread
+    ODBG(ODT_Tool) << "Updated id " << flush_id << " cursor " << cursor
+                   << " buf " << flush_md_itr->second.FlushBuf->Start;
+  }
+  flush_lock.unlock();
+  buf_lock.unlock();
+
+  // Wake up a helper thread to invoke the buffer-completion callback
+  FlushCv.notify_one();
+}
+
+// This is the driver routine for the completion thread
+void OmptTracingBufferMgr::driveCompletion() {
+  while (true) {
+    bool should_signal_workers = false;
+    std::unique_lock<std::mutex> flush_lock(FlushMutex);
+    if (DoneTracing) {
+      // An upper layer serializes flush_trace and stop_trace. In
+      // addition, before DoneTracing is set, a flush is performed as
+      // part of stop_trace. So assert that no flush is in progress.
+      assert(ThreadFlushTracker == 0);
+      break;
+    }
+    FlushCv.wait(flush_lock, [this] {
+      return DoneTracing ||
+             (!Id2FlushMdMap.empty() &&
+              llvm::omp::target::ompt::TracingActive) ||
+             isThisThreadFlushWaitedUpon();
+    });
+    if (isThisThreadFlushWaitedUpon()) {
+      resetThisThreadFlush();
+      if (ThreadFlushTracker == 0)
+        should_signal_workers = true;
+    }
+    flush_lock.unlock();
+
+    invokeCallbacks();
+
+    if (should_signal_workers)
+      ThreadFlushCv.notify_all();
+
+    // There is a scenario where a buffer was processed but not full
+    // or owned, so it was put back in waiting state. So this thread
+    // would not wait but keep on looping without having any actual
+    // work until new trace records are added and this thread
+    // signaled. Hence, this thread yields.
+    std::this_thread::yield();
+  }
+  bool is_last_helper = false;
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  assert(DoneTracing && "Helper thread exiting but not yet done");
+  assert(isThisThreadShutdownWaitedUpon() &&
+         "Helper thread exiting but not waited upon");
+  resetThisThreadShutdown();
+  if (ThreadShutdownTracker == 0)
+    is_last_helper = true;
+  flush_lock.unlock();
+  if (is_last_helper)
+    ThreadShutdownCv.notify_all();
+
+  // Note that some trace records may have been written but not
+  // delivered to the tool. If flush/stop APIs are not called by the
+  // tool, those trace records may never be delivered to the tool and
+  // the corresponding buffers not reclaimed. TODO Explore whether
+  // this cleanup must be done.
+}
+
+/*
+ * Called by a buffer-completion helper thread. This function examines
+ * the flushed buffers in flush order and dispatches
+ * callbacks. Lock holding is minimized by reserving a buffer,
+ * processing it, and then unreserving it if there are more trace
+ * records to flush later. If all trace records are flushed, a
+ * callback is dispatched informing the tool that the buffer can be
+ * deallocated. If the buffer can be deallocated, all metadata is
+ * destroyed.
+ * Note that this function must be called without holding any locks.
+ */
+void OmptTracingBufferMgr::invokeCallbacks() {
+  ODBG(ODT_Tool) << "Looking for callbacks to invoke";
+  auto max_id = std::numeric_limits<uint64_t>::max();
+  auto curr_id = max_id;
+  auto end_id = get_flush_id();
+  ODBG(ODT_Tool) << "End id is " << end_id;
+  while (true) {
+    // Set the status of the flushed buffer to in-processing so that
+    // another helper thread does not process it concurrently. An
+    // OpenMP worker thread may, however, populate a trace record in a
+    // reserved buffer concurrently.
+    FlushInfo flush_info = findAndReserveFlushedBuf(curr_id);
+
+    // no entry found, nothing to process
+    if (curr_id == max_id && flush_info.FlushCursor == nullptr)
+      return;
+
+    if (flush_info.FlushCursor != nullptr) {
+      // increment curr_id to get the candidate for the next iteration
+      curr_id = flush_info.FlushId + 1;
+    } else {
+      assert(curr_id != max_id && "Cannot increment max id");
+      ++curr_id;
+    }
+
+    ODBG(ODT_Tool) << "Next id will be " << curr_id;
+
+    if (flush_info.FlushCursor == nullptr) {
+      // This buffer must have been processed already
+      if (curr_id < end_id)
+        continue;
+      else
+        return; // nothing else to process
+    }
+
+    ODBG(ODT_Tool) << "Buf " << flush_info.FlushBuf->Start << " Cursor "
+                   << flush_info.FlushCursor << " Id " << flush_info.FlushId
+                   << " will be flushed";
+
+    // Examine the status of the trace records and dispatch
+    // buffer-completion callbacks as appropriate.
+    flushBuffer(flush_info);
+
+    // TODO optimize to set buffer-owned in the same pass above.
+    // Currently, this is the only way a buffer is deallocated
+    if (isBufferFull(flush_info)) {
+      // All trace records have been delivered to the tool
+      if (isBufferOwned(flush_info)) {
+        // erase element from buffer and flush maps
+        destroyFlushedBuf(flush_info);
+
+        // dispatch callback with a null range and have the tool
+        // deallocate the buffer
+        dispatchBufferOwnedCallback(flush_info);
+      } else {
+        unreserveFlushedBuf(flush_info);
+      }
+    } else {
+      unreserveFlushedBuf(flush_info);
+    }
+    if (curr_id >= end_id)
+      return;
+  }
+}
+
+/*
+ * This function is called on a buffer that is already reserved by
+ * this thread. Buffer-completion callbacks are dispatched for every
+ * range of trace records that are ready.
+ * This routine must be called without holding locks
+ */
+void OmptTracingBufferMgr::flushBuffer(FlushInfo flush_info) {
+  assert(flush_info.FlushBuf && "Cannot flush an empty buffer");
+  assert(flush_info.FlushCursor && "Cannot flush upto a null cursor");
+
+  void *curr_tr = flush_info.FlushBuf->Start;
+  void *last_tr = flush_info.FlushCursor;
+  // Compute a range [first_cursor,last_cursor] to flush
+  void *first_cursor = nullptr;
+  void *last_cursor = nullptr;
+  while (curr_tr <= last_tr) {
+    TRStatus tr_status = getTRStatus(curr_tr);
+    if (tr_status == TR_init || tr_status == TR_released) {
+      if (first_cursor == nullptr) {
+        // This TR won't be part of a range
+        assert(last_cursor == nullptr &&
+               "Begin/last cursors mutually inconsistent");
+      } else {
+        // End the current interval
+        dispatchCallback(flush_info.FlushBuf->DeviceId,
+                         flush_info.FlushBuf->Start, first_cursor, last_cursor);
+        first_cursor = last_cursor = nullptr;
+      }
+    } else {
+      assert(tr_status == TR_ready && "Unknown trace record status");
+      setTRStatus(curr_tr, TR_released);
+      if (first_cursor == nullptr)
+        first_cursor = curr_tr;
+      last_cursor = curr_tr;
+    }
+    curr_tr = getNextTR(curr_tr);
+  }
+  if (first_cursor != nullptr) {
+    assert(last_cursor != nullptr);
+    dispatchCallback(flush_info.FlushBuf->DeviceId, flush_info.FlushBuf->Start,
+                     first_cursor, last_cursor);
+  }
+}
+
+// Given a range of trace records, dispatch a buffer-completion callback
+void OmptTracingBufferMgr::dispatchCallback(int64_t DeviceId, void *Buffer,
+                                            void *FirstCursor,
+                                            void *LastCursor) {
+  assert(FirstCursor != nullptr && LastCursor != nullptr &&
+         "Callback with nullptr");
+  addLastCursor(LastCursor);
+
+  // This is best effort.
+  // There is a small window when the buffer-completion callback may
+  // be invoked even after tracing has been disabled.
+  // Note that we don't want to hold a lock when dispatching the callback.
+  if (llvm::omp::target::ompt::isTracedDevice(DeviceId)) {
+    ODBG(ODT_Tool) << "Dispatch callback w/ range (inclusive) to be flushed: "
+                   << FirstCursor << " -> " << LastCursor;
+    llvm::omp::target::ompt::ompt_callback_buffer_complete(
+        DeviceId, Buffer,
+        /* bytes returned in this callback */
+        (char *)getNextTR(LastCursor) - (char *)FirstCursor,
+        (ompt_buffer_cursor_t)FirstCursor, false /* buffer_owned */);
+  }
+
+  removeLastCursor(LastCursor);
+}
+
+// Dispatch a buffer-completion callback with buffer_owned set so that
+// the tool can deallocate the buffer
+void OmptTracingBufferMgr::dispatchBufferOwnedCallback(
+    const FlushInfo &flush_info) {
+  // This is best effort.
+  // There is a small window when the buffer-completion callback may
+  // be invoked even after tracing has been disabled.
+  // Note that we don't want to hold a lock when dispatching the callback.
+  if (llvm::omp::target::ompt::isTracedDevice(flush_info.FlushBuf->DeviceId)) {
+    ODBG(ODT_Tool) << "Dispatch callback with buffer "
+                   << flush_info.FlushBuf->Start << " owned";
+    llvm::omp::target::ompt::ompt_callback_buffer_complete(
+        flush_info.FlushBuf->DeviceId, flush_info.FlushBuf->Start, 0,
+        (ompt_buffer_cursor_t)0, true /* buffer owned */);
+  }
+}
+
+void OmptTracingBufferMgr::initTraceRecordMetaData(void *Rec) {
+  setTRStatus(Rec, TR_init);
+}
+
+OmptTracingBufferMgr::BufPtr
+OmptTracingBufferMgr::getDeviceSpecificBuffer(int64_t DeviceId) {
+  if (DeviceId < 0 || DeviceId > MAX_NUM_DEVICES - 1) {
+    REPORT() << "getDeviceSpecificBuffer: Device id " << DeviceId
+             << " invalid or exceeds supported max: " << MAX_NUM_DEVICES - 1;
+    return nullptr;
+  }
+  return ArrayOfBufPtr[DeviceId];
+}
+
+void OmptTracingBufferMgr::setDeviceSpecificBuffer(int64_t DeviceId,
+                                                   BufPtr Buf) {
+  if (DeviceId < 0 || DeviceId > MAX_NUM_DEVICES - 1) {
+    REPORT() << "setDeviceSpecificBuffer: Device id " << DeviceId
+             << " invalid or exceeds supported max: " << MAX_NUM_DEVICES - 1;
+    return;
+  }
+  ArrayOfBufPtr[DeviceId] = Buf;
+}
+
+void OmptTracingBufferMgr::setTRStatus(void *Rec, TRStatus Status) {
+  TraceRecord *TR = static_cast<TraceRecord *>(Rec);
+  TR->TRState.store(Status, std::memory_order_release);
+}
+
+OmptTracingBufferMgr::TRStatus OmptTracingBufferMgr::getTRStatus(void *Rec) {
+  return static_cast<TraceRecord *>(Rec)->TRState.load(
+      std::memory_order_acquire);
+}
+
+void *OmptTracingBufferMgr::getNextTR(void *TR) {
+  size_t RecSize = getTRSize();
+  // warning: no overflow check done
+  return (char *)TR + RecSize;
+}
+
+bool OmptTracingBufferMgr::isBufferFull(const FlushInfo &flush_info) {
+  std::unique_lock<std::mutex> buf_lock(BufferMgrMutex);
+  return flush_info.FlushBuf->IsFull;
+}
+
+void *OmptTracingBufferMgr::getBufferCursor(BufPtr buf) {
+  return buf->Cursor.load(std::memory_order_acquire);
+}
+
+/*
+ * Traverse all the trace records of a buffer and return true if all
+ * of them have been released to the tool, otherwise return false
+ */
+bool OmptTracingBufferMgr::isBufferOwned(const FlushInfo &flush_info) {
+  assert(isBufferFull(flush_info) && "Compute buffer-owned when it is full");
+  void *curr_tr = flush_info.FlushBuf->Start;
+  // Since the buffer is full, the cursor must be the last valid
+  // TR. Note that this may be more up-to-date than the cursor in the
+  // flush_info. Use the last valid TR to avoid dropping trace records
+  void *last_tr = getBufferCursor(flush_info.FlushBuf);
+  while (curr_tr <= last_tr) {
+    if (getTRStatus(curr_tr) != TR_released)
+      return false;
+    curr_tr = getNextTR(curr_tr);
+  }
+  return true;
+}
+
+/*
+ * A buffer must be reserved by a thread before it can be processed
+ * and callbacks dispatched for that buffer. Reservation is done by
+ * setting the status to in-processing.
+ *
+ * If a buffer is found in the flush metadata for the given id and it
+ * is not in in-processing mode, reserve it by setting its mode to
+ * in-processing and return the corresponding flush metadata. If the
+ * given id is set to max, return the first waiting buffer in the
+ * list of buffers to be flushed.
+ */
+OmptTracingBufferMgr::FlushInfo
+OmptTracingBufferMgr::findAndReserveFlushedBuf(uint64_t FlushId) {
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  MapId2Md::iterator flush_itr;
+  if (FlushId == std::numeric_limits<uint64_t>::max()) {
+    // Reserve the first waiting buffer and return it
+    if (Id2FlushMdMap.empty())
+      return FlushInfo();
+    for (flush_itr = Id2FlushMdMap.begin(); flush_itr != Id2FlushMdMap.end();
+         ++flush_itr) {
+      // Reserve only if waiting
+      if (flush_itr->second.FlushStatus == Flush_waiting)
+        break;
+    }
+    if (flush_itr == Id2FlushMdMap.end())
+      return FlushInfo();
+  } else {
+    flush_itr = Id2FlushMdMap.find(FlushId);
+    if (flush_itr == Id2FlushMdMap.end() ||
+        flush_itr->second.FlushStatus == Flush_processing)
+      return FlushInfo();
+  }
+  assert(flush_itr->second.FlushStatus == Flush_waiting);
+  flush_itr->second.FlushStatus = Flush_processing;
+  // Update the metadata cursor since more trace records may have been
+  // generated.
+  flush_itr->second.FlushCursor =
+      flush_itr->second.FlushBuf->Cursor.load(std::memory_order_acquire);
+
+  FlushInfo flush_info(flush_itr->first, flush_itr->second.FlushCursor,
+                       flush_itr->second.FlushBuf);
+  ODBG(ODT_Tool) << "Reserved buffer: flush_id:" << flush_itr->first
+                 << ", cursor:" << flush_itr->second.FlushCursor
+                 << ", buf:" << flush_itr->second.FlushBuf->Start;
+  return flush_info;
+}
+
+/*
+ * Given a buffer, verify that it is in processing state and set its
+ * status to waiting, removing the reservation. The same thread that
+ * reserved it should be unreserving it but currently there is no such
+ * check.
+ */
+void OmptTracingBufferMgr::unreserveFlushedBuf(const FlushInfo &flush_info) {
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  auto itr = Id2FlushMdMap.find(flush_info.FlushId);
+  assert(itr != Id2FlushMdMap.end() &&
+         itr->second.FlushStatus == Flush_processing);
+  itr->second.FlushStatus = Flush_waiting;
+  ODBG(ODT_Tool) << "Unreserved buffer: flush_id:" << flush_info.FlushId
+                 << ", cursor:" << flush_info.FlushCursor
+                 << ", buf:" << flush_info.FlushBuf->Start;
+}
+
+/*
+ * This function must be called after all of the trace records in the
+ * buffer have been released to the tool. The buffer is removed from
+ * all metadata maps.
+ * Note lock order: buf_lock -> flush_lock
+ */
+void OmptTracingBufferMgr::destroyFlushedBuf(const FlushInfo &flush_info) {
+  ODBG(ODT_Tool) << "Destroying buffer: flush_id:" << flush_info.FlushId
+                 << ", cursor:" << flush_info.FlushCursor
+                 << ", buf:" << flush_info.FlushBuf->Start;
+
+  BufPtr buf = flush_info.FlushBuf;
+
+  std::unique_lock<std::mutex> buf_lock(BufferMgrMutex);
+  Id2BufferMap.erase(buf->Id);
+
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  auto flush_itr = Id2FlushMdMap.find(flush_info.FlushId);
+  assert(flush_itr != Id2FlushMdMap.end());
+  assert(flush_itr->second.FlushBuf == buf);
+  Id2FlushMdMap.erase(flush_itr);
+  FlushBufPtr2IdMap.erase(buf);
+}
+
+/*
+ * Generate a new flush id and add the buffer to the flush metadata
+ * maps. This function must be called while holding the flush lock.
+ */
+uint64_t OmptTracingBufferMgr::addNewFlushEntry(BufPtr Buf, void *Cursor) {
+  assert(FlushBufPtr2IdMap.find(Buf) == FlushBufPtr2IdMap.end());
+  uint64_t FlushId = get_and_inc_flush_id();
+  FlushBufPtr2IdMap.emplace(Buf, FlushId);
+  assert(Id2FlushMdMap.find(FlushId) == Id2FlushMdMap.end());
+  Id2FlushMdMap.emplace(FlushId, FlushMd(Cursor, Buf, Flush_waiting));
+
+  ODBG(ODT_Tool) << "Added new flush id " << FlushId << " cursor " << Cursor
+                 << " buf " << Buf->Start;
+
+  return FlushId;
+}
+
+/*
+ * Called by ompt_flush_trace and ompt_stop_trace. Traverse the
+ * existing buffers in creation order and flush all the ready TRs
+ */
+int OmptTracingBufferMgr::flushAllBuffers(int DeviceId) {
+  ODBG(ODT_Tool) << "Flushing buffers for device " << DeviceId << " :: START";
+  // Overloading MAX_NUM_DEVICES to mean all devices.
+  if (DeviceId < 0 || DeviceId > MAX_NUM_DEVICES)
+    return 0; // failed to flush
+
+  if (!areHelperThreadsAvailable())
+    return 0; // failed to flush
+
+  // If flush is called from a helper thread, just bail out
+  if (amIHelperThread())
+    return 0; // failed to flush
+
+  // To avoid holding the mutex for too long, get the ids of the first
+  // and the last TRs under lock, and then go through that range,
+  // holding the mutex for an individual TR
+  std::unique_lock<std::mutex> buf_lock(BufferMgrMutex);
+  if (Id2BufferMap.empty())
+    return 1; // no trace records to flush
+  uint64_t curr_buf_id = Id2BufferMap.begin()->first;
+  uint64_t last_buf_id = Id2BufferMap.rbegin()->first;
+  buf_lock.unlock();
+
+  while (curr_buf_id <= last_buf_id) {
+    std::unique_lock<std::mutex> buf_lock(BufferMgrMutex);
+    // Another thread may have deleted this buffer by now
+    auto buf_itr = Id2BufferMap.find(curr_buf_id);
+    if (buf_itr == Id2BufferMap.end()) {
+      ++curr_buf_id;
+      continue;
+    }
+    BufPtr curr_buf = buf_itr->second;
+
+    // If the device-id does not match, skip it. A device-id of MAX_NUM_DEVICES
+    // indicates flushing for all devices.
+    if (DeviceId != MAX_NUM_DEVICES && curr_buf->DeviceId != DeviceId) {
+      ++curr_buf_id;
+      continue;
+    }
+
+    // If this buffer is in the flush-map, skip it. It is either in
+    // process by another thread or will be processed
+    std::unique_lock<std::mutex> flush_lock(FlushMutex);
+    auto flush_itr = FlushBufPtr2IdMap.find(curr_buf);
+    if (flush_itr != FlushBufPtr2IdMap.end()) {
+      ++curr_buf_id;
+      continue;
+    }
+    // This buffer has not been flushed yet
+    void *CurrBufCursor = getBufferCursor(curr_buf);
+    uint64_t flush_id = addNewFlushEntry(curr_buf, CurrBufCursor);
+    (void)flush_id; // Silence warning.
+    ODBG(ODT_Tool) << "flushAllBuffers: Added new id " << flush_id << " cursor "
+                   << CurrBufCursor << " buf " << curr_buf->Start;
+
+    flush_lock.unlock();
+    buf_lock.unlock();
+
+    ++curr_buf_id;
+  }
+
+  ODBG(ODT_Tool) << "Flushing buffers for device " << DeviceId << " :: WAIT";
+
+  // This is best effort. It is possible that some trace records are
+  // not flushed when the wait is done.
+  waitForFlushCompletion();
+
+  ODBG(ODT_Tool) << "Flushing buffers for device " << DeviceId << " :: STOP";
+
+  return 1; // success
+}
+
+void OmptTracingBufferMgr::waitForFlushCompletion() {
+  {
+    std::unique_lock<std::mutex> flush_lock(FlushMutex);
+    // Setting the flush bit for a given helper thread indicates that the worker
+    // thread is ready for the helper thread to do some work.
+    for (uint32_t i = 0; i < OMPT_NUM_HELPER_THREADS; ++i)
+      setThreadFlush(i);
+  }
+
+  // Wake up all helper threads to invoke buffer-completion callbacks.
+  FlushCv.notify_all();
+
+  // Now wait for all helper threads  to complete flushing.
+  {
+    std::unique_lock<std::mutex> flush_lock(FlushMutex);
+    ThreadFlushCv.wait(flush_lock, [this] { return ThreadFlushTracker == 0; });
+  }
+}
+
+void OmptTracingBufferMgr::init() {
+  for (int i = 0; i < MAX_NUM_DEVICES; ++i)
+    ArrayOfBufPtr[i] = nullptr;
+  ThreadFlushTracker = 0;
+  ThreadShutdownTracker = 0;
+  DoneTracing = false; // TODO make it a class member
+}
+
+void OmptTracingBufferMgr::startHelperThreads() {
+  // All helper threads are stopped while holding FlushMutex. So if
+  // any helper thread is present, just return. This takes care of
+  // repeated calls to start-trace.
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  if (!HelperThreadIdMap.empty()) {
+    assert(!DoneTracing && "Helper threads exist but tracing is done");
+    return;
+  }
+  init();
+  createHelperThreads();
+}
+
+bool OmptTracingBufferMgr::areHelperThreadsAvailable() {
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  if (DoneTracing // If another thread called stop, assume there are no threads
+      || HelperThreadIdMap.empty() // Threads were never started
+  ) {
+    // Don't assert on HelperThreadIdMap since shutdown by another
+    // thread may be in progress
+    return false;
+  }
+  return true;
+}
+
+void OmptTracingBufferMgr::shutdownHelperThreads() {
+  if (!areHelperThreadsAvailable())
+    return;
+
+  std::unique_lock<std::mutex> flush_lock(FlushMutex);
+  // If I am destroying the threads, then at least one thread must be present
+  assert(!CompletionThreads.empty());
+  assert(!HelperThreadIdMap.empty());
+  assert(ThreadShutdownTracker == 0);
+
+  // Set the done flag which helper threads will look at
+  DoneTracing = true;
+  // Wait to make sure all helper threads exit
+  for (uint32_t i = 0; i < OMPT_NUM_HELPER_THREADS; ++i)
+    setThreadShutdown(i);
+  // Signal indicating that DoneTracing is set
+  FlushCv.notify_all();
+  ThreadShutdownCv.wait(flush_lock,
+                        [this] { return ThreadShutdownTracker == 0; });
+
+  // Now destroy all the helper threads
+  destroyHelperThreads();
+}
+
+void OmptTracingBufferMgr::flushAndShutdownHelperThreads() {
+  std::unique_lock<std::mutex> Lock(llvm::omp::target::ompt::TraceControlMutex);
+  // Flush buffers for all devices.
+  if (OMPX_FlushOnShutdown)
+    flushAllBuffers(MAX_NUM_DEVICES);
+  else
+    waitForFlushCompletion(); // Dont initiate but wait for outstanding flushes.
+  shutdownHelperThreads();
+}
+
+void OmptTracingBufferMgr::createHelperThreads() {
+  for (uint32_t i = 0; i < OMPT_NUM_HELPER_THREADS; ++i) {
+    CompletionThreads.emplace_back(
+        std::thread(&OmptTracingBufferMgr::driveCompletion, this));
+    HelperThreadIdMap[CompletionThreads.back().get_id()] = i;
+  }
+}
+
+void OmptTracingBufferMgr::destroyHelperThreads() {
+  for (auto &thd : CompletionThreads)
+    thd.join();
+  CompletionThreads.clear();
+  HelperThreadIdMap.clear();
+}
+#endif

>From fbb1953aaefd0360bf4eedd49862b571e4379339 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 2 Apr 2026 07:52:40 -0500
Subject: [PATCH 07/12] [Offload][OMPT] Add tracing orchestration and
 libomptarget integration

Wire the OMPT device tracing subsystem into libomptarget, completing the tracing pipeline from record production through buffer management to tool delivery. Include the synchronous fallback and initialization fixes in the integration that requires them.

Co-Authored-By: Claude <noreply at anthropic.com>
---
 offload/libomptarget/CMakeLists.txt           |   1 +
 offload/libomptarget/OpenMP/OMPT/Callback.cpp |  12 +
 .../libomptarget/OpenMP/OMPT/OmptTracing.cpp  | 891 ++++++++++++++++++
 offload/libomptarget/PluginManager.cpp        |  22 +
 offload/libomptarget/device.cpp               |  58 +-
 offload/libomptarget/exports                  |   8 +
 offload/libomptarget/interface.cpp            |  44 +-
 offload/libomptarget/omptarget.cpp            |   6 +-
 8 files changed, 1014 insertions(+), 28 deletions(-)
 create mode 100644 offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp

diff --git a/offload/libomptarget/CMakeLists.txt b/offload/libomptarget/CMakeLists.txt
index 33b32d21e3c69..1481f01497f92 100644
--- a/offload/libomptarget/CMakeLists.txt
+++ b/offload/libomptarget/CMakeLists.txt
@@ -17,6 +17,7 @@ add_library(omptarget SHARED
   OpenMP/Mapping.cpp
   OpenMP/InteropAPI.cpp
   OpenMP/OMPT/Callback.cpp
+  OpenMP/OMPT/OmptTracing.cpp
   OpenMP/OMPT/OmptTracingBuffer.cpp
 
   KernelLanguage/API.cpp
diff --git a/offload/libomptarget/OpenMP/OMPT/Callback.cpp b/offload/libomptarget/OpenMP/OMPT/Callback.cpp
index 1e03f1455d1b2..071276b3903ca 100644
--- a/offload/libomptarget/OpenMP/OMPT/Callback.cpp
+++ b/offload/libomptarget/OpenMP/OMPT/Callback.cpp
@@ -12,9 +12,13 @@
 
 #ifdef OMPT_SUPPORT
 
+#include <atomic>
+#include <cassert>
 #include <cstdlib>
 #include <cstring>
 #include <memory>
+#include <mutex>
+#include <thread>
 
 #include "Shared/Debug.h"
 
@@ -23,7 +27,9 @@
 #include "OpenMP/OMPT/Interface.h"
 
 #include "llvm/Support/DynamicLibrary.h"
+#include "llvm/Support/ErrorHandling.h"
 
+#pragma push_macro("DEBUG_PREFIX")
 #undef DEBUG_PREFIX
 #define DEBUG_PREFIX "OMPT"
 
@@ -441,6 +447,11 @@ void Interface::endTarget(int64_t DeviceId, void *Code) {
   endTargetRegion();
 }
 
+void Interface::announceTargetRegion(const char *RegionName) {
+  ODBG(ODT_Tool) << "in Interface::target_region_" << RegionName
+                 << " target_id=" << TargetData.value;
+}
+
 void Interface::beginTargetDataOperation() {
   ODBG(ODT_Tool) << "in ompt_target_region_begin (TargetRegionId = "
                  << TargetData.value << ")";
@@ -562,4 +573,5 @@ void llvm::omp::target::ompt::connectLibrary() {
   ODBG(ODT_Tool) << "Exiting connectLibrary";
 }
 
+#pragma pop_macro("DEBUG_PREFIX")
 #endif // OMPT_SUPPORT
diff --git a/offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp b/offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp
new file mode 100644
index 0000000000000..77c771786f29b
--- /dev/null
+++ b/offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp
@@ -0,0 +1,891 @@
+//===-- OmptTracing.cpp - Target independent OpenMP target RTL --- C++ -*--===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementation of OMPT tracing interfaces for target independent layer
+//
+//===----------------------------------------------------------------------===//
+
+#ifdef OMPT_SUPPORT
+
+#include "OpenMP/OMPT/OmptTracing.h"
+#include "OpenMP/OMPT/Callback.h"
+#include "OpenMP/OMPT/Interface.h"
+#include "OpenMP/OMPT/OmptTracingBuffer.h"
+#include "PluginManager.h"
+#include "Shared/Debug.h"
+#include "omp-tools.h"
+
+#include "llvm/Support/DynamicLibrary.h"
+
+#include <atomic>
+#include <cassert>
+#include <cstdlib>
+#include <cstring>
+#include <memory>
+#include <mutex>
+#include <thread>
+
+#pragma push_macro("DEBUG_PREFIX")
+#undef DEBUG_PREFIX
+#define DEBUG_PREFIX "OMPT"
+
+using namespace llvm::omp::target::ompt;
+
+std::mutex llvm::omp::target::ompt::DeviceAccessMutex;
+std::mutex llvm::omp::target::ompt::TraceAccessMutex;
+std::mutex llvm::omp::target::ompt::TraceControlMutex;
+std::mutex llvm::omp::target::ompt::TraceHashThreadMutex;
+std::mutex llvm::omp::target::ompt::BufferManagementFnMutex;
+
+std::unordered_map<int /*DeviceId*/, std::pair<ompt_callback_buffer_request_t,
+                                               ompt_callback_buffer_complete_t>>
+    llvm::omp::target::ompt::BufferManagementFns;
+
+thread_local uint32_t llvm::omp::target::ompt::TraceRecordNumGrantedTeams = 0;
+thread_local uint64_t llvm::omp::target::ompt::TraceRecordStartTime = 0;
+thread_local uint64_t llvm::omp::target::ompt::TraceRecordStopTime = 0;
+thread_local uint64_t llvm::omp::target::ompt::ThreadId =
+    std::numeric_limits<uint64_t>::max();
+
+std::map<int32_t, uint64_t> llvm::omp::target::ompt::TracedDevices;
+
+bool llvm::omp::target::ompt::TracingActive = false;
+
+void llvm::omp::target::ompt::resetTimestamp(uint64_t *T) { *T = 0; }
+
+ompt_callback_buffer_request_t
+llvm::omp::target::ompt::getBufferRequestFn(int DeviceId) {
+  std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
+  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
+  if (BufferMgrItr == BufferManagementFns.end()) {
+    return nullptr;
+  }
+  return BufferMgrItr->second.first;
+}
+
+ompt_callback_buffer_complete_t
+llvm::omp::target::ompt::getBufferCompleteFn(int DeviceId) {
+  std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
+  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
+  if (BufferMgrItr == BufferManagementFns.end()) {
+    return nullptr;
+  }
+  return BufferMgrItr->second.second;
+}
+
+void llvm::omp::target::ompt::setBufferManagementFns(
+    int DeviceId, ompt_callback_buffer_request_t ReqFn,
+    ompt_callback_buffer_complete_t CmpltFn) {
+  std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
+  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
+  if (BufferMgrItr != BufferManagementFns.end()) {
+    REPORT()
+        << "Buffer request and complete functions already exist for device  "
+        << DeviceId << "ignoring ...";
+    return;
+  }
+  BufferManagementFns[DeviceId] = std::make_pair(ReqFn, CmpltFn);
+}
+
+void llvm::omp::target::ompt::removeBufferManagementFns(int DeviceId) {
+  std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
+  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
+  if (BufferMgrItr == BufferManagementFns.end()) {
+    REPORT() << "Buffer request and complete functions don't exist for device  "
+             << DeviceId << "ignoring ...";
+    return;
+  }
+  BufferManagementFns.erase(BufferMgrItr);
+}
+
+bool llvm::omp::target::ompt::isAllDeviceTracingStopped() {
+  std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
+  return BufferManagementFns.empty();
+}
+
+void llvm::omp::target::ompt::ompt_callback_buffer_request(
+    int DeviceId, ompt_buffer_t **BufferPtr, size_t *Bytes) {
+  if (auto Fn = getBufferRequestFn(DeviceId))
+    Fn(DeviceId, BufferPtr, Bytes);
+}
+
+void llvm::omp::target::ompt::ompt_callback_buffer_complete(
+    int DeviceId, ompt_buffer_t *Buffer, size_t Bytes,
+    ompt_buffer_cursor_t BeginCursor, int BufferOwned) {
+  if (auto Fn = getBufferCompleteFn(DeviceId))
+    Fn(DeviceId, Buffer, Bytes, BeginCursor, BufferOwned);
+}
+
+inline void setDeviceTracing(uint64_t &TracingTypes) {
+  // Set bit 0 to indicate generally enabled device tracing.
+  TracingTypes |= 1UL;
+}
+
+inline void resetDeviceTracing(uint64_t &TracingTypes) {
+  // Reset bit 0 to indicate generally disabled device tracing.
+  TracingTypes &= ~(1UL);
+}
+
+inline bool checkDeviceTracingState(const uint64_t &TracingTypes) {
+  // Return state of bit 0 to indicate if device is actively traced.
+  return TracingTypes & 1UL;
+}
+
+void llvm::omp::target::ompt::enableDeviceTracing(int DeviceId) {
+  std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
+  auto Device = TracedDevices.find(DeviceId);
+  if (Device == TracedDevices.end()) {
+    uint64_t TracingTypes{0};
+    setDeviceTracing(TracingTypes);
+    TracedDevices.emplace(DeviceId, TracingTypes);
+  } else
+    setDeviceTracing(Device->second);
+  // In any case: at least one device is traced
+  TracingActive = true;
+}
+
+void llvm::omp::target::ompt::disableDeviceTracing(int DeviceId) {
+  std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
+  auto Device = TracedDevices.find(DeviceId);
+  if (Device == TracedDevices.end()) {
+    uint64_t TracingTypes{0};
+    resetDeviceTracing(TracingTypes);
+    TracedDevices.emplace(DeviceId, TracingTypes);
+  } else
+    resetDeviceTracing(Device->second);
+
+  // Check for actively traced devices
+  for (auto &Dev : TracedDevices)
+    if (checkDeviceTracingState(Dev.second))
+      return;
+
+  // If no device is currently traced: set global tracing flag to false
+  TracingActive = false;
+}
+
+bool llvm::omp::target::ompt::isTracingEnabled(int DeviceId,
+                                               unsigned int EventTy) {
+  return TracingActive && isTracedDevice(DeviceId) &&
+         isTracingTypeGroupEnabled(DeviceId, EventTy);
+}
+
+bool llvm::omp::target::ompt::isTracedDevice(int DeviceId) {
+  std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
+  auto Device = TracedDevices.find(DeviceId);
+  if (Device != TracedDevices.end())
+    return checkDeviceTracingState(Device->second);
+
+  return false;
+}
+
+bool llvm::omp::target::ompt::isTracingTypeEnabled(int DeviceId,
+                                                   unsigned int EventTy) {
+  std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
+  // Make sure we do not shift more than std::numeric_limits<uint64_t>::digits
+  assert(EventTy < 64 && "Shift limit exceeded: EventTy must be less than 64");
+  auto Device = TracedDevices.find(DeviceId);
+  if (Device != TracedDevices.end() && EventTy < 64)
+    return (Device->second & (1UL << EventTy));
+  return false;
+}
+
+bool llvm::omp::target::ompt::isTracingTypeGroupEnabled(int DeviceId,
+                                                        unsigned int EventTy) {
+  std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
+  // Make sure we do not shift more than std::numeric_limits<uint64_t>::digits
+  assert(EventTy < 64 && "Shift limit exceeded: EventTy must be less than 64");
+  auto Device = TracedDevices.find(DeviceId);
+  if (Device != TracedDevices.end() && EventTy < 64) {
+    auto TracedEvents = Device->second;
+    switch (EventTy) {
+    case ompt_callbacks_t::ompt_callback_target:
+    case ompt_callbacks_t::ompt_callback_target_emi:
+      return ((TracedEvents & (1UL << ompt_callback_target))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_emi)));
+    case ompt_callbacks_t::ompt_callback_target_data_op:
+    case ompt_callbacks_t::ompt_callback_target_data_op_emi:
+      return ((TracedEvents & (1UL << ompt_callback_target_data_op))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_data_op_emi)));
+    case ompt_callbacks_t::ompt_callback_target_submit:
+    case ompt_callbacks_t::ompt_callback_target_submit_emi:
+      return ((TracedEvents & (1UL << ompt_callback_target_submit))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_submit_emi)));
+    // Special case: EventTy == 0 -> Check all EventTy
+    case 0:
+      return ((TracedEvents & (1UL << ompt_callback_target))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_emi))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_data_op))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_data_op_emi))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_submit))) ||
+             ((TracedEvents & (1UL << ompt_callback_target_submit_emi)));
+    }
+  }
+  return false;
+}
+
+void llvm::omp::target::ompt::setTracingTypeEnabled(uint64_t &TracedEventTy,
+                                                    bool Enable,
+                                                    unsigned int EventTy) {
+  // Make sure we do not shift more than std::numeric_limits<uint64_t>::digits
+  assert(EventTy < 64 && "Shift limit exceeded: EventTy must be less than 64");
+  if (EventTy < 64) {
+    if (Enable)
+      TracedEventTy |= (1UL << EventTy);
+    else
+      TracedEventTy &= ~(1UL << EventTy);
+  }
+}
+
+ompt_set_result_t
+llvm::omp::target::ompt::setTraceEventTy(int DeviceId, unsigned int Enable,
+                                         unsigned int EventTy) {
+  if (DeviceId < 0) {
+    REPORT() << "Failed to set trace event type for DeviceId=" << DeviceId;
+    return ompt_set_never;
+  }
+
+  ODBG(debug::ODT_Tool) << "Executing setTraceEventTy: DeviceId=" << DeviceId
+                        << " Enable=" << Enable << " EventTy=" << EventTy;
+
+  std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
+  if (TracedDevices.find(DeviceId) == TracedDevices.end())
+    TracedDevices.emplace(DeviceId, 0UL);
+
+  auto &TracedEventTy = TracedDevices[DeviceId];
+  bool Enabled = Enable > 0;
+  if (EventTy == 0) {
+    // Set / reset all supported types
+    setTracingTypeEnabled(TracedEventTy, Enabled,
+                          ompt_callbacks_t::ompt_callback_target);
+    setTracingTypeEnabled(TracedEventTy, Enabled,
+                          ompt_callbacks_t::ompt_callback_target_data_op);
+    setTracingTypeEnabled(TracedEventTy, Enabled,
+                          ompt_callbacks_t::ompt_callback_target_submit);
+    setTracingTypeEnabled(TracedEventTy, Enabled,
+                          ompt_callbacks_t::ompt_callback_target_emi);
+    setTracingTypeEnabled(TracedEventTy, Enabled,
+                          ompt_callbacks_t::ompt_callback_target_data_op_emi);
+    setTracingTypeEnabled(TracedEventTy, Enabled,
+                          ompt_callbacks_t::ompt_callback_target_submit_emi);
+
+    if (Enabled) {
+      // Event subset is enabled
+      return ompt_set_sometimes;
+    } else {
+      // All events are disabled
+      return ompt_set_always;
+    }
+  }
+
+  switch (EventTy) {
+  case ompt_callbacks_t::ompt_callback_target:
+  case ompt_callbacks_t::ompt_callback_target_data_op:
+  case ompt_callbacks_t::ompt_callback_target_submit:
+  case ompt_callbacks_t::ompt_callback_target_emi:
+  case ompt_callbacks_t::ompt_callback_target_data_op_emi:
+  case ompt_callbacks_t::ompt_callback_target_submit_emi: {
+    setTracingTypeEnabled(TracedEventTy, Enabled, EventTy);
+    return ompt_set_always;
+  }
+  default: {
+    if (Enabled) {
+      // Unimplemented
+      return ompt_set_never;
+    } else {
+      // Always disabled anyways
+      return ompt_set_always;
+    }
+  }
+  }
+}
+
+uint64_t llvm::omp::target::ompt::getThreadId() {
+  // Grab the value from thread local storage, if valid.
+  if (ThreadId != std::numeric_limits<uint64_t>::max())
+    return ThreadId;
+  // Otherwise set it, protecting the hash with a lock.
+  std::unique_lock<std::mutex> Lock(TraceHashThreadMutex);
+  ThreadId = std::hash<std::thread::id>()(std::this_thread::get_id());
+  return ThreadId;
+}
+
+void Interface::setTraceRecordCommon(ompt_record_ompt_t *DataPtr,
+                                     ompt_callbacks_t CallbackType) {
+  DataPtr->type = CallbackType;
+
+  if (CallbackType == ompt_callback_target)
+    DataPtr->time = 0; // Currently, no consumer, so no need to set it
+  else {
+    DataPtr->time = TraceRecordStartTime;
+    resetTimestamp(&TraceRecordStartTime);
+  }
+
+  DataPtr->thread_id = getThreadId();
+  DataPtr->target_id = TargetData.value;
+}
+
+void Interface::setTraceRecordTargetDataOp(ompt_record_target_data_op_t *Record,
+                                           ompt_target_data_op_t DataOpType,
+                                           void *SrcAddr, int64_t SrcDeviceNum,
+                                           void *DstAddr, int64_t DstDeviceNum,
+                                           size_t Bytes, void *CodePtr) {
+  Record->host_op_id = HostOpId;
+  Record->optype = DataOpType;
+  Record->src_addr = SrcAddr;
+  Record->src_device_num = SrcDeviceNum;
+  Record->dest_addr = DstAddr;
+  Record->dest_device_num = DstDeviceNum;
+  Record->bytes = Bytes;
+
+  Record->end_time = TraceRecordStopTime;
+  resetTimestamp(&TraceRecordStopTime);
+
+  Record->codeptr_ra = CodePtr;
+}
+
+void Interface::setTraceRecordTargetKernel(ompt_record_target_kernel_t *Record,
+                                           unsigned int NumTeams) {
+  Record->host_op_id = HostOpId;
+  Record->requested_num_teams = NumTeams;
+  Record->granted_num_teams = TraceRecordNumGrantedTeams;
+
+  Record->end_time = TraceRecordStopTime;
+  resetTimestamp(&TraceRecordStopTime);
+}
+
+void Interface::setTraceRecordTarget(ompt_record_target_t *Record,
+                                     int64_t DeviceId, ompt_target_t TargetKind,
+                                     ompt_scope_endpoint_t Endpoint,
+                                     void *CodePtr) {
+  Record->kind = TargetKind;
+  Record->endpoint = Endpoint;
+  Record->device_num = DeviceId;
+  assert(TaskData);
+  Record->task_id = TaskData->value;
+  Record->target_id = TargetData.value;
+  Record->codeptr_ra = CodePtr;
+}
+
+void Interface::startTargetDataAllocTrace(int64_t DeviceId, void *HstPtrBegin,
+                                          void **TgtPtrBegin, size_t Size,
+                                          void *Code) {}
+
+ompt_record_ompt_t *Interface::stopTargetDataAllocTrace(int64_t DeviceId,
+                                                        void *HstPtrBegin,
+                                                        void **TgtPtrBegin,
+                                                        size_t Size,
+                                                        void *Code) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target_data_op))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr = (ompt_record_ompt_t *)TRM->assignCursor(
+      ompt_callback_target_data_op, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target_data_op);
+  setTraceRecordTargetDataOp(&DataPtr->record.target_data_op,
+                             ompt_target_data_alloc, HstPtrBegin,
+                             /*SrcDeviceNum=*/omp_get_initial_device(),
+                             *TgtPtrBegin, DeviceId, Size, Code);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Generated trace record: " << DataPtr
+                        << " (ompt_target_data_alloc)";
+  return DataPtr;
+}
+
+void Interface::startTargetDataDeleteTrace(int64_t DeviceId, void *TgtPtrBegin,
+                                           void *Code) {}
+
+ompt_record_ompt_t *Interface::stopTargetDataDeleteTrace(int64_t DeviceId,
+                                                         void *TgtPtrBegin,
+                                                         void *Code) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target_data_op))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr = (ompt_record_ompt_t *)TRM->assignCursor(
+      ompt_callback_target_data_op, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target_data_op);
+  setTraceRecordTargetDataOp(&DataPtr->record.target_data_op,
+                             ompt_target_data_delete, TgtPtrBegin, DeviceId,
+                             /*DstAddr=*/nullptr,
+                             /*DstDeviceNum=*/-1, /*Bytes=*/0, Code);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Generated trace record: " << DataPtr
+                        << " (ompt_target_data_delete)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *
+Interface::startTargetDataSubmitTrace(int64_t SrcDeviceId, void *SrcPtrBegin,
+                                      int64_t DstDeviceId, void *DstPtrBegin,
+                                      size_t Size, void *Code) {
+  if (!isTracingEnabled(DstDeviceId, ompt_callback_target_data_op))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr = (ompt_record_ompt_t *)TRM->assignCursor(
+      ompt_callback_target_data_op, DstDeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target_data_op);
+  DataPtr->time = 0; // Set to sanity value and let "stop" function fix it
+
+  // Set some of the data-op specific fields here
+  setTraceRecordTargetDataOp(&DataPtr->record.target_data_op,
+                             ompt_target_data_transfer_to_device, SrcPtrBegin,
+                             SrcDeviceId, DstPtrBegin, DstDeviceId, Size, Code);
+
+  ODBG(debug::ODT_Tool) << "OMPT-Async: Returning data trace record buf ptr "
+                        << DataPtr;
+  return DataPtr;
+}
+
+ompt_record_ompt_t *
+Interface::startTargetDataRetrieveTrace(int64_t SrcDeviceId, void *SrcPtrBegin,
+                                        int64_t DstDeviceId, void *DstPtrBegin,
+                                        size_t Size, void *Code) {
+  if (!isTracingEnabled(SrcDeviceId, ompt_callback_target_data_op))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr = (ompt_record_ompt_t *)TRM->assignCursor(
+      ompt_callback_target_data_op, SrcDeviceId);
+
+  if (!DataPtr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target_data_op);
+  DataPtr->time = 0; // Set to sanity value and let "stop" function fix it
+
+  // Set some of the data-op specific fields here
+  setTraceRecordTargetDataOp(&DataPtr->record.target_data_op,
+                             ompt_target_data_transfer_from_device, SrcPtrBegin,
+                             SrcDeviceId, DstPtrBegin, DstDeviceId, Size, Code);
+
+  ODBG(debug::ODT_Tool) << "OMPT-Async: Returning data trace record buf ptr "
+                        << DataPtr;
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::stopTargetDataMovementTraceAsync(
+    ompt_record_ompt_t *DataPtr, uint64_t NanosStart, uint64_t NanosEnd) {
+  // Finalize the data that comes from the plugin.
+  DataPtr->time = NanosStart;
+  auto Record = static_cast<ompt_record_target_data_op_t *>(
+      &DataPtr->record.target_data_op);
+  Record->end_time = NanosEnd;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "OMPT-Async: Completed target_data trace record "
+                        << DataPtr;
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::startTargetSubmitTrace(int64_t DeviceId,
+                                                      unsigned int NumTeams) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target_submit))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr = (ompt_record_ompt_t *)TRM->assignCursor(
+      ompt_callback_target_submit, DeviceId);
+
+  // Set all known entries and leave remaining to the stop function
+  setTraceRecordCommon(DataPtr, ompt_callback_target_submit);
+  DataPtr->time = 0; // Set to sanity value and let "stop" function fix it
+  // Kernel specific things
+  DataPtr->record.target_kernel.requested_num_teams = NumTeams;
+  DataPtr->record.target_kernel.host_op_id = getHostOpId();
+
+  // May be null if event is not traced
+  ODBG(debug::ODT_Tool) << "OMPT-Async: Returning kernel trace record buf ptr "
+                        << DataPtr;
+  return DataPtr;
+}
+
+ompt_record_ompt_t *
+Interface::stopTargetSubmitTraceAsync(ompt_record_ompt_t *DataPtr,
+                                      unsigned int NumTeams,
+                                      uint64_t NanosStart, uint64_t NanosStop) {
+  // Common fields
+  DataPtr->time = NanosStart;
+  // Submit specific
+  DataPtr->record.target_kernel.end_time = NanosStop;
+  DataPtr->record.target_kernel.granted_num_teams = NumTeams;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  // Ready Record
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "OMPT-Async: Completed trace record buf ptr "
+                        << DataPtr;
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::startTargetDataEnterTrace(int64_t DeviceId,
+                                                         void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId,
+                       ompt_target_enter_data, ompt_scope_begin, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Returning trace record buf ptr " << DataPtr
+                        << " (ompt_target_enter_data)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::stopTargetDataEnterTrace(int64_t DeviceId,
+                                                        void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId,
+                       ompt_target_enter_data, ompt_scope_end, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Generated trace record " << DataPtr
+                        << " (ompt_target_enter_data)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::startTargetDataExitTrace(int64_t DeviceId,
+                                                        void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId, ompt_target_exit_data,
+                       ompt_scope_begin, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Returning trace record buf ptr " << DataPtr
+                        << " (ompt_target_exit_data)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::stopTargetDataExitTrace(int64_t DeviceId,
+                                                       void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId, ompt_target_exit_data,
+                       ompt_scope_end, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Generated trace record " << DataPtr
+                        << " (ompt_target_exit_data)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::startTargetUpdateTrace(int64_t DeviceId,
+                                                      void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId, ompt_target_update,
+                       ompt_scope_begin, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Returning trace record buf ptr " << DataPtr
+                        << " (ompt_target_update)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::stopTargetUpdateTrace(int64_t DeviceId,
+                                                     void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId, ompt_target_update,
+                       ompt_scope_end, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Generated trace record " << DataPtr
+                        << " (ompt_target_update)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::startTargetTrace(int64_t DeviceId,
+                                                void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId, ompt_target,
+                       ompt_scope_begin, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+  ODBG(debug::ODT_Tool) << "Returning trace record buf ptr " << DataPtr
+                        << " (ompt_target)";
+  return DataPtr;
+}
+
+ompt_record_ompt_t *Interface::stopTargetTrace(int64_t DeviceId,
+                                               void *CodePtr) {
+  if (!isTracingEnabled(DeviceId, ompt_callback_target))
+    return nullptr;
+
+  assert(PM && "Plugin manager not initialized");
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  ompt_record_ompt_t *DataPtr =
+      (ompt_record_ompt_t *)TRM->assignCursor(ompt_callback_target, DeviceId);
+
+  // This event will not be traced
+  if (DataPtr == nullptr)
+    return nullptr;
+
+  setTraceRecordCommon(DataPtr, ompt_callback_target);
+  setTraceRecordTarget(&DataPtr->record.target, DeviceId, ompt_target,
+                       ompt_scope_end, CodePtr);
+
+  // The trace record has been created, mark it ready for delivery to the tool
+  TRM->setTRStatus(DataPtr, OmptTracingBufferMgr::TR_ready);
+
+  ODBG(debug::ODT_Tool) << "Generated trace record " << DataPtr
+                        << " (ompt_target)";
+  return DataPtr;
+}
+
+extern "C" {
+// Device-independent entry point for ompt_set_trace_ompt
+ompt_set_result_t libomptarget_ompt_set_trace_ompt(int DeviceId,
+                                                   unsigned int Enable,
+                                                   unsigned int EventTy) {
+  std::unique_lock<std::mutex> Lock(TraceAccessMutex);
+  return llvm::omp::target::ompt::setTraceEventTy(DeviceId, Enable, EventTy);
+}
+
+// Device-independent entry point for ompt_start_trace
+int libomptarget_ompt_start_trace(int DeviceId,
+                                  ompt_callback_buffer_request_t Request,
+                                  ompt_callback_buffer_complete_t Complete) {
+  if (!PM) {
+    REPORT() << "Failed to start trace for DeviceId=" << DeviceId
+             << " (invalid plugin manager)";
+    // Indicate failure
+    return 0;
+  }
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  std::unique_lock<std::mutex> Lock(TraceControlMutex);
+  if (Request && Complete) {
+    // Set buffer related functions
+    llvm::omp::target::ompt::setBufferManagementFns(DeviceId, Request,
+                                                    Complete);
+    llvm::omp::target::ompt::enableDeviceTracing(DeviceId);
+    TRM->startHelperThreads();
+    // Success
+    return 1;
+  }
+  // Failure
+  return 0;
+}
+
+// Device-independent entry point for ompt_flush_trace
+int libomptarget_ompt_flush_trace(int DeviceId) {
+  if (!PM) {
+    REPORT() << "Failed to flush trace for DeviceId=" << DeviceId
+             << " (invalid plugin manager)";
+    // Indicate failure
+    return 0;
+  }
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  std::unique_lock<std::mutex> Lock(TraceControlMutex);
+  return TRM->flushAllBuffers(DeviceId);
+}
+
+// Device independent entry point for ompt_stop_trace
+int libomptarget_ompt_stop_trace(int DeviceId) {
+  if (!PM) {
+    REPORT() << "Failed to stop trace for DeviceId=" << DeviceId
+             << " (invalid plugin manager)";
+    // Indicate failure
+    return 0;
+  }
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  std::unique_lock<std::mutex> Lock(TraceControlMutex);
+
+  // Schedule flushing of trace records for this device
+  int Status = TRM->flushAllBuffers(DeviceId);
+
+  // De-register this device so that no more traces are collected
+  // or delivered for this device until an ompt_start_trace is
+  // invoked for this device.
+  removeBufferManagementFns(DeviceId);
+
+  // If no device is being traced, shut down the helper threads. A
+  // subsequent ompt_start_trace will start up the helper threads.
+  if (isAllDeviceTracingStopped()) {
+    // TODO shutdown should perhaps return a status
+    TRM->shutdownHelperThreads();
+    llvm::omp::target::ompt::disableDeviceTracing(DeviceId);
+  }
+  return Status;
+}
+
+// Device independent entry point for ompt_advance_buffer_cursor
+// Note: The input parameter size is unused here. It refers to the
+// bytes returned in the corresponding callback.
+int libomptarget_ompt_advance_buffer_cursor(ompt_device_t *Device,
+                                            ompt_buffer_t *Buffer, size_t Size,
+                                            ompt_buffer_cursor_t CurrentPos,
+                                            ompt_buffer_cursor_t *NextPos) {
+  if (!PM) {
+    REPORT() << "Failed to advance buffer cursor for Device=" << Device
+             << " (invalid plugin manager)";
+    // Indicate failure
+    return false;
+  }
+  OmptTracingBufferMgr *TRM = PM->getTraceRecordManager();
+  char *TraceRecord = (char *)CurrentPos;
+  // Don't assert if CurrentPos is null, just indicate end of buffer
+  if (TraceRecord == nullptr || TRM->isLastCursor(TraceRecord)) {
+    *NextPos = 0;
+    return false;
+  }
+  // TODO In debug mode, assert that the metadata points to the
+  // input parameter buffer
+
+  size_t TRSize = TRM->getTRSize();
+  *NextPos = (ompt_buffer_cursor_t)(TraceRecord + TRSize);
+  ODBG(llvm::omp::target::debug::ODT_Tool)
+      << "Advanced buffer pointer by " << TRSize << " bytes to "
+      << TraceRecord + TRSize;
+  return true;
+}
+
+// This function is invoked before the kernel launch. So, when the trace record
+// is populated after kernel completion, TraceRecordNumGrantedTeams is already
+// updated.
+void libomptarget_ompt_set_granted_teams(uint32_t NumTeams) {
+  TraceRecordNumGrantedTeams = NumTeams;
+}
+
+// Assume a synchronous implementation and set thread local variables to track
+// timestamps. The thread local variables can then be used to populate trace
+// records.
+void libomptarget_ompt_set_timestamp(uint64_t Start, uint64_t Stop) {
+  TraceRecordStartTime = Start;
+  TraceRecordStopTime = Stop;
+}
+
+// Device-independent entry point to query for the trace format used.
+// Currently, only OMPT format is supported.
+ompt_record_t
+libomptarget_ompt_get_record_type(ompt_buffer_t *Buffer,
+                                  ompt_buffer_cursor_t CurrentPos) {
+  // TODO: When different OMPT trace buffer formats supported, this needs to be
+  // fixed.
+  return ompt_record_t::ompt_record_ompt;
+}
+} // extern "C"
+
+#pragma pop_macro("DEBUG_PREFIX")
+
+#endif // OMPT_SUPPORT
diff --git a/offload/libomptarget/PluginManager.cpp b/offload/libomptarget/PluginManager.cpp
index 41b653a60adfd..5ef8200c7006b 100644
--- a/offload/libomptarget/PluginManager.cpp
+++ b/offload/libomptarget/PluginManager.cpp
@@ -12,6 +12,9 @@
 
 #include "PluginManager.h"
 #include "OffloadPolicy.h"
+#include "OpenMP/OMPT/Callback.h"
+#include "OpenMP/OMPT/OmptCommonDefs.h"
+#include "OpenMP/OMPT/OmptTracing.h"
 #include "Shared/Debug.h"
 #include "Shared/Profile.h"
 #include "device.h"
@@ -47,6 +50,12 @@ void PluginManager::init() {
   } while (false);
 #include "Shared/Targets.def"
 
+#ifdef OMPT_SUPPORT
+  assert(TraceRecordManager == nullptr &&
+         "Expected trace record manager to be null");
+  TraceRecordManager = new OmptTracingBufferMgr();
+#endif
+
   ODBG(ODT_Init) << "RTLs loaded!";
 }
 
@@ -54,6 +63,16 @@ void PluginManager::deinit() {
   TIMESCOPE();
   ODBG(ODT_Deinit) << "Unloading RTLs...";
 
+#ifdef OMPT_SUPPORT
+  // When offloading is disabled, the TRM is not initialized.
+  if (!OffloadPolicy::isOffloadDisabled()) {
+    assert(TraceRecordManager != nullptr &&
+           "Trace record manager should have been non-null");
+    delete TraceRecordManager;
+    TraceRecordManager = nullptr;
+  }
+#endif
+
   for (auto &Plugin : Plugins) {
     if (!Plugin->is_initialized())
       continue;
@@ -323,6 +342,9 @@ int target(ident_t *Loc, DeviceTy &Device, void *HostPtr,
 void PluginManager::unregisterLib(__tgt_bin_desc *Desc) {
   ODBG(ODT_Deinit) << "Unloading target library!";
 
+  OMPT_IF_TRACING_ENABLED(
+      PM->getTraceRecordManager()->flushAndShutdownHelperThreads(););
+
   Desc = upgradeLegacyEntries(Desc);
 
   PM->RTLsMtx.lock();
diff --git a/offload/libomptarget/device.cpp b/offload/libomptarget/device.cpp
index 18339e9afe975..770b74ca35597 100644
--- a/offload/libomptarget/device.cpp
+++ b/offload/libomptarget/device.cpp
@@ -15,6 +15,8 @@
 #include "OpenMP/Mapping.h"
 #include "OpenMP/OMPT/Callback.h"
 #include "OpenMP/OMPT/Interface.h"
+#include "OpenMP/OMPT/OmptCommonDefs.h"
+#include "OpenMP/OMPT/OmptTracing.h"
 #include "PluginManager.h"
 #include "Shared/APITypes.h"
 #include "Shared/Debug.h"
@@ -34,7 +36,8 @@
 #include <thread>
 
 #ifdef OMPT_SUPPORT
-using namespace llvm::omp::target::ompt;
+using namespace llvm::omp::target;
+using namespace ompt;
 #endif
 
 using namespace llvm::omp::target::plugin;
@@ -249,10 +252,15 @@ DeviceTy::loadBinary(__tgt_device_image *Img) {
 void *DeviceTy::allocData(int64_t Size, void *HstPtr, int32_t Kind) {
   /// RAII to establish tool anchors before and after data allocation
   void *TargetPtr = nullptr;
-  OMPT_IF_BUILT(InterfaceRAII TargetDataAllocRAII(
-                    RegionInterface.getCallbacks<ompt_target_data_alloc>(),
-                    DeviceID, HstPtr, &TargetPtr, Size,
-                    /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
+  OMPT_IF_BUILT(
+      InterfaceRAII TargetDataAllocRAII(
+          RegionInterface.getCallbacks<ompt_target_data_alloc>(), DeviceID,
+          HstPtr, &TargetPtr, Size,
+          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+      InterfaceRAII TargetDataAllocTraceRAII(
+          RegionInterface.getTraceGenerators<ompt_target_data_alloc>(),
+          RTLDeviceID, HstPtr, &TargetPtr, Size,
+          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
 
   TargetPtr = RTL->data_alloc(RTLDeviceID, Size, HstPtr, Kind);
   return TargetPtr;
@@ -260,11 +268,15 @@ void *DeviceTy::allocData(int64_t Size, void *HstPtr, int32_t Kind) {
 
 int32_t DeviceTy::deleteData(void *TgtAllocBegin, int32_t Kind) {
   /// RAII to establish tool anchors before and after data deletion
-  OMPT_IF_BUILT(InterfaceRAII TargetDataDeleteRAII(
-                    RegionInterface.getCallbacks<ompt_target_data_delete>(),
-                    DeviceID, TgtAllocBegin,
-                    /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
-
+  OMPT_IF_BUILT(
+      InterfaceRAII TargetDataDeleteRAII(
+          RegionInterface.getCallbacks<ompt_target_data_delete>(), DeviceID,
+          TgtAllocBegin,
+          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+      InterfaceRAII TargetDataDeleteTraceRAII(
+          RegionInterface.getTraceGenerators<ompt_target_data_delete>(),
+          DeviceID, TgtAllocBegin,
+          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
   return RTL->data_delete(RTLDeviceID, TgtAllocBegin, Kind);
 }
 
@@ -281,6 +293,13 @@ int32_t DeviceTy::submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size,
       InterfaceRAII TargetDataSubmitRAII(
           RegionInterface.getCallbacks<ompt_target_data_transfer_to_device>(),
           omp_initial_device, HstPtrBegin, DeviceID, TgtPtrBegin, Size,
+          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+      TracerInterfaceRAII TargetDataSubmitTraceRAII(
+          RegionInterface
+              .getTraceGenerators<ompt_target_data_transfer_to_device>(),
+          AsyncInfo, RTL->getProfiler(), /*TracedDeviceId=*/DeviceID,
+          /*EventType=*/ompt_callback_target_data_op, omp_initial_device,
+          HstPtrBegin, DeviceID, TgtPtrBegin, Size,
           /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
 
   return RTL->data_submit_async(RTLDeviceID, TgtPtrBegin, HstPtrBegin, Size,
@@ -301,6 +320,13 @@ int32_t DeviceTy::retrieveData(void *HstPtrBegin, void *TgtPtrBegin,
       InterfaceRAII TargetDataRetrieveRAII(
           RegionInterface.getCallbacks<ompt_target_data_transfer_from_device>(),
           DeviceID, TgtPtrBegin, omp_initial_device, HstPtrBegin, Size,
+          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+      TracerInterfaceRAII TargetDataRetrieveTraceRAII(
+          RegionInterface
+              .getTraceGenerators<ompt_target_data_transfer_from_device>(),
+          AsyncInfo, RTL->getProfiler(), /*TracedDeviceId=*/DeviceID,
+          /*EventType=*/ompt_callback_target_data_op, DeviceID, TgtPtrBegin,
+          omp_initial_device, HstPtrBegin, Size,
           /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
 
   return RTL->data_retrieve_async(RTLDeviceID, HstPtrBegin, TgtPtrBegin, Size,
@@ -320,11 +346,15 @@ int32_t DeviceTy::dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr,
       InterfaceRAII TargetDataExchangeRAII(
           RegionInterface.getCallbacks<ompt_target_data_transfer_from_device>(),
           RTLDeviceID, SrcPtr, DstDev.RTLDeviceID, DstPtr, Size,
+          /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+      TracerInterfaceRAII TargetDataExchangeTraceRAII(
+          RegionInterface
+              .getTraceGenerators<ompt_target_data_transfer_from_device>(),
+          AsyncInfo, RTL->getProfiler(), /*TracedDeviceId=*/RTLDeviceID,
+          /*EventType=*/ompt_callback_target_data_op, RTLDeviceID, SrcPtr,
+          DstDev.RTLDeviceID, DstPtr, Size,
           /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
-  if (!AsyncInfo) {
-    return RTL->data_exchange(RTLDeviceID, SrcPtr, DstDev.RTLDeviceID, DstPtr,
-                              Size);
-  }
+
   return RTL->data_exchange_async(RTLDeviceID, SrcPtr, DstDev.RTLDeviceID,
                                   DstPtr, Size, AsyncInfo);
 }
diff --git a/offload/libomptarget/exports b/offload/libomptarget/exports
index 1831c43cc5f29..6e404ed059096 100644
--- a/offload/libomptarget/exports
+++ b/offload/libomptarget/exports
@@ -67,6 +67,14 @@ VERS1.0 {
     llvm_omp_target_unlock_mem;
     __tgt_set_info_flag;
     __tgt_print_device_info;
+    libomptarget_ompt_set_trace_ompt;
+    libomptarget_ompt_start_trace;
+    libomptarget_ompt_flush_trace;
+    libomptarget_ompt_stop_trace;
+    libomptarget_ompt_set_granted_teams;
+    libomptarget_ompt_set_timestamp;
+    libomptarget_ompt_advance_buffer_cursor;
+    libomptarget_ompt_get_record_type;
     omp_get_interop_ptr;
     omp_get_interop_str;
     omp_get_interop_int;
diff --git a/offload/libomptarget/interface.cpp b/offload/libomptarget/interface.cpp
index 5d7d948711b99..1e04d10dab8e4 100644
--- a/offload/libomptarget/interface.cpp
+++ b/offload/libomptarget/interface.cpp
@@ -14,6 +14,7 @@
 #include "OpenMP/OMPT/Interface.h"
 #include "OffloadPolicy.h"
 #include "OpenMP/OMPT/Callback.h"
+#include "OpenMP/OMPT/OmptCommonDefs.h"
 #include "OpenMP/omp.h"
 #include "PluginManager.h"
 #include "omptarget.h"
@@ -153,19 +154,30 @@ targetData(ident_t *Loc, int64_t DeviceId, int32_t ArgNum, void **ArgsBase,
   AsyncInfoTy &AsyncInfo = TargetAsyncInfo;
 
   /// RAII to establish tool anchors before and after data begin / end / update
-  OMPT_IF_BUILT(assert((TargetDataFunction == targetDataBegin ||
-                        TargetDataFunction == targetDataEnd ||
-                        TargetDataFunction == targetDataUpdate) &&
-                       "Encountered unexpected TargetDataFunction during "
-                       "execution of targetData");
-                auto CallbackFunctions =
-                    (TargetDataFunction == targetDataBegin)
-                        ? RegionInterface.getCallbacks<ompt_target_enter_data>()
-                    : (TargetDataFunction == targetDataEnd)
-                        ? RegionInterface.getCallbacks<ompt_target_exit_data>()
-                        : RegionInterface.getCallbacks<ompt_target_update>();
-                InterfaceRAII TargetDataRAII(CallbackFunctions, DeviceId,
-                                             OMPT_GET_RETURN_ADDRESS);)
+  OMPT_IF_BUILT(
+      assert((TargetDataFunction == targetDataBegin ||
+              TargetDataFunction == targetDataEnd ||
+              TargetDataFunction == targetDataUpdate) &&
+             "Encountered unexpected TargetDataFunction during "
+             "execution of targetData");
+      auto CallbackFunctions =
+          (TargetDataFunction == targetDataBegin)
+              ? RegionInterface.getCallbacks<ompt_target_enter_data>()
+          : (TargetDataFunction == targetDataEnd)
+              ? RegionInterface.getCallbacks<ompt_target_exit_data>()
+              : RegionInterface.getCallbacks<ompt_target_update>();
+
+      auto TraceGenerators =
+          (TargetDataFunction == targetDataBegin)
+              ? RegionInterface.getTraceGenerators<ompt_target_enter_data>()
+          : (TargetDataFunction == targetDataEnd)
+              ? RegionInterface.getTraceGenerators<ompt_target_exit_data>()
+              : RegionInterface.getTraceGenerators<ompt_target_update>();
+
+      InterfaceRAII TargetDataRAII(CallbackFunctions, DeviceId,
+                                   /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+      InterfaceRAII TargetDataTraceRAII(TraceGenerators, DeviceId,
+                                        /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
 
   int Rc = OFFLOAD_SUCCESS;
 
@@ -433,6 +445,9 @@ static inline int targetKernel(ident_t *Loc, int64_t DeviceId, int32_t NumTeams,
   /// RAII to establish tool anchors before and after target region
   OMPT_IF_BUILT(InterfaceRAII TargetRAII(
                     RegionInterface.getCallbacks<ompt_target>(), DeviceId,
+                    /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+                InterfaceRAII TargetTraceRAII(
+                    RegionInterface.getTraceGenerators<ompt_target>(), DeviceId,
                     /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
 
   int Rc = OFFLOAD_SUCCESS;
@@ -542,6 +557,9 @@ EXTERN int __tgt_target_kernel_replay(
   /// RAII to establish tool anchors before and after target region
   OMPT_IF_BUILT(InterfaceRAII TargetRAII(
                     RegionInterface.getCallbacks<ompt_target>(), DeviceId,
+                    /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);
+                InterfaceRAII TargetTraceRAII(
+                    RegionInterface.getTraceGenerators<ompt_target>(), DeviceId,
                     /*CodePtr=*/OMPT_GET_RETURN_ADDRESS);)
 
   AsyncInfoTy AsyncInfo(*DeviceOrErr);
diff --git a/offload/libomptarget/omptarget.cpp b/offload/libomptarget/omptarget.cpp
index 24169e68a29c7..187e7b5351d4b 100644
--- a/offload/libomptarget/omptarget.cpp
+++ b/offload/libomptarget/omptarget.cpp
@@ -2363,8 +2363,12 @@ int target(ident_t *Loc, DeviceTy &Device, void *HostPtr,
     // No need to guard this with OMPT_IF_BUILT
     InterfaceRAII TargetSubmitRAII(
         RegionInterface.getCallbacks<ompt_callback_target_submit>(), NumTeams);
-#endif
 
+    TracerInterfaceRAII TargetTraceRAII(
+        RegionInterface.getTraceGenerators<ompt_callback_target_submit>(),
+        AsyncInfo, Device.RTL->getProfiler(), /*TracedDeviceId=*/DeviceId,
+        /*EventType=*/ompt_callback_target_submit, DeviceId, NumTeams);
+#endif
     Ret = Device.launchKernel(TgtEntryPtr, TgtArgs.data(), TgtOffsets.data(),
                               KernelArgs, nullptr, AsyncInfo);
   }

>From a096d1d0aac52f225786ff23eced1c75a8b9354d Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Mon, 13 Apr 2026 06:26:35 -0500
Subject: [PATCH 08/12] [Offload][OMPT] Fix global destructor warnings in OMPT
 tracing globals

Replace five globals with non-trivial destructors with function-local
statics, following the LLVM canonical pattern (see Signals.cpp). This
resolves -Werror=global-constructors failures that were triggered by
the destructor of std::map, std::unordered_map, std::shared_ptr, and
a thread_local array of shared_ptr.

Affected globals:
- ParentLibrary (shared_ptr) in plugin OmptTracing.cpp -> file-local
  getParentLibraryStorage() helper
- Devices (map) in plugin OmptTracing.cpp -> getDevices()
- BufferManagementFns (unordered_map) in libomptarget OmptTracing.cpp
  -> file-local getBufferMgmt() helper
- TracedDevices (map) in libomptarget OmptTracing.cpp -> getTracedDevices()
- ArrayOfBufPtr (thread_local shared_ptr array) in OmptTracingBuffer.cpp
  -> getArrayOfBufPtr()

extern declarations are replaced with getter function declarations in
the corresponding headers (OmptDeviceTracing.h, OmptTracing.h,
OmptTracingBuffer.h). All use-sites are updated accordingly.

Made-with: Cursor
---
 offload/include/OpenMP/OMPT/OmptTracing.h     | 11 +--
 .../include/OpenMP/OMPT/OmptTracingBuffer.h   |  5 +-
 .../libomptarget/OpenMP/OMPT/OmptTracing.cpp  | 72 ++++++++++---------
 .../OpenMP/OMPT/OmptTracingBuffer.cpp         | 12 ++--
 .../common/OMPT/OmptDeviceTracing.h           | 20 +++---
 .../common/OMPT/OmptTracing.cpp               | 41 ++++++-----
 6 files changed, 85 insertions(+), 76 deletions(-)

diff --git a/offload/include/OpenMP/OMPT/OmptTracing.h b/offload/include/OpenMP/OMPT/OmptTracing.h
index 2a892582923d5..4c8c049a9ae48 100644
--- a/offload/include/OpenMP/OMPT/OmptTracing.h
+++ b/offload/include/OpenMP/OMPT/OmptTracing.h
@@ -102,8 +102,9 @@ ompt_set_result_t setTraceEventTy(int DeviceId, unsigned int Enable,
 /// Return thread id
 uint64_t getThreadId();
 
-/// See TracedDevices in OmptDeviceTracing.h
-extern std::map<int32_t, uint64_t> TracedDevices;
+/// Access the map of RTL device IDs to their currently enabled tracing event
+/// types. See also OmptDeviceTracing.h.
+std::map<int32_t, uint64_t> &getTracedDevices();
 /// Activate tracing on the given device
 void enableDeviceTracing(int DeviceId);
 /// Deactivate tracing on the given device
@@ -123,12 +124,6 @@ extern std::mutex TraceHashThreadMutex;
 /// buffer-completion callback functions.
 extern std::mutex BufferManagementFnMutex;
 
-/// Map from device-id to the corresponding buffer-request and buffer-completion
-/// callback functions.
-extern std::unordered_map<int, std::pair<ompt_callback_buffer_request_t,
-                                         ompt_callback_buffer_complete_t>>
-    BufferManagementFns;
-
 /// Thread local variables used by the plugin to communicate OMPT information
 /// that are then used to populate trace records. This method assumes a
 /// synchronous implementation, otherwise it won't work.
diff --git a/offload/include/OpenMP/OMPT/OmptTracingBuffer.h b/offload/include/OpenMP/OMPT/OmptTracingBuffer.h
index 5c9f4bf33dae8..a9b7f8d569fcb 100644
--- a/offload/include/OpenMP/OMPT/OmptTracingBuffer.h
+++ b/offload/include/OpenMP/OMPT/OmptTracingBuffer.h
@@ -133,8 +133,9 @@ class OmptTracingBufferMgr {
   // Thread-specific array of pointers to a buffer. The buffer pointed to
   // is the last one allocated by this thread for a given device. The ith
   // element points to the buffer for the ith device. At most MAX_NUM_DEVICES
-  // devices are supported.
-  static thread_local BufPtr ArrayOfBufPtr[MAX_NUM_DEVICES];
+  // devices are supported. Stored as a function-local thread_local to avoid
+  // a global destructor.
+  static BufPtr *getArrayOfBufPtr();
 
   /*
    * A buffer is flushed when it fills up or when the tool invokes
diff --git a/offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp b/offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp
index 77c771786f29b..ad4bb58ff1701 100644
--- a/offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp
+++ b/offload/libomptarget/OpenMP/OMPT/OmptTracing.cpp
@@ -42,17 +42,25 @@ std::mutex llvm::omp::target::ompt::TraceControlMutex;
 std::mutex llvm::omp::target::ompt::TraceHashThreadMutex;
 std::mutex llvm::omp::target::ompt::BufferManagementFnMutex;
 
-std::unordered_map<int /*DeviceId*/, std::pair<ompt_callback_buffer_request_t,
-                                               ompt_callback_buffer_complete_t>>
-    llvm::omp::target::ompt::BufferManagementFns;
-
 thread_local uint32_t llvm::omp::target::ompt::TraceRecordNumGrantedTeams = 0;
 thread_local uint64_t llvm::omp::target::ompt::TraceRecordStartTime = 0;
 thread_local uint64_t llvm::omp::target::ompt::TraceRecordStopTime = 0;
 thread_local uint64_t llvm::omp::target::ompt::ThreadId =
     std::numeric_limits<uint64_t>::max();
 
-std::map<int32_t, uint64_t> llvm::omp::target::ompt::TracedDevices;
+// File-local helper returning a reference to the buffer-management map stored
+// as a function-local static, avoiding a global destructor.
+static auto &getBufferMgmt() {
+  static std::unordered_map<int, std::pair<ompt_callback_buffer_request_t,
+                                           ompt_callback_buffer_complete_t>>
+      M;
+  return M;
+}
+
+std::map<int32_t, uint64_t> &llvm::omp::target::ompt::getTracedDevices() {
+  static std::map<int32_t, uint64_t> TracedDevices;
+  return TracedDevices;
+}
 
 bool llvm::omp::target::ompt::TracingActive = false;
 
@@ -61,8 +69,8 @@ void llvm::omp::target::ompt::resetTimestamp(uint64_t *T) { *T = 0; }
 ompt_callback_buffer_request_t
 llvm::omp::target::ompt::getBufferRequestFn(int DeviceId) {
   std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
-  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
-  if (BufferMgrItr == BufferManagementFns.end()) {
+  auto BufferMgrItr = getBufferMgmt().find(DeviceId);
+  if (BufferMgrItr == getBufferMgmt().end()) {
     return nullptr;
   }
   return BufferMgrItr->second.first;
@@ -71,8 +79,8 @@ llvm::omp::target::ompt::getBufferRequestFn(int DeviceId) {
 ompt_callback_buffer_complete_t
 llvm::omp::target::ompt::getBufferCompleteFn(int DeviceId) {
   std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
-  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
-  if (BufferMgrItr == BufferManagementFns.end()) {
+  auto BufferMgrItr = getBufferMgmt().find(DeviceId);
+  if (BufferMgrItr == getBufferMgmt().end()) {
     return nullptr;
   }
   return BufferMgrItr->second.second;
@@ -82,30 +90,30 @@ void llvm::omp::target::ompt::setBufferManagementFns(
     int DeviceId, ompt_callback_buffer_request_t ReqFn,
     ompt_callback_buffer_complete_t CmpltFn) {
   std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
-  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
-  if (BufferMgrItr != BufferManagementFns.end()) {
+  auto BufferMgrItr = getBufferMgmt().find(DeviceId);
+  if (BufferMgrItr != getBufferMgmt().end()) {
     REPORT()
         << "Buffer request and complete functions already exist for device  "
         << DeviceId << "ignoring ...";
     return;
   }
-  BufferManagementFns[DeviceId] = std::make_pair(ReqFn, CmpltFn);
+  getBufferMgmt()[DeviceId] = std::make_pair(ReqFn, CmpltFn);
 }
 
 void llvm::omp::target::ompt::removeBufferManagementFns(int DeviceId) {
   std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
-  auto BufferMgrItr = BufferManagementFns.find(DeviceId);
-  if (BufferMgrItr == BufferManagementFns.end()) {
+  auto BufferMgrItr = getBufferMgmt().find(DeviceId);
+  if (BufferMgrItr == getBufferMgmt().end()) {
     REPORT() << "Buffer request and complete functions don't exist for device  "
              << DeviceId << "ignoring ...";
     return;
   }
-  BufferManagementFns.erase(BufferMgrItr);
+  getBufferMgmt().erase(BufferMgrItr);
 }
 
 bool llvm::omp::target::ompt::isAllDeviceTracingStopped() {
   std::unique_lock<std::mutex> Lock(BufferManagementFnMutex);
-  return BufferManagementFns.empty();
+  return getBufferMgmt().empty();
 }
 
 void llvm::omp::target::ompt::ompt_callback_buffer_request(
@@ -138,11 +146,11 @@ inline bool checkDeviceTracingState(const uint64_t &TracingTypes) {
 
 void llvm::omp::target::ompt::enableDeviceTracing(int DeviceId) {
   std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
-  auto Device = TracedDevices.find(DeviceId);
-  if (Device == TracedDevices.end()) {
+  auto Device = getTracedDevices().find(DeviceId);
+  if (Device == getTracedDevices().end()) {
     uint64_t TracingTypes{0};
     setDeviceTracing(TracingTypes);
-    TracedDevices.emplace(DeviceId, TracingTypes);
+    getTracedDevices().emplace(DeviceId, TracingTypes);
   } else
     setDeviceTracing(Device->second);
   // In any case: at least one device is traced
@@ -151,16 +159,16 @@ void llvm::omp::target::ompt::enableDeviceTracing(int DeviceId) {
 
 void llvm::omp::target::ompt::disableDeviceTracing(int DeviceId) {
   std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
-  auto Device = TracedDevices.find(DeviceId);
-  if (Device == TracedDevices.end()) {
+  auto Device = getTracedDevices().find(DeviceId);
+  if (Device == getTracedDevices().end()) {
     uint64_t TracingTypes{0};
     resetDeviceTracing(TracingTypes);
-    TracedDevices.emplace(DeviceId, TracingTypes);
+    getTracedDevices().emplace(DeviceId, TracingTypes);
   } else
     resetDeviceTracing(Device->second);
 
   // Check for actively traced devices
-  for (auto &Dev : TracedDevices)
+  for (auto &Dev : getTracedDevices())
     if (checkDeviceTracingState(Dev.second))
       return;
 
@@ -176,8 +184,8 @@ bool llvm::omp::target::ompt::isTracingEnabled(int DeviceId,
 
 bool llvm::omp::target::ompt::isTracedDevice(int DeviceId) {
   std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
-  auto Device = TracedDevices.find(DeviceId);
-  if (Device != TracedDevices.end())
+  auto Device = getTracedDevices().find(DeviceId);
+  if (Device != getTracedDevices().end())
     return checkDeviceTracingState(Device->second);
 
   return false;
@@ -188,8 +196,8 @@ bool llvm::omp::target::ompt::isTracingTypeEnabled(int DeviceId,
   std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
   // Make sure we do not shift more than std::numeric_limits<uint64_t>::digits
   assert(EventTy < 64 && "Shift limit exceeded: EventTy must be less than 64");
-  auto Device = TracedDevices.find(DeviceId);
-  if (Device != TracedDevices.end() && EventTy < 64)
+  auto Device = getTracedDevices().find(DeviceId);
+  if (Device != getTracedDevices().end() && EventTy < 64)
     return (Device->second & (1UL << EventTy));
   return false;
 }
@@ -199,8 +207,8 @@ bool llvm::omp::target::ompt::isTracingTypeGroupEnabled(int DeviceId,
   std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
   // Make sure we do not shift more than std::numeric_limits<uint64_t>::digits
   assert(EventTy < 64 && "Shift limit exceeded: EventTy must be less than 64");
-  auto Device = TracedDevices.find(DeviceId);
-  if (Device != TracedDevices.end() && EventTy < 64) {
+  auto Device = getTracedDevices().find(DeviceId);
+  if (Device != getTracedDevices().end() && EventTy < 64) {
     auto TracedEvents = Device->second;
     switch (EventTy) {
     case ompt_callbacks_t::ompt_callback_target:
@@ -253,10 +261,10 @@ llvm::omp::target::ompt::setTraceEventTy(int DeviceId, unsigned int Enable,
                         << " Enable=" << Enable << " EventTy=" << EventTy;
 
   std::unique_lock<std::mutex> Lock(DeviceAccessMutex);
-  if (TracedDevices.find(DeviceId) == TracedDevices.end())
-    TracedDevices.emplace(DeviceId, 0UL);
+  if (getTracedDevices().find(DeviceId) == getTracedDevices().end())
+    getTracedDevices().emplace(DeviceId, 0UL);
 
-  auto &TracedEventTy = TracedDevices[DeviceId];
+  auto &TracedEventTy = getTracedDevices()[DeviceId];
   bool Enabled = Enable > 0;
   if (EventTy == 0) {
     // Set / reset all supported types
diff --git a/offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp b/offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp
index 5ac1974cea03b..92609e6e690be 100644
--- a/offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp
+++ b/offload/libomptarget/OpenMP/OMPT/OmptTracingBuffer.cpp
@@ -32,8 +32,10 @@ static std::atomic<uint64_t> BufId{0};
 // Unique id in buffer flush order
 static std::atomic<uint64_t> FlushId{0};
 
-thread_local OmptTracingBufferMgr::BufPtr
-    OmptTracingBufferMgr::ArrayOfBufPtr[MAX_NUM_DEVICES];
+OmptTracingBufferMgr::BufPtr *OmptTracingBufferMgr::getArrayOfBufPtr() {
+  thread_local BufPtr Array[MAX_NUM_DEVICES]{};
+  return Array;
+}
 
 static uint64_t get_and_inc_buf_id() { return BufId++; }
 
@@ -416,7 +418,7 @@ OmptTracingBufferMgr::getDeviceSpecificBuffer(int64_t DeviceId) {
              << " invalid or exceeds supported max: " << MAX_NUM_DEVICES - 1;
     return nullptr;
   }
-  return ArrayOfBufPtr[DeviceId];
+  return getArrayOfBufPtr()[DeviceId];
 }
 
 void OmptTracingBufferMgr::setDeviceSpecificBuffer(int64_t DeviceId,
@@ -426,7 +428,7 @@ void OmptTracingBufferMgr::setDeviceSpecificBuffer(int64_t DeviceId,
              << " invalid or exceeds supported max: " << MAX_NUM_DEVICES - 1;
     return;
   }
-  ArrayOfBufPtr[DeviceId] = Buf;
+  getArrayOfBufPtr()[DeviceId] = Buf;
 }
 
 void OmptTracingBufferMgr::setTRStatus(void *Rec, TRStatus Status) {
@@ -676,7 +678,7 @@ void OmptTracingBufferMgr::waitForFlushCompletion() {
 
 void OmptTracingBufferMgr::init() {
   for (int i = 0; i < MAX_NUM_DEVICES; ++i)
-    ArrayOfBufPtr[i] = nullptr;
+    getArrayOfBufPtr()[i] = nullptr;
   ThreadFlushTracker = 0;
   ThreadShutdownTracker = 0;
   DoneTracing = false; // TODO make it a class member
diff --git a/offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h b/offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h
index 4ffd1976e9730..f93686eaa1d13 100644
--- a/offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h
+++ b/offload/plugins-nextgen/common/OMPT/OmptDeviceTracing.h
@@ -86,19 +86,17 @@ extern double HostToDeviceSlope;
 /// Host to device constant clock offset
 extern double HostToDeviceOffset;
 
-/// Mapping of device pointers to their corresponding RTL device ID
-extern std::map<ompt_device_t *, int32_t> Devices;
+/// Access the map of device pointers to their corresponding RTL device ID
+std::map<ompt_device_t *, int32_t> &getDevices();
 
-/// Mapping of RTL device IDs to their currently enabled tracing event types.
-/// Note: Event type '0' (bit position) indicates if this device is traced.
-extern std::map<int32_t, uint64_t> TracedDevices;
+/// Access the map of RTL device IDs to their currently enabled tracing event
+/// types. Note: Event type '0' (bit position) indicates if this device is
+/// traced.
+std::map<int32_t, uint64_t> &getTracedDevices();
 
 /// OMPT global tracing status. Indicates if at least one device is traced.
 extern bool TracingActive;
 
-/// Parent library pointer
-extern std::shared_ptr<llvm::sys::DynamicLibrary> ParentLibrary;
-
 /// Get the parent library by pointer. If it is not already set, it will set the
 /// parent library pointer.
 std::shared_ptr<llvm::sys::DynamicLibrary> getParentLibrary();
@@ -112,10 +110,10 @@ void setParentLibrary(const char *Filename);
 template <typename FT>
 void ensureFuncPtrLoaded(const std::string &FuncName, FT *FuncPtr) {
   if (*FuncPtr == nullptr) {
-    if ((ParentLibrary == nullptr && getParentLibrary() == nullptr) ||
-        !ParentLibrary->isValid())
+    auto Lib = getParentLibrary();
+    if (!Lib || !Lib->isValid())
       return;
-    void *SymbolPtr = ParentLibrary->getAddressOfSymbol(FuncName.c_str());
+    void *SymbolPtr = Lib->getAddressOfSymbol(FuncName.c_str());
     if (SymbolPtr == nullptr)
       return;
     *FuncPtr = reinterpret_cast<FT>(SymbolPtr);
diff --git a/offload/plugins-nextgen/common/OMPT/OmptTracing.cpp b/offload/plugins-nextgen/common/OMPT/OmptTracing.cpp
index 7a8e312b3d476..637452e535695 100644
--- a/offload/plugins-nextgen/common/OMPT/OmptTracing.cpp
+++ b/offload/plugins-nextgen/common/OMPT/OmptTracing.cpp
@@ -46,39 +46,44 @@ std::mutex llvm::omp::target::ompt::DeviceIdWritingMutex;
 using namespace llvm::omp::target::ompt;
 using namespace llvm::omp::target::debug;
 
-std::shared_ptr<llvm::sys::DynamicLibrary>
-    llvm::omp::target::ompt::ParentLibrary(nullptr);
-
 double llvm::omp::target::ompt::HostToDeviceSlope = .0;
 double llvm::omp::target::ompt::HostToDeviceOffset = .0;
 
-std::map<ompt_device_t *, int32_t> llvm::omp::target::ompt::Devices;
+// File-local helper to hold the parent library shared_ptr as a function-local
+// static, avoiding a global destructor.
+static std::shared_ptr<llvm::sys::DynamicLibrary> &getParentLibraryStorage() {
+  static std::shared_ptr<llvm::sys::DynamicLibrary> Lib(nullptr);
+  return Lib;
+}
 
 std::shared_ptr<llvm::sys::DynamicLibrary>
 llvm::omp::target::ompt::getParentLibrary() {
-  static bool ParentLibraryAssigned = false;
-  if (!ParentLibraryAssigned) {
+  if (!getParentLibraryStorage()) {
     setParentLibrary("libomptarget.so");
-    ParentLibraryAssigned = true;
   }
-  return ParentLibrary;
+  return getParentLibraryStorage();
 }
 
 void llvm::omp::target::ompt::setParentLibrary(const char *Filename) {
-  if (ParentLibrary)
+  if (getParentLibraryStorage())
     return;
   std::string ErrorMsg;
-  ParentLibrary = std::make_shared<llvm::sys::DynamicLibrary>(
+  getParentLibraryStorage() = std::make_shared<llvm::sys::DynamicLibrary>(
       llvm::sys::DynamicLibrary::getPermanentLibrary(Filename, &ErrorMsg));
-  if ((ParentLibrary == nullptr) || (!ParentLibrary->isValid()))
+  if (!getParentLibraryStorage() || !getParentLibraryStorage()->isValid())
     REPORT() << "Failed to set parent library: " << ErrorMsg.c_str();
 }
 
+std::map<ompt_device_t *, int32_t> &llvm::omp::target::ompt::getDevices() {
+  static std::map<ompt_device_t *, int32_t> Devices;
+  return Devices;
+}
+
 int llvm::omp::target::ompt::getDeviceId(ompt_device_t *Device) {
   // Block other threads, which might trigger an erase (for the same device)
   std::unique_lock<std::mutex> Lock(DeviceIdWritingMutex);
-  auto DeviceIterator = Devices.find(Device);
-  if (Device == nullptr || DeviceIterator == Devices.end()) {
+  auto DeviceIterator = getDevices().find(Device);
+  if (Device == nullptr || DeviceIterator == getDevices().end()) {
     REPORT() << "Failed to get ID for Device=" << Device;
     return -1;
   }
@@ -93,8 +98,8 @@ void llvm::omp::target::ompt::setDeviceId(ompt_device_t *Device,
     return;
   }
   std::unique_lock<std::mutex> Lock(DeviceIdWritingMutex);
-  auto DeviceIterator = Devices.find(Device);
-  if (DeviceIterator != Devices.end()) {
+  auto DeviceIterator = getDevices().find(Device);
+  if (DeviceIterator != getDevices().end()) {
     auto CurrentDeviceId = DeviceIterator->second;
     if (DeviceId == CurrentDeviceId) {
       REPORT() << "Tried to duplicate OMPT Device= " << Device
@@ -105,7 +110,7 @@ void llvm::omp::target::ompt::setDeviceId(ompt_device_t *Device,
     }
     return;
   }
-  Devices.emplace(Device, DeviceId);
+  getDevices().emplace(Device, DeviceId);
 }
 
 void llvm::omp::target::ompt::removeDeviceId(ompt_device_t *Device) {
@@ -115,8 +120,8 @@ void llvm::omp::target::ompt::removeDeviceId(ompt_device_t *Device) {
     return;
   }
   std::unique_lock<std::mutex> Lock(DeviceIdWritingMutex);
-  Devices.erase(Device);
-  TracedDevices.erase(DeviceId);
+  getDevices().erase(Device);
+  getTracedDevices().erase(DeviceId);
 }
 
 OMPT_API_ROUTINE ompt_set_result_t ompt_set_trace_ompt(ompt_device_t *Device,

>From 3e1ad579baa24b4b2453684c9dada5ced435a969 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Wed, 5 Aug 2026 14:02:14 -0500
Subject: [PATCH 09/12] [Offload][OMPT] Route device callbacks through
 OmptProfilerTy

Compile and activate the complete OMPT profiler backend now that its tracing dependencies are available, and remove the superseded direct callback dispatch from PluginInterface in the same transition.

Co-Authored-By: Claude <noreply at anthropic.com>
---
 offload/libomptarget/CMakeLists.txt           | 16 ++++++
 offload/plugins-nextgen/common/CMakeLists.txt | 23 ++++++++
 .../common/include/PluginInterface.h          | 10 ----
 .../common/src/PluginInterface.cpp            | 53 -------------------
 4 files changed, 39 insertions(+), 63 deletions(-)

diff --git a/offload/libomptarget/CMakeLists.txt b/offload/libomptarget/CMakeLists.txt
index 1481f01497f92..7f2ff7329a536 100644
--- a/offload/libomptarget/CMakeLists.txt
+++ b/offload/libomptarget/CMakeLists.txt
@@ -49,6 +49,22 @@ endforeach()
 
 target_compile_options(omptarget PRIVATE ${offload_compile_flags})
 target_link_options(omptarget PRIVATE ${offload_link_flags})
+if (OMPT_TARGET_DEFAULT AND LIBOMPTARGET_OMPT_SUPPORT)
+  # OmptProfiler.cpp provides the strong getProfilerToAttach() that routes
+  # plugin device events through OmptProfilerTy. It must be compiled directly
+  # into omptarget so its strong definition unconditionally overrides the weak
+  # fallback in GenericProfiler.cpp (which is embedded into each plugin archive
+  # via the PluginCommon OBJECT library and would otherwise win the link).
+  target_sources(omptarget PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}/../plugins-nextgen/common/OMPT/OmptProfiler.cpp
+  )
+  target_include_directories(omptarget PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}/../plugins-nextgen/common/OMPT
+    ${CMAKE_CURRENT_SOURCE_DIR}/../plugins-nextgen/common/include
+  )
+  # PluginOmpt provides OmptTracing.cpp (plugin-side GPU tracing).
+  target_link_libraries(omptarget PRIVATE PluginOmpt)
+endif()
 
 # libomptarget.so needs to be aware of where the plugins live as they
 # are now separated in the build directory.
diff --git a/offload/plugins-nextgen/common/CMakeLists.txt b/offload/plugins-nextgen/common/CMakeLists.txt
index 556b8ddf3311c..c3dcb0f53d218 100644
--- a/offload/plugins-nextgen/common/CMakeLists.txt
+++ b/offload/plugins-nextgen/common/CMakeLists.txt
@@ -37,6 +37,29 @@ endif()
 include(FindLibcCommonUtils)
 target_link_libraries(PluginCommon PRIVATE llvm-libc-common-utilities)
 
+if (OMPT_TARGET_DEFAULT AND LIBOMPTARGET_OMPT_SUPPORT)
+  # OmptTracing.cpp calls plugin-specific symbols (HSA profiling etc.) and
+  # is linked per-plugin. OmptProfiler.cpp is added directly to omptarget
+  # in offload/libomptarget/CMakeLists.txt to ensure its strong
+  # getProfilerToAttach() override is unconditionally included.
+  add_library(PluginOmpt STATIC OMPT/OmptTracing.cpp)
+  target_include_directories(PluginOmpt PUBLIC
+    OMPT
+    ${CMAKE_CURRENT_SOURCE_DIR}/include
+    ${CMAKE_CURRENT_BINARY_DIR}/include
+    ${LIBOMPTARGET_LLVM_INCLUDE_DIRS}
+    ${LIBOMPTARGET_BINARY_INCLUDE_DIR}
+    ${LIBOMPTARGET_INCLUDE_DIR}
+  )
+  target_compile_options(PluginOmpt PUBLIC ${offload_compile_flags} -fPIC)
+  target_link_options(PluginOmpt PUBLIC ${offload_link_flags})
+  target_compile_definitions(PluginOmpt PRIVATE
+    TARGET_NAME="Profiler"
+    DEBUG_PREFIX="OMPT"
+  )
+  add_dependencies(PluginOmpt PluginErrcodes)
+endif()
+
 if (TARGET flang_rt.runtime.static)
   target_link_libraries(PluginCommon PRIVATE flang_rt.runtime.static)
   target_compile_definitions(PluginCommon PRIVATE OFFLOAD_HAS_FLANG_RT)
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index b6f19cd503137..ab2f3b2643648 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -1460,16 +1460,6 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
   /// This is used to run the RPC server during task synchronization.
   RPCServerTy *RPCServer;
 
-#ifdef OMPT_SUPPORT
-  /// OMPT callback functions
-#define defineOmptCallback(Name, Type, Code) Name##_t Name##_fn = nullptr;
-  FOREACH_OMPT_DEVICE_EVENT(defineOmptCallback)
-#undef defineOmptCallback
-
-  /// Internal representation for OMPT device (initialize & finalize)
-  std::atomic<bool> OmptInitialized;
-#endif
-
   /// The total per-block native shared memory that a kernel may use.
   size_t MaxBlockSharedMemSize = 0;
 };
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index d6f9d135a0091..f3fc96cce5a8d 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -21,11 +21,6 @@
 #include "Utils/ELF.h"
 #include "omptarget.h"
 
-#ifdef OMPT_SUPPORT
-#include "OpenMP/OMPT/Callback.h"
-#include "omp-tools.h"
-#endif
-
 #include "GenericProfiler.h"
 
 #include "llvm/Bitcode/BitcodeReader.h"
@@ -521,22 +516,6 @@ GenericDeviceTy::GenericDeviceTy(GenericPluginTy &Plugin, int32_t DeviceId,
   // vendor (u)uid will become available later.
   setDeviceUidFromVendorUid(std::to_string(static_cast<uint64_t>(DeviceId)));
 
-#ifdef OMPT_SUPPORT
-  OmptInitialized.store(false);
-  // Bind the callbacks to this device's member functions
-#define bindOmptCallback(Name, Type, Code)                                     \
-  if (ompt::Initialized && ompt::lookupCallbackByCode) {                       \
-    ompt::lookupCallbackByCode((ompt_callbacks_t)(Code),                       \
-                               ((ompt_callback_t *)&(Name##_fn)));             \
-    ODBG(OLDT_Tool) << "OMPT: class bound " << #Name << "="                    \
-                    << ((void *)(uint64_t)Name##_fn);                          \
-  }
-
-  FOREACH_OMPT_DEVICE_EVENT(bindOmptCallback);
-#undef bindOmptCallback
-
-#endif
-
   // Envar that indicates whether mapped host buffers should be locked
   // automatically. The possible values are boolean (on/off) and a special:
   //   off:       Mapped host buffers are not locked.
@@ -568,18 +547,6 @@ Error GenericDeviceTy::init(GenericPluginTy &Plugin) {
   if (auto Err = initImpl(Plugin))
     return Err;
 
-#ifdef OMPT_SUPPORT
-  if (ompt::Initialized) {
-    bool ExpectedStatus = false;
-    if (OmptInitialized.compare_exchange_strong(ExpectedStatus, true))
-      performOmptCallback(device_initialize, Plugin.getUserId(DeviceId),
-                          /*type=*/getComputeUnitKind().c_str(),
-                          /*device=*/reinterpret_cast<ompt_device_t *>(this),
-                          /*lookup=*/ompt::lookupCallbackByName,
-                          /*documentation=*/nullptr);
-  }
-#endif
-
   Plugin.getProfiler()->handleInit(this, &Plugin);
 
   // Read and reinitialize the envars that depend on the device initialization.
@@ -675,14 +642,6 @@ Error GenericDeviceTy::deinit(GenericPluginTy &Plugin) {
     RecordReplay = nullptr;
   }
 
-#ifdef OMPT_SUPPORT
-  if (ompt::Initialized) {
-    bool ExpectedStatus = true;
-    if (OmptInitialized.compare_exchange_strong(ExpectedStatus, false))
-      performOmptCallback(device_finalize, Plugin.getUserId(DeviceId));
-  }
-#endif
-
   Plugin.getProfiler()->handleDeinit(this, &Plugin);
 
   return deinitImpl();
@@ -730,18 +689,6 @@ Expected<DeviceImageTy *> GenericDeviceTy::loadBinary(GenericPluginTy &Plugin,
   if (auto Err = setupRPCServer(Plugin, *Image))
     return std::move(Err);
 
-#ifdef OMPT_SUPPORT
-  if (ompt::Initialized) {
-    size_t Bytes = InputTgtImage.size();
-    performOmptCallback(
-        device_load, Plugin.getUserId(DeviceId),
-        /*FileName=*/nullptr, /*FileOffset=*/0, /*VmaInFile=*/nullptr,
-        /*ImgSize=*/Bytes,
-        /*HostAddr=*/const_cast<unsigned char *>(InputTgtImage.bytes_begin()),
-        /*DeviceAddr=*/nullptr, /* FIXME: ModuleId */ 0);
-  }
-#endif
-
   Plugin.getProfiler()->handleLoadBinary(this, &Plugin, InputTgtImage);
 
   // Call any global constructors present on the device.

>From a08130e74014b6c959cfc096228cb1515d82d368 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 21 May 2026 08:43:39 -0500
Subject: [PATCH 10/12] [OpenMP] Add OMPT device tracing tests

---
 .../callbacks.h                               | 340 ++++++++++++++++++
 .../veccopy_ompt_selective_tracing_dataop.cpp |  66 ++++
 .../callbacks.h                               | 340 ++++++++++++++++++
 .../veccopy_ompt_selective_tracing_kernel.cpp |  70 ++++
 .../callbacks.h                               | 332 +++++++++++++++++
 .../veccopy-ompt-target-data-tracing-emi.cpp  | 146 ++++++++
 .../callbacks.h                               | 333 +++++++++++++++++
 .../veccopy-ompt-target-emi-tracing-dag.cpp   |  81 +++++
 .../callbacks.h                               | 332 +++++++++++++++++
 .../veccopy-ompt-target-emi-tracing.cpp       |  78 ++++
 .../callbacks.h                               | 332 +++++++++++++++++
 .../veccopy-ompt-target-emi-tracing.cpp       |  67 ++++
 .../callbacks.h                               | 332 +++++++++++++++++
 .../veccopy-ompt-target-emi-tracing.cpp       |  66 ++++
 .../veccopy-ompt-target-tracing/callbacks.h   | 292 +++++++++++++++
 .../veccopy-ompt-target-tracing.cpp           |  78 ++++
 16 files changed, 3285 insertions(+)
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/veccopy_ompt_selective_tracing_dataop.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/veccopy_ompt_selective_tracing_kernel.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/veccopy-ompt-target-data-tracing-emi.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/veccopy-ompt-target-emi-tracing-dag.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/veccopy-ompt-target-emi-tracing.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/veccopy-ompt-target-emi-tracing.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/veccopy-ompt-target-emi-tracing.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-tracing/veccopy-ompt-target-tracing.cpp

diff --git a/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/callbacks.h
new file mode 100644
index 0000000000000..5cd611990d32b
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/callbacks.h
@@ -0,0 +1,340 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+  // Only enable device tracing for dataop events explicitly
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d filename:%s host_adddr:%p device_addr:%p"
+         " bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+  assert(DeviceMapPtr != nullptr && "DeviceMapPtr must be valid");
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d optype=%d target_task_data=%p "
+         "(0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) src=%p "
+         "src_device_num=%d dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, optype, target_task_data, target_task_data_value,
+         target_data, target_data->value, host_op_id, *host_op_id, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d req_num_teams=%d target_data=%p "
+         "(0x%lx) host_op_id=%p (0x%lx)\n",
+         endpoint, requested_num_teams, target_data, target_data->value,
+         host_op_id, *host_op_id);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+#if EMI
+  register_ompt_callback(ompt_callback_target_data_op_emi);
+  register_ompt_callback(ompt_callback_target_emi);
+  register_ompt_callback(ompt_callback_target_submit_emi);
+#else
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+#endif
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/veccopy_ompt_selective_tracing_dataop.cpp b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/veccopy_ompt_selective_tracing_dataop.cpp
new file mode 100644
index 0000000000000..c07bff4230688
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-dataop/veccopy_ompt_selective_tracing_dataop.cpp
@@ -0,0 +1,66 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
+/// CHECK: rec={{0x[0-9a-fA-F]+}} type=9
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=10
+/// CHECK: rec={{0x[0-9a-fA-F]+}} type=9
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
+
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
+/// CHECK: rec={{0x[0-9a-fA-F]+}} type=9
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=10
+/// CHECK: rec={{0x[0-9a-fA-F]+}} type=9
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
diff --git a/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/callbacks.h
new file mode 100644
index 0000000000000..9655d718c7c48
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/callbacks.h
@@ -0,0 +1,340 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+  // Only enable device tracing for kernel submit events
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d filename:%s host_adddr:%p device_addr:%p"
+         " bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+  assert(DeviceMapPtr != nullptr && "DeviceMapPtr must be valid");
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d optype=%d target_task_data=%p "
+         "(0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) src=%p "
+         "src_device_num=%d dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, optype, target_task_data, target_task_data_value,
+         target_data, target_data->value, host_op_id, *host_op_id, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d req_num_teams=%d target_data=%p "
+         "(0x%lx) host_op_id=%p (0x%lx)\n",
+         endpoint, requested_num_teams, target_data, target_data->value,
+         host_op_id, *host_op_id);
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+#if EMI
+  register_ompt_callback(ompt_callback_target_data_op_emi);
+  register_ompt_callback(ompt_callback_target_emi);
+  register_ompt_callback(ompt_callback_target_submit_emi);
+#else
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+#endif
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/veccopy_ompt_selective_tracing_kernel.cpp b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/veccopy_ompt_selective_tracing_kernel.cpp
new file mode 100644
index 0000000000000..285f1f1011268
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-selective-tracing-kernel/veccopy_ompt_selective_tracing_kernel.cpp
@@ -0,0 +1,70 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// XFAIL: *
+// clang-format on
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  for (auto Device : *DeviceMapPtr)
+    flush_trace(Device);
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=9
+/// CHECK: rec={{0x[0-9a-fA-F]+}} type=10
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=9
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
+
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=9
+/// CHECK: rec={{0x[0-9a-fA-F]+}} type=10
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=8
+/// CHECK-NOT: rec={{0x[0-9a-fA-F]+}} type=9
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/callbacks.h
new file mode 100644
index 0000000000000..d05f05a4a4501
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/callbacks.h
@@ -0,0 +1,332 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+#define EMI 1
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d filename:%s host_adddr:%p device_addr:%p"
+         " bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d optype=%d target_task_data=%p "
+         "(0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) src=%p "
+         "src_device_num=%d dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, optype, target_task_data, target_task_data_value,
+         target_data, target_data->value, host_op_id, *host_op_id, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d req_num_teams=%d target_data=%p "
+         "(0x%lx) host_op_id=%p (0x%lx)\n",
+         endpoint, requested_num_teams, target_data, target_data->value,
+         host_op_id, *host_op_id);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+#if EMI
+  register_ompt_callback(ompt_callback_target_data_op_emi);
+  register_ompt_callback(ompt_callback_target_emi);
+  register_ompt_callback(ompt_callback_target_submit_emi);
+#else
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+#endif
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/veccopy-ompt-target-data-tracing-emi.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/veccopy-ompt-target-data-tracing-emi.cpp
new file mode 100644
index 0000000000000..35a38f4eba739
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-data-tracing-emi/veccopy-ompt-target-data-tracing-emi.cpp
@@ -0,0 +1,146 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+#define N 100000
+
+#pragma omp declare target
+int c[N];
+#pragma omp end declare target
+
+int main() {
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+  for (i = 0; i < N; i++)
+    c[i] = 0;
+
+#pragma omp target enter data map(to : a)
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+#pragma omp target exit data map(from : a)
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+#pragma omp target parallel for map(alloc : c)
+  {
+    for (int j = 0; j < N; j++)
+      c[j] = 2 * j + 1;
+  }
+#pragma omp target update from(c) nowait
+#pragma omp barrier
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+  int rc = 0;
+  for (i = 0; i < N; i++) {
+    if (a[i] != i) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+  }
+
+  for (i = 0; i < N; i++) {
+    if (c[i] != 2 * i + 1) {
+      rc++;
+      printf("Wrong value: c[%d]=%d\n", i, c[i]);
+    }
+  }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+// > OMPT device callback related checks below. <
+
+/// CHECK-NOT: Callback Target EMI:
+/// CHECK-NOT: device_num=-1
+
+/// CHECK: Callback Init:
+/// CHECK: Callback Load:
+
+/// CHECK-DAG: Callback Target EMI: kind=2 endpoint=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=2
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=2
+/// CHECK-DAG: Callback Target EMI: kind=2 endpoint=2
+
+/// CHECK-DAG: Callback Target EMI: kind=1 endpoint=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=2
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=2
+/// CHECK-DAG: Callback Submit EMI: endpoint=1 req_num_teams=1
+/// CHECK-DAG: Callback Submit EMI: endpoint=2 req_num_teams=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=3
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=3
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=4
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=4
+/// CHECK-DAG: Callback Target EMI: kind=1 endpoint=2
+/// CHECK-DAG: Callback Target EMI: kind=3 endpoint=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=3
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=3
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=4
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=4
+/// CHECK-DAG: Callback Target EMI: kind=3 endpoint=2
+
+/// CHECK-DAG: Callback Target EMI: kind=1 endpoint=1
+/// CHECK-DAG: Callback Submit EMI: endpoint=1 req_num_teams=1
+/// CHECK-DAG: Callback Submit EMI: endpoint=2 req_num_teams=1
+/// CHECK-DAG: Callback Target EMI: kind=1 endpoint=2
+/// CHECK-DAG: Callback Target EMI: kind=4 endpoint=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=3
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=3
+/// CHECK-DAG: Callback Target EMI: kind=4 endpoint=2
+
+/// CHECK-DAG: Success
+/// CHECK-DAG: Callback Fini:
+
+// > OMPT device tracing related checks below. <
+
+
+
+// Note: Split checks for record address and content. That way we do not imply
+//       any order. Records 01-06 and 12-17 occur interleaved and belong to the
+//       first target region. 07-11 occur interleaved with 18-22 and belong to
+//       the second target region.
+
+// Note: These addresses will only occur once. They are only captured to
+//       indicate their existence.
+
+/// CHECK-DAG: type=8 (Target task)
+/// CHECK-DAG: type=9 (Target data op)
+
+// Note: ADDRX_11 may not trigger a final callback.
+
+// Note: ADDRX_11 may not be deallocated.
+
+/// CHECK-NOT: host_op_id=0x0
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/callbacks.h
new file mode 100644
index 0000000000000..f5724fa7a6852
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/callbacks.h
@@ -0,0 +1,333 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// Enable / disable OMPT's 'External Monitoring Interface'
+#define EMI 1
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d filename:%s host_adddr:%p device_addr:%p"
+         " bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d optype=%d target_task_data=%p "
+         "(0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) src=%p "
+         "src_device_num=%d dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, optype, target_task_data, target_task_data_value,
+         target_data, target_data->value, host_op_id, *host_op_id, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d req_num_teams=%d target_data=%p "
+         "(0x%lx) host_op_id=%p (0x%lx)\n",
+         endpoint, requested_num_teams, target_data, target_data->value,
+         host_op_id, *host_op_id);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+#if EMI
+  register_ompt_callback(ompt_callback_target_data_op_emi);
+  register_ompt_callback(ompt_callback_target_emi);
+  register_ompt_callback(ompt_callback_target_submit_emi);
+#else
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+#endif
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/veccopy-ompt-target-emi-tracing-dag.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/veccopy-ompt-target-emi-tracing-dag.cpp
new file mode 100644
index 0000000000000..0fdae19017ab2
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-emi-tracing-dag/veccopy-ompt-target-emi-tracing-dag.cpp
@@ -0,0 +1,81 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+/// CHECK-NOT: host_op_id=0x0
+
+
+
+// Note: Split checks for record address and content. That way we do not imply
+//       any order. Records 01-06 and 12-17 occur interleaved and belong to the
+//       first target region. 07-11 occur interleaved with 18-22 and belong to
+//       the second target region.
+
+/// CHECK-DAG: type=8 (Target task)
+/// CHECK-DAG: type=9 (Target data op)
+
+// Note: These addresses will only occur once. They are only captured to
+//       indicate their existence.
+
+
+// Note: ADDRX_11 may not trigger a final callback.
+
+// Note: ADDRX_11 may not be deallocated.
+
+/// CHECK-DAG: Success
+
+/// CHECK-NOT: host_op_id=0x0
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/callbacks.h
new file mode 100644
index 0000000000000..d05f05a4a4501
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/callbacks.h
@@ -0,0 +1,332 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+#define EMI 1
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d filename:%s host_adddr:%p device_addr:%p"
+         " bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d optype=%d target_task_data=%p "
+         "(0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) src=%p "
+         "src_device_num=%d dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, optype, target_task_data, target_task_data_value,
+         target_data, target_data->value, host_op_id, *host_op_id, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d req_num_teams=%d target_data=%p "
+         "(0x%lx) host_op_id=%p (0x%lx)\n",
+         endpoint, requested_num_teams, target_data, target_data->value,
+         host_op_id, *host_op_id);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+#if EMI
+  register_ompt_callback(ompt_callback_target_data_op_emi);
+  register_ompt_callback(ompt_callback_target_emi);
+  register_ompt_callback(ompt_callback_target_submit_emi);
+#else
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+#endif
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/veccopy-ompt-target-emi-tracing.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/veccopy-ompt-target-emi-tracing.cpp
new file mode 100644
index 0000000000000..f9dfb47ffff43
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-api-use/veccopy-ompt-target-emi-tracing.cpp
@@ -0,0 +1,78 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-generic && env LIBOMPTARGET_OMPT_FLUSH_ON_SHUTDOWN=false LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL=false %libomptarget-run-generic | %fcheck-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+/*
+ * This test is run with LIBOMPTARGET_OMPT_FLUSH_ON_SHUTDOWN=false and
+ * LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL=false while doing explicit flushing
+ * using ompt_flush_trace. The intention is to check whether trace records are
+ * properly flushed when the program/tool uses the ompt_flush_trace API. There
+ * should be 23 trace records returned to the tool.
+ */
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+/// CHECK-NOT: host_op_id=0x0
+
+/// CHECK: rec=
+
+/// CHECK-DAG: Success
+
+/// The user calls flush before printing success, so
+/// no more records should be returned here.
+
+/// CHECK-NOT: host_op_id=0x0
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/callbacks.h
new file mode 100644
index 0000000000000..d05f05a4a4501
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/callbacks.h
@@ -0,0 +1,332 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+#define EMI 1
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d filename:%s host_adddr:%p device_addr:%p"
+         " bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d optype=%d target_task_data=%p "
+         "(0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) src=%p "
+         "src_device_num=%d dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, optype, target_task_data, target_task_data_value,
+         target_data, target_data->value, host_op_id, *host_op_id, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d req_num_teams=%d target_data=%p "
+         "(0x%lx) host_op_id=%p (0x%lx)\n",
+         endpoint, requested_num_teams, target_data, target_data->value,
+         host_op_id, *host_op_id);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+#if EMI
+  register_ompt_callback(ompt_callback_target_data_op_emi);
+  register_ompt_callback(ompt_callback_target_emi);
+  register_ompt_callback(ompt_callback_target_submit_emi);
+#else
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+#endif
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/veccopy-ompt-target-emi-tracing.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/veccopy-ompt-target-emi-tracing.cpp
new file mode 100644
index 0000000000000..abf2426d2eee9
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-buffer-full/veccopy-ompt-target-emi-tracing.cpp
@@ -0,0 +1,67 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-generic && env LIBOMPTARGET_OMPT_FLUSH_ON_SHUTDOWN=false %libomptarget-run-generic | %fcheck-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+/*
+ * This test is run with LIBOMPTARGET_OMPT_FLUSH_ON_SHUTDOWN=false and
+ * ompt_flush_trace is not invoked by the user/tool. The intention is to check
+ * whether trace records are properly flushed as a buffer fills up. Currently,
+ * the buffer size in callbacks.h implies that every buffer holds 2 trace
+ * records. With this assumption, there should be 22 trace records. The last
+ * trace record is not flushed because the last buffer is not yet full.
+ */
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+/// CHECK-NOT: host_op_id=0x0
+
+/// CHECK: rec=
+/// CHECK-NOT: host_op_id=0x0
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/callbacks.h
new file mode 100644
index 0000000000000..d05f05a4a4501
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/callbacks.h
@@ -0,0 +1,332 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+#define EMI 1
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d filename:%s host_adddr:%p device_addr:%p"
+         " bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d optype=%d target_task_data=%p "
+         "(0x%lx) target_data=%p (0x%lx) host_op_id=%p (0x%lx) src=%p "
+         "src_device_num=%d dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, optype, target_task_data, target_task_data_value,
+         target_data, target_data->value, host_op_id, *host_op_id, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d req_num_teams=%d target_data=%p "
+         "(0x%lx) host_op_id=%p (0x%lx)\n",
+         endpoint, requested_num_teams, target_data, target_data->value,
+         host_op_id, *host_op_id);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+#if EMI
+  register_ompt_callback(ompt_callback_target_data_op_emi);
+  register_ompt_callback(ompt_callback_target_emi);
+  register_ompt_callback(ompt_callback_target_submit_emi);
+#else
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+#endif
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/veccopy-ompt-target-emi-tracing.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/veccopy-ompt-target-emi-tracing.cpp
new file mode 100644
index 0000000000000..7f4e99b83e4e3
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing-flush-only-on-shutdown/veccopy-ompt-target-emi-tracing.cpp
@@ -0,0 +1,66 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-generic && env LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL=false %libomptarget-run-generic | %fcheck-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+/*
+ * This test is run with LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL=false and
+ * ompt_flush_trace is not invoked by the user/tool. The intention is to check
+ * whether trace records are properly flushed on shutdown. 23 trace records
+ * should be flushed during shutdown.
+ */
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+/// CHECK-NOT: host_op_id=0x0
+
+/// CHECK: rec=
+
+/// CHECK-NOT: host_op_id=0x0
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-tracing/callbacks.h
new file mode 100644
index 0000000000000..cfb5fd4147ebf
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing/callbacks.h
@@ -0,0 +1,292 @@
+// Copyright © Advanced Micro Devices, Inc., or its affiliates.
+//
+// SPDX-License-Identifier:  MIT
+
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// Enable / disable OMPT's 'External Monitoring Interface'
+#define EMI 0
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE (4 * 1024)
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+static ompt_get_record_type_t ompt_get_record_type_fn = 0;
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+// Note: This callback must handle a null begin cursor. Currently,
+// ompt_get_record_ompt, print_record_ompt, and
+// ompt_advance_buffer_cursor handle a null cursor.
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+
+    if (ompt_get_record_type_fn(buffer, current) != ompt_record_ompt) {
+      printf("WARNING: received non-ompt type buffer object\n");
+    }
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n", device_num,
+         type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  ompt_get_record_type_fn =
+      (ompt_get_record_type_t)lookup("ompt_get_record_type");
+  if (!ompt_get_record_type_fn) {
+    printf("WARNING: No function ompt_get_record_type found in device "
+           "callbacks\n");
+  }
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Load: device_num:%d filename:%s host_adddr:%p device_addr:%p "
+         "bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: target_id=%lu host_op_id=%lu optype=%d src=%p "
+         "src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         target_id, host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: target_id=%lu kind=%d endpoint=%d device_num=%d "
+         "code=%p\n",
+         target_id, kind, endpoint, device_num, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-tracing/veccopy-ompt-target-tracing.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-tracing/veccopy-ompt-target-tracing.cpp
new file mode 100644
index 0000000000000..d393bc63b2e14
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-tracing/veccopy-ompt-target-tracing.cpp
@@ -0,0 +1,78 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+// Copyright © Advanced Micro Devices, Inc., or its affiliates.
+//
+// SPDX-License-Identifier:  MIT
+
+#include <assert.h>
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+// > OMPT device tracing related checks below. <
+
+// Note: This test will allocate one buffer, big enough to hold all trace
+//       records, hence there will be only one allocation.
+
+
+// Note: Split checks for record address and content. That way we do not imply
+//       any order. Records may / will occur interleaved.
+
+// Note: These addresses will only occur once. They are only captured to
+//       indicate their existence.
+
+/// CHECK-DAG: type=8 (Target task)
+/// CHECK-DAG: type=9 (Target data op)
+
+// Note: ADDRX_01 may not trigger a final callback.
+// Note: ADDRX_01 may not be deallocated.
+
+/// CHECK-NOT: host_op_id=0x0

>From edb41205e384bffa47df5db3b30b0025d2a8b7b0 Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 21 May 2026 08:44:24 -0500
Subject: [PATCH 11/12] [Openmp] Add OMPT device tracing support tests

These tests concern functionality used by parts of the OMPT device
tracing implementation.
---
 .../veccopy-ompt-ctor-1/Inputs/callbacks.h    | 322 +++++++++++++++++
 .../Inputs/veccopy-ctor-1.cpp                 |  35 ++
 .../tracing/veccopy-ompt-ctor-1/veccopy.cpp   |  51 +++
 .../callbacks.h                               | 286 ++++++++++++++++
 .../veccopy-ompt-target-default-device.cpp    |  87 +++++
 .../veccopy-ompt-target-devices/callbacks.h   | 324 ++++++++++++++++++
 .../veccopy-ompt-target-devices.cpp           |  87 +++++
 .../callbacks.h                               | 136 ++++++++
 .../veccopy-ompt-target.c                     |  77 +++++
 .../callbacks.h                               | 131 +++++++
 .../veccopy-ompt-target.c                     |  77 +++++
 11 files changed, 1613 insertions(+)
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/veccopy-ctor-1.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-ctor-1/veccopy.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-default-device/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-default-device/veccopy-ompt-target-default-device.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-devices/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-devices/veccopy-ompt-target-devices.cpp
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-translate-time/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-translate-time/veccopy-ompt-target.c
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/callbacks.h
 create mode 100644 offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/veccopy-ompt-target.c

diff --git a/offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/callbacks.h
new file mode 100644
index 0000000000000..78bd4fe39b87b
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/callbacks.h
@@ -0,0 +1,322 @@
+#include <assert.h>
+#include <memory>
+#include <unordered_set>
+
+#define EMI 1
+
+// Tool related code below
+#include <omp-tools.h>
+
+ompt_id_t next_op_id = 0x8000000000000001;
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  printf("rec=%p type=%2d time=%lu thread_id=%lu target_id=%lu\n", rec,
+         rec->type, rec->time, rec->thread_id, rec->target_id);
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("\tRecord Target task: target_id=0x%lx kind=%d endpoint=%d "
+           "device=%d task_id=%lu codeptr=%p\n",
+           target_rec.target_id, target_rec.kind, target_rec.endpoint,
+           target_rec.device_num, target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("\t  Record Target data op: target_id=0x%lx host_op_id=0x%lx "
+           "optype=%d src_addr=%p src_device=%d "
+           "dest_addr=%p dest_device=%d bytes=%lu end_time=%lu duration=%lu ns "
+           "codeptr=%p\n",
+           rec->target_id, target_data_op_rec.host_op_id,
+           target_data_op_rec.optype, target_data_op_rec.src_addr,
+           target_data_op_rec.src_device_num, target_data_op_rec.dest_addr,
+           target_data_op_rec.dest_device_num, target_data_op_rec.bytes,
+           target_data_op_rec.end_time, target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("\t  Record Target kernel:  target_id=0x%lx host_op_id=0x%lx "
+           "requested_num_teams=%u granted_num_teams=%u "
+           "end_time=%lu duration=%lu ns\n",
+           rec->target_id, target_kernel_rec.host_op_id,
+           target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback\n", *bytes,
+         *buffer);
+}
+
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: %d %p %lu %p %d\n", device_num,
+         buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+#if EMI
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op_emi);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit_emi);
+#else
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, 1, ompt_callback_target_submit);
+#endif
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n", device_num,
+         type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  // In many scenarios, this will be a good place to start the
+  // trace. If start_trace is called from the main program before this
+  // callback is dispatched, the start_trace handle will be null. This
+  // is because this device_init callback is invoked during the first
+  // target construct implementation.
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Load: device_num:%d filename:%s host_adddr:%p device_addr:%p "
+         "bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: host_op_id=%lu optype=%d src=%p src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target_data_op_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_task_data,
+    ompt_data_t *target_data, ompt_id_t *host_op_id,
+    ompt_target_data_op_t optype, void *src_addr, int src_device_num,
+    void *dest_addr, int dest_device_num, size_t bytes,
+    const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    *host_op_id = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("  Callback DataOp EMI: endpoint=%d target_task_data=%p (0x%lx) "
+         "target_data=%p (0x%lx) host_op_id=%p (0x%lx) optype=%d src=%p "
+         "src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         endpoint, target_task_data, target_task_data_value, target_data,
+         target_data->value, host_op_id, *host_op_id, optype, src_addr,
+         src_device_num, dest_addr, dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: kind=%d endpoint=%d device_num=%d target_id=%lu "
+         "code=%p\n",
+         kind, endpoint, device_num, target_id, codeptr_ra);
+}
+
+static void on_ompt_callback_target_emi(ompt_target_t kind,
+                                        ompt_scope_endpoint_t endpoint,
+                                        int device_num, ompt_data_t *task_data,
+                                        ompt_data_t *target_task_data,
+                                        ompt_data_t *target_data,
+                                        const void *codeptr_ra) {
+  if (endpoint == ompt_scope_begin)
+    target_data->value = next_op_id++;
+  // target_task_data may be null, avoid dereferencing it
+  uint64_t target_task_data_value =
+      (target_task_data) ? target_task_data->value : 0;
+  printf("Callback Target EMI: kind=%d endpoint=%d device_num=%d task_data=%p "
+         "(0x%lx) target_task_data=%p (0x%lx) target_data=%p (0x%lx) code=%p\n",
+         kind, endpoint, device_num, task_data, task_data->value,
+         target_task_data, target_task_data_value, target_data,
+         target_data->value, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+static void on_ompt_callback_target_submit_emi(
+    ompt_scope_endpoint_t endpoint, ompt_data_t *target_data,
+    ompt_id_t *host_op_id, unsigned int requested_num_teams) {
+  printf("  Callback Submit EMI: endpoint=%d target_data=%p (0x%lx) "
+         "host_op_id=%p (0x%lx) req_num_teams=%d\n",
+         endpoint, target_data, target_data->value, host_op_id, *host_op_id,
+         requested_num_teams);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  ompt_set_callback(ompt_callback_device_initialize,
+                    (ompt_callback_t)&on_ompt_callback_device_initialize);
+  ompt_set_callback(ompt_callback_device_load,
+                    (ompt_callback_t)&on_ompt_callback_device_load);
+#if EMI
+  ompt_set_callback(ompt_callback_target_submit_emi,
+                    (ompt_callback_t)&on_ompt_callback_target_submit_emi);
+  ompt_set_callback(ompt_callback_target_data_op_emi,
+                    (ompt_callback_t)&on_ompt_callback_target_data_op_emi);
+  ompt_set_callback(ompt_callback_target_emi,
+                    (ompt_callback_t)&on_ompt_callback_target_emi);
+#else
+  ompt_set_callback(ompt_callback_target_submit,
+                    (ompt_callback_t)&on_ompt_callback_target_submit);
+  ompt_set_callback(ompt_callback_target_data_op,
+                    (ompt_callback_t)&on_ompt_callback_target_data_op);
+  ompt_set_callback(ompt_callback_target,
+                    (ompt_callback_t)&on_ompt_callback_target);
+#endif
+  return 1; //success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/veccopy-ctor-1.cpp b/offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/veccopy-ctor-1.cpp
new file mode 100644
index 0000000000000..137a60735f345
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-ctor-1/Inputs/veccopy-ctor-1.cpp
@@ -0,0 +1,35 @@
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int status;
+
+__attribute__((constructor(101))) void veccopy_init() {
+  int N = 10;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  if (a[5] != b[5])
+    status = 1;
+  else
+    status = 0;
+}
diff --git a/offload/test/ompt/tracing/veccopy-ompt-ctor-1/veccopy.cpp b/offload/test/ompt/tracing/veccopy-ompt-ctor-1/veccopy.cpp
new file mode 100644
index 0000000000000..76af6a9b6c33f
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-ctor-1/veccopy.cpp
@@ -0,0 +1,51 @@
+// clang-format off
+// RUN: %clangxx-generic %s %S/Inputs/veccopy-ctor-1.cpp -o %t -Xoffload-linker -lompdevice && %libomptarget-run-generic | %fcheck-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+/*
+ * Test whether OMPT support works for OpenMP directives
+ * in a global constructor without using a shared library.
+ * This currently succeeds.
+ */
+
+#include <omp.h>
+#include <stdio.h>
+
+extern int status;
+
+int main() {
+  int N = 10;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong varlue: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc && !status)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+/// CHECK: Record Target task
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-default-device/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-default-device/callbacks.h
new file mode 100644
index 0000000000000..5df9985ced0c3
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-default-device/callbacks.h
@@ -0,0 +1,286 @@
+#include <cassert>
+#include <memory>
+#include <unordered_set>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// Enable / disable OMPT's 'External Monitoring Interface'
+#define EMI 0
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+static ompt_get_record_type_t ompt_get_record_type_fn = 0;
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request_default_device(
+    int device_num, ompt_buffer_t **buffer, size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback for default "
+         "device %d\n",
+         *bytes, *buffer, device_num);
+}
+
+// Note: This callback must handle a null begin cursor. Currently,
+// ompt_get_record_ompt, print_record_ompt, and
+// ompt_advance_buffer_cursor handle a null cursor.
+static void on_ompt_callback_buffer_complete_default_device(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback for default device: \
+  device_num=%d, buffer=%p, num_bytes=%lu, begin=%p, buffer_owned=%d\n",
+         device_num, buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+
+    if (ompt_get_record_type_fn(buffer, current) != ompt_record_ompt) {
+      printf("WARNING: received non-ompt type buffer object\n");
+    }
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+  ompt_set_trace_ompt(Device, /*enable=*/1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, /*enable=*/1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, /*enable=*/1, ompt_callback_target_submit);
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+  if (device_num != omp_get_default_device())
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  return ompt_start_trace(Device,
+                          &on_ompt_callback_buffer_request_default_device,
+                          &on_ompt_callback_buffer_complete_default_device);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n", device_num,
+         type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  ompt_get_record_type_fn =
+      (ompt_get_record_type_t)lookup("ompt_get_record_type");
+  if (!ompt_get_record_type_fn) {
+    printf("WARNING: No function ompt_get_record_type found in device "
+           "callbacks\n");
+  }
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Load: device_num:%d filename:%s host_adddr:%p device_addr:%p "
+         "bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: target_id=%lu host_op_id=%lu optype=%d src=%p "
+         "src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         target_id, host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: target_id=%lu kind=%d endpoint=%d device_num=%d "
+         "code=%p\n",
+         target_id, kind, endpoint, device_num, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-default-device/veccopy-ompt-target-default-device.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-default-device/veccopy-ompt-target-default-device.cpp
new file mode 100644
index 0000000000000..6446cda7e7fbb
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-default-device/veccopy-ompt-target-default-device.cpp
@@ -0,0 +1,87 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+#include <cassert>
+#include <omp.h>
+#include <stdio.h>
+
+// This test starts device tracing on the default device only (see
+// start_trace in callbacks.h). However, if more devices are
+// available, it calls flush and stop on the other devices as
+// well. The intention is to check correct runtime behavior if a tool
+// invokes flush or stop on a device that was not started. The runtime
+// should just return without doing anything.
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+  // Warm up
+#pragma omp target
+  {
+  }
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+  for (int dev = 0; dev < omp_get_num_devices(); ++dev) {
+#pragma omp target teams distribute parallel for device(dev)
+    {
+      for (int j = 0; j < N; j++)
+        a[j] = b[j];
+    }
+  }
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+// > OMPT device tracing related checks below. <
+
+
+
+// Note: Split checks for record address and content. That way we do not imply
+//       any order. Records may / will occur interleaved.
+
+// Note: These addresses will only occur once. They are only captured to
+//       indicate their existence.
+
+/// CHECK-DAG: type=8 (Target task)
+/// CHECK-DAG: type=9 (Target data op)
+
+// Note: ADDRX_07 may not trigger a final callback.
+
+// Note: ADDRX_07 may not be deallocated.
+
+/// CHECK-NOT: host_op_id=0x0
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-devices/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-devices/callbacks.h
new file mode 100644
index 0000000000000..0625b1a7b1522
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-devices/callbacks.h
@@ -0,0 +1,324 @@
+#include <cassert>
+#include <memory>
+#include <unordered_set>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// Enable / disable OMPT's 'External Monitoring Interface'
+#define EMI 0
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+#define OMPT_BUFFER_REQUEST_SIZE 256
+
+// Map of devices traced
+typedef std::unordered_set<ompt_device_t *> DeviceMap_t;
+typedef std::unique_ptr<DeviceMap_t> DeviceMapPtr_t;
+extern DeviceMapPtr_t DeviceMapPtr;
+
+// Utilities
+static void print_record_ompt(ompt_record_ompt_t *rec) {
+  if (rec == NULL)
+    return;
+
+  switch (rec->type) {
+  case ompt_callback_target:
+  case ompt_callback_target_emi: {
+    ompt_record_target_t target_rec = rec->record.target;
+    printf("rec=%p type=%d (Target task) time=%lu thread_id=%lu "
+           "target_id=0x%lx kind=%d endpoint=%d device=%d task_id=%lu "
+           "codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_rec.kind, target_rec.endpoint, target_rec.device_num,
+           target_rec.task_id, target_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_data_op:
+  case ompt_callback_target_data_op_emi: {
+    ompt_record_target_data_op_t target_data_op_rec =
+        rec->record.target_data_op;
+    printf("rec=%p type=%d (Target data op) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx optype=%d "
+           "src_addr=%p src_device=%d dest_addr=%p dest_device=%d bytes=%lu "
+           "end_time=%lu duration=%lu ns codeptr=%p\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_data_op_rec.host_op_id, target_data_op_rec.optype,
+           target_data_op_rec.src_addr, target_data_op_rec.src_device_num,
+           target_data_op_rec.dest_addr, target_data_op_rec.dest_device_num,
+           target_data_op_rec.bytes, target_data_op_rec.end_time,
+           target_data_op_rec.end_time - rec->time,
+           target_data_op_rec.codeptr_ra);
+    break;
+  }
+  case ompt_callback_target_submit:
+  case ompt_callback_target_submit_emi: {
+    ompt_record_target_kernel_t target_kernel_rec = rec->record.target_kernel;
+    printf("rec=%p type=%d (Target kernel) time=%lu thread_id=%lu "
+           "target_id=0x%lx host_op_id=0x%lx requested_num_teams=%u "
+           "granted_num_teams=%u end_time=%lu duration=%lu ns\n",
+           rec, rec->type, rec->time, rec->thread_id, rec->target_id,
+           target_kernel_rec.host_op_id, target_kernel_rec.requested_num_teams,
+           target_kernel_rec.granted_num_teams, target_kernel_rec.end_time,
+           target_kernel_rec.end_time - rec->time);
+    break;
+  }
+  default:
+    assert(0);
+    break;
+  }
+}
+
+static void delete_buffer_ompt(ompt_buffer_t *buffer) {
+  free(buffer);
+  printf("Deallocated %p\n", buffer);
+}
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_set_trace_ompt_t ompt_set_trace_ompt = 0;
+static ompt_start_trace_t ompt_start_trace = 0;
+static ompt_flush_trace_t ompt_flush_trace = 0;
+static ompt_stop_trace_t ompt_stop_trace = 0;
+static ompt_get_record_ompt_t ompt_get_record_ompt = 0;
+static ompt_advance_buffer_cursor_t ompt_advance_buffer_cursor = 0;
+static ompt_get_record_type_t ompt_get_record_type_fn = 0;
+// OMPT callbacks
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request_default_device(
+    int device_num, ompt_buffer_t **buffer, size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback for default "
+         "device %d\n",
+         *bytes, *buffer, device_num);
+}
+
+// Trace record callbacks
+static void on_ompt_callback_buffer_request(int device_num,
+                                            ompt_buffer_t **buffer,
+                                            size_t *bytes) {
+  *bytes = OMPT_BUFFER_REQUEST_SIZE;
+  *buffer = malloc(*bytes);
+  printf("Allocated %lu bytes at %p in buffer request callback for device %d\n",
+         *bytes, *buffer, device_num);
+}
+
+// Note: This callback must handle a null begin cursor. Currently,
+// ompt_get_record_ompt, print_record_ompt, and
+// ompt_advance_buffer_cursor handle a null cursor.
+static void on_ompt_callback_buffer_complete_default_device(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback for default device: \
+  device_num=%d, buffer=%p, num_bytes=%lu, begin=%p, buffer_owned=%d\n",
+         device_num, buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+
+    if (ompt_get_record_type_fn(buffer, current) != ompt_record_ompt) {
+      printf("WARNING: received non-ompt type buffer object\n");
+    }
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+// Note: This callback must handle a null begin cursor. Currently,
+// ompt_get_record_ompt, print_record_ompt, and
+// ompt_advance_buffer_cursor handle a null cursor.
+static void on_ompt_callback_buffer_complete(
+    int device_num, ompt_buffer_t *buffer,
+    size_t bytes, /* bytes returned in this callback */
+    ompt_buffer_cursor_t begin, int buffer_owned) {
+  printf("Executing buffer complete callback: \
+  device_num=%d, buffer=%p, num_bytes=%lu, begin=%p, buffer_owned=%d\n",
+         device_num, buffer, bytes, (void *)begin, buffer_owned);
+
+  int status = 1;
+  ompt_buffer_cursor_t current = begin;
+  while (status) {
+    ompt_record_ompt_t *rec = ompt_get_record_ompt(buffer, current);
+
+    if (ompt_get_record_type_fn(buffer, current) != ompt_record_ompt) {
+      printf("WARNING: received non-ompt type buffer object\n");
+    }
+    print_record_ompt(rec);
+    status = ompt_advance_buffer_cursor(NULL, /* TODO device */
+                                        buffer, bytes, current, &current);
+  }
+  if (buffer_owned)
+    delete_buffer_ompt(buffer);
+}
+
+static ompt_set_result_t set_trace_ompt(ompt_device_t *Device) {
+  if (!ompt_set_trace_ompt)
+    return ompt_set_error;
+
+  ompt_set_trace_ompt(Device, /*enable=*/1, ompt_callback_target);
+  ompt_set_trace_ompt(Device, /*enable=*/1, ompt_callback_target_data_op);
+  ompt_set_trace_ompt(Device, /*enable=*/1, ompt_callback_target_submit);
+
+  return ompt_set_always;
+}
+
+static int start_trace(int device_num, ompt_device_t *Device) {
+  if (!ompt_start_trace)
+    return 0;
+
+  // This device will be traced.
+  assert(DeviceMapPtr->find(Device) == DeviceMapPtr->end() &&
+         "Device already present in the map");
+  DeviceMapPtr->insert(Device);
+
+  if (device_num == omp_get_default_device())
+    return ompt_start_trace(Device,
+                            &on_ompt_callback_buffer_request_default_device,
+                            &on_ompt_callback_buffer_complete_default_device);
+  return ompt_start_trace(Device, &on_ompt_callback_buffer_request,
+                          &on_ompt_callback_buffer_complete);
+}
+
+static int flush_trace(ompt_device_t *Device) {
+  if (!ompt_flush_trace)
+    return 0;
+  return ompt_flush_trace(Device);
+}
+
+static int stop_trace(ompt_device_t *Device) {
+  if (!ompt_stop_trace)
+    return 0;
+  return ompt_stop_trace(Device);
+}
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n", device_num,
+         type, device, lookup, documentation);
+  if (!lookup) {
+    printf("Trace collection disabled on device %d\n", device_num);
+    return;
+  }
+
+  ompt_set_trace_ompt = (ompt_set_trace_ompt_t)lookup("ompt_set_trace_ompt");
+  ompt_start_trace = (ompt_start_trace_t)lookup("ompt_start_trace");
+  ompt_flush_trace = (ompt_flush_trace_t)lookup("ompt_flush_trace");
+  ompt_stop_trace = (ompt_stop_trace_t)lookup("ompt_stop_trace");
+  ompt_get_record_ompt = (ompt_get_record_ompt_t)lookup("ompt_get_record_ompt");
+  ompt_advance_buffer_cursor =
+      (ompt_advance_buffer_cursor_t)lookup("ompt_advance_buffer_cursor");
+
+  ompt_get_record_type_fn =
+      (ompt_get_record_type_t)lookup("ompt_get_record_type");
+  if (!ompt_get_record_type_fn) {
+    printf("WARNING: No function ompt_get_record_type found in device "
+           "callbacks\n");
+  }
+
+  // DeviceMap must be initialized only once. Ensure this logic does not
+  // depend on external data structures because this init function may be
+  // called before main.
+  static bool IsDeviceMapInitialized = false;
+  if (!IsDeviceMapInitialized) {
+    DeviceMapPtr = std::make_unique<DeviceMap_t>();
+    IsDeviceMapInitialized = true;
+  }
+
+  set_trace_ompt(device);
+
+  start_trace(device_num, device);
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Load: device_num:%d filename:%s host_adddr:%p device_addr:%p "
+         "bytes:%lu\n",
+         device_num, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  printf("  Callback DataOp: target_id=%lu host_op_id=%lu optype=%d src=%p "
+         "src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         target_id, host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  printf("Callback Target: target_id=%lu kind=%d endpoint=%d device_num=%d "
+         "code=%p\n",
+         target_id, kind, endpoint, device_num, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+
+  return 1; // success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-devices/veccopy-ompt-target-devices.cpp b/offload/test/ompt/tracing/veccopy-ompt-target-devices/veccopy-ompt-target-devices.cpp
new file mode 100644
index 0000000000000..0c6c0b001cff9
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-devices/veccopy-ompt-target-devices.cpp
@@ -0,0 +1,87 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+#include <cassert>
+#include <omp.h>
+#include <stdio.h>
+
+// This test starts device tracing on all available devices (see
+// start_trace in callbacks.h). It subsequently calls flush and stop
+// on all devices like a tool would normally do.
+
+#include "callbacks.h"
+
+// Map of devices traced
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+  // Warm up
+#pragma omp target
+  {
+  }
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+  for (int dev = 0; dev < omp_get_num_devices(); ++dev) {
+#pragma omp target teams distribute parallel for device(dev)
+    {
+      for (int j = 0; j < N; j++)
+        a[j] = b[j];
+    }
+  }
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+// clang-format off
+
+// > OMPT device tracing related checks below. <
+
+// Note: Since this test may (theoretically) run on an arbitrary number of
+//       devices, we may only check for the minimal set of records.
+
+
+
+// Note: Split checks for record address and content. That way we do not imply
+//       any order. Records may / will occur interleaved.
+
+// Note: These addresses will only occur once. They are only captured to
+//       indicate their existence.
+
+/// CHECK-DAG: type=8 (Target task)
+/// CHECK-DAG: type=9 (Target data op)
+
+// Note: ADDRX_07 may not trigger a final callback.
+
+// Note: ADDRX_07 may not be deallocated.
+
+/// CHECK-NOT: host_op_id=0x0
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-translate-time/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-translate-time/callbacks.h
new file mode 100644
index 0000000000000..0bb080d863a9b
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-translate-time/callbacks.h
@@ -0,0 +1,136 @@
+#include <assert.h>
+#include <math.h>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_translate_time_t ompt_translate_time_fn = 0;
+static ompt_get_device_time_t ompt_get_device_time_fn = 0;
+
+// OMPT callbacks
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+
+  ompt_device_time_t device_time = 0;
+
+  ompt_translate_time_fn = (ompt_translate_time_t)lookup("ompt_translate_time");
+  ompt_get_device_time_fn =
+      (ompt_get_device_time_t)lookup("ompt_get_device_time");
+
+  if (ompt_get_device_time_fn) {
+    uint64_t time = ompt_get_device_time_fn(NULL);
+    printf("The device time can be queried: %lu\n", time);
+    device_time = time;
+  } else {
+    printf("Could not determine the device time\n");
+  }
+
+  if (ompt_translate_time_fn) {
+    assert(device_time > 0 && "Obtain device time was successfull");
+    printf("Translate time available\n");
+    double omp_wtime = omp_get_wtime();
+    device_time = ompt_get_device_time_fn(NULL);
+    double omp_device_wtime = ompt_translate_time_fn(NULL, device_time);
+    printf("OMPT: Host time: %f, Device time: %f\n", omp_wtime,
+           omp_device_wtime);
+  } else {
+    printf("Could not translate time.\n");
+  }
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d module_id:%lu filename:%s host_adddr:%p "
+         "device_addr:%p bytes:%lu\n",
+         device_num, module_id, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  assert(codeptr_ra != 0);
+  // Both src and dest must not be null
+  assert(src_addr != 0 || dest_addr != 0);
+  printf("  Callback DataOp: target_id=%lu host_op_id=%lu optype=%d src=%p "
+         "src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         target_id, host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  assert(codeptr_ra != 0);
+  printf("Callback Target: target_id=%lu kind=%d endpoint=%d device_num=%d "
+         "code=%p\n",
+         target_id, kind, endpoint, device_num, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+
+  return 1; //success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-translate-time/veccopy-ompt-target.c b/offload/test/ompt/tracing/veccopy-ompt-target-translate-time/veccopy-ompt-target.c
new file mode 100644
index 0000000000000..97936f987afc2
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-translate-time/veccopy-ompt-target.c
@@ -0,0 +1,77 @@
+// clang-format off
+// RUN: %libomptarget-compile-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+/// CHECK: Callback Init:
+/// CHECK: The device time can be queried: [[X:[0-9]+]]
+/// CHECK: Translate time available
+/// CHECK: Callback Load:
+/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2
+
+/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2
+/// CHECK: Callback Fini:
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/callbacks.h b/offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/callbacks.h
new file mode 100644
index 0000000000000..66495a7f45f6b
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/callbacks.h
@@ -0,0 +1,131 @@
+#include <assert.h>
+
+// Tool related code below
+#include <omp-tools.h>
+
+// From openmp/runtime/test/ompt/callback.h
+#define register_ompt_callback_t(name, type)                                   \
+  do {                                                                         \
+    type f_##name = &on_##name;                                                \
+    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+      printf("0: Could not register callback '" #name "'\n");                  \
+  } while (0)
+
+#define register_ompt_callback(name) register_ompt_callback_t(name, name##_t)
+
+// OMPT entry point handles
+static ompt_set_callback_t ompt_set_callback = 0;
+static ompt_get_record_type_t ompt_get_record_type_fn = 0;
+static ompt_get_device_time_t ompt_get_device_time_fn = 0;
+
+// OMPT callbacks
+
+// Synchronous callbacks
+static void on_ompt_callback_device_initialize(int device_num, const char *type,
+                                               ompt_device_t *device,
+                                               ompt_function_lookup_t lookup,
+                                               const char *documentation) {
+  printf("Callback Init: device_num=%d type=%s device=%p lookup=%p doc=%p\n",
+         device_num, type, device, lookup, documentation);
+
+  ompt_get_record_type_fn =
+      (ompt_get_record_type_t)lookup("ompt_get_record_type");
+  ompt_get_device_time_fn =
+      (ompt_get_device_time_t)lookup("ompt_get_device_time");
+
+  if (ompt_get_record_type_fn) {
+    ompt_buffer_cursor_t buf;
+    // FIXME: For now, we pass a NULL buffer
+    ompt_record_t rec_type = ompt_get_record_type_fn(NULL, buf);
+    printf("Record Type: %s\n",
+           ((rec_type == ompt_record_ompt) ? "OMPT" : "Unknown"));
+  } else {
+    printf("Could not determine Record Type");
+  }
+
+  if (ompt_get_device_time_fn) {
+    uint64_t time = ompt_get_device_time_fn(NULL);
+    printf("The device time can be queried: %lu\n", time);
+  } else {
+    printf("Could not determine the device time \n");
+  }
+}
+
+static void on_ompt_callback_device_finalize(int device_num) {
+  printf("Callback Fini: device_num=%d\n", device_num);
+}
+
+static void on_ompt_callback_device_load(int device_num, const char *filename,
+                                         int64_t offset_in_file,
+                                         void *vma_in_file, size_t bytes,
+                                         void *host_addr, void *device_addr,
+                                         uint64_t module_id) {
+  printf("Callback Load: device_num:%d module_id:%lu filename:%s host_adddr:%p "
+         "device_addr:%p bytes:%lu\n",
+         device_num, module_id, filename, host_addr, device_addr, bytes);
+}
+
+static void on_ompt_callback_target_data_op(
+    ompt_id_t target_id, ompt_id_t host_op_id, ompt_target_data_op_t optype,
+    void *src_addr, int src_device_num, void *dest_addr, int dest_device_num,
+    size_t bytes, const void *codeptr_ra) {
+  assert(codeptr_ra != 0);
+  // Both src and dest must not be null
+  assert(src_addr != 0 || dest_addr != 0);
+  printf("  Callback DataOp: target_id=%lu host_op_id=%lu optype=%d src=%p "
+         "src_device_num=%d "
+         "dest=%p dest_device_num=%d bytes=%lu code=%p\n",
+         target_id, host_op_id, optype, src_addr, src_device_num, dest_addr,
+         dest_device_num, bytes, codeptr_ra);
+}
+
+static void on_ompt_callback_target(ompt_target_t kind,
+                                    ompt_scope_endpoint_t endpoint,
+                                    int device_num, ompt_data_t *task_data,
+                                    ompt_id_t target_id,
+                                    const void *codeptr_ra) {
+  assert(codeptr_ra != 0);
+  printf("Callback Target: target_id=%lu kind=%d endpoint=%d device_num=%d "
+         "code=%p\n",
+         target_id, kind, endpoint, device_num, codeptr_ra);
+}
+
+static void on_ompt_callback_target_submit(ompt_id_t target_id,
+                                           ompt_id_t host_op_id,
+                                           unsigned int requested_num_teams) {
+  printf("  Callback Submit: target_id=%lu host_op_id=%lu req_num_teams=%d\n",
+         target_id, host_op_id, requested_num_teams);
+}
+
+// Init functions
+int ompt_initialize(ompt_function_lookup_t lookup, int initial_device_num,
+                    ompt_data_t *tool_data) {
+  ompt_set_callback = (ompt_set_callback_t)lookup("ompt_set_callback");
+
+  if (!ompt_set_callback)
+    return 0; // failed
+
+  register_ompt_callback(ompt_callback_device_initialize);
+  register_ompt_callback(ompt_callback_device_finalize);
+  register_ompt_callback(ompt_callback_device_load);
+  register_ompt_callback(ompt_callback_target_data_op);
+  register_ompt_callback(ompt_callback_target);
+  register_ompt_callback(ompt_callback_target_submit);
+
+  return 1; //success
+}
+
+void ompt_finalize(ompt_data_t *tool_data) {}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+ompt_start_tool_result_t *ompt_start_tool(unsigned int omp_version,
+                                          const char *runtime_version) {
+  static ompt_start_tool_result_t ompt_start_tool_result = {&ompt_initialize,
+                                                            &ompt_finalize, 0};
+  return &ompt_start_tool_result;
+}
+#ifdef __cplusplus
+}
+#endif
diff --git a/offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/veccopy-ompt-target.c b/offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/veccopy-ompt-target.c
new file mode 100644
index 0000000000000..e636072826bb0
--- /dev/null
+++ b/offload/test/ompt/tracing/veccopy-ompt-target-type-device-time/veccopy-ompt-target.c
@@ -0,0 +1,77 @@
+// clang-format off
+// RUN: %libomptarget-compile-run-and-check-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+#include <omp.h>
+#include <stdio.h>
+
+#include "callbacks.h"
+
+int main() {
+  int N = 100000;
+
+  int a[N];
+  int b[N];
+
+  int i;
+
+  for (i = 0; i < N; i++)
+    a[i] = 0;
+
+  for (i = 0; i < N; i++)
+    b[i] = i;
+
+#pragma omp target parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+#pragma omp target teams distribute parallel for
+  {
+    for (int j = 0; j < N; j++)
+      a[j] = b[j];
+  }
+
+  int rc = 0;
+  for (i = 0; i < N; i++)
+    if (a[i] != b[i]) {
+      rc++;
+      printf("Wrong value: a[%d]=%d\n", i, a[i]);
+    }
+
+  if (!rc)
+    printf("Success\n");
+
+  return rc;
+}
+
+/// CHECK: Callback Init:
+/// CHECK: Record Type: OMPT
+/// CHECK: The device time can be queried: [[X:[0-9]+]]
+/// CHECK: Callback Load:
+/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2
+
+/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=1
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=1
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=2
+/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=3
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+/// CHECK: Callback DataOp: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] optype=4
+/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=1 endpoint=2
+/// CHECK: Callback Fini:

>From 6a671c884682d1b186f891ac61cd58d6bc5b47ee Mon Sep 17 00:00:00 2001
From: JP Lehr <JanPatrick.Lehr at amd.com>
Date: Thu, 21 May 2026 09:11:16 -0500
Subject: [PATCH 12/12] [OMPT] Add test for synchronous path w/ OMPT

---
 .../large_mapping_sync_tracing.cpp            | 81 +++++++++++++++++++
 1 file changed, 81 insertions(+)
 create mode 100644 offload/test/ompt/tracing/large-mapping-sync-tracing/large_mapping_sync_tracing.cpp

diff --git a/offload/test/ompt/tracing/large-mapping-sync-tracing/large_mapping_sync_tracing.cpp b/offload/test/ompt/tracing/large-mapping-sync-tracing/large_mapping_sync_tracing.cpp
new file mode 100644
index 0000000000000..44939536d30f3
--- /dev/null
+++ b/offload/test/ompt/tracing/large-mapping-sync-tracing/large_mapping_sync_tracing.cpp
@@ -0,0 +1,81 @@
+// clang-format off
+// RUN: %libomptarget-compilexx-generic && env LIBOMPTARGET_AMDGPU_MAX_ASYNC_COPY_BYTES=1 LIBOMPTARGET_OMPT_FLUSH_ON_SHUTDOWN=false LIBOMPTARGET_OMPT_FLUSH_ON_BUFFER_FULL=false %libomptarget-run-generic | %fcheck-generic
+// REQUIRES: ompt
+// REQUIRES: amdgpu
+// clang-format on
+
+// Exercises OMPT device tracing for AMDGPU's synchronous large-copy path.
+// The transfer size is intentionally above the plugin's default async-copy
+// threshold, while the RUN line also lowers the threshold to keep the test
+// independent of future default changes.
+
+#include <omp.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "../veccopy-ompt-target-data-tracing-emi/callbacks.h"
+
+#define COPY_BYTES (50 * 1024 * 1024)
+
+DeviceMapPtr_t DeviceMapPtr;
+
+int main() {
+  if (omp_get_num_devices() == 0) {
+    printf("Success\n");
+    return 0;
+  }
+
+  const size_t N = COPY_BYTES / sizeof(int);
+  int *Input = (int *)malloc(COPY_BYTES);
+  int *Output = (int *)malloc(COPY_BYTES);
+
+  if (!Input || !Output) {
+    free(Input);
+    free(Output);
+    printf("Failure\n");
+    return 1;
+  }
+
+  for (size_t I = 0; I < N; ++I) {
+    Input[I] = (int)(I & 0x7fff);
+    Output[I] = 0;
+  }
+
+#pragma omp target teams distribute parallel for map(to : Input[0 : N])        \
+    map(from : Output[0 : N])
+  for (size_t I = 0; I < N; ++I)
+    Output[I] = Input[I] + 1;
+
+  for (auto Dev : *DeviceMapPtr)
+    flush_trace(Dev);
+
+  int Failed = 0;
+  for (size_t I = 0; I < N; ++I) {
+    if (Output[I] != Input[I] + 1) {
+      Failed = 1;
+      break;
+    }
+  }
+
+  free(Input);
+  free(Output);
+
+  printf("%s\n", Failed ? "Failure" : "Success");
+  return Failed;
+}
+
+// clang-format off
+
+/// CHECK: Callback Init:
+/// CHECK: Callback Load:
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=1
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=2 {{.*}} bytes=52428800
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=2 {{.*}} bytes=52428800
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=3 {{.*}} bytes=52428800
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=3 {{.*}} bytes=52428800
+/// CHECK-DAG: Callback DataOp EMI: endpoint=1 optype=4
+/// CHECK-DAG: Callback DataOp EMI: endpoint=2 optype=4
+
+/// CHECK-DAG: type=9 (Target data op) {{.*}} optype=1 {{.*}} bytes=52428800
+/// CHECK: Success


From openmp-commits at lists.llvm.org  Fri Aug  7 03:42:04 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Fri, 07 Aug 2026 03:42:04 -0700 (PDT)
Subject: [Openmp-commits] [openmp] 09375e3 - [OpenMP][OMPT] Error when
 registering EMI and non-EMI callbacks (#213697)
Message-ID: <6a75b67c.92d2873c.701b.a167@mx.google.com>


Author: Jan André Reuter
Date: 2026-08-07T12:41:59+02:00
New Revision: 09375e336d7cdbe96018300ef69d2be7d3c441a2

URL: https://github.com/llvm/llvm-project/commit/09375e336d7cdbe96018300ef69d2be7d3c441a2
DIFF: https://github.com/llvm/llvm-project/commit/09375e336d7cdbe96018300ef69d2be7d3c441a2.diff

LOG: [OpenMP][OMPT] Error when registering EMI and non-EMI callbacks (#213697)

The OpenMP specification includes a restriction for registering both EMI
and non-EMI callbacks (e.g. `ompt_callback_target`, OpenMP v5.2, p. 503,
l. 18-20):

  These callbacks must not be registered at the same time.

However, this was not respected in the runtime. A tool was able to
register both, receiving `ompt_set_always` for each `ompt_set_callback`
call. When an event associated with the EMI or non-EMI callbacks was
dispatched and both callbacks were registered, only the EMI version was
dispatched.

To align the runtime behavior with the OpenMP specification, return
`ompt_set_error` when a tool tries to register both the EMI and non-EMI
variant for a callback. With this, only the tool's first registration is
used.

Update the offload `veccopy_disallow_both.c` test, which already tested
the prior, non-compliant, behavior.

Closes #212959

Signed-off-by: Jan André Reuter <jan at zyten.de>

Added: 
    

Modified: 
    offload/test/ompt/register_both.h
    offload/test/ompt/veccopy_disallow_both.c
    openmp/runtime/src/include/omp-tools.h.var
    openmp/runtime/src/ompt-general.cpp

Removed: 
    


################################################################################
diff  --git a/offload/test/ompt/register_both.h b/offload/test/ompt/register_both.h
index 77cb430aa1f97..a20b42bad2196 100644
--- a/offload/test/ompt/register_both.h
+++ b/offload/test/ompt/register_both.h
@@ -4,7 +4,9 @@
 #define register_ompt_callback_t(name, type)                                   \
   do {                                                                         \
     type f_##name = &on_##name;                                                \
-    if (ompt_set_callback(name, (ompt_callback_t)f_##name) == ompt_set_never)  \
+    ompt_set_result_t result =                                                 \
+        ompt_set_callback(name, (ompt_callback_t)f_##name);                    \
+    if (result == ompt_set_never || result == ompt_set_error)                  \
       printf("0: Could not register callback '" #name "'\n");                  \
   } while (0)
 

diff  --git a/offload/test/ompt/veccopy_disallow_both.c b/offload/test/ompt/veccopy_disallow_both.c
index bfc67c5f4d274..2e24529042b37 100644
--- a/offload/test/ompt/veccopy_disallow_both.c
+++ b/offload/test/ompt/veccopy_disallow_both.c
@@ -7,8 +7,9 @@
 /*
  * Example OpenMP program that shows that both EMI and non-EMI
  * callbacks cannot be registered for the same type. In the
- * current implementation, the EMI callback overrides the non-EMI
- * callback.
+ * current implementation, the first registered callback takes
+ * precedence. Trying to register the other callback results in
+ * ompt_set_error for this callback.
  */
 
 #include <omp.h>
@@ -57,9 +58,11 @@ int main() {
 }
 
 // clang-format off
+/// CHECK: [[THREAD_ID:[0-9]+]]: Could not register callback 'ompt_callback_target_data_op'
+/// CHECK: [[THREAD_ID]]: Could not register callback 'ompt_callback_target_emi'
 /// CHECK: Callback Init:
 /// CHECK: Callback Load:
-/// CHECK: Callback Target EMI: kind=ompt_target endpoint=ompt_scope_begin
+/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=ompt_target endpoint=ompt_scope_begin
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_alloc
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_alloc
 /// CHECK-NOT: dest=(nil)
@@ -70,7 +73,7 @@ int main() {
 /// CHECK-NOT: dest=(nil)
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_transfer_to_device
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_transfer_to_device
-/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1
+/// CHECK: Callback Submit: target_id=[[TARGET_ID]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=1
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_transfer_from_device
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_transfer_from_device
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_transfer_from_device
@@ -79,8 +82,8 @@ int main() {
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_delete
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_delete
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_delete
-/// CHECK: Callback Target EMI: kind=ompt_target endpoint=ompt_scope_end
-/// CHECK: Callback Target EMI: kind=ompt_target endpoint=ompt_scope_begin
+/// CHECK: Callback Target: target_id=[[TARGET_ID]] kind=ompt_target endpoint=ompt_scope_end
+/// CHECK: Callback Target: target_id=[[TARGET_ID:[0-9]+]] kind=ompt_target endpoint=ompt_scope_begin
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_alloc
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_alloc
 /// CHECK-NOT: dest=(nil)
@@ -91,7 +94,7 @@ int main() {
 /// CHECK-NOT: dest=(nil)
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_transfer_to_device
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_transfer_to_device
-/// CHECK: Callback Submit: target_id=[[TARGET_ID:[0-9]+]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0
+/// CHECK: Callback Submit: target_id=[[TARGET_ID]] host_op_id=[[HOST_OP_ID:[0-9]+]] req_num_teams=0
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_transfer_from_device
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_transfer_from_device
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_transfer_from_device
@@ -100,5 +103,5 @@ int main() {
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_delete
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_begin optype=ompt_target_data_delete
 /// CHECK: Callback DataOp EMI: endpoint=ompt_scope_end optype=ompt_target_data_delete
-/// CHECK: Callback Target EMI: kind=ompt_target endpoint=ompt_scope_end
+/// CHECK: Callback Target: target_id=[[TARGET_ID]] kind=ompt_target endpoint=ompt_scope_end
 /// CHECK: Callback Fini:

diff  --git a/openmp/runtime/src/include/omp-tools.h.var b/openmp/runtime/src/include/omp-tools.h.var
index af9213a7290e7..46643dff5feab 100644
--- a/openmp/runtime/src/include/omp-tools.h.var
+++ b/openmp/runtime/src/include/omp-tools.h.var
@@ -185,6 +185,19 @@
     /* Optional Events */                                                                                                \
     macro (ompt_callback_target_map_emi,    ompt_callback_target_map_emi_t,    36) /* target map                      */
 
+#define FOREACH_MAPPED_OMPT_TARGET_EVENT(macro)                                                                          \
+    /* macro including correlated callback enum (e.g. target -- target_emi) for comparisons between both */              \
+    /*--- Mandatory Events ---*/                                                                                         \
+    macro (ompt_callback_target,             ompt_callback_target_emi,         ompt_callback_target_t,              8)   \
+    macro (ompt_callback_target_data_op,     ompt_callback_target_data_op_emi, ompt_callback_target_data_op_t,      9)   \
+    macro (ompt_callback_target_submit,      ompt_callback_target_submit_emi,  ompt_callback_target_submit_t,      10)   \
+    macro (ompt_callback_target_emi,         ompt_callback_target,             ompt_callback_target_emi_t,         33)   \
+    macro (ompt_callback_target_data_op_emi, ompt_callback_target_data_op,     ompt_callback_target_data_op_emi_t, 34)   \
+    macro (ompt_callback_target_submit_emi,  ompt_callback_target_submit,      ompt_callback_target_submit_emi_t,  35)   \
+    /* Optional Events */                                                                                                \
+    macro (ompt_callback_target_map,         ompt_callback_target_map_emi,     ompt_callback_target_map_t,         22)   \
+    macro (ompt_callback_target_map_emi,     ompt_callback_target_map,         ompt_callback_target_map_emi_t,     36)
+
 #define FOREACH_OMPT_50_TARGET_EVENT(macro)                                                                              \
     FOREACH_OMPT_DEVICE_EVENT(macro)                                                                                     \
     FOREACH_OMPT_NOEMI_EVENT(macro) 

diff  --git a/openmp/runtime/src/ompt-general.cpp b/openmp/runtime/src/ompt-general.cpp
index 1682f7e74bd14..8f1ba4910815e 100644
--- a/openmp/runtime/src/ompt-general.cpp
+++ b/openmp/runtime/src/ompt-general.cpp
@@ -597,10 +597,35 @@ OMPT_API_ROUTINE ompt_set_result_t ompt_set_callback(ompt_callbacks_t which,
     else                                                                       \
       return ompt_set_always;
 
-    FOREACH_OMPT_EVENT(ompt_event_macro)
+    FOREACH_OMPT_HOST_EVENT(ompt_event_macro)
+    FOREACH_OMPT_DEVICE_EVENT(ompt_event_macro)
 
 #undef ompt_event_macro
 
+    // OpenMP v5.2, p. 503, l. 18-20:
+    // Restrictions to the ompt_callback_target_emi and ompt_callback_target
+    // callbacks are as follows:
+    //   These callbacks must not be registered at the same time.
+    //
+    // Similar restrictions apply to target_data_op, target_submit,
+    // and target_map. Hence, handle registration of them separately to ensure
+    // only one can be registered at the same time.
+#define ompt_target_event_macro(event_name, comparison_name, callback_type,    \
+                                event_id)                                      \
+  case event_name:                                                             \
+    if (ompt_callbacks.ompt_callback(comparison_name))                         \
+      return ompt_set_error;                                                   \
+    ompt_callbacks.ompt_callback(event_name) = (callback_type)callback;        \
+    ompt_enabled.event_name = (callback != 0);                                 \
+    if (callback)                                                              \
+      return ompt_event_implementation_status(event_name);                     \
+    else                                                                       \
+      return ompt_set_always;
+
+    FOREACH_MAPPED_OMPT_TARGET_EVENT(ompt_target_event_macro);
+
+#undef ompt_target_event_macro
+
   default:
     return ompt_set_error;
   }


        

From openmp-commits at lists.llvm.org  Fri Aug  7 03:42:07 2026
From: openmp-commits at lists.llvm.org (=?UTF-8?Q?Jan_Andr=C3=A9_Reuter?= via Openmp-commits)
Date: Fri, 07 Aug 2026 03:42:07 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMP][OMPT] Error when
 registering EMI and non-EMI callbacks (PR #213697)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213697@github.com>
Message-ID: <6a75b67f.21a19891.1fa7d.8b2f@mx.google.com>


https://github.com/Thyre closed https://github.com/llvm/llvm-project/pull/213697

From openmp-commits at lists.llvm.org  Fri Aug  7 03:43:30 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Fri, 07 Aug 2026 03:43:30 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OMPT] Upstream device tracing
	implementation (PR #200165)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/200165@github.com>
Message-ID: <6a75b6d2.c4d3d80a.23fb9f.77df@mx.google.com>


github-actions[bot] wrote:

<!--PREMERGE ADVISOR COMMENT: Linux-->
# :penguin: Linux x64 Test Results

The build failed before running any tests. Click on a failure below to see the details.

<details>
<summary>offload/libomptarget/CMakeFiles/omptarget.dir/home/gha/actions-runner/_work/llvm-project/llvm-project/offload/plugins-nextgen/common/OMPT/OmptProfiler.cpp.o</summary>

```
FAILED: offload/libomptarget/CMakeFiles/omptarget.dir/home/gha/actions-runner/_work/llvm-project/llvm-project/offload/plugins-nextgen/common/OMPT/OmptProfiler.cpp.o
/home/gha/actions-runner/_work/llvm-project/llvm-project/build/./bin/clang++ --target=x86_64-unknown-linux-gnu -DDEBUG_PREFIX=\"omptarget\" -DOMPT_SUPPORT=1 -DTARGET_NAME=omptarget -D_DEBUG -D_GLIBCXX_ASSERTIONS -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS -Domptarget_EXPORTS -I/home/gha/actions-runner/_work/llvm-project/llvm-project/llvm/include -I/home/gha/actions-runner/_work/llvm-project/llvm-project/build/include -I/home/gha/actions-runner/_work/llvm-project/llvm-project/build/runtimes/runtimes-bins/openmp/runtime/src -I/home/gha/actions-runner/_work/llvm-project/llvm-project/offload/include -I/home/gha/actions-runner/_work/llvm-project/llvm-project/build/runtimes/runtimes-bins/offload/include -I/home/gha/actions-runner/_work/llvm-project/llvm-project/offload/libomptarget/../plugins-nextgen/common/OMPT -I/home/gha/actions-runner/_work/llvm-project/llvm-project/offload/libomptarget/../plugins-nextgen/common/include -I/home/gha/actions-runner/_work/llvm-project/llvm-project/offload/plugins-nextgen/common/include -I/home/gha/actions-runner/_work/llvm-project/llvm-project/build/runtimes/runtimes-bins/offload/plugins-nextgen/common/include -I/home/gha/actions-runner/_work/llvm-project/llvm-project/offload/plugins-nextgen/common/OMPT -fPIC -fno-semantic-interposition -fvisibility-inlines-hidden -Werror -Werror=date-time -Werror=unguarded-availability-new -Wall -Wextra -Wno-unused-parameter -Wwrite-strings -Wcast-qual -Wmissing-field-initializers -Wimplicit-fallthrough -Wcovered-switch-default -Wno-noexcept-type -Wnon-virtual-dtor -Wdelete-non-virtual-dtor -Wsuggest-override -Wstring-conversion -Wno-pass-failed -Wmisleading-indentation -Wctad-maybe-unsupported -fdiagnostics-color -ffunction-sections -fdata-sections -Xclang -fno-pch-timestamp -O3 -DNDEBUG -std=c++17 -fPIC -UNDEBUG -fno-exceptions -fno-rtti -Werror=global-constructors -fPIC -MD -MT offload/libomptarget/CMakeFiles/omptarget.dir/home/gha/actions-runner/_work/llvm-project/llvm-project/offload/plugins-nextgen/common/OMPT/OmptProfiler.cpp.o -MF offload/libomptarget/CMakeFiles/omptarget.dir/home/gha/actions-runner/_work/llvm-project/llvm-project/offload/plugins-nextgen/common/OMPT/OmptProfiler.cpp.o.d -o offload/libomptarget/CMakeFiles/omptarget.dir/home/gha/actions-runner/_work/llvm-project/llvm-project/offload/plugins-nextgen/common/OMPT/OmptProfiler.cpp.o -c /home/gha/actions-runner/_work/llvm-project/llvm-project/offload/plugins-nextgen/common/OMPT/OmptProfiler.cpp
In file included from /home/gha/actions-runner/_work/llvm-project/llvm-project/offload/plugins-nextgen/common/OMPT/OmptProfiler.cpp:13:
/home/gha/actions-runner/_work/llvm-project/llvm-project/offload/plugins-nextgen/common/OMPT/OmptProfiler.h:119:7: error: 'std::scoped_lock' may not intend to support class template argument deduction [-Werror,-Wctad-maybe-unsupported]
119 |       std::scoped_lock Lock(ProfilerDataMutex);
|       ^
/usr/lib/gcc/x86_64-linux-gnu/13/../../../../include/c++/13/mutex:740:11: note: add a deduction guide to suppress this warning
740 |     class scoped_lock
|           ^
In file included from /home/gha/actions-runner/_work/llvm-project/llvm-project/offload/plugins-nextgen/common/OMPT/OmptProfiler.cpp:13:
/home/gha/actions-runner/_work/llvm-project/llvm-project/offload/plugins-nextgen/common/OMPT/OmptProfiler.h:126:5: error: 'std::scoped_lock' may not intend to support class template argument deduction [-Werror,-Wctad-maybe-unsupported]
126 |     std::scoped_lock Lock(ProfilerDataMutex);
|     ^
/usr/lib/gcc/x86_64-linux-gnu/13/../../../../include/c++/13/mutex:740:11: note: add a deduction guide to suppress this warning
740 |     class scoped_lock
|           ^
2 errors generated.
```
</details>

If these failures are unrelated to your changes (for example tests are broken or flaky at HEAD), please open an issue at https://github.com/llvm/llvm-project/issues and add the `infrastructure` label.

https://github.com/llvm/llvm-project/pull/200165

From openmp-commits at lists.llvm.org  Fri Aug  7 03:46:50 2026
From: openmp-commits at lists.llvm.org (Jan Patrick Lehr via Openmp-commits)
Date: Fri, 07 Aug 2026 03:46:50 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [Offload][OMPT][NFCI] Call
 ompt_libomp_connect instead of dlopen libomp (PR #214556)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/214556@github.com>
Message-ID: <6a75b79a.492e23e4.1b03cd.5483@mx.google.com>


jplehr wrote:

This is for sure the right direction.

https://github.com/llvm/llvm-project/pull/214556

From openmp-commits at lists.llvm.org  Fri Aug  7 03:47:46 2026
From: openmp-commits at lists.llvm.org (Jan Patrick Lehr via Openmp-commits)
Date: Fri, 07 Aug 2026 03:47:46 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [Offload][OMPT][NFCI] Call
 ompt_libomp_connect instead of dlopen libomp (PR #214556)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/214556@github.com>
Message-ID: <6a75b7d2.824e11a6.3cb239.ab53@mx.google.com>


https://github.com/jplehr approved this pull request.


https://github.com/llvm/llvm-project/pull/214556

From openmp-commits at lists.llvm.org  Fri Aug  7 05:45:57 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Fri, 07 Aug 2026 05:45:57 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [Offload][OMPT][NFCI] Call
 ompt_libomp_connect instead of dlopen libomp (PR #214556)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/214556@github.com>
Message-ID: <6a75d385.4a342cbc.f12fe.10c6@mx.google.com>


https://github.com/jprotze approved this pull request.

I was wondering why this overly complicated approach was introduced in the first place and why "libomp" is explicitly passed into the connector. It looks like no one ever used this connector with something different than "libomp". Is some downstream implementation using liboffload with a different OpenMP runtime?

I'm fine with the change.

https://github.com/llvm/llvm-project/pull/214556

From openmp-commits at lists.llvm.org  Fri Aug  7 05:55:08 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Fri, 07 Aug 2026 05:55:08 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMP][OMPT] Error when
 registering EMI and non-EMI callbacks (PR #213697)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213697@github.com>
Message-ID: <6a75d5ac.8e75e0cb.39924.03ce@mx.google.com>


jprotze wrote:

A tool violating the restriction makes the tool invalid and we are in unspecified behavior land. I agree that the runtime can easily catch this error and return the error code. So I'm fine with the change.

https://github.com/llvm/llvm-project/pull/213697

From openmp-commits at lists.llvm.org  Fri Aug  7 06:00:30 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Fri, 07 Aug 2026 06:00:30 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP][OMPT] Add ompt_set_frame_enter
 support to libomptarget (PR #201127)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/201127@github.com>
Message-ID: <6a75d6ee.d9d6bf9e.2db05.7ab4@mx.google.com>


================
@@ -481,6 +481,21 @@ int __ompt_get_task_memory_internal(void **addr, size_t *size, int blocknum) {
   return 0;
 }
 
+//----------------------------------------------------------
+// target region support
+//----------------------------------------------------------
+
+int __ompt_set_frame_enter_internal(void *addr, int flags, int state) {
+  int gtid = __kmp_entry_gtid();
+  kmp_info_t *thr = __kmp_threads[gtid];
+
+  ompt_frame_t *ompt_frame = &OMPT_CUR_TASK_INFO(thr)->frame;
+  OMPT_FRAME_SET(ompt_frame, enter, addr, flags);
+  int old_state = thr->th.ompt_thread_info.state;
+  thr->th.ompt_thread_info.state = ompt_state_work_parallel;
----------------
jprotze wrote:

A sampling tool can query the state when taking a timer-based sample.
Similarly, sampling tools are the typical consumers of frame information. 

https://github.com/llvm/llvm-project/pull/201127

From openmp-commits at lists.llvm.org  Fri Aug  7 06:05:28 2026
From: openmp-commits at lists.llvm.org (=?UTF-8?Q?Jan_Andr=C3=A9_Reuter?= via Openmp-commits)
Date: Fri, 07 Aug 2026 06:05:28 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP][OMPT] Add ompt_set_frame_enter
 support to libomptarget (PR #201127)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/201127@github.com>
Message-ID: <6a75d818.881e6cae.4accd.29e0@mx.google.com>


================
@@ -481,6 +481,21 @@ int __ompt_get_task_memory_internal(void **addr, size_t *size, int blocknum) {
   return 0;
 }
 
+//----------------------------------------------------------
+// target region support
+//----------------------------------------------------------
+
+int __ompt_set_frame_enter_internal(void *addr, int flags, int state) {
+  int gtid = __kmp_entry_gtid();
+  kmp_info_t *thr = __kmp_threads[gtid];
+
+  ompt_frame_t *ompt_frame = &OMPT_CUR_TASK_INFO(thr)->frame;
+  OMPT_FRAME_SET(ompt_frame, enter, addr, flags);
+  int old_state = thr->th.ompt_thread_info.state;
+  thr->th.ompt_thread_info.state = ompt_state_work_parallel;
----------------
Thyre wrote:

Completely agreed. I was more confused by setting the state to `work_parallel` when we're entering a target region.
But I guess `ompt_state_wait_target` doesn't apply until the kernel has been dispatched.
I don't think there's a flag to really indicate the runtime entering a target region. A sampling tool would probably see something like `work_serial`/ `work_parallel`, depending on the outer context.

https://github.com/llvm/llvm-project/pull/201127

From openmp-commits at lists.llvm.org  Fri Aug  7 07:35:56 2026
From: openmp-commits at lists.llvm.org (Robert Imschweiler via Openmp-commits)
Date: Fri, 07 Aug 2026 07:35:56 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [offload] Use pinned memory for
	KLE (PR #213767)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213767@github.com>
Message-ID: <6a75ed4c.90c9f43d.32e5d2.b21a@mx.google.com>


https://github.com/ro-i updated https://github.com/llvm/llvm-project/pull/213767

>From 022265c76bd74cc16714960cdb91961fd6fb42a7 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Wed, 5 Aug 2026 10:24:00 -0500
Subject: [PATCH 1/5] [offload] Do not pool memory while allocation traces are
 requested

Would otherwise hide use-after-free because memory stays valid if it's
in the pool.
---
 offload/plugins-nextgen/common/src/PluginInterface.cpp | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index e539cbfc55324..d4f3d8b62a272 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -598,9 +598,13 @@ Error GenericDeviceTy::init(GenericPluginTy &Plugin) {
     GridValues.GV_Max_WG_Size =
         std::min(GridValues.GV_Max_WG_Size, uint32_t(OMP_TeamsThreadLimit));
 
-  // Enable the memory manager if required.
+  // Enable the memory manager if required. Memory that the manager pools is
+  // not returned to the driver when it is freed, so a later access to it
+  // neither faults nor is reported by the allocation tracker. Leave the pool
+  // disabled while allocation traces are requested, so that we don't mask
+  // use-after-free (since they don't fault if the memory is still in the pool).
   auto [ThresholdMM, EnableMM] = MemoryManagerTy::getSizeThresholdFromEnv();
-  if (EnableMM) {
+  if (EnableMM && !OMPX_TrackAllocationTraces) {
     if (ThresholdMM == 0)
       ThresholdMM = getMemoryManagerSizeThreshold();
     MemoryManager = new MemoryManagerTy(*this, ThresholdMM);

>From 35c52f4f5d6ba85ffdf5ec1438780bc7caf883e2 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Thu, 6 Aug 2026 11:51:52 -0500
Subject: [PATCH 2/5] [offload] Respect alignment in memory manager

Also fix alignment comparison in AMDGPU rtl.cpp.
---
 offload/plugins-nextgen/amdgpu/src/rtl.cpp             | 2 +-
 offload/plugins-nextgen/common/include/MemoryManager.h | 9 +++++++--
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index 29b5006f0d7b6..0f0de67b20c7c 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -339,7 +339,7 @@ struct AMDGPUMemoryPoolTy {
     // compared with the alignment of the memory allocated using the given pool.
     // If the default alignment is greater than or equal to the alignment
     // requested by the user, it would still meet the user's requirements.
-    if (Alignment > 0 && Alignment >= PoolAllocationAlignment) {
+    if (Alignment > 0 && Alignment > PoolAllocationAlignment) {
       return Plugin::error(ErrorCode::UNSUPPORTED,
                            "requested alignment (%lu) larger than maximum "
                            "supported pool alignment (%lu)",
diff --git a/offload/plugins-nextgen/common/include/MemoryManager.h b/offload/plugins-nextgen/common/include/MemoryManager.h
index 4b57be45e7551..883ac40c269ea 100644
--- a/offload/plugins-nextgen/common/include/MemoryManager.h
+++ b/offload/plugins-nextgen/common/include/MemoryManager.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_OPENMP_LIBOMPTARGET_PLUGINS_COMMON_MEMORYMANAGER_H
 #define LLVM_OPENMP_LIBOMPTARGET_PLUGINS_COMMON_MEMORYMANAGER_H
 
+#include <algorithm>
 #include <cassert>
 #include <functional>
 #include <list>
@@ -25,6 +26,7 @@
 #include "Shared/Utils.h"
 #include "omptarget.h"
 
+#include "llvm/Support/Alignment.h"
 #include "llvm/Support/Error.h"
 
 using namespace llvm::offload::debug;
@@ -268,9 +270,12 @@ class MemoryManagerTy {
 
       NodeTy TempNode(Size, nullptr);
       std::lock_guard<std::mutex> LG(FreeListLocks[B]);
-      const auto Itr = List.find(TempNode);
+      auto [First, Last] = List.equal_range(TempNode);
 
-      if (Itr != List.end()) {
+      auto Itr = std::find_if(First, Last, [Alignment](const NodeTy &N) {
+        return Alignment == 0 || isAddrAligned(Align(Alignment), N.Ptr);
+      });
+      if (Itr != Last) {
         NodePtr = &Itr->get();
         List.erase(Itr);
       }

>From 0692ee2164dc82135cafd645c0860a36257b93c9 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Wed, 5 Aug 2026 10:24:27 -0500
Subject: [PATCH 3/5] [offload] Pool host and shared allocations

Route them through a memory manager, like the device allocations. Also,
move the registration as pinned memory to the plugin site since only the
plugin knows if the corresponding host/shared memory is pinned.
---
 offload/plugins-nextgen/amdgpu/src/rtl.cpp    | 12 ++++
 .../common/include/MemoryManager.h            | 20 ++++--
 .../common/include/PluginInterface.h          | 20 +++++-
 .../common/src/PluginInterface.cpp            | 69 ++++++++-----------
 openmp/docs/design/Runtimes.rst               | 11 +--
 5 files changed, 80 insertions(+), 52 deletions(-)

diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index 0f0de67b20c7c..e74cf03071fa7 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -2728,6 +2728,12 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
       return Plugin::error(ErrorCode::OUT_OF_RESOURCES,
                            "no memory pool for the specified allocation kind");
 
+    // See allocate() for the registration of host / shared memory as pinned
+    // memory.
+    if (Kind == TARGET_ALLOC_HOST || Kind == TARGET_ALLOC_SHARED)
+      if (auto Err = PinnedAllocs.unregisterHostBuffer(TgtPtr))
+        return Err;
+
     if (auto Err = MemoryPool->deallocate(TgtPtr))
       return Err;
 
@@ -4529,6 +4535,12 @@ Expected<void *> AMDGPUDeviceTy::allocate(size_t Size, void *,
     // Enable all valid kernel agents to access the buffer.
     if (auto Err = MemoryPool->enableAccess(Alloc, Size, Agents))
       return std::move(Err);
+
+    // Register host / shared memory as pinned memory, so that transfers reading
+    // from it can take a device-accessible path.
+    if (Kind == TARGET_ALLOC_HOST || Kind == TARGET_ALLOC_SHARED)
+      if (auto Err = PinnedAllocs.registerHostBuffer(Alloc, Alloc, Size))
+        return std::move(Err);
   }
 
   return Alloc;
diff --git a/offload/plugins-nextgen/common/include/MemoryManager.h b/offload/plugins-nextgen/common/include/MemoryManager.h
index 883ac40c269ea..9712a461ad974 100644
--- a/offload/plugins-nextgen/common/include/MemoryManager.h
+++ b/offload/plugins-nextgen/common/include/MemoryManager.h
@@ -138,21 +138,24 @@ class MemoryManagerTy {
 
   /// The reference to a device allocator
   DeviceAllocatorTy &DeviceAllocator;
+  /// The kind of device for which the memory manager is allocated
+  TargetAllocTy DeviceKind;
 
   /// The threshold to manage memory using memory manager. If the request size
   /// is larger than \p SizeThreshold, the allocation will not be managed by the
   /// memory manager.
-  size_t SizeThreshold = 1U << 13;
+  size_t SizeThreshold = DefaultSizeThreshold;
 
   /// Request memory from target device
   Expected<void *> allocateOnDevice(size_t Size, void *HstPtr,
                                     size_t Alignment) const {
-    return DeviceAllocator.allocate(Size, HstPtr, TARGET_ALLOC_DEVICE,
-                                    Alignment);
+    return DeviceAllocator.allocate(Size, HstPtr, DeviceKind, Alignment);
   }
 
   /// Deallocate data on device
-  Error deleteOnDevice(void *Ptr) const { return DeviceAllocator.free(Ptr); }
+  Error deleteOnDevice(void *Ptr) const {
+    return DeviceAllocator.free(Ptr, DeviceKind);
+  }
 
   /// This function is called when it tries to allocate memory on device but the
   /// device returns out of memory. It will first free all memory in the
@@ -216,11 +219,14 @@ class MemoryManagerTy {
   }
 
 public:
+  static constexpr size_t DefaultSizeThreshold = 1U << 13;
+
   /// Constructor. If \p Threshold is non-zero, then the default threshold will
   /// be overwritten by \p Threshold.
-  MemoryManagerTy(DeviceAllocatorTy &DeviceAllocator, size_t Threshold = 0)
+  MemoryManagerTy(DeviceAllocatorTy &DeviceAllocator, size_t Threshold = 0,
+                  TargetAllocTy DeviceKind = TARGET_ALLOC_DEVICE)
       : FreeLists(NumBuckets), FreeListLocks(NumBuckets),
-        DeviceAllocator(DeviceAllocator) {
+        DeviceAllocator(DeviceAllocator), DeviceKind(DeviceKind) {
     if (Threshold)
       SizeThreshold = Threshold;
   }
@@ -359,6 +365,8 @@ class MemoryManagerTy {
   /// threshold and the second element represents whether user disables memory
   /// manager explicitly by setting the var to 0. If user doesn't specify
   /// anything, returns <0, true>.
+  /// Note that this only affects the device memory manager, not the manager for
+  /// host or shared memory.
   static std::pair<size_t, bool> getSizeThresholdFromEnv() {
     static UInt64Envar MemoryManagerThreshold(
         "LIBOMPTARGET_MEMORY_MANAGER_THRESHOLD", 0);
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index b3675e5a8700f..50be67886c527 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -1368,8 +1368,12 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
   /// only necessary for unhosted targets like the GPU.
   virtual bool shouldSetupRPCServer() const { return false; }
 
-  /// Pointer to the memory manager or nullptr if not available.
+  /// Pointer to the device memory manager or nullptr if not available.
   MemoryManagerTy *MemoryManager;
+  /// Memory managers for the host and shared allocation kinds or nullptr if not
+  /// available.
+  MemoryManagerTy *HostMemoryManager;
+  MemoryManagerTy *SharedMemoryManager;
 
   /// Per device setting of MemoryManager's Threshold
   virtual size_t getMemoryManagerSizeThreshold() { return 0; }
@@ -1406,6 +1410,20 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
   /// Record and replay manager.
   RecordReplayTy *RecordReplay = nullptr;
 
+  /// Return the memory manager for the given allocation kind.
+  MemoryManagerTy *getMemoryManagerFor(TargetAllocTy Kind) {
+    switch (Kind) {
+    case TARGET_ALLOC_DEFAULT:
+    case TARGET_ALLOC_DEVICE:
+      return MemoryManager;
+    case TARGET_ALLOC_HOST:
+      return HostMemoryManager;
+    case TARGET_ALLOC_SHARED:
+      return SharedMemoryManager;
+    }
+    return nullptr;
+  }
+
 protected:
   /// Environment variables defined by the LLVM OpenMP implementation
   /// regarding the initial number of streams and events.
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index d4f3d8b62a272..389d2ffbf4395 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -490,7 +490,8 @@ uint32_t GenericKernelTy::getEffectiveNumBlocks(
 GenericDeviceTy::GenericDeviceTy(GenericPluginTy &Plugin, int32_t DeviceId,
                                  int32_t NumDevices,
                                  const llvm::omp::GV &OMPGridValues)
-    : Plugin(Plugin), MemoryManager(nullptr), OMP_TeamLimit("OMP_TEAM_LIMIT"),
+    : Plugin(Plugin), MemoryManager(nullptr), HostMemoryManager(nullptr),
+      SharedMemoryManager(nullptr), OMP_TeamLimit("OMP_TEAM_LIMIT"),
       OMP_NumTeams("OMP_NUM_TEAMS"),
       OMP_TeamsThreadLimit("OMP_TEAMS_THREAD_LIMIT"),
       OMPX_DebugKind("LIBOMPTARGET_DEVICE_RTL_DEBUG"),
@@ -609,6 +610,14 @@ Error GenericDeviceTy::init(GenericPluginTy &Plugin) {
       ThresholdMM = getMemoryManagerSizeThreshold();
     MemoryManager = new MemoryManagerTy(*this, ThresholdMM);
   }
+  if (!OMPX_TrackAllocationTraces) {
+    // Keep the threshold for pooling sizes conservative since we're dealing
+    // with pinned memory for the host.
+    HostMemoryManager = new MemoryManagerTy(
+        *this, MemoryManagerTy::DefaultSizeThreshold, TARGET_ALLOC_HOST);
+    SharedMemoryManager = new MemoryManagerTy(
+        *this, MemoryManagerTy::DefaultSizeThreshold, TARGET_ALLOC_SHARED);
+  }
 
   return Plugin::success();
 }
@@ -656,6 +665,12 @@ Error GenericDeviceTy::deinit(GenericPluginTy &Plugin) {
   if (MemoryManager)
     delete MemoryManager;
   MemoryManager = nullptr;
+  if (HostMemoryManager)
+    delete HostMemoryManager;
+  HostMemoryManager = nullptr;
+  if (SharedMemoryManager)
+    delete SharedMemoryManager;
+  SharedMemoryManager = nullptr;
 
   if (RecordReplay) {
     if (auto Err = RecordReplay->deinit())
@@ -1016,22 +1031,15 @@ Expected<void *> GenericDeviceTy::dataAlloc(int64_t Size, void *HostPtr,
   if (RecordReplay && RecordReplay->isRecordingOrReplaying())
     return RecordReplay->allocate(Size);
 
-  switch (Kind) {
-  case TARGET_ALLOC_DEFAULT:
-  case TARGET_ALLOC_DEVICE:
-    if (MemoryManager) {
-      auto AllocOrErr = MemoryManager->allocate(Size, HostPtr, Alignment);
-      if (!AllocOrErr)
-        return AllocOrErr.takeError();
-      Alloc = *AllocOrErr;
-      if (!Alloc)
-        return Plugin::error(ErrorCode::OUT_OF_RESOURCES,
-                             "failed to allocate from memory manager");
-      break;
-    }
-    [[fallthrough]];
-  case TARGET_ALLOC_HOST:
-  case TARGET_ALLOC_SHARED: {
+  if (MemoryManagerTy *MM = getMemoryManagerFor(Kind)) {
+    auto AllocOrErr = MM->allocate(Size, HostPtr, Alignment);
+    if (!AllocOrErr)
+      return AllocOrErr.takeError();
+    Alloc = *AllocOrErr;
+    if (!Alloc)
+      return Plugin::error(ErrorCode::OUT_OF_RESOURCES,
+                           "failed to allocate from memory manager");
+  } else {
     auto AllocOrErr = allocate(Size, HostPtr, Kind, Alignment);
     if (!AllocOrErr)
       return AllocOrErr.takeError();
@@ -1040,7 +1048,6 @@ Expected<void *> GenericDeviceTy::dataAlloc(int64_t Size, void *HostPtr,
       return Plugin::error(ErrorCode::OUT_OF_RESOURCES,
                            "failed to allocate from device allocator");
   }
-  }
 
   // Report error if the memory manager or the device allocator did not return
   // any memory buffer.
@@ -1049,11 +1056,6 @@ Expected<void *> GenericDeviceTy::dataAlloc(int64_t Size, void *HostPtr,
                          "invalid target data allocation kind or requested "
                          "allocator not implemented yet");
 
-  // Register allocated buffer as pinned memory if the type is host memory.
-  if (Kind == TARGET_ALLOC_HOST)
-    if (auto Err = PinnedAllocs.registerHostBuffer(Alloc, Alloc, Size))
-      return std::move(Err);
-
   // Keep track of the allocation stack if we track allocation traces.
   if (OMPX_TrackAllocationTraces) {
     std::string StackTrace;
@@ -1110,26 +1112,13 @@ Error GenericDeviceTy::dataDelete(void *TgtPtr, TargetAllocTy Kind) {
     ATI->DeallocationTrace = StackTrace;
   }
 
-  switch (Kind) {
-  case TARGET_ALLOC_DEFAULT:
-  case TARGET_ALLOC_DEVICE:
-    if (MemoryManager) {
-      if (auto Err = MemoryManager->free(TgtPtr))
-        return Err;
-      break;
-    }
-    [[fallthrough]];
-  case TARGET_ALLOC_HOST:
-  case TARGET_ALLOC_SHARED:
-    if (auto Err = free(TgtPtr, Kind))
+  if (MemoryManagerTy *MM = getMemoryManagerFor(Kind)) {
+    if (auto Err = MM->free(TgtPtr))
       return Err;
+  } else if (auto Err = free(TgtPtr, Kind)) {
+    return Err;
   }
 
-  // Unregister deallocated pinned memory buffer if the type is host memory.
-  if (Kind == TARGET_ALLOC_HOST)
-    if (auto Err = PinnedAllocs.unregisterHostBuffer(TgtPtr))
-      return Err;
-
   return Plugin::success();
 }
 
diff --git a/openmp/docs/design/Runtimes.rst b/openmp/docs/design/Runtimes.rst
index 14e300a0f531f..0e75925c78742 100644
--- a/openmp/docs/design/Runtimes.rst
+++ b/openmp/docs/design/Runtimes.rst
@@ -784,11 +784,12 @@ LIBOMPTARGET_MEMORY_MANAGER_THRESHOLD
 """""""""""""""""""""""""""""""""""""
 
 ``LIBOMPTARGET_MEMORY_MANAGER_THRESHOLD`` sets the threshold size for which the
-``libomptarget`` memory manager will handle the allocation. Any allocations
-larger than this threshold will not use the memory manager and be freed after
-the device kernel exits. The default threshold value is ``8KB``. If
-``LIBOMPTARGET_MEMORY_MANAGER_THRESHOLD`` is set to ``0`` the memory manager
-will be completely disabled.
+``libomptarget`` device memory manager will handle the allocation. Any
+allocations larger than this threshold will not use the memory manager and be
+freed after the device kernel exits. The default threshold value is ``8KB``. If
+``LIBOMPTARGET_MEMORY_MANAGER_THRESHOLD`` is set to ``0`` the device memory
+manager will be completely disabled.
+This has no effect on the host or shared memory managers.
 
 .. _libomptarget_info:
 

>From bfb4fbea1bd328fd51df913b7dc515f1fba969e8 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Fri, 7 Aug 2026 02:43:12 -0500
Subject: [PATCH 4/5] [offload] Thread allocation kind through async info

Allocations registered for release after finalization were all freed as
TARGET_ALLOC_DEVICE, so only device memory could be registered. Store the
kind alongside the pointer and free with it.

Release them in queryAsync as well. An async info that belongs to a nowait
task is never synchronized, so queryAsync reporting the queue as complete is
the only notification it gets and its allocations were leaked. Collect them
under the async info's mutex, as synchronize does, so that an operation
issued between the query and the release does not have its allocations freed
while it is still using them.

Claude assisted with this patch.
---
 offload/include/Shared/APITypes.h             |  7 ++-
 .../common/include/PluginInterface.h          |  4 +-
 .../common/src/PluginInterface.cpp            | 44 ++++++++++++++++---
 3 files changed, 44 insertions(+), 11 deletions(-)

diff --git a/offload/include/Shared/APITypes.h b/offload/include/Shared/APITypes.h
index 47d8c49bf7ef2..67edc7c9f60b7 100644
--- a/offload/include/Shared/APITypes.h
+++ b/offload/include/Shared/APITypes.h
@@ -23,6 +23,9 @@
 #include <cstddef>
 #include <cstdint>
 #include <mutex>
+#include <utility>
+
+enum TargetAllocTy : int32_t;
 
 extern "C" {
 
@@ -75,8 +78,8 @@ struct __tgt_async_info {
   void *Queue = nullptr;
 
   /// A collection of allocations that are associated with this stream and that
-  /// should be freed after finalization.
-  llvm::SmallVector<void *, 2> AssociatedAllocations;
+  /// should be freed after finalization. Also store the type of the allocation.
+  llvm::SmallVector<std::pair<void *, TargetAllocTy>, 2> AssociatedAllocations;
 
   /// Mutex to guard access to AssociatedAllocations and the Queue.
   std::mutex Mutex;
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index 50be67886c527..0b6a2223a75ce 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -168,9 +168,9 @@ struct AsyncInfoWrapperTy {
 
   /// Register \p Ptr as an associated allocation that is freed after
   /// finalization.
-  void freeAllocationAfterSynchronization(void *Ptr) {
+  void freeAllocationAfterSynchronization(void *Ptr, TargetAllocTy Kind) {
     std::lock_guard<std::mutex> AllocationGuard(AsyncInfoPtr->Mutex);
-    AsyncInfoPtr->AssociatedAllocations.push_back(Ptr);
+    AsyncInfoPtr->AssociatedAllocations.push_back({Ptr, Kind});
   }
 
 private:
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index 389d2ffbf4395..f4dcff3879ccd 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -137,7 +137,8 @@ GenericKernelTy::getKernelLaunchEnvironment(
     return AllocOrErr.takeError();
 
   // Remember to free the memory later.
-  AsyncInfoWrapper.freeAllocationAfterSynchronization(*AllocOrErr);
+  AsyncInfoWrapper.freeAllocationAfterSynchronization(
+      *AllocOrErr, TargetAllocTy::TARGET_ALLOC_DEVICE);
 
   /// Use the KLE in the __tgt_async_info to ensure a stable address for the
   /// async data transfer.
@@ -159,7 +160,8 @@ GenericKernelTy::getKernelLaunchEnvironment(
       return AllocOrErr.takeError();
     LocalKLE.ReductionBuffer = *AllocOrErr;
     // Remember to free the memory later.
-    AsyncInfoWrapper.freeAllocationAfterSynchronization(*AllocOrErr);
+    AsyncInfoWrapper.freeAllocationAfterSynchronization(
+        *AllocOrErr, TargetAllocTy::TARGET_ALLOC_DEVICE);
   }
 
   INFO(OMP_INFOTYPE_DATA_TRANSFER, GenericDevice.getDeviceId(),
@@ -289,7 +291,7 @@ Error GenericKernelTy::launch(GenericDeviceTy &GenericDevice, void **ArgPtrs,
   DynBlockMemConfTy &DynBlockMemConf = *DynBlockMemConfOrErr;
   if (DynBlockMemConf.FallbackPtr)
     AsyncInfoWrapper.freeAllocationAfterSynchronization(
-        DynBlockMemConf.FallbackPtr);
+        DynBlockMemConf.FallbackPtr, TargetAllocTy::TARGET_ALLOC_DEVICE);
 
   auto KernelLaunchEnvOrErr =
       getKernelLaunchEnvironment(GenericDevice, KernelArgs, DynBlockMemConf,
@@ -976,7 +978,7 @@ Error GenericDeviceTy::synchronize(__tgt_async_info *AsyncInfo,
     return Plugin::error(ErrorCode::INVALID_ARGUMENT,
                          "invalid async info queue");
 
-  SmallVector<void *> AllocsToDelete{};
+  SmallVector<std::pair<void *, TargetAllocTy>> AllocsToDelete{};
   {
     std::lock_guard<std::mutex> AllocationGuard{AsyncInfo->Mutex};
 
@@ -989,8 +991,8 @@ Error GenericDeviceTy::synchronize(__tgt_async_info *AsyncInfo,
     std::swap(AllocsToDelete, AsyncInfo->AssociatedAllocations);
   }
 
-  for (auto *Ptr : AllocsToDelete)
-    if (auto Err = dataDelete(Ptr, TargetAllocTy::TARGET_ALLOC_DEVICE))
+  for (auto [Ptr, Kind] : AllocsToDelete)
+    if (auto Err = dataDelete(Ptr, Kind))
       return Err;
 
   return Plugin::success();
@@ -1003,7 +1005,35 @@ Error GenericDeviceTy::queryAsync(__tgt_async_info *AsyncInfo,
     return Plugin::error(ErrorCode::INVALID_ARGUMENT,
                          "invalid async info queue");
 
-  return queryAsyncImpl(*AsyncInfo, ReleaseQueue, IsQueueWorkCompleted);
+  bool WorkCompleted = false;
+  SmallVector<std::pair<void *, TargetAllocTy>> AllocsToDelete{};
+
+  {
+    // Query and collect under the mutex, as synchronize does. Querying outside
+    // it would let an operation issued in between have its allocations freed
+    // here while it is still using them.
+    std::lock_guard<std::mutex> AllocationGuard{AsyncInfo->Mutex};
+    if (auto Err = queryAsyncImpl(*AsyncInfo, ReleaseQueue, &WorkCompleted)) {
+      if (IsQueueWorkCompleted)
+        *IsQueueWorkCompleted = WorkCompleted;
+      return Err;
+    }
+
+    // An async info belonging to a nowait task is never synchronized, so this
+    // is the only completion notification it ever gets. Without releasing its
+    // allocations here they are never freed at all.
+    if (WorkCompleted)
+      std::swap(AllocsToDelete, AsyncInfo->AssociatedAllocations);
+  }
+
+  for (auto [Ptr, Kind] : AllocsToDelete)
+    if (auto Err = dataDelete(Ptr, Kind))
+      return Err;
+
+  if (IsQueueWorkCompleted)
+    *IsQueueWorkCompleted = WorkCompleted;
+
+  return Plugin::success();
 }
 
 Error GenericDeviceTy::memoryVAMap(void **Addr, void *VAddr, size_t *RSize) {

>From fefb28de440ab9b98163665e7d366bd972402ba3 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Mon, 3 Aug 2026 16:38:02 -0500
Subject: [PATCH 5/5] [offload] Use pinned memory for KLE

Reduce kernel launch latency by using the fast path "pinned host memory
-> device memory" for submitting the kernel launch environment to the
device.

Claude assisted with this patch.
---
 offload/plugins-nextgen/amdgpu/src/rtl.cpp    |  2 +
 .../common/include/PluginInterface.h          | 14 ++++
 .../common/src/PluginInterface.cpp            | 49 ++++++++++++-
 .../offloading/kernel_launch_environment.c    | 71 +++++++++++++++++++
 4 files changed, 134 insertions(+), 2 deletions(-)
 create mode 100644 offload/test/offloading/kernel_launch_environment.c

diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index e74cf03071fa7..327764102c43d 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -2839,6 +2839,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     return true;
   }
 
+  bool hasFastSubmitFromPinnedMemory() const override { return true; }
+
   /// Submit data to the device (host to device transfer).
   Error dataSubmitImpl(void *TgtPtr, const void *HstPtr, int64_t Size,
                        AsyncInfoWrapperTy &AsyncInfoWrapper) override {
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index 0b6a2223a75ce..1092a6ced6a6d 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -982,6 +982,20 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
   virtual Error queryAsyncImpl(__tgt_async_info &AsyncInfo, bool ReleaseQueue,
                                bool *IsQueueWorkCompleted) = 0;
 
+  /// Indicate whether dataSubmitImpl has a faster path for host buffers that
+  /// are registered as pinned memory. If a plugin returns true, the kernel
+  /// launch environment is copied into a pinned host buffer before it is
+  /// submitted, so that its transfer takes that path.
+  virtual bool hasFastSubmitFromPinnedMemory() const { return false; }
+
+  /// Allocate a pinned host buffer to stage a kernel launch environment. The
+  /// caller owns it until it registers it with
+  /// AsyncInfoWrapperTy::freeAllocationAfterSynchronization, which releases it
+  /// once the transfer reading it has completed. Returns nullptr if staging is
+  /// unavailable, in which case the caller must submit the launch environment
+  /// from ordinary host memory.
+  KernelLaunchEnvironmentTy *getPinnedLaunchEnvBuffer();
+
   /// Check whether the architecture supports VA management
   virtual bool supportVAManagement() const { return false; }
 
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index f4dcff3879ccd..32357ac678b2d 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -164,17 +164,36 @@ GenericKernelTy::getKernelLaunchEnvironment(
         *AllocOrErr, TargetAllocTy::TARGET_ALLOC_DEVICE);
   }
 
+  // Copy into a pinned buffer if the plugin transfers those faster: dataAlloc
+  // registers TARGET_ALLOC_HOST memory in PinnedAllocs, so the dataSubmit
+  // below can take the one-step copy path.
+  const void *LaunchEnvSrc = &LocalKLE;
+  auto *PinnedKLE = GenericDevice.getPinnedLaunchEnvBuffer();
+  if (PinnedKLE) {
+    *PinnedKLE = LocalKLE;
+    LaunchEnvSrc = PinnedKLE;
+  }
+
   INFO(OMP_INFOTYPE_DATA_TRANSFER, GenericDevice.getDeviceId(),
        "Copying data from host to device, HstPtr=" DPxMOD ", TgtPtr=" DPxMOD
        ", Size=%" PRId64 ", Name=KernelLaunchEnv\n",
-       DPxPTR(&LocalKLE), DPxPTR(*AllocOrErr),
+       DPxPTR(LaunchEnvSrc), DPxPTR(*AllocOrErr),
        sizeof(KernelLaunchEnvironmentTy));
 
-  auto Err = GenericDevice.dataSubmit(*AllocOrErr, &LocalKLE,
+  auto Err = GenericDevice.dataSubmit(*AllocOrErr, LaunchEnvSrc,
                                       sizeof(KernelLaunchEnvironmentTy),
                                       AsyncInfoWrapper);
   if (Err)
     return Err;
+
+  // Register the staging buffer only now that the transfer reading it has
+  // been issued. Registering it earlier would let a concurrent finalization
+  // that observes the queue as complete release it while the transfer is
+  // still in flight.
+  if (PinnedKLE)
+    AsyncInfoWrapper.freeAllocationAfterSynchronization(
+        PinnedKLE, TargetAllocTy::TARGET_ALLOC_HOST);
+
   return static_cast<KernelLaunchEnvironmentTy *>(*AllocOrErr);
 }
 
@@ -1036,6 +1055,32 @@ Error GenericDeviceTy::queryAsync(__tgt_async_info *AsyncInfo,
   return Plugin::success();
 }
 
+KernelLaunchEnvironmentTy *GenericDeviceTy::getPinnedLaunchEnvBuffer() {
+  if (!hasFastSubmitFromPinnedMemory())
+    return nullptr;
+
+  // While recording or replaying, dataAlloc serves every allocation kind from
+  // the record-replay device memory pool, so it cannot give us host memory.
+  if (RecordReplay && RecordReplay->isRecordingOrReplaying())
+    return nullptr;
+
+  auto AllocOrErr =
+      dataAlloc(sizeof(KernelLaunchEnvironmentTy), /*HostPtr=*/nullptr,
+                TargetAllocTy::TARGET_ALLOC_HOST, /*Alignment=*/0);
+  if (!AllocOrErr) {
+    // Staging is optional, so fall back to unpinned memory. Consume the
+    // error unconditionally: ODBG does not evaluate its operands unless
+    // debugging is enabled.
+    std::string ErrStr = toString(AllocOrErr.takeError());
+    ODBG(OLDT_Alloc) << "Failed to allocate a pinned buffer for the kernel "
+                        "launch environment, submitting it unstaged: "
+                     << ErrStr;
+    return nullptr;
+  }
+
+  return static_cast<KernelLaunchEnvironmentTy *>(*AllocOrErr);
+}
+
 Error GenericDeviceTy::memoryVAMap(void **Addr, void *VAddr, size_t *RSize) {
   return Plugin::error(ErrorCode::UNSUPPORTED,
                        "device does not support VA Management");
diff --git a/offload/test/offloading/kernel_launch_environment.c b/offload/test/offloading/kernel_launch_environment.c
new file mode 100644
index 0000000000000..71a31e9d1bc6d
--- /dev/null
+++ b/offload/test/offloading/kernel_launch_environment.c
@@ -0,0 +1,71 @@
+// Stress the kernel launch environment (KLE) transfer.
+//
+// A cross-team reduction gives the launch a KLE, which the plugin stages in a
+// host buffer and copies to the device asynchronously. The KLE carries that
+// launch's own reduction buffer, so overlapping launches must neither share a
+// staging buffer nor release one while its transfer is still in flight: either
+// makes a launch reduce into another launch's buffer. Every launch here
+// accumulates a distinct value, so that shows up as a wrong sum.
+//
+// RUN: %libomptarget-compile-generic -fopenmp-offload-mandatory
+// RUN: %libomptarget-run-generic
+// RUN: %libomptarget-compileopt-generic -fopenmp-offload-mandatory
+// RUN: %libomptarget-run-generic
+//
+// REQUIRES: gpu
+
+#include <omp.h>
+#include <stdio.h>
+
+#define NUM_LAUNCHES 32
+#define N 4096
+
+// Launch k accumulates k + 1 per element.
+static long expected(int k) { return (long)(k + 1) * N; }
+
+static int check(const char *Phase, long *Results) {
+  int Errors = 0;
+  for (int k = 0; k < NUM_LAUNCHES; k++)
+    if (Results[k] != expected(k)) {
+      fprintf(stderr, "%s: launch %d reduced to %ld, expected %ld\n", Phase, k,
+              Results[k], expected(k));
+      Errors++;
+    }
+  return Errors;
+}
+
+int main(void) {
+  static long Results[NUM_LAUNCHES];
+  int Errors = 0;
+
+  // Launches issued back to back without an intervening synchronization, so
+  // that several KLE transfers are outstanding at once.
+  for (int k = 0; k < NUM_LAUNCHES; k++) {
+    Results[k] = 0;
+#pragma omp target teams distribute parallel for map(tofrom : Results[k : 1])  \
+    reduction(+ : Results[k]) firstprivate(k) nowait
+    for (int i = 0; i < N; i++)
+      Results[k] += k + 1;
+  }
+#pragma omp taskwait
+  Errors += check("nowait", Results);
+
+  // Same, but with the launches and the synchronizations spread over several
+  // host threads: a thread finalizing its queue must not release a staging
+  // buffer that another thread's launch is still using.
+#pragma omp parallel for num_threads(8)
+  for (int k = 0; k < NUM_LAUNCHES; k++) {
+    long Sum = 0;
+#pragma omp target teams distribute parallel for map(tofrom : Sum)             \
+    reduction(+ : Sum) firstprivate(k)
+    for (int i = 0; i < N; i++)
+      Sum += k + 1;
+    Results[k] = Sum;
+  }
+  Errors += check("threaded", Results);
+
+  if (Errors)
+    return 1;
+  printf("PASS\n");
+  return 0;
+}


From openmp-commits at lists.llvm.org  Fri Aug  7 07:36:22 2026
From: openmp-commits at lists.llvm.org (Robert Imschweiler via Openmp-commits)
Date: Fri, 07 Aug 2026 07:36:22 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [offload] Use pinned memory for
	KLE (PR #213767)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213767@github.com>
Message-ID: <6a75ed66.44484dd4.3bb80e.bffa@mx.google.com>


https://github.com/ro-i updated https://github.com/llvm/llvm-project/pull/213767

>From 022265c76bd74cc16714960cdb91961fd6fb42a7 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Wed, 5 Aug 2026 10:24:00 -0500
Subject: [PATCH 1/5] [offload] Do not pool memory while allocation traces are
 requested

Would otherwise hide use-after-free because memory stays valid if it's
in the pool.
---
 offload/plugins-nextgen/common/src/PluginInterface.cpp | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index e539cbfc55324..d4f3d8b62a272 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -598,9 +598,13 @@ Error GenericDeviceTy::init(GenericPluginTy &Plugin) {
     GridValues.GV_Max_WG_Size =
         std::min(GridValues.GV_Max_WG_Size, uint32_t(OMP_TeamsThreadLimit));
 
-  // Enable the memory manager if required.
+  // Enable the memory manager if required. Memory that the manager pools is
+  // not returned to the driver when it is freed, so a later access to it
+  // neither faults nor is reported by the allocation tracker. Leave the pool
+  // disabled while allocation traces are requested, so that we don't mask
+  // use-after-free (since they don't fault if the memory is still in the pool).
   auto [ThresholdMM, EnableMM] = MemoryManagerTy::getSizeThresholdFromEnv();
-  if (EnableMM) {
+  if (EnableMM && !OMPX_TrackAllocationTraces) {
     if (ThresholdMM == 0)
       ThresholdMM = getMemoryManagerSizeThreshold();
     MemoryManager = new MemoryManagerTy(*this, ThresholdMM);

>From 35c52f4f5d6ba85ffdf5ec1438780bc7caf883e2 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Thu, 6 Aug 2026 11:51:52 -0500
Subject: [PATCH 2/5] [offload] Respect alignment in memory manager

Also fix alignment comparison in AMDGPU rtl.cpp.
---
 offload/plugins-nextgen/amdgpu/src/rtl.cpp             | 2 +-
 offload/plugins-nextgen/common/include/MemoryManager.h | 9 +++++++--
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index 29b5006f0d7b6..0f0de67b20c7c 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -339,7 +339,7 @@ struct AMDGPUMemoryPoolTy {
     // compared with the alignment of the memory allocated using the given pool.
     // If the default alignment is greater than or equal to the alignment
     // requested by the user, it would still meet the user's requirements.
-    if (Alignment > 0 && Alignment >= PoolAllocationAlignment) {
+    if (Alignment > 0 && Alignment > PoolAllocationAlignment) {
       return Plugin::error(ErrorCode::UNSUPPORTED,
                            "requested alignment (%lu) larger than maximum "
                            "supported pool alignment (%lu)",
diff --git a/offload/plugins-nextgen/common/include/MemoryManager.h b/offload/plugins-nextgen/common/include/MemoryManager.h
index 4b57be45e7551..883ac40c269ea 100644
--- a/offload/plugins-nextgen/common/include/MemoryManager.h
+++ b/offload/plugins-nextgen/common/include/MemoryManager.h
@@ -13,6 +13,7 @@
 #ifndef LLVM_OPENMP_LIBOMPTARGET_PLUGINS_COMMON_MEMORYMANAGER_H
 #define LLVM_OPENMP_LIBOMPTARGET_PLUGINS_COMMON_MEMORYMANAGER_H
 
+#include <algorithm>
 #include <cassert>
 #include <functional>
 #include <list>
@@ -25,6 +26,7 @@
 #include "Shared/Utils.h"
 #include "omptarget.h"
 
+#include "llvm/Support/Alignment.h"
 #include "llvm/Support/Error.h"
 
 using namespace llvm::offload::debug;
@@ -268,9 +270,12 @@ class MemoryManagerTy {
 
       NodeTy TempNode(Size, nullptr);
       std::lock_guard<std::mutex> LG(FreeListLocks[B]);
-      const auto Itr = List.find(TempNode);
+      auto [First, Last] = List.equal_range(TempNode);
 
-      if (Itr != List.end()) {
+      auto Itr = std::find_if(First, Last, [Alignment](const NodeTy &N) {
+        return Alignment == 0 || isAddrAligned(Align(Alignment), N.Ptr);
+      });
+      if (Itr != Last) {
         NodePtr = &Itr->get();
         List.erase(Itr);
       }

>From 73db29cd0403c6f01c0b838e24310653cfe6321a Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Wed, 5 Aug 2026 10:24:27 -0500
Subject: [PATCH 3/5] [offload] Pool host and shared allocations

Route them through a memory manager, like the device allocations. Also,
move the registration as pinned memory to the plugin site since only the
plugin knows if the corresponding host/shared memory is pinned.
Improves performance of affected allocations by ballpark 1,000x.
---
 offload/plugins-nextgen/amdgpu/src/rtl.cpp    | 12 ++++
 .../common/include/MemoryManager.h            | 20 ++++--
 .../common/include/PluginInterface.h          | 20 +++++-
 .../common/src/PluginInterface.cpp            | 69 ++++++++-----------
 openmp/docs/design/Runtimes.rst               | 11 +--
 5 files changed, 80 insertions(+), 52 deletions(-)

diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index 0f0de67b20c7c..e74cf03071fa7 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -2728,6 +2728,12 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
       return Plugin::error(ErrorCode::OUT_OF_RESOURCES,
                            "no memory pool for the specified allocation kind");
 
+    // See allocate() for the registration of host / shared memory as pinned
+    // memory.
+    if (Kind == TARGET_ALLOC_HOST || Kind == TARGET_ALLOC_SHARED)
+      if (auto Err = PinnedAllocs.unregisterHostBuffer(TgtPtr))
+        return Err;
+
     if (auto Err = MemoryPool->deallocate(TgtPtr))
       return Err;
 
@@ -4529,6 +4535,12 @@ Expected<void *> AMDGPUDeviceTy::allocate(size_t Size, void *,
     // Enable all valid kernel agents to access the buffer.
     if (auto Err = MemoryPool->enableAccess(Alloc, Size, Agents))
       return std::move(Err);
+
+    // Register host / shared memory as pinned memory, so that transfers reading
+    // from it can take a device-accessible path.
+    if (Kind == TARGET_ALLOC_HOST || Kind == TARGET_ALLOC_SHARED)
+      if (auto Err = PinnedAllocs.registerHostBuffer(Alloc, Alloc, Size))
+        return std::move(Err);
   }
 
   return Alloc;
diff --git a/offload/plugins-nextgen/common/include/MemoryManager.h b/offload/plugins-nextgen/common/include/MemoryManager.h
index 883ac40c269ea..9712a461ad974 100644
--- a/offload/plugins-nextgen/common/include/MemoryManager.h
+++ b/offload/plugins-nextgen/common/include/MemoryManager.h
@@ -138,21 +138,24 @@ class MemoryManagerTy {
 
   /// The reference to a device allocator
   DeviceAllocatorTy &DeviceAllocator;
+  /// The kind of device for which the memory manager is allocated
+  TargetAllocTy DeviceKind;
 
   /// The threshold to manage memory using memory manager. If the request size
   /// is larger than \p SizeThreshold, the allocation will not be managed by the
   /// memory manager.
-  size_t SizeThreshold = 1U << 13;
+  size_t SizeThreshold = DefaultSizeThreshold;
 
   /// Request memory from target device
   Expected<void *> allocateOnDevice(size_t Size, void *HstPtr,
                                     size_t Alignment) const {
-    return DeviceAllocator.allocate(Size, HstPtr, TARGET_ALLOC_DEVICE,
-                                    Alignment);
+    return DeviceAllocator.allocate(Size, HstPtr, DeviceKind, Alignment);
   }
 
   /// Deallocate data on device
-  Error deleteOnDevice(void *Ptr) const { return DeviceAllocator.free(Ptr); }
+  Error deleteOnDevice(void *Ptr) const {
+    return DeviceAllocator.free(Ptr, DeviceKind);
+  }
 
   /// This function is called when it tries to allocate memory on device but the
   /// device returns out of memory. It will first free all memory in the
@@ -216,11 +219,14 @@ class MemoryManagerTy {
   }
 
 public:
+  static constexpr size_t DefaultSizeThreshold = 1U << 13;
+
   /// Constructor. If \p Threshold is non-zero, then the default threshold will
   /// be overwritten by \p Threshold.
-  MemoryManagerTy(DeviceAllocatorTy &DeviceAllocator, size_t Threshold = 0)
+  MemoryManagerTy(DeviceAllocatorTy &DeviceAllocator, size_t Threshold = 0,
+                  TargetAllocTy DeviceKind = TARGET_ALLOC_DEVICE)
       : FreeLists(NumBuckets), FreeListLocks(NumBuckets),
-        DeviceAllocator(DeviceAllocator) {
+        DeviceAllocator(DeviceAllocator), DeviceKind(DeviceKind) {
     if (Threshold)
       SizeThreshold = Threshold;
   }
@@ -359,6 +365,8 @@ class MemoryManagerTy {
   /// threshold and the second element represents whether user disables memory
   /// manager explicitly by setting the var to 0. If user doesn't specify
   /// anything, returns <0, true>.
+  /// Note that this only affects the device memory manager, not the manager for
+  /// host or shared memory.
   static std::pair<size_t, bool> getSizeThresholdFromEnv() {
     static UInt64Envar MemoryManagerThreshold(
         "LIBOMPTARGET_MEMORY_MANAGER_THRESHOLD", 0);
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index b3675e5a8700f..50be67886c527 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -1368,8 +1368,12 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
   /// only necessary for unhosted targets like the GPU.
   virtual bool shouldSetupRPCServer() const { return false; }
 
-  /// Pointer to the memory manager or nullptr if not available.
+  /// Pointer to the device memory manager or nullptr if not available.
   MemoryManagerTy *MemoryManager;
+  /// Memory managers for the host and shared allocation kinds or nullptr if not
+  /// available.
+  MemoryManagerTy *HostMemoryManager;
+  MemoryManagerTy *SharedMemoryManager;
 
   /// Per device setting of MemoryManager's Threshold
   virtual size_t getMemoryManagerSizeThreshold() { return 0; }
@@ -1406,6 +1410,20 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
   /// Record and replay manager.
   RecordReplayTy *RecordReplay = nullptr;
 
+  /// Return the memory manager for the given allocation kind.
+  MemoryManagerTy *getMemoryManagerFor(TargetAllocTy Kind) {
+    switch (Kind) {
+    case TARGET_ALLOC_DEFAULT:
+    case TARGET_ALLOC_DEVICE:
+      return MemoryManager;
+    case TARGET_ALLOC_HOST:
+      return HostMemoryManager;
+    case TARGET_ALLOC_SHARED:
+      return SharedMemoryManager;
+    }
+    return nullptr;
+  }
+
 protected:
   /// Environment variables defined by the LLVM OpenMP implementation
   /// regarding the initial number of streams and events.
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index d4f3d8b62a272..389d2ffbf4395 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -490,7 +490,8 @@ uint32_t GenericKernelTy::getEffectiveNumBlocks(
 GenericDeviceTy::GenericDeviceTy(GenericPluginTy &Plugin, int32_t DeviceId,
                                  int32_t NumDevices,
                                  const llvm::omp::GV &OMPGridValues)
-    : Plugin(Plugin), MemoryManager(nullptr), OMP_TeamLimit("OMP_TEAM_LIMIT"),
+    : Plugin(Plugin), MemoryManager(nullptr), HostMemoryManager(nullptr),
+      SharedMemoryManager(nullptr), OMP_TeamLimit("OMP_TEAM_LIMIT"),
       OMP_NumTeams("OMP_NUM_TEAMS"),
       OMP_TeamsThreadLimit("OMP_TEAMS_THREAD_LIMIT"),
       OMPX_DebugKind("LIBOMPTARGET_DEVICE_RTL_DEBUG"),
@@ -609,6 +610,14 @@ Error GenericDeviceTy::init(GenericPluginTy &Plugin) {
       ThresholdMM = getMemoryManagerSizeThreshold();
     MemoryManager = new MemoryManagerTy(*this, ThresholdMM);
   }
+  if (!OMPX_TrackAllocationTraces) {
+    // Keep the threshold for pooling sizes conservative since we're dealing
+    // with pinned memory for the host.
+    HostMemoryManager = new MemoryManagerTy(
+        *this, MemoryManagerTy::DefaultSizeThreshold, TARGET_ALLOC_HOST);
+    SharedMemoryManager = new MemoryManagerTy(
+        *this, MemoryManagerTy::DefaultSizeThreshold, TARGET_ALLOC_SHARED);
+  }
 
   return Plugin::success();
 }
@@ -656,6 +665,12 @@ Error GenericDeviceTy::deinit(GenericPluginTy &Plugin) {
   if (MemoryManager)
     delete MemoryManager;
   MemoryManager = nullptr;
+  if (HostMemoryManager)
+    delete HostMemoryManager;
+  HostMemoryManager = nullptr;
+  if (SharedMemoryManager)
+    delete SharedMemoryManager;
+  SharedMemoryManager = nullptr;
 
   if (RecordReplay) {
     if (auto Err = RecordReplay->deinit())
@@ -1016,22 +1031,15 @@ Expected<void *> GenericDeviceTy::dataAlloc(int64_t Size, void *HostPtr,
   if (RecordReplay && RecordReplay->isRecordingOrReplaying())
     return RecordReplay->allocate(Size);
 
-  switch (Kind) {
-  case TARGET_ALLOC_DEFAULT:
-  case TARGET_ALLOC_DEVICE:
-    if (MemoryManager) {
-      auto AllocOrErr = MemoryManager->allocate(Size, HostPtr, Alignment);
-      if (!AllocOrErr)
-        return AllocOrErr.takeError();
-      Alloc = *AllocOrErr;
-      if (!Alloc)
-        return Plugin::error(ErrorCode::OUT_OF_RESOURCES,
-                             "failed to allocate from memory manager");
-      break;
-    }
-    [[fallthrough]];
-  case TARGET_ALLOC_HOST:
-  case TARGET_ALLOC_SHARED: {
+  if (MemoryManagerTy *MM = getMemoryManagerFor(Kind)) {
+    auto AllocOrErr = MM->allocate(Size, HostPtr, Alignment);
+    if (!AllocOrErr)
+      return AllocOrErr.takeError();
+    Alloc = *AllocOrErr;
+    if (!Alloc)
+      return Plugin::error(ErrorCode::OUT_OF_RESOURCES,
+                           "failed to allocate from memory manager");
+  } else {
     auto AllocOrErr = allocate(Size, HostPtr, Kind, Alignment);
     if (!AllocOrErr)
       return AllocOrErr.takeError();
@@ -1040,7 +1048,6 @@ Expected<void *> GenericDeviceTy::dataAlloc(int64_t Size, void *HostPtr,
       return Plugin::error(ErrorCode::OUT_OF_RESOURCES,
                            "failed to allocate from device allocator");
   }
-  }
 
   // Report error if the memory manager or the device allocator did not return
   // any memory buffer.
@@ -1049,11 +1056,6 @@ Expected<void *> GenericDeviceTy::dataAlloc(int64_t Size, void *HostPtr,
                          "invalid target data allocation kind or requested "
                          "allocator not implemented yet");
 
-  // Register allocated buffer as pinned memory if the type is host memory.
-  if (Kind == TARGET_ALLOC_HOST)
-    if (auto Err = PinnedAllocs.registerHostBuffer(Alloc, Alloc, Size))
-      return std::move(Err);
-
   // Keep track of the allocation stack if we track allocation traces.
   if (OMPX_TrackAllocationTraces) {
     std::string StackTrace;
@@ -1110,26 +1112,13 @@ Error GenericDeviceTy::dataDelete(void *TgtPtr, TargetAllocTy Kind) {
     ATI->DeallocationTrace = StackTrace;
   }
 
-  switch (Kind) {
-  case TARGET_ALLOC_DEFAULT:
-  case TARGET_ALLOC_DEVICE:
-    if (MemoryManager) {
-      if (auto Err = MemoryManager->free(TgtPtr))
-        return Err;
-      break;
-    }
-    [[fallthrough]];
-  case TARGET_ALLOC_HOST:
-  case TARGET_ALLOC_SHARED:
-    if (auto Err = free(TgtPtr, Kind))
+  if (MemoryManagerTy *MM = getMemoryManagerFor(Kind)) {
+    if (auto Err = MM->free(TgtPtr))
       return Err;
+  } else if (auto Err = free(TgtPtr, Kind)) {
+    return Err;
   }
 
-  // Unregister deallocated pinned memory buffer if the type is host memory.
-  if (Kind == TARGET_ALLOC_HOST)
-    if (auto Err = PinnedAllocs.unregisterHostBuffer(TgtPtr))
-      return Err;
-
   return Plugin::success();
 }
 
diff --git a/openmp/docs/design/Runtimes.rst b/openmp/docs/design/Runtimes.rst
index 14e300a0f531f..0e75925c78742 100644
--- a/openmp/docs/design/Runtimes.rst
+++ b/openmp/docs/design/Runtimes.rst
@@ -784,11 +784,12 @@ LIBOMPTARGET_MEMORY_MANAGER_THRESHOLD
 """""""""""""""""""""""""""""""""""""
 
 ``LIBOMPTARGET_MEMORY_MANAGER_THRESHOLD`` sets the threshold size for which the
-``libomptarget`` memory manager will handle the allocation. Any allocations
-larger than this threshold will not use the memory manager and be freed after
-the device kernel exits. The default threshold value is ``8KB``. If
-``LIBOMPTARGET_MEMORY_MANAGER_THRESHOLD`` is set to ``0`` the memory manager
-will be completely disabled.
+``libomptarget`` device memory manager will handle the allocation. Any
+allocations larger than this threshold will not use the memory manager and be
+freed after the device kernel exits. The default threshold value is ``8KB``. If
+``LIBOMPTARGET_MEMORY_MANAGER_THRESHOLD`` is set to ``0`` the device memory
+manager will be completely disabled.
+This has no effect on the host or shared memory managers.
 
 .. _libomptarget_info:
 

>From d4ce6a6138bc4af6cb85156ab8945493cb1228ef Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Fri, 7 Aug 2026 02:43:12 -0500
Subject: [PATCH 4/5] [offload] Thread allocation kind through async info

Claude assisted with this patch.
---
 offload/include/Shared/APITypes.h             |  7 ++-
 .../common/include/PluginInterface.h          |  4 +-
 .../common/src/PluginInterface.cpp            | 44 ++++++++++++++++---
 3 files changed, 44 insertions(+), 11 deletions(-)

diff --git a/offload/include/Shared/APITypes.h b/offload/include/Shared/APITypes.h
index 47d8c49bf7ef2..67edc7c9f60b7 100644
--- a/offload/include/Shared/APITypes.h
+++ b/offload/include/Shared/APITypes.h
@@ -23,6 +23,9 @@
 #include <cstddef>
 #include <cstdint>
 #include <mutex>
+#include <utility>
+
+enum TargetAllocTy : int32_t;
 
 extern "C" {
 
@@ -75,8 +78,8 @@ struct __tgt_async_info {
   void *Queue = nullptr;
 
   /// A collection of allocations that are associated with this stream and that
-  /// should be freed after finalization.
-  llvm::SmallVector<void *, 2> AssociatedAllocations;
+  /// should be freed after finalization. Also store the type of the allocation.
+  llvm::SmallVector<std::pair<void *, TargetAllocTy>, 2> AssociatedAllocations;
 
   /// Mutex to guard access to AssociatedAllocations and the Queue.
   std::mutex Mutex;
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index 50be67886c527..0b6a2223a75ce 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -168,9 +168,9 @@ struct AsyncInfoWrapperTy {
 
   /// Register \p Ptr as an associated allocation that is freed after
   /// finalization.
-  void freeAllocationAfterSynchronization(void *Ptr) {
+  void freeAllocationAfterSynchronization(void *Ptr, TargetAllocTy Kind) {
     std::lock_guard<std::mutex> AllocationGuard(AsyncInfoPtr->Mutex);
-    AsyncInfoPtr->AssociatedAllocations.push_back(Ptr);
+    AsyncInfoPtr->AssociatedAllocations.push_back({Ptr, Kind});
   }
 
 private:
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index 389d2ffbf4395..f4dcff3879ccd 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -137,7 +137,8 @@ GenericKernelTy::getKernelLaunchEnvironment(
     return AllocOrErr.takeError();
 
   // Remember to free the memory later.
-  AsyncInfoWrapper.freeAllocationAfterSynchronization(*AllocOrErr);
+  AsyncInfoWrapper.freeAllocationAfterSynchronization(
+      *AllocOrErr, TargetAllocTy::TARGET_ALLOC_DEVICE);
 
   /// Use the KLE in the __tgt_async_info to ensure a stable address for the
   /// async data transfer.
@@ -159,7 +160,8 @@ GenericKernelTy::getKernelLaunchEnvironment(
       return AllocOrErr.takeError();
     LocalKLE.ReductionBuffer = *AllocOrErr;
     // Remember to free the memory later.
-    AsyncInfoWrapper.freeAllocationAfterSynchronization(*AllocOrErr);
+    AsyncInfoWrapper.freeAllocationAfterSynchronization(
+        *AllocOrErr, TargetAllocTy::TARGET_ALLOC_DEVICE);
   }
 
   INFO(OMP_INFOTYPE_DATA_TRANSFER, GenericDevice.getDeviceId(),
@@ -289,7 +291,7 @@ Error GenericKernelTy::launch(GenericDeviceTy &GenericDevice, void **ArgPtrs,
   DynBlockMemConfTy &DynBlockMemConf = *DynBlockMemConfOrErr;
   if (DynBlockMemConf.FallbackPtr)
     AsyncInfoWrapper.freeAllocationAfterSynchronization(
-        DynBlockMemConf.FallbackPtr);
+        DynBlockMemConf.FallbackPtr, TargetAllocTy::TARGET_ALLOC_DEVICE);
 
   auto KernelLaunchEnvOrErr =
       getKernelLaunchEnvironment(GenericDevice, KernelArgs, DynBlockMemConf,
@@ -976,7 +978,7 @@ Error GenericDeviceTy::synchronize(__tgt_async_info *AsyncInfo,
     return Plugin::error(ErrorCode::INVALID_ARGUMENT,
                          "invalid async info queue");
 
-  SmallVector<void *> AllocsToDelete{};
+  SmallVector<std::pair<void *, TargetAllocTy>> AllocsToDelete{};
   {
     std::lock_guard<std::mutex> AllocationGuard{AsyncInfo->Mutex};
 
@@ -989,8 +991,8 @@ Error GenericDeviceTy::synchronize(__tgt_async_info *AsyncInfo,
     std::swap(AllocsToDelete, AsyncInfo->AssociatedAllocations);
   }
 
-  for (auto *Ptr : AllocsToDelete)
-    if (auto Err = dataDelete(Ptr, TargetAllocTy::TARGET_ALLOC_DEVICE))
+  for (auto [Ptr, Kind] : AllocsToDelete)
+    if (auto Err = dataDelete(Ptr, Kind))
       return Err;
 
   return Plugin::success();
@@ -1003,7 +1005,35 @@ Error GenericDeviceTy::queryAsync(__tgt_async_info *AsyncInfo,
     return Plugin::error(ErrorCode::INVALID_ARGUMENT,
                          "invalid async info queue");
 
-  return queryAsyncImpl(*AsyncInfo, ReleaseQueue, IsQueueWorkCompleted);
+  bool WorkCompleted = false;
+  SmallVector<std::pair<void *, TargetAllocTy>> AllocsToDelete{};
+
+  {
+    // Query and collect under the mutex, as synchronize does. Querying outside
+    // it would let an operation issued in between have its allocations freed
+    // here while it is still using them.
+    std::lock_guard<std::mutex> AllocationGuard{AsyncInfo->Mutex};
+    if (auto Err = queryAsyncImpl(*AsyncInfo, ReleaseQueue, &WorkCompleted)) {
+      if (IsQueueWorkCompleted)
+        *IsQueueWorkCompleted = WorkCompleted;
+      return Err;
+    }
+
+    // An async info belonging to a nowait task is never synchronized, so this
+    // is the only completion notification it ever gets. Without releasing its
+    // allocations here they are never freed at all.
+    if (WorkCompleted)
+      std::swap(AllocsToDelete, AsyncInfo->AssociatedAllocations);
+  }
+
+  for (auto [Ptr, Kind] : AllocsToDelete)
+    if (auto Err = dataDelete(Ptr, Kind))
+      return Err;
+
+  if (IsQueueWorkCompleted)
+    *IsQueueWorkCompleted = WorkCompleted;
+
+  return Plugin::success();
 }
 
 Error GenericDeviceTy::memoryVAMap(void **Addr, void *VAddr, size_t *RSize) {

>From b5bd0cd9b2f25c2b3659cae942ff2fcdb6f826f5 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Mon, 3 Aug 2026 16:38:02 -0500
Subject: [PATCH 5/5] [offload] Use pinned memory for KLE

Reduce kernel launch latency by using the fast path "pinned host memory
-> device memory" for submitting the kernel launch environment to the
device.

Claude assisted with this patch.
---
 offload/plugins-nextgen/amdgpu/src/rtl.cpp    |  2 +
 .../common/include/PluginInterface.h          | 14 ++++
 .../common/src/PluginInterface.cpp            | 49 ++++++++++++-
 .../offloading/kernel_launch_environment.c    | 71 +++++++++++++++++++
 4 files changed, 134 insertions(+), 2 deletions(-)
 create mode 100644 offload/test/offloading/kernel_launch_environment.c

diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index e74cf03071fa7..327764102c43d 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -2839,6 +2839,8 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     return true;
   }
 
+  bool hasFastSubmitFromPinnedMemory() const override { return true; }
+
   /// Submit data to the device (host to device transfer).
   Error dataSubmitImpl(void *TgtPtr, const void *HstPtr, int64_t Size,
                        AsyncInfoWrapperTy &AsyncInfoWrapper) override {
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index 0b6a2223a75ce..1092a6ced6a6d 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -982,6 +982,20 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
   virtual Error queryAsyncImpl(__tgt_async_info &AsyncInfo, bool ReleaseQueue,
                                bool *IsQueueWorkCompleted) = 0;
 
+  /// Indicate whether dataSubmitImpl has a faster path for host buffers that
+  /// are registered as pinned memory. If a plugin returns true, the kernel
+  /// launch environment is copied into a pinned host buffer before it is
+  /// submitted, so that its transfer takes that path.
+  virtual bool hasFastSubmitFromPinnedMemory() const { return false; }
+
+  /// Allocate a pinned host buffer to stage a kernel launch environment. The
+  /// caller owns it until it registers it with
+  /// AsyncInfoWrapperTy::freeAllocationAfterSynchronization, which releases it
+  /// once the transfer reading it has completed. Returns nullptr if staging is
+  /// unavailable, in which case the caller must submit the launch environment
+  /// from ordinary host memory.
+  KernelLaunchEnvironmentTy *getPinnedLaunchEnvBuffer();
+
   /// Check whether the architecture supports VA management
   virtual bool supportVAManagement() const { return false; }
 
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index f4dcff3879ccd..32357ac678b2d 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -164,17 +164,36 @@ GenericKernelTy::getKernelLaunchEnvironment(
         *AllocOrErr, TargetAllocTy::TARGET_ALLOC_DEVICE);
   }
 
+  // Copy into a pinned buffer if the plugin transfers those faster: dataAlloc
+  // registers TARGET_ALLOC_HOST memory in PinnedAllocs, so the dataSubmit
+  // below can take the one-step copy path.
+  const void *LaunchEnvSrc = &LocalKLE;
+  auto *PinnedKLE = GenericDevice.getPinnedLaunchEnvBuffer();
+  if (PinnedKLE) {
+    *PinnedKLE = LocalKLE;
+    LaunchEnvSrc = PinnedKLE;
+  }
+
   INFO(OMP_INFOTYPE_DATA_TRANSFER, GenericDevice.getDeviceId(),
        "Copying data from host to device, HstPtr=" DPxMOD ", TgtPtr=" DPxMOD
        ", Size=%" PRId64 ", Name=KernelLaunchEnv\n",
-       DPxPTR(&LocalKLE), DPxPTR(*AllocOrErr),
+       DPxPTR(LaunchEnvSrc), DPxPTR(*AllocOrErr),
        sizeof(KernelLaunchEnvironmentTy));
 
-  auto Err = GenericDevice.dataSubmit(*AllocOrErr, &LocalKLE,
+  auto Err = GenericDevice.dataSubmit(*AllocOrErr, LaunchEnvSrc,
                                       sizeof(KernelLaunchEnvironmentTy),
                                       AsyncInfoWrapper);
   if (Err)
     return Err;
+
+  // Register the staging buffer only now that the transfer reading it has
+  // been issued. Registering it earlier would let a concurrent finalization
+  // that observes the queue as complete release it while the transfer is
+  // still in flight.
+  if (PinnedKLE)
+    AsyncInfoWrapper.freeAllocationAfterSynchronization(
+        PinnedKLE, TargetAllocTy::TARGET_ALLOC_HOST);
+
   return static_cast<KernelLaunchEnvironmentTy *>(*AllocOrErr);
 }
 
@@ -1036,6 +1055,32 @@ Error GenericDeviceTy::queryAsync(__tgt_async_info *AsyncInfo,
   return Plugin::success();
 }
 
+KernelLaunchEnvironmentTy *GenericDeviceTy::getPinnedLaunchEnvBuffer() {
+  if (!hasFastSubmitFromPinnedMemory())
+    return nullptr;
+
+  // While recording or replaying, dataAlloc serves every allocation kind from
+  // the record-replay device memory pool, so it cannot give us host memory.
+  if (RecordReplay && RecordReplay->isRecordingOrReplaying())
+    return nullptr;
+
+  auto AllocOrErr =
+      dataAlloc(sizeof(KernelLaunchEnvironmentTy), /*HostPtr=*/nullptr,
+                TargetAllocTy::TARGET_ALLOC_HOST, /*Alignment=*/0);
+  if (!AllocOrErr) {
+    // Staging is optional, so fall back to unpinned memory. Consume the
+    // error unconditionally: ODBG does not evaluate its operands unless
+    // debugging is enabled.
+    std::string ErrStr = toString(AllocOrErr.takeError());
+    ODBG(OLDT_Alloc) << "Failed to allocate a pinned buffer for the kernel "
+                        "launch environment, submitting it unstaged: "
+                     << ErrStr;
+    return nullptr;
+  }
+
+  return static_cast<KernelLaunchEnvironmentTy *>(*AllocOrErr);
+}
+
 Error GenericDeviceTy::memoryVAMap(void **Addr, void *VAddr, size_t *RSize) {
   return Plugin::error(ErrorCode::UNSUPPORTED,
                        "device does not support VA Management");
diff --git a/offload/test/offloading/kernel_launch_environment.c b/offload/test/offloading/kernel_launch_environment.c
new file mode 100644
index 0000000000000..71a31e9d1bc6d
--- /dev/null
+++ b/offload/test/offloading/kernel_launch_environment.c
@@ -0,0 +1,71 @@
+// Stress the kernel launch environment (KLE) transfer.
+//
+// A cross-team reduction gives the launch a KLE, which the plugin stages in a
+// host buffer and copies to the device asynchronously. The KLE carries that
+// launch's own reduction buffer, so overlapping launches must neither share a
+// staging buffer nor release one while its transfer is still in flight: either
+// makes a launch reduce into another launch's buffer. Every launch here
+// accumulates a distinct value, so that shows up as a wrong sum.
+//
+// RUN: %libomptarget-compile-generic -fopenmp-offload-mandatory
+// RUN: %libomptarget-run-generic
+// RUN: %libomptarget-compileopt-generic -fopenmp-offload-mandatory
+// RUN: %libomptarget-run-generic
+//
+// REQUIRES: gpu
+
+#include <omp.h>
+#include <stdio.h>
+
+#define NUM_LAUNCHES 32
+#define N 4096
+
+// Launch k accumulates k + 1 per element.
+static long expected(int k) { return (long)(k + 1) * N; }
+
+static int check(const char *Phase, long *Results) {
+  int Errors = 0;
+  for (int k = 0; k < NUM_LAUNCHES; k++)
+    if (Results[k] != expected(k)) {
+      fprintf(stderr, "%s: launch %d reduced to %ld, expected %ld\n", Phase, k,
+              Results[k], expected(k));
+      Errors++;
+    }
+  return Errors;
+}
+
+int main(void) {
+  static long Results[NUM_LAUNCHES];
+  int Errors = 0;
+
+  // Launches issued back to back without an intervening synchronization, so
+  // that several KLE transfers are outstanding at once.
+  for (int k = 0; k < NUM_LAUNCHES; k++) {
+    Results[k] = 0;
+#pragma omp target teams distribute parallel for map(tofrom : Results[k : 1])  \
+    reduction(+ : Results[k]) firstprivate(k) nowait
+    for (int i = 0; i < N; i++)
+      Results[k] += k + 1;
+  }
+#pragma omp taskwait
+  Errors += check("nowait", Results);
+
+  // Same, but with the launches and the synchronizations spread over several
+  // host threads: a thread finalizing its queue must not release a staging
+  // buffer that another thread's launch is still using.
+#pragma omp parallel for num_threads(8)
+  for (int k = 0; k < NUM_LAUNCHES; k++) {
+    long Sum = 0;
+#pragma omp target teams distribute parallel for map(tofrom : Sum)             \
+    reduction(+ : Sum) firstprivate(k)
+    for (int i = 0; i < N; i++)
+      Sum += k + 1;
+    Results[k] = Sum;
+  }
+  Errors += check("threaded", Results);
+
+  if (Errors)
+    return 1;
+  printf("PASS\n");
+  return 0;
+}


From openmp-commits at lists.llvm.org  Fri Aug  7 07:36:27 2026
From: openmp-commits at lists.llvm.org (Robert Imschweiler via Openmp-commits)
Date: Fri, 07 Aug 2026 07:36:27 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [offload] Use pinned memory for
	KLE (PR #213767)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213767@github.com>
Message-ID: <6a75ed6b.307fd207.3541e0.1d74@mx.google.com>


https://github.com/ro-i ready_for_review https://github.com/llvm/llvm-project/pull/213767

From openmp-commits at lists.llvm.org  Fri Aug  7 07:53:18 2026
From: openmp-commits at lists.llvm.org (=?UTF-8?Q?Michael_Halkenh=C3=A4user?= via Openmp-commits)
Date: Fri, 07 Aug 2026 07:53:18 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [Offload][OMPT][NFCI] Call
 ompt_libomp_connect instead of dlopen libomp (PR #214556)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/214556@github.com>
Message-ID: <6a75f15e.7d702ed2.13dab5.e818@mx.google.com>


mhalk wrote:

> Is some downstream implementation using liboffload with a different OpenMP runtime?
IDK for sure, but to the best of my knowledge I'd say: No, w.r.t. ROCm.

Thanks for the feedback and approvals :)

https://github.com/llvm/llvm-project/pull/214556

From openmp-commits at lists.llvm.org  Fri Aug  7 08:20:07 2026
From: openmp-commits at lists.llvm.org (=?UTF-8?Q?Michael_Halkenh=C3=A4user?= via Openmp-commits)
Date: Fri, 07 Aug 2026 08:20:07 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [Offload][OMPT][NFCI] Call
 ompt_libomp_connect instead of dlopen libomp (PR #214556)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/214556@github.com>
Message-ID: <6a75f7a7.e5907290.b65ee.9c7f@mx.google.com>


https://github.com/mhalk updated https://github.com/llvm/llvm-project/pull/214556

>From c987dd8e317ad9f9c556d492febf6588f35fc151 Mon Sep 17 00:00:00 2001
From: Michael Halkenhaeuser <MichaelGerald.Halkenhauser at amd.com>
Date: Thu, 6 Aug 2026 16:44:22 +0000
Subject: [PATCH] [Offload][OMPT][NFCI] Call ompt_libomp_connect instead of
 dlopen libomp

connectLibrary() reached libomp's ompt_libomp_connect through
OmptLibraryConnectorTy, which dlopens "libomp.so" and resolves the symbol at
run time. That cannot do anything the linker has not already done:
libomptarget links libomp and imports seven __kmpc_* symbols from it, so
libomp.so is a DT_NEEDED entry, and libomp defines and exports
ompt_libomp_connect unconditionally -- as a stub in kmp_utility.cpp when
built without OMPT support. Declare it, call it, and delete Connector.h.

AI-assisted.
---
 offload/include/OpenMP/OMPT/Callback.h        |   8 +-
 offload/include/OpenMP/OMPT/Connector.h       | 112 ------------------
 offload/libomptarget/OpenMP/OMPT/Callback.cpp |  17 ++-
 openmp/runtime/src/ompt-general.cpp           |   4 +-
 4 files changed, 11 insertions(+), 130 deletions(-)
 delete mode 100644 offload/include/OpenMP/OMPT/Connector.h

diff --git a/offload/include/OpenMP/OMPT/Callback.h b/offload/include/OpenMP/OMPT/Callback.h
index 9d545c643223f..2bc3f59dcd3ee 100644
--- a/offload/include/OpenMP/OMPT/Callback.h
+++ b/offload/include/OpenMP/OMPT/Callback.h
@@ -69,18 +69,14 @@ extern ompt_get_callback_t lookupCallbackByCode;
 /// \p InterfaceFunctionName the name of the OMPT callback function to look up
 extern ompt_function_lookup_t lookupCallbackByName;
 
-/// This is the function called by the higher layer (libomp / libomtarget)
-/// responsible for initializing OMPT in this library. This is passed to libomp
-/// as part of the OMPT connector object.
+/// Initializes OMPT in this library. Passed to libomp via ompt_libomp_connect.
 /// \p lookup to be used to query callbacks registered with libomp
 /// \p initial_device_num initial device num (id) provided by libomp
 /// \p tool_data as provided by the tool
 int initializeLibrary(ompt_function_lookup_t lookup, int initial_device_num,
                       ompt_data_t *tool_data);
 
-/// This function is passed to libomp / libomtarget as part of the OMPT
-/// connector object. It is called by libomp during finalization of OMPT in
-/// libomptarget -OR- by libomptarget during finalization of OMPT in the plugin.
+/// Finalizes OMPT in this library. Passed to libomp via ompt_libomp_connect.
 /// \p tool_data as provided by the tool
 void finalizeLibrary(ompt_data_t *tool_data);
 
diff --git a/offload/include/OpenMP/OMPT/Connector.h b/offload/include/OpenMP/OMPT/Connector.h
deleted file mode 100644
index add8941cc4905..0000000000000
--- a/offload/include/OpenMP/OMPT/Connector.h
+++ /dev/null
@@ -1,112 +0,0 @@
-//===-- OpenMP/OMPT/Connector.h - OpenMP Tooling lib connector -*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Support used by OMPT implementation to establish communication between
-// various OpenMP runtime libraries: host openmp library, target-independent
-// runtime library, and device-dependent runtime libraries.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef OMPTARGET_OPENMP_OMPT_CONNECTOR_H
-#define OMPTARGET_OPENMP_OMPT_CONNECTOR_H
-
-#ifdef OMPT_SUPPORT
-
-#include "llvm/Support/DynamicLibrary.h"
-
-#include <memory>
-#include <string>
-
-#include "omp-tools.h"
-#include "omptarget.h"
-
-#include "Shared/Debug.h"
-
-using namespace llvm::omp::target::debug;
-
-#pragma push_macro("DEBUG_PREFIX")
-#undef DEBUG_PREFIX
-#define DEBUG_PREFIX "OMPT"
-
-/// Type for the function to be invoked for connecting two libraries.
-typedef void (*OmptConnectRtnTy)(ompt_start_tool_result_t *result);
-
-/// Establish connection between openmp runtime libraries
-///
-/// This class is used to communicate between an OMPT implementation in
-/// libomptarget and libomp. It is also used to communicate between an
-/// OMPT implementation in a device-specific plugin and
-/// libomptarget. The decision whether OMPT is enabled or not needs to
-/// be made when the library is loaded before any functions in the
-/// library are invoked. For that reason, an instance of this class is
-/// intended to be defined in the constructor for libomptarget or a
-/// plugin so that the decision about whether OMPT is supposed to be
-/// enabled is known before any interface function in the library is
-/// invoked.
-class OmptLibraryConnectorTy {
-public:
-  /// Use \p LibName as the prefix of the global function used for connecting
-  /// two libraries, the source indicated by \p LibName and the destination
-  /// being the one that creates this object.
-  OmptLibraryConnectorTy(const char *Ident) {
-    LibIdent.append(Ident);
-    IsInitialized = false;
-  }
-  OmptLibraryConnectorTy() = delete;
-  /// Use \p OmptResult init to connect the two libraries denoted by this
-  /// object. The init function of \p OmptResult will be used during connection
-  /// and the fini function of \p OmptResult will be used during teardown.
-  void connect(ompt_start_tool_result_t *OmptResult) {
-    initialize();
-    if (!LibConnHandle)
-      return;
-    // Call the function provided by the source library for connect
-    LibConnHandle(OmptResult);
-  }
-
-private:
-  void initialize() {
-    if (IsInitialized)
-      return;
-
-    std::string ErrMsg;
-    std::string LibName = LibIdent;
-    LibName += ".so";
-
-    ODBG(ODT_Tool) << "OMPT: Trying to load library " << LibName;
-    auto DynLibHandle = std::make_unique<llvm::sys::DynamicLibrary>(
-        llvm::sys::DynamicLibrary::getPermanentLibrary(LibName.c_str(),
-                                                       &ErrMsg));
-    if (!DynLibHandle->isValid()) {
-      // The upper layer will bail out if the handle is null.
-      LibConnHandle = nullptr;
-    } else {
-      auto LibConnRtn = "ompt_" + LibIdent + "_connect";
-      ODBG(ODT_Tool) << "OMPT: Trying to get address of connection routine "
-                     << LibConnRtn;
-      LibConnHandle = reinterpret_cast<OmptConnectRtnTy>(
-          DynLibHandle->getAddressOfSymbol(LibConnRtn.c_str()));
-    }
-    ODBG(ODT_Tool) << "OMPT: Library connection handle = "
-                   << reinterpret_cast<void *>(LibConnHandle);
-    IsInitialized = true;
-  }
-
-  /// Ensure initialization occurs only once
-  bool IsInitialized;
-  /// Handle of connect routine provided by source library
-  OmptConnectRtnTy LibConnHandle;
-  /// Name of connect routine provided by source library
-  std::string LibIdent;
-};
-
-#endif // OMPT_SUPPORT
-
-#pragma pop_macro("DEBUG_PREFIX")
-
-#endif // OMPTARGET_OPENMP_OMPT_CONNECTOR_H
diff --git a/offload/libomptarget/OpenMP/OMPT/Callback.cpp b/offload/libomptarget/OpenMP/OMPT/Callback.cpp
index 1e03f1455d1b2..150fcb85015f7 100644
--- a/offload/libomptarget/OpenMP/OMPT/Callback.cpp
+++ b/offload/libomptarget/OpenMP/OMPT/Callback.cpp
@@ -19,14 +19,15 @@
 #include "Shared/Debug.h"
 
 #include "OpenMP/OMPT/Callback.h"
-#include "OpenMP/OMPT/Connector.h"
 #include "OpenMP/OMPT/Interface.h"
 
-#include "llvm/Support/DynamicLibrary.h"
-
 #undef DEBUG_PREFIX
 #define DEBUG_PREFIX "OMPT"
 
+/// Registers this library's initialize and finalize functions with libomp,
+/// which always defines this entry point (a stub if built without OMPT).
+extern "C" void ompt_libomp_connect(ompt_start_tool_result_t *);
+
 // Define OMPT callback functions (bound to actual callbacks later on)
 #define defineOmptCallback(Name, Type, Code)                                   \
   Name##_t llvm::omp::target::ompt::Name##_fn = nullptr;
@@ -537,18 +538,14 @@ void llvm::omp::target::ompt::finalizeLibrary(ompt_data_t *data) {
 
 void llvm::omp::target::ompt::connectLibrary() {
   ODBG(ODT_Tool) << "Entering connectLibrary";
-  // Connect with libomp
-  static OmptLibraryConnectorTy LibompConnector("libomp");
+  // libomp retains this pointer to run the finalizer
   static ompt_start_tool_result_t OmptResult;
-
-  // Initialize OmptResult with the init and fini functions that will be
-  // called by the connector
   OmptResult.initialize = ompt::initializeLibrary;
   OmptResult.finalize = ompt::finalizeLibrary;
   OmptResult.tool_data.value = 0;
 
-  // Now call connect that causes the above init/fini functions to be called
-  LibompConnector.connect(&OmptResult);
+  // Calls initializeLibrary if a tool enabled OMPT
+  ompt_libomp_connect(&OmptResult);
 
 #define bindOmptCallback(Name, Type, Code)                                     \
   if (lookupCallbackByCode)                                                    \
diff --git a/openmp/runtime/src/ompt-general.cpp b/openmp/runtime/src/ompt-general.cpp
index 8f1ba4910815e..79b56f35fb5be 100644
--- a/openmp/runtime/src/ompt-general.cpp
+++ b/openmp/runtime/src/ompt-general.cpp
@@ -948,8 +948,8 @@ static ompt_interface_fn_t ompt_libomp_target_fn_lookup(const char *s) {
   return (ompt_interface_fn_t)0;
 }
 
-/// This function is called by the libomptarget connector to assign
-/// callbacks already registered with libomp.
+/// This function is called by libomptarget to assign callbacks already
+/// registered with libomp.
 _OMP_EXTERN void ompt_libomp_connect(ompt_start_tool_result_t *result) {
   OMPT_VERBOSE_INIT_PRINT("libomp --> OMPT: Enter ompt_libomp_connect\n");
 


From openmp-commits at lists.llvm.org  Fri Aug  7 08:20:53 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Fri, 07 Aug 2026 08:20:53 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [Offload][OMPT][NFCI] Call
 ompt_libomp_connect instead of dlopen libomp (PR #214556)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/214556@github.com>
Message-ID: <6a75f7d5.e7e80b80.beaff.b3e3@mx.google.com>


llvmorg-github-actions[bot] wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-offload

Author: Michael Halkenhäuser (mhalk)

<details>
<summary>Changes</summary>

connectLibrary() reached libomp's ompt_libomp_connect through OmptLibraryConnectorTy, which dlopens "libomp.so" and resolves the symbol at run time. That cannot do anything the linker has not already done: libomptarget links libomp and imports seven __kmpc_* symbols from it, so libomp.so is a DT_NEEDED entry, and libomp defines and exports ompt_libomp_connect unconditionally -- as a stub in kmp_utility.cpp when built without OMPT support. Declare it, call it, and delete Connector.h.

AI-assisted.

---
Full diff: https://github.com/llvm/llvm-project/pull/214556.diff


4 Files Affected:

- (modified) offload/include/OpenMP/OMPT/Callback.h (+2-6) 
- (removed) offload/include/OpenMP/OMPT/Connector.h (-112) 
- (modified) offload/libomptarget/OpenMP/OMPT/Callback.cpp (+7-10) 
- (modified) openmp/runtime/src/ompt-general.cpp (+2-2) 


``````````diff
diff --git a/offload/include/OpenMP/OMPT/Callback.h b/offload/include/OpenMP/OMPT/Callback.h
index 9d545c643223f..2bc3f59dcd3ee 100644
--- a/offload/include/OpenMP/OMPT/Callback.h
+++ b/offload/include/OpenMP/OMPT/Callback.h
@@ -69,18 +69,14 @@ extern ompt_get_callback_t lookupCallbackByCode;
 /// \p InterfaceFunctionName the name of the OMPT callback function to look up
 extern ompt_function_lookup_t lookupCallbackByName;
 
-/// This is the function called by the higher layer (libomp / libomtarget)
-/// responsible for initializing OMPT in this library. This is passed to libomp
-/// as part of the OMPT connector object.
+/// Initializes OMPT in this library. Passed to libomp via ompt_libomp_connect.
 /// \p lookup to be used to query callbacks registered with libomp
 /// \p initial_device_num initial device num (id) provided by libomp
 /// \p tool_data as provided by the tool
 int initializeLibrary(ompt_function_lookup_t lookup, int initial_device_num,
                       ompt_data_t *tool_data);
 
-/// This function is passed to libomp / libomtarget as part of the OMPT
-/// connector object. It is called by libomp during finalization of OMPT in
-/// libomptarget -OR- by libomptarget during finalization of OMPT in the plugin.
+/// Finalizes OMPT in this library. Passed to libomp via ompt_libomp_connect.
 /// \p tool_data as provided by the tool
 void finalizeLibrary(ompt_data_t *tool_data);
 
diff --git a/offload/include/OpenMP/OMPT/Connector.h b/offload/include/OpenMP/OMPT/Connector.h
deleted file mode 100644
index add8941cc4905..0000000000000
--- a/offload/include/OpenMP/OMPT/Connector.h
+++ /dev/null
@@ -1,112 +0,0 @@
-//===-- OpenMP/OMPT/Connector.h - OpenMP Tooling lib connector -*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Support used by OMPT implementation to establish communication between
-// various OpenMP runtime libraries: host openmp library, target-independent
-// runtime library, and device-dependent runtime libraries.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef OMPTARGET_OPENMP_OMPT_CONNECTOR_H
-#define OMPTARGET_OPENMP_OMPT_CONNECTOR_H
-
-#ifdef OMPT_SUPPORT
-
-#include "llvm/Support/DynamicLibrary.h"
-
-#include <memory>
-#include <string>
-
-#include "omp-tools.h"
-#include "omptarget.h"
-
-#include "Shared/Debug.h"
-
-using namespace llvm::omp::target::debug;
-
-#pragma push_macro("DEBUG_PREFIX")
-#undef DEBUG_PREFIX
-#define DEBUG_PREFIX "OMPT"
-
-/// Type for the function to be invoked for connecting two libraries.
-typedef void (*OmptConnectRtnTy)(ompt_start_tool_result_t *result);
-
-/// Establish connection between openmp runtime libraries
-///
-/// This class is used to communicate between an OMPT implementation in
-/// libomptarget and libomp. It is also used to communicate between an
-/// OMPT implementation in a device-specific plugin and
-/// libomptarget. The decision whether OMPT is enabled or not needs to
-/// be made when the library is loaded before any functions in the
-/// library are invoked. For that reason, an instance of this class is
-/// intended to be defined in the constructor for libomptarget or a
-/// plugin so that the decision about whether OMPT is supposed to be
-/// enabled is known before any interface function in the library is
-/// invoked.
-class OmptLibraryConnectorTy {
-public:
-  /// Use \p LibName as the prefix of the global function used for connecting
-  /// two libraries, the source indicated by \p LibName and the destination
-  /// being the one that creates this object.
-  OmptLibraryConnectorTy(const char *Ident) {
-    LibIdent.append(Ident);
-    IsInitialized = false;
-  }
-  OmptLibraryConnectorTy() = delete;
-  /// Use \p OmptResult init to connect the two libraries denoted by this
-  /// object. The init function of \p OmptResult will be used during connection
-  /// and the fini function of \p OmptResult will be used during teardown.
-  void connect(ompt_start_tool_result_t *OmptResult) {
-    initialize();
-    if (!LibConnHandle)
-      return;
-    // Call the function provided by the source library for connect
-    LibConnHandle(OmptResult);
-  }
-
-private:
-  void initialize() {
-    if (IsInitialized)
-      return;
-
-    std::string ErrMsg;
-    std::string LibName = LibIdent;
-    LibName += ".so";
-
-    ODBG(ODT_Tool) << "OMPT: Trying to load library " << LibName;
-    auto DynLibHandle = std::make_unique<llvm::sys::DynamicLibrary>(
-        llvm::sys::DynamicLibrary::getPermanentLibrary(LibName.c_str(),
-                                                       &ErrMsg));
-    if (!DynLibHandle->isValid()) {
-      // The upper layer will bail out if the handle is null.
-      LibConnHandle = nullptr;
-    } else {
-      auto LibConnRtn = "ompt_" + LibIdent + "_connect";
-      ODBG(ODT_Tool) << "OMPT: Trying to get address of connection routine "
-                     << LibConnRtn;
-      LibConnHandle = reinterpret_cast<OmptConnectRtnTy>(
-          DynLibHandle->getAddressOfSymbol(LibConnRtn.c_str()));
-    }
-    ODBG(ODT_Tool) << "OMPT: Library connection handle = "
-                   << reinterpret_cast<void *>(LibConnHandle);
-    IsInitialized = true;
-  }
-
-  /// Ensure initialization occurs only once
-  bool IsInitialized;
-  /// Handle of connect routine provided by source library
-  OmptConnectRtnTy LibConnHandle;
-  /// Name of connect routine provided by source library
-  std::string LibIdent;
-};
-
-#endif // OMPT_SUPPORT
-
-#pragma pop_macro("DEBUG_PREFIX")
-
-#endif // OMPTARGET_OPENMP_OMPT_CONNECTOR_H
diff --git a/offload/libomptarget/OpenMP/OMPT/Callback.cpp b/offload/libomptarget/OpenMP/OMPT/Callback.cpp
index 1e03f1455d1b2..150fcb85015f7 100644
--- a/offload/libomptarget/OpenMP/OMPT/Callback.cpp
+++ b/offload/libomptarget/OpenMP/OMPT/Callback.cpp
@@ -19,14 +19,15 @@
 #include "Shared/Debug.h"
 
 #include "OpenMP/OMPT/Callback.h"
-#include "OpenMP/OMPT/Connector.h"
 #include "OpenMP/OMPT/Interface.h"
 
-#include "llvm/Support/DynamicLibrary.h"
-
 #undef DEBUG_PREFIX
 #define DEBUG_PREFIX "OMPT"
 
+/// Registers this library's initialize and finalize functions with libomp,
+/// which always defines this entry point (a stub if built without OMPT).
+extern "C" void ompt_libomp_connect(ompt_start_tool_result_t *);
+
 // Define OMPT callback functions (bound to actual callbacks later on)
 #define defineOmptCallback(Name, Type, Code)                                   \
   Name##_t llvm::omp::target::ompt::Name##_fn = nullptr;
@@ -537,18 +538,14 @@ void llvm::omp::target::ompt::finalizeLibrary(ompt_data_t *data) {
 
 void llvm::omp::target::ompt::connectLibrary() {
   ODBG(ODT_Tool) << "Entering connectLibrary";
-  // Connect with libomp
-  static OmptLibraryConnectorTy LibompConnector("libomp");
+  // libomp retains this pointer to run the finalizer
   static ompt_start_tool_result_t OmptResult;
-
-  // Initialize OmptResult with the init and fini functions that will be
-  // called by the connector
   OmptResult.initialize = ompt::initializeLibrary;
   OmptResult.finalize = ompt::finalizeLibrary;
   OmptResult.tool_data.value = 0;
 
-  // Now call connect that causes the above init/fini functions to be called
-  LibompConnector.connect(&OmptResult);
+  // Calls initializeLibrary if a tool enabled OMPT
+  ompt_libomp_connect(&OmptResult);
 
 #define bindOmptCallback(Name, Type, Code)                                     \
   if (lookupCallbackByCode)                                                    \
diff --git a/openmp/runtime/src/ompt-general.cpp b/openmp/runtime/src/ompt-general.cpp
index 8f1ba4910815e..79b56f35fb5be 100644
--- a/openmp/runtime/src/ompt-general.cpp
+++ b/openmp/runtime/src/ompt-general.cpp
@@ -948,8 +948,8 @@ static ompt_interface_fn_t ompt_libomp_target_fn_lookup(const char *s) {
   return (ompt_interface_fn_t)0;
 }
 
-/// This function is called by the libomptarget connector to assign
-/// callbacks already registered with libomp.
+/// This function is called by libomptarget to assign callbacks already
+/// registered with libomp.
 _OMP_EXTERN void ompt_libomp_connect(ompt_start_tool_result_t *result) {
   OMPT_VERBOSE_INIT_PRINT("libomp --> OMPT: Enter ompt_libomp_connect\n");
 

``````````

</details>


https://github.com/llvm/llvm-project/pull/214556

From openmp-commits at lists.llvm.org  Fri Aug  7 08:35:09 2026
From: openmp-commits at lists.llvm.org (Robert Imschweiler via Openmp-commits)
Date: Fri, 07 Aug 2026 08:35:09 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [offload] Pool host and shared
	allocations (PR #214754)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/214754@github.com>
Message-ID: <6a75fb2d.0f490953.1944de.e2f4@mx.google.com>


https://github.com/ro-i edited https://github.com/llvm/llvm-project/pull/214754

From openmp-commits at lists.llvm.org  Fri Aug  7 08:35:13 2026
From: openmp-commits at lists.llvm.org (Robert Imschweiler via Openmp-commits)
Date: Fri, 07 Aug 2026 08:35:13 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [offload] Pool host and shared
	allocations (PR #214754)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/214754@github.com>
Message-ID: <6a75fb31.f06f4994.311f07.7127@mx.google.com>


https://github.com/ro-i updated https://github.com/llvm/llvm-project/pull/214754

>From 9ecc43d40d73fb6f614a1d1fba6b46bdfab2438f Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Wed, 5 Aug 2026 10:24:27 -0500
Subject: [PATCH] [offload] Pool host and shared allocations

Route them through a memory manager, like the device allocations. Also,
move the registration as pinned memory to the plugin site since only the
plugin knows if the corresponding host/shared memory is pinned.
Improves performance of affected allocations by ballpark 1,000x.
---
 offload/plugins-nextgen/amdgpu/src/rtl.cpp    | 12 ++++
 .../common/include/MemoryManager.h            | 20 ++++--
 .../common/include/PluginInterface.h          | 20 +++++-
 .../common/src/PluginInterface.cpp            | 69 ++++++++-----------
 openmp/docs/design/Runtimes.rst               | 11 +--
 5 files changed, 80 insertions(+), 52 deletions(-)

diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index 0f0de67b20c7c..e74cf03071fa7 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -2728,6 +2728,12 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
       return Plugin::error(ErrorCode::OUT_OF_RESOURCES,
                            "no memory pool for the specified allocation kind");
 
+    // See allocate() for the registration of host / shared memory as pinned
+    // memory.
+    if (Kind == TARGET_ALLOC_HOST || Kind == TARGET_ALLOC_SHARED)
+      if (auto Err = PinnedAllocs.unregisterHostBuffer(TgtPtr))
+        return Err;
+
     if (auto Err = MemoryPool->deallocate(TgtPtr))
       return Err;
 
@@ -4529,6 +4535,12 @@ Expected<void *> AMDGPUDeviceTy::allocate(size_t Size, void *,
     // Enable all valid kernel agents to access the buffer.
     if (auto Err = MemoryPool->enableAccess(Alloc, Size, Agents))
       return std::move(Err);
+
+    // Register host / shared memory as pinned memory, so that transfers reading
+    // from it can take a device-accessible path.
+    if (Kind == TARGET_ALLOC_HOST || Kind == TARGET_ALLOC_SHARED)
+      if (auto Err = PinnedAllocs.registerHostBuffer(Alloc, Alloc, Size))
+        return std::move(Err);
   }
 
   return Alloc;
diff --git a/offload/plugins-nextgen/common/include/MemoryManager.h b/offload/plugins-nextgen/common/include/MemoryManager.h
index 883ac40c269ea..9712a461ad974 100644
--- a/offload/plugins-nextgen/common/include/MemoryManager.h
+++ b/offload/plugins-nextgen/common/include/MemoryManager.h
@@ -138,21 +138,24 @@ class MemoryManagerTy {
 
   /// The reference to a device allocator
   DeviceAllocatorTy &DeviceAllocator;
+  /// The kind of device for which the memory manager is allocated
+  TargetAllocTy DeviceKind;
 
   /// The threshold to manage memory using memory manager. If the request size
   /// is larger than \p SizeThreshold, the allocation will not be managed by the
   /// memory manager.
-  size_t SizeThreshold = 1U << 13;
+  size_t SizeThreshold = DefaultSizeThreshold;
 
   /// Request memory from target device
   Expected<void *> allocateOnDevice(size_t Size, void *HstPtr,
                                     size_t Alignment) const {
-    return DeviceAllocator.allocate(Size, HstPtr, TARGET_ALLOC_DEVICE,
-                                    Alignment);
+    return DeviceAllocator.allocate(Size, HstPtr, DeviceKind, Alignment);
   }
 
   /// Deallocate data on device
-  Error deleteOnDevice(void *Ptr) const { return DeviceAllocator.free(Ptr); }
+  Error deleteOnDevice(void *Ptr) const {
+    return DeviceAllocator.free(Ptr, DeviceKind);
+  }
 
   /// This function is called when it tries to allocate memory on device but the
   /// device returns out of memory. It will first free all memory in the
@@ -216,11 +219,14 @@ class MemoryManagerTy {
   }
 
 public:
+  static constexpr size_t DefaultSizeThreshold = 1U << 13;
+
   /// Constructor. If \p Threshold is non-zero, then the default threshold will
   /// be overwritten by \p Threshold.
-  MemoryManagerTy(DeviceAllocatorTy &DeviceAllocator, size_t Threshold = 0)
+  MemoryManagerTy(DeviceAllocatorTy &DeviceAllocator, size_t Threshold = 0,
+                  TargetAllocTy DeviceKind = TARGET_ALLOC_DEVICE)
       : FreeLists(NumBuckets), FreeListLocks(NumBuckets),
-        DeviceAllocator(DeviceAllocator) {
+        DeviceAllocator(DeviceAllocator), DeviceKind(DeviceKind) {
     if (Threshold)
       SizeThreshold = Threshold;
   }
@@ -359,6 +365,8 @@ class MemoryManagerTy {
   /// threshold and the second element represents whether user disables memory
   /// manager explicitly by setting the var to 0. If user doesn't specify
   /// anything, returns <0, true>.
+  /// Note that this only affects the device memory manager, not the manager for
+  /// host or shared memory.
   static std::pair<size_t, bool> getSizeThresholdFromEnv() {
     static UInt64Envar MemoryManagerThreshold(
         "LIBOMPTARGET_MEMORY_MANAGER_THRESHOLD", 0);
diff --git a/offload/plugins-nextgen/common/include/PluginInterface.h b/offload/plugins-nextgen/common/include/PluginInterface.h
index b3675e5a8700f..50be67886c527 100644
--- a/offload/plugins-nextgen/common/include/PluginInterface.h
+++ b/offload/plugins-nextgen/common/include/PluginInterface.h
@@ -1368,8 +1368,12 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
   /// only necessary for unhosted targets like the GPU.
   virtual bool shouldSetupRPCServer() const { return false; }
 
-  /// Pointer to the memory manager or nullptr if not available.
+  /// Pointer to the device memory manager or nullptr if not available.
   MemoryManagerTy *MemoryManager;
+  /// Memory managers for the host and shared allocation kinds or nullptr if not
+  /// available.
+  MemoryManagerTy *HostMemoryManager;
+  MemoryManagerTy *SharedMemoryManager;
 
   /// Per device setting of MemoryManager's Threshold
   virtual size_t getMemoryManagerSizeThreshold() { return 0; }
@@ -1406,6 +1410,20 @@ struct GenericDeviceTy : public DeviceAllocatorTy {
   /// Record and replay manager.
   RecordReplayTy *RecordReplay = nullptr;
 
+  /// Return the memory manager for the given allocation kind.
+  MemoryManagerTy *getMemoryManagerFor(TargetAllocTy Kind) {
+    switch (Kind) {
+    case TARGET_ALLOC_DEFAULT:
+    case TARGET_ALLOC_DEVICE:
+      return MemoryManager;
+    case TARGET_ALLOC_HOST:
+      return HostMemoryManager;
+    case TARGET_ALLOC_SHARED:
+      return SharedMemoryManager;
+    }
+    return nullptr;
+  }
+
 protected:
   /// Environment variables defined by the LLVM OpenMP implementation
   /// regarding the initial number of streams and events.
diff --git a/offload/plugins-nextgen/common/src/PluginInterface.cpp b/offload/plugins-nextgen/common/src/PluginInterface.cpp
index 70e289060a446..0d6aede7a1426 100644
--- a/offload/plugins-nextgen/common/src/PluginInterface.cpp
+++ b/offload/plugins-nextgen/common/src/PluginInterface.cpp
@@ -490,7 +490,8 @@ uint32_t GenericKernelTy::getEffectiveNumBlocks(
 GenericDeviceTy::GenericDeviceTy(GenericPluginTy &Plugin, int32_t DeviceId,
                                  int32_t NumDevices,
                                  const llvm::omp::GV &OMPGridValues)
-    : Plugin(Plugin), MemoryManager(nullptr), OMP_TeamLimit("OMP_TEAM_LIMIT"),
+    : Plugin(Plugin), MemoryManager(nullptr), HostMemoryManager(nullptr),
+      SharedMemoryManager(nullptr), OMP_TeamLimit("OMP_TEAM_LIMIT"),
       OMP_NumTeams("OMP_NUM_TEAMS"),
       OMP_TeamsThreadLimit("OMP_TEAMS_THREAD_LIMIT"),
       OMPX_DebugKind("LIBOMPTARGET_DEVICE_RTL_DEBUG"),
@@ -607,6 +608,14 @@ Error GenericDeviceTy::init(GenericPluginTy &Plugin) {
       ThresholdMM = getMemoryManagerSizeThreshold();
     MemoryManager = new MemoryManagerTy(*this, ThresholdMM);
   }
+  if (!OMPX_TrackAllocationTraces) {
+    // Keep the threshold for pooling sizes conservative since we're dealing
+    // with pinned memory for the host.
+    HostMemoryManager = new MemoryManagerTy(
+        *this, MemoryManagerTy::DefaultSizeThreshold, TARGET_ALLOC_HOST);
+    SharedMemoryManager = new MemoryManagerTy(
+        *this, MemoryManagerTy::DefaultSizeThreshold, TARGET_ALLOC_SHARED);
+  }
 
   return Plugin::success();
 }
@@ -654,6 +663,12 @@ Error GenericDeviceTy::deinit(GenericPluginTy &Plugin) {
   if (MemoryManager)
     delete MemoryManager;
   MemoryManager = nullptr;
+  if (HostMemoryManager)
+    delete HostMemoryManager;
+  HostMemoryManager = nullptr;
+  if (SharedMemoryManager)
+    delete SharedMemoryManager;
+  SharedMemoryManager = nullptr;
 
   if (RecordReplay) {
     if (auto Err = RecordReplay->deinit())
@@ -1014,22 +1029,15 @@ Expected<void *> GenericDeviceTy::dataAlloc(int64_t Size, void *HostPtr,
   if (RecordReplay && RecordReplay->isRecordingOrReplaying())
     return RecordReplay->allocate(Size);
 
-  switch (Kind) {
-  case TARGET_ALLOC_DEFAULT:
-  case TARGET_ALLOC_DEVICE:
-    if (MemoryManager) {
-      auto AllocOrErr = MemoryManager->allocate(Size, HostPtr, Alignment);
-      if (!AllocOrErr)
-        return AllocOrErr.takeError();
-      Alloc = *AllocOrErr;
-      if (!Alloc)
-        return Plugin::error(ErrorCode::OUT_OF_RESOURCES,
-                             "failed to allocate from memory manager");
-      break;
-    }
-    [[fallthrough]];
-  case TARGET_ALLOC_HOST:
-  case TARGET_ALLOC_SHARED: {
+  if (MemoryManagerTy *MM = getMemoryManagerFor(Kind)) {
+    auto AllocOrErr = MM->allocate(Size, HostPtr, Alignment);
+    if (!AllocOrErr)
+      return AllocOrErr.takeError();
+    Alloc = *AllocOrErr;
+    if (!Alloc)
+      return Plugin::error(ErrorCode::OUT_OF_RESOURCES,
+                           "failed to allocate from memory manager");
+  } else {
     auto AllocOrErr = allocate(Size, HostPtr, Kind, Alignment);
     if (!AllocOrErr)
       return AllocOrErr.takeError();
@@ -1038,7 +1046,6 @@ Expected<void *> GenericDeviceTy::dataAlloc(int64_t Size, void *HostPtr,
       return Plugin::error(ErrorCode::OUT_OF_RESOURCES,
                            "failed to allocate from device allocator");
   }
-  }
 
   // Report error if the memory manager or the device allocator did not return
   // any memory buffer.
@@ -1047,11 +1054,6 @@ Expected<void *> GenericDeviceTy::dataAlloc(int64_t Size, void *HostPtr,
                          "invalid target data allocation kind or requested "
                          "allocator not implemented yet");
 
-  // Register allocated buffer as pinned memory if the type is host memory.
-  if (Kind == TARGET_ALLOC_HOST)
-    if (auto Err = PinnedAllocs.registerHostBuffer(Alloc, Alloc, Size))
-      return std::move(Err);
-
   // Keep track of the allocation stack if we track allocation traces.
   if (OMPX_TrackAllocationTraces) {
     std::string StackTrace;
@@ -1108,26 +1110,13 @@ Error GenericDeviceTy::dataDelete(void *TgtPtr, TargetAllocTy Kind) {
     ATI->DeallocationTrace = StackTrace;
   }
 
-  switch (Kind) {
-  case TARGET_ALLOC_DEFAULT:
-  case TARGET_ALLOC_DEVICE:
-    if (MemoryManager) {
-      if (auto Err = MemoryManager->free(TgtPtr))
-        return Err;
-      break;
-    }
-    [[fallthrough]];
-  case TARGET_ALLOC_HOST:
-  case TARGET_ALLOC_SHARED:
-    if (auto Err = free(TgtPtr, Kind))
+  if (MemoryManagerTy *MM = getMemoryManagerFor(Kind)) {
+    if (auto Err = MM->free(TgtPtr))
       return Err;
+  } else if (auto Err = free(TgtPtr, Kind)) {
+    return Err;
   }
 
-  // Unregister deallocated pinned memory buffer if the type is host memory.
-  if (Kind == TARGET_ALLOC_HOST)
-    if (auto Err = PinnedAllocs.unregisterHostBuffer(TgtPtr))
-      return Err;
-
   return Plugin::success();
 }
 
diff --git a/openmp/docs/design/Runtimes.rst b/openmp/docs/design/Runtimes.rst
index 14e300a0f531f..0e75925c78742 100644
--- a/openmp/docs/design/Runtimes.rst
+++ b/openmp/docs/design/Runtimes.rst
@@ -784,11 +784,12 @@ LIBOMPTARGET_MEMORY_MANAGER_THRESHOLD
 """""""""""""""""""""""""""""""""""""
 
 ``LIBOMPTARGET_MEMORY_MANAGER_THRESHOLD`` sets the threshold size for which the
-``libomptarget`` memory manager will handle the allocation. Any allocations
-larger than this threshold will not use the memory manager and be freed after
-the device kernel exits. The default threshold value is ``8KB``. If
-``LIBOMPTARGET_MEMORY_MANAGER_THRESHOLD`` is set to ``0`` the memory manager
-will be completely disabled.
+``libomptarget`` device memory manager will handle the allocation. Any
+allocations larger than this threshold will not use the memory manager and be
+freed after the device kernel exits. The default threshold value is ``8KB``. If
+``LIBOMPTARGET_MEMORY_MANAGER_THRESHOLD`` is set to ``0`` the device memory
+manager will be completely disabled.
+This has no effect on the host or shared memory managers.
 
 .. _libomptarget_info:
 


From openmp-commits at lists.llvm.org  Sat Aug  8 00:39:42 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Sat, 08 Aug 2026 00:39:42 -0700 (PDT)
Subject: [Openmp-commits] [flang] [llvm] [openmp] [OpenMP][Flang] Add
 integer-kind wrappers to omp_lib (PR #213505)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213505@github.com>
Message-ID: <6a76dd3e.7270d2fd.2c6a17.c725@mx.google.com>


shivaramaarao wrote:

@mjklemm  could you check if these changes are good?. thank you

https://github.com/llvm/llvm-project/pull/213505

From openmp-commits at lists.llvm.org  Sat Aug  8 12:56:40 2026
From: openmp-commits at lists.llvm.org (=?UTF-8?Q?Michael_Halkenh=C3=A4user?= via Openmp-commits)
Date: Sat, 08 Aug 2026 12:56:40 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [Offload][OMPT][NFCI] Call
 ompt_libomp_connect instead of dlopen libomp (PR #214556)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/214556@github.com>
Message-ID: <6a7789f8.4b9e17a8.129179.85b0@mx.google.com>


https://github.com/mhalk closed https://github.com/llvm/llvm-project/pull/214556

From openmp-commits at lists.llvm.org  Sun Aug  9 15:40:52 2026
From: openmp-commits at lists.llvm.org (Alexey Bataev via Openmp-commits)
Date: Sun, 09 Aug 2026 15:40:52 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [Clang][OpenMP] Add
 `#pragma omp flatten` loop directive + `depth` clause (PR #206977)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206977@github.com>
Message-ID: <6a7901f4.510f576d.49693.e026@mx.google.com>


================
@@ -16382,6 +16395,277 @@ StmtResult SemaOpenMP::ActOnOpenMPInterchangeDirective(
                                          buildPreInits(Context, PreInits));
 }
 
+/// Counts the perfectly nested canonical loop depth at \p AStmt. Returns
+/// std::nullopt if a loop-generating transformation prevents a static count.
+static std::optional<unsigned> getCanonicalLoopNestDepth(Stmt *AStmt) {
+  unsigned Depth = 0;
+  Stmt *CurStmt = AStmt ? AStmt->IgnoreContainers() : nullptr;
+  while (CurStmt) {
+    if (isa<OMPLoopTransformationDirective>(CurStmt))
+      return std::nullopt;
+    if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(CurStmt))
+      CurStmt = CanonLoop->getLoopStmt();
+    Stmt *Body = nullptr;
+    if (auto *For = dyn_cast<ForStmt>(CurStmt))
+      Body = For->getBody();
+    else if (auto *RangeFor = dyn_cast<CXXForRangeStmt>(CurStmt))
+      Body = RangeFor->getBody();
+    else
+      break;
+    ++Depth;
+    CurStmt = Body ? Body->IgnoreContainers() : nullptr;
+  }
+  return Depth;
+}
+
+StmtResult
+SemaOpenMP::ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
+                                        Stmt *AStmt, SourceLocation StartLoc,
+                                        SourceLocation EndLoc) {
+  ASTContext &Context = getASTContext();
+  DeclContext *CurContext = SemaRef.CurContext;
+  Scope *CurScope = SemaRef.getCurScope();
+
+  // Empty statement should only be possible if there already was an error.
+  if (!AStmt)
+    return StmtError();
+
+  // flatten without 'depth' clause combines two loops; 'depth(k)' selects k.
+  unsigned NumLoops = 2;
+  bool DepthIsValueDependent = false;
+  const auto *DepthClause =
+      OMPExecutableDirective::getSingleClause<OMPDepthClause>(Clauses);
+  if (DepthClause) {
+    Expr *DepthExpr = DepthClause->getDepth();
+    if (DepthExpr && DepthExpr->isValueDependent()) {
+      DepthIsValueDependent = true;
+    } else if (DepthExpr) {
+      Expr::EvalResult EvalResult;
+      if (DepthExpr->EvaluateAsInt(EvalResult, Context))
+        NumLoops = EvalResult.Val.getInt().getZExtValue();
----------------
alexey-bataev wrote:

```suggestion
        NumLoops = EvalResult.Val.getInt(). getLimitedValue();
```

https://github.com/llvm/llvm-project/pull/206977

From openmp-commits at lists.llvm.org  Sun Aug  9 15:40:52 2026
From: openmp-commits at lists.llvm.org (Alexey Bataev via Openmp-commits)
Date: Sun, 09 Aug 2026 15:40:52 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [Clang][OpenMP] Add
 `#pragma omp flatten` loop directive + `depth` clause (PR #206977)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206977@github.com>
Message-ID: <6a7901f4.cf1b4e83.b695d.ba3c@mx.google.com>


================
@@ -1380,6 +1380,74 @@ class OMPPartialClause final : public OMPClause {
   }
 };
 
+/// This represents the 'depth' clause on the '#pragma omp flatten' (and
+/// '#pragma omp fuse') loop-transformation directives.
+///
+/// \code
+/// #pragma omp flatten depth(3)
+/// \endcode
+/// In this example the 'flatten' directive has a 'depth' clause whose argument
+/// '3' specifies how many perfectly nested loops are combined into one.
+/// The argument must be a positive integer constant expression that evaluates
+/// to at most the loop nest depth of the associated loop nest.
+class OMPDepthClause final : public OMPClause {
+  friend class OMPClauseReader;
+
+  /// Location of '('.
+  SourceLocation LParenLoc;
+
+  /// The depth expression (number of loops to flatten).
+  Stmt *Depth = nullptr;
+
+  /// Build an empty clause.
+  explicit OMPDepthClause() : OMPClause(llvm::omp::OMPC_depth, {}, {}) {}
+
+  /// Set the depth expression.
+  void setDepth(Expr *E) { Depth = E; }
+
+  /// Sets the location of '('.
+  void setLParenLoc(SourceLocation Loc) { LParenLoc = Loc; }
+
+public:
+  /// Build an AST node for a 'depth' clause.
+  ///
+  /// \param C         Context of the AST.
+  /// \param StartLoc  Location of the 'depth' identifier.
+  /// \param LParenLoc Location of '('.
+  /// \param EndLoc    Location of ')'.
+  /// \param Depth     The depth expression.
+  static OMPDepthClause *Create(const ASTContext &C, SourceLocation StartLoc,
+                                SourceLocation LParenLoc, SourceLocation EndLoc,
+                                Expr *Depth);
+
+  /// Build an empty 'depth' AST node for deserialization.
+  ///
+  /// \param C     Context of the AST.
+  static OMPDepthClause *CreateEmpty(const ASTContext &C);
----------------
alexey-bataev wrote:

Usually, we declare this pseudo-constructors, if the default constructor does not work =, like if we have tail allocated elements. Otherwise, just regular constructors should be good enough

https://github.com/llvm/llvm-project/pull/206977

From openmp-commits at lists.llvm.org  Sun Aug  9 15:40:52 2026
From: openmp-commits at lists.llvm.org (Alexey Bataev via Openmp-commits)
Date: Sun, 09 Aug 2026 15:40:52 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [Clang][OpenMP] Add
 `#pragma omp flatten` loop directive + `depth` clause (PR #206977)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206977@github.com>
Message-ID: <6a7901f4.21a19891.1fa7d.9e3b@mx.google.com>


================

----------------
alexey-bataev wrote:

It does not look like it supports this new clause

https://github.com/llvm/llvm-project/pull/206977

From openmp-commits at lists.llvm.org  Sun Aug  9 15:40:52 2026
From: openmp-commits at lists.llvm.org (Alexey Bataev via Openmp-commits)
Date: Sun, 09 Aug 2026 15:40:52 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [Clang][OpenMP] Add
 `#pragma omp flatten` loop directive + `depth` clause (PR #206977)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206977@github.com>
Message-ID: <6a7901f4.492e23e4.1b03cd.a44e@mx.google.com>


================
@@ -16382,6 +16395,277 @@ StmtResult SemaOpenMP::ActOnOpenMPInterchangeDirective(
                                          buildPreInits(Context, PreInits));
 }
 
+/// Counts the perfectly nested canonical loop depth at \p AStmt. Returns
+/// std::nullopt if a loop-generating transformation prevents a static count.
+static std::optional<unsigned> getCanonicalLoopNestDepth(Stmt *AStmt) {
+  unsigned Depth = 0;
+  Stmt *CurStmt = AStmt ? AStmt->IgnoreContainers() : nullptr;
+  while (CurStmt) {
+    if (isa<OMPLoopTransformationDirective>(CurStmt))
+      return std::nullopt;
+    if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(CurStmt))
+      CurStmt = CanonLoop->getLoopStmt();
+    Stmt *Body = nullptr;
+    if (auto *For = dyn_cast<ForStmt>(CurStmt))
+      Body = For->getBody();
+    else if (auto *RangeFor = dyn_cast<CXXForRangeStmt>(CurStmt))
+      Body = RangeFor->getBody();
+    else
+      break;
+    ++Depth;
+    CurStmt = Body ? Body->IgnoreContainers() : nullptr;
+  }
+  return Depth;
+}
+
+StmtResult
+SemaOpenMP::ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
+                                        Stmt *AStmt, SourceLocation StartLoc,
+                                        SourceLocation EndLoc) {
+  ASTContext &Context = getASTContext();
+  DeclContext *CurContext = SemaRef.CurContext;
+  Scope *CurScope = SemaRef.getCurScope();
+
+  // Empty statement should only be possible if there already was an error.
+  if (!AStmt)
+    return StmtError();
+
+  // flatten without 'depth' clause combines two loops; 'depth(k)' selects k.
+  unsigned NumLoops = 2;
+  bool DepthIsValueDependent = false;
+  const auto *DepthClause =
+      OMPExecutableDirective::getSingleClause<OMPDepthClause>(Clauses);
+  if (DepthClause) {
+    Expr *DepthExpr = DepthClause->getDepth();
+    if (DepthExpr && DepthExpr->isValueDependent()) {
----------------
alexey-bataev wrote:

```suggestion
    if (DepthExpr && DepthExpr-> isInstantiationDependent()) {
```

https://github.com/llvm/llvm-project/pull/206977

From openmp-commits at lists.llvm.org  Mon Aug 10 01:20:49 2026
From: openmp-commits at lists.llvm.org (Amit Tiwari via Openmp-commits)
Date: Mon, 10 Aug 2026 01:20:49 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [mlir] [openmp] [Clang][OpenMP]
 Canonicalize Intra-tiles in Loop Tiling (PR #191114)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/191114@github.com>
Message-ID: <6a7989e1.824e11a6.3cb239.d909@mx.google.com>


================
@@ -0,0 +1,521 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
+// Regression tests for standalone `#pragma omp tile` lowering (no consuming
+// directive). The intra-tile loop is emitted in its min-bounded form
+//   for (.tile.iv = .floor.iv; .tile.iv < min(.floor.iv + tilesize, N); ++.tile.iv)
+// so the upper bound is materialized as a `min()` (cond.true/cond.false/phi over
+// the two `.tile_*.iv`/`.floor_*.iv` counters). There is no `.tile.cnt`
+// rectangular counter and no body-guard predicate here; those only appear when a
+// directive (e.g. `collapse`) consumes the tile -- see
+// tile_collapse_reinterpret_codegen.cpp.
+//
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fopenmp -emit-llvm -o - %s | FileCheck %s --check-prefix=IR
+// expected-no-diagnostics
+
+extern "C" void body(int);
+
+// Trip count 6, tile 4 → the last tile is partial; the min() upper bound stops
+// the tile loop at the trip count (no separate body guard).
+// IR-LABEL: define dso_local void @_Z18remainder_6_tile_4v(
+// IR-SAME: ) #[[ATTR0:[0-9]+]] {
+// IR-NEXT:  [[ENTRY:.*:]]
+// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    store i32 0, ptr [[I]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND:.*]]
+// IR:       [[FOR_COND]]:
+// IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 6
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END9:.*]]
+// IR:       [[FOR_BODY]]:
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND1:.*]]
+// IR:       [[FOR_COND1]]:
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 6, [[ADD]]
+// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 4
+// IR-NEXT:    br label %[[COND_END]]
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ 6, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
+// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
+// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY5]]:
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
+// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void @body(i32 noundef [[TMP6]])
+// IR-NEXT:    br label %[[FOR_INC:.*]]
+// IR:       [[FOR_INC]]:
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP7]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP2:![0-9]+]]
+// IR:       [[FOR_END]]:
+// IR-NEXT:    br label %[[FOR_INC7:.*]]
+// IR:       [[FOR_INC7]]:
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP8]], 4
+// IR-NEXT:    store i32 [[ADD8]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP4:![0-9]+]]
+// IR:       [[FOR_END9]]:
+// IR-NEXT:    ret void
+//
+void remainder_6_tile_4(void) {
+  // Rectangular tile counter and constant tile-size bound.
+#pragma omp tile sizes(4)
----------------
loopacino wrote:

This is intentional. It is independently checking the cases where canonicalisation is not required (the other side of the functionality). I consider it is redundant with `tile_codegen.cpp` and `tile_ast_print.cpp`, and I can trim and maybe add  `collapse(2)` case here.    

https://github.com/llvm/llvm-project/pull/191114

From openmp-commits at lists.llvm.org  Mon Aug 10 10:56:01 2026
From: openmp-commits at lists.llvm.org (CHANDRA GHALE via Openmp-commits)
Date: Mon, 10 Aug 2026 10:56:01 -0700 (PDT)
Subject: [Openmp-commits] [flang] [llvm] [mlir] [openmp] [Flang][OpenMP]
 Lower scan directive and inscan reduction modifier (PR #206747)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206747@github.com>
Message-ID: <6a7a10b1.abe55a58.2a7581.6193@mx.google.com>


https://github.com/chandraghale updated https://github.com/llvm/llvm-project/pull/206747

>From 82416bc775c4befdbcce1f999a5f5b97ef954cd8 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Tue, 30 Jun 2026 10:00:56 -0500
Subject: [PATCH 1/9] Lower scan directive and inscan reduction modifier

---
 flang/lib/Lower/OpenMP/OpenMP.cpp             |  57 ++-
 .../Lower/OpenMP/Todo/nested-wsloop-scan.f90  |  34 ++
 .../OpenMP/Todo/wsloop-scan-collapse.f90      |  29 ++
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      | 476 +++++++++++++++---
 .../Target/LLVMIR/openmp-reduction-scan.mlir  | 123 +++++
 mlir/test/Target/LLVMIR/openmp-todo.mlir      |  41 +-
 openmp/runtime/test/scan/scan.f90             |  38 ++
 7 files changed, 726 insertions(+), 72 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/Todo/nested-wsloop-scan.f90
 create mode 100644 flang/test/Lower/OpenMP/Todo/wsloop-scan-collapse.f90
 create mode 100644 mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
 create mode 100644 openmp/runtime/test/scan/scan.f90

diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index ea8c279962508..e77584a142eea 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -2972,12 +2972,59 @@ genParallelOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
 
 static mlir::omp::ScanOp
 genScanOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
-          semantics::SemanticsContext &semaCtx, mlir::Location loc,
-          const ConstructQueue &queue, ConstructQueue::const_iterator item) {
+          semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+          mlir::Location loc, const ConstructQueue &queue,
+          ConstructQueue::const_iterator item) {
   mlir::omp::ScanOperands clauseOps;
   genScanClauses(converter, semaCtx, item->clauses, loc, clauseOps);
-  return mlir::omp::ScanOp::create(converter.getFirOpBuilder(),
-                                   converter.getCurrentLocation(), clauseOps);
+  mlir::omp::ScanOp scanOp = mlir::omp::ScanOp::create(
+      converter.getFirOpBuilder(), converter.getCurrentLocation(), clauseOps);
+
+  /// Scan reduction is not implemented with nested workshare loops, linear
+  /// clause, tiling
+  mlir::omp::LoopNestOp loopNestOp =
+      scanOp->getParentOfType<mlir::omp::LoopNestOp>();
+  llvm::SmallVector<mlir::omp::LoopWrapperInterface> loopWrappers;
+  loopNestOp.gatherWrappers(loopWrappers);
+  mlir::Operation *loopWrapperOp = loopWrappers.front().getOperation();
+  if (llvm::isa<mlir::omp::SimdOp>(loopWrapperOp))
+    TODO(loc, "unsupported simd");
+  if (loopWrappers.size() > 1)
+    TODO(loc, "unsupported composite");
+  mlir::omp::WsloopOp wsLoopOp = llvm::cast<mlir::omp::WsloopOp>(loopWrapperOp);
+  bool isNested =
+      (loopNestOp.getNumLoops() > 1) ||
+      (wsLoopOp && (wsLoopOp->getParentOfType<mlir::omp::WsloopOp>()));
+  if (isNested)
+    TODO(loc, "Scan directive inside nested workshare loops");
+  if (wsLoopOp && !wsLoopOp.getLinearVars().empty())
+    TODO(loc, "Scan directive with linear clause");
+  if (loopNestOp.getTileSizes())
+    TODO(loc, "Scan directive with loop tiling");
+
+  // All loop indices should be loaded after the scan construct as otherwise,
+  // it would result in using the index variable across scan directive.
+  // (`Intra-iteration dependences from a statement in the structured
+  // block sequence that precede a scan directive to a statement in the
+  // structured block sequence that follows a scan directive must not exist,
+  // except for dependences for the list items specified in an inclusive or
+  // exclusive clause.`).
+  // TODO: Nested loops are not handled.
+  mlir::Region &region = loopNestOp->getRegion(0);
+  mlir::Value indexVal = fir::getBase(region.getArgument(0));
+  lower::pft::Evaluation *doConstructEval = eval.parentConstruct;
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+  lower::pft::Evaluation *doLoop = &doConstructEval->getFirstNestedEvaluation();
+  auto *doStmt = doLoop->getIf<parser::NonLabelDoStmt>();
+  assert(doStmt && "Expected do loop to be in the nested evaluation");
+  const auto &loopControl =
+      std::get<std::optional<parser::LoopControl>>(doStmt->t);
+  const parser::LoopControl::Bounds *bounds =
+      std::get_if<parser::LoopControl::Bounds>(&loopControl->u);
+  mlir::Operation *storeOp =
+      setLoopVar(converter, loc, indexVal, bounds->Name().thing.symbol);
+  firOpBuilder.setInsertionPointAfter(storeOp);
+  return scanOp;
 }
 
 static mlir::omp::SectionsOp
@@ -4305,7 +4352,7 @@ static void genOMPDispatch(lower::AbstractConverter &converter,
                                   loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_scan:
-    newOp = genScanOp(converter, symTable, semaCtx, loc, queue, item);
+    newOp = genScanOp(converter, symTable, semaCtx, eval, loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_section:
     llvm_unreachable("genOMPDispatch: OMPD_section");
diff --git a/flang/test/Lower/OpenMP/Todo/nested-wsloop-scan.f90 b/flang/test/Lower/OpenMP/Todo/nested-wsloop-scan.f90
new file mode 100644
index 0000000000000..414e2ef2d5aed
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/nested-wsloop-scan.f90
@@ -0,0 +1,34 @@
+! Tests scan reduction behavior when used in nested workshare loops
+
+! RUN: %not_todo_cmd %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+program nested_scan_example
+  implicit none
+  integer, parameter :: n = 4, m = 5
+  integer :: a(n, m), b(n, m)
+  integer :: i, j
+  integer :: row_sum, col_sum
+
+  do i = 1, n
+     do j = 1, m
+        a(i, j) = i + j
+     end do
+  end do
+
+  !$omp parallel do reduction(inscan, +: row_sum) private(col_sum, j)
+  do i = 1, n
+     row_sum = row_sum + i
+     !$omp scan inclusive(row_sum)
+
+     col_sum = 0
+     !$omp parallel do reduction(inscan, +: col_sum)
+     do j = 1, m
+        col_sum = col_sum + a(i, j)
+        !CHECK: not yet implemented: Scan directive inside nested workshare loops
+        !$omp scan inclusive(col_sum)
+        b(i, j) = col_sum + row_sum 
+     end do
+     !$omp end parallel do
+  end do
+  !$omp end parallel do
+end program nested_scan_example
diff --git a/flang/test/Lower/OpenMP/Todo/wsloop-scan-collapse.f90 b/flang/test/Lower/OpenMP/Todo/wsloop-scan-collapse.f90
new file mode 100644
index 0000000000000..b8e6e831884ab
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/wsloop-scan-collapse.f90
@@ -0,0 +1,29 @@
+! Tests scan reduction behavior when used in nested workshare loops
+
+! RUN: %not_todo_cmd %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+program nested_loop_example
+  implicit none
+  integer :: i, j, x
+  integer, parameter :: N = 100, M = 200
+  real :: A(N, M), B(N, M)
+  x = 0
+
+  do i = 1, N
+    do j = 1, M
+      A(i, j) = i * j
+    end do
+  end do
+  
+  !$omp parallel do collapse(2) reduction(inscan, +:x)
+  do i = 1, N
+    do j = 1, M
+      x = x + A(i,j)
+      !CHECK: not yet implemented: Scan directive inside nested workshare loops
+      !$omp scan inclusive(x)
+      B(i, j) = x
+    end do
+  end do
+  !$omp end parallel do
+
+end program nested_loop_example
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index d860c408e0fd2..297ebac51de88 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -41,6 +41,8 @@
 #include "llvm/TargetParser/Triple.h"
 #include "llvm/Transforms/Utils/ModuleUtils.h"
 
+#include <cassert>
+#include <cstddef>
 #include <cstdint>
 #include <iterator>
 #include <numeric>
@@ -84,6 +86,10 @@ class OpenMPAllocStackFrame
       : allocInsertPoint(allocaIP), deallocBlocks(deallocBlocks) {}
   llvm::OpenMPIRBuilder::InsertPointTy allocInsertPoint;
   llvm::SmallVector<llvm::BasicBlock *> deallocBlocks;
+  /// Set to true when this alloca frame encloses an omp.parallel operation.
+  /// The alloca insertion point of a function in which a parallel op is
+  /// defined may be used to allocate the temporary buffer for scan reductions.
+  bool containsParallelOp = false;
 };
 
 /// Stack frame to hold a \see llvm::CanonicalLoopInfo representing the
@@ -93,7 +99,17 @@ class OpenMPLoopInfoStackFrame
     : public StateStackFrameBase<OpenMPLoopInfoStackFrame> {
 public:
   MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(OpenMPLoopInfoStackFrame)
+  /// For constructs like `scan`, a single `omp.loop_nest` is split into an
+  /// input loop and a scan loop. In that case `loopInfo` holds the input loop
+  /// info and `scanloopInfo` holds the scan loop info.
   llvm::CanonicalLoopInfo *loopInfo = nullptr;
+  llvm::CanonicalLoopInfo *scanloopInfo = nullptr;
+  llvm::ScanInfo *scanInfo = nullptr;
+  /// Map reduction variables to their LLVM types. Populated when the reduction
+  /// clause is processed and used when a `scan` directive is encountered in the
+  /// loop body.
+  std::unique_ptr<llvm::DenseMap<llvm::Value *, llvm::Type *>>
+      reductionVarToType = nullptr;
 };
 
 /// Custom error class to signal translation errors that don't need reporting,
@@ -359,6 +375,10 @@ static LogicalResult checkImplementationStatus(Operation &op) {
     if (!op.getDependVars().empty() || op.getDependKinds())
       result = todo("depend");
   };
+  auto checkExclusive = [&todo](auto op, LogicalResult &result) {
+    if (!op.getExclusiveVars().empty())
+      result = todo("exclusive");
+  };
   auto checkHint = [](auto op, LogicalResult &) {
     if (op.getHint())
       op.emitWarning("hint clause discarded");
@@ -389,21 +409,27 @@ static LogicalResult checkImplementationStatus(Operation &op) {
         op.getReductionMod().value() != omp::ReductionModifier::defaultmod) {
       omp::ReductionModifier mod = op.getReductionMod().value();
       // The `task` reduction modifier is supported on the parallel and
-      // worksharing (do/for and sections) constructs. Other modifiers, and the
-      // `task` modifier on other constructs, are not yet implemented.
+      // worksharing (do/for and sections) constructs. The `inscan` modifier is
+      // supported on the worksharing-loop construct (it is translated as a scan
+      // reduction). Other modifiers, and these modifiers on other constructs,
+      // are not yet implemented.
       bool taskModifierSupported =
           mod == omp::ReductionModifier::task &&
           isa<omp::ParallelOp, omp::WsloopOp, omp::SectionsOp>(op);
-      if (!taskModifierSupported) {
+      bool inscanModifierSupported =
+          mod == omp::ReductionModifier::inscan && isa<omp::WsloopOp>(op);
+      if (!taskModifierSupported && !inscanModifierSupported) {
         result = todo("reduction with modifier");
-      } else if (auto byref = op.getReductionByref()) {
-        // The task reduction modifier lowering only handles non-byref
-        // reductions for now.
-        for (bool isByRef : *byref)
-          if (isByRef) {
-            result = todo("task reduction modifier with by-ref reduction");
-            break;
-          }
+      } else if (taskModifierSupported) {
+        if (auto byref = op.getReductionByref()) {
+          // The task reduction modifier lowering only handles non-byref
+          // reductions for now.
+          for (bool isByRef : *byref)
+            if (isByRef) {
+              result = todo("task reduction modifier with by-ref reduction");
+              break;
+            }
+        }
       }
     }
   };
@@ -460,6 +486,7 @@ static LogicalResult checkImplementationStatus(Operation &op) {
         checkAllocate(op, result);
         checkOrder(op, result);
       })
+      .Case([&](omp::ScanOp op) { checkExclusive(op, result); })
       .Case([&](omp::SectionsOp op) {
         checkAllocate(op, result);
         checkPrivate(op, result);
@@ -650,6 +677,97 @@ findCurrentLoopInfo(LLVM::ModuleTranslation &moduleTranslation) {
   return loopInfo;
 }
 
+/// Find the scan loop information structure for the scan loop nest being
+/// translated. It will return a `null` value unless called from the
+/// translation function for a loop wrapper operation after successfully
+/// translating its body.
+static llvm::CanonicalLoopInfo *
+findCurrentScanLoopInfo(LLVM::ModuleTranslation &moduleTranslation) {
+  llvm::CanonicalLoopInfo *scanLoopInfo = nullptr;
+  moduleTranslation.stackWalk<OpenMPLoopInfoStackFrame>(
+      [&](OpenMPLoopInfoStackFrame &frame) {
+        scanLoopInfo = frame.scanloopInfo;
+        return WalkResult::interrupt();
+      });
+  return scanLoopInfo;
+}
+
+/// Find the `ScanInfo` stored on the loop stack frame. Upon encountering an
+/// `inscan` reduction modifier, `scanInfoInitialize` initializes the
+/// `ScanInfo`, which is then used when a `scan` directive is encountered in the
+/// body of the loop nest.
+static llvm::ScanInfo *
+findScanInfo(LLVM::ModuleTranslation &moduleTranslation) {
+  llvm::ScanInfo *scanInfo = nullptr;
+  moduleTranslation.stackWalk<OpenMPLoopInfoStackFrame>(
+      [&](OpenMPLoopInfoStackFrame &frame) {
+        scanInfo = frame.scanInfo;
+        return WalkResult::interrupt();
+      });
+  return scanInfo;
+}
+
+/// The types of reduction variables are used for lowering a `scan` directive
+/// that appears in the body of the loop. The types are stored in the loop frame
+/// when the reduction clause is encountered and used when the `scan` directive
+/// is encountered.
+static llvm::DenseMap<llvm::Value *, llvm::Type *> *
+findReductionVarTypes(LLVM::ModuleTranslation &moduleTranslation) {
+  llvm::DenseMap<llvm::Value *, llvm::Type *> *reductionVarToType = nullptr;
+  moduleTranslation.stackWalk<OpenMPLoopInfoStackFrame>(
+      [&](OpenMPLoopInfoStackFrame &frame) {
+        if (!frame.reductionVarToType)
+          frame.reductionVarToType =
+              std::make_unique<llvm::DenseMap<llvm::Value *, llvm::Type *>>();
+        reductionVarToType = frame.reductionVarToType.get();
+        return WalkResult::interrupt();
+      });
+  return reductionVarToType;
+}
+
+/// Scan reduction requires a shared buffer to be allocated to perform the
+/// reduction. The allocation needs to be done outside the parallel region in
+/// which the scan operation is used.
+static llvm::OpenMPIRBuilder::InsertPointTy
+findParallelAllocaIP(llvm::IRBuilderBase &builder,
+                     LLVM::ModuleTranslation &moduleTranslation) {
+  // If there is an alloca insertion point on the stack belonging to a frame
+  // that encloses a parallel op, use it.
+  llvm::OpenMPIRBuilder::InsertPointTy allocaInsertPoint;
+  WalkResult walkResult = moduleTranslation.stackWalk<OpenMPAllocStackFrame>(
+      [&](OpenMPAllocStackFrame &frame) {
+        if (frame.containsParallelOp) {
+          allocaInsertPoint = frame.allocInsertPoint;
+          return WalkResult::interrupt();
+        }
+        return WalkResult::skip();
+      });
+  if (walkResult.wasInterrupted())
+    return allocaInsertPoint;
+  // Otherwise, insert into the entry block of the surrounding function.
+  // If the current IRBuilder InsertPoint is the function's entry, it cannot
+  // also be used for alloca insertion which would result in insertion order
+  // confusion. Create a new BasicBlock for the Builder and use the entry block
+  // for the allocs.
+  // TODO: Create a dedicated alloca BasicBlock at function creation such that
+  // we do not need to move the current InsertPoint here.
+  if (builder.GetInsertBlock() ==
+      &builder.GetInsertBlock()->getParent()->getEntryBlock()) {
+    assert(builder.GetInsertPoint() == builder.GetInsertBlock()->end() &&
+           "Assuming end of basic block");
+    llvm::BasicBlock *entryBB = llvm::BasicBlock::Create(
+        builder.getContext(), "entry", builder.GetInsertBlock()->getParent(),
+        builder.GetInsertBlock()->getNextNode());
+    builder.CreateBr(entryBB);
+    builder.SetInsertPoint(entryBB);
+  }
+
+  llvm::BasicBlock &funcEntryBlock =
+      builder.GetInsertBlock()->getParent()->getEntryBlock();
+  return llvm::OpenMPIRBuilder::InsertPointTy(
+      &funcEntryBlock, funcEntryBlock.getFirstInsertionPt());
+}
+
 /// Converts the given region that appears within an OpenMP dialect operation to
 /// LLVM IR, creating a branch from the `sourceBlock` to the entry block of the
 /// region, and a branch from any block with an successor-less OpenMP terminator
@@ -1414,7 +1532,8 @@ initReductionVars(OP op, ArrayRef<BlockArgument> reductionArgs,
                   SmallVectorImpl<llvm::Value *> &privateReductionVariables,
                   DenseMap<Value, llvm::Value *> &reductionVariableMap,
                   llvm::ArrayRef<bool> isByRef,
-                  SmallVectorImpl<DeferredStore> &deferredStores) {
+                  SmallVectorImpl<DeferredStore> &deferredStores,
+                  bool isInScanRegion = false) {
   if (op.getNumReductionVars() == 0)
     return success();
 
@@ -1451,11 +1570,17 @@ initReductionVars(OP op, ArrayRef<BlockArgument> reductionArgs,
   for (auto [data, addr] : deferredStores)
     builder.CreateStore(data, addr);
 
+  llvm::DenseMap<llvm::Value *, llvm::Type *> *reductionVarToType =
+      findReductionVarTypes(moduleTranslation);
   // Before the loop, store the initial values of reductions into reduction
   // variables. Although this could be done after allocas, we don't want to mess
   // up with the alloca insertion point.
   for (unsigned i = 0; i < op.getNumReductionVars(); ++i) {
     SmallVector<llvm::Value *, 1> phis;
+    llvm::Type *reductionType =
+        moduleTranslation.convertType(reductionDecls[i].getType());
+    if (isInScanRegion && reductionVarToType != nullptr)
+      (*reductionVarToType)[privateReductionVariables[i]] = reductionType;
 
     // map block argument to initializer region
     mapInitializationArgs(op, moduleTranslation, builder, reductionDecls,
@@ -4406,11 +4531,15 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
     return failure();
 
   assert(afterAllocas.get()->getSinglePredecessor());
+  bool isInScanRegion =
+      wsloopOp.getReductionMod() && (wsloopOp.getReductionMod().value() ==
+                                     mlir::omp::ReductionModifier::inscan);
   if (failed(initReductionVars(wsloopOp, reductionArgs, builder,
                                moduleTranslation,
                                afterAllocas.get()->getSinglePredecessor(),
                                reductionDecls, privateReductionVariables,
-                               reductionVariableMap, isByRef, deferredStores)))
+                               reductionVariableMap, isByRef, deferredStores,
+                               isInScanRegion)))
     return failure();
 
   // For `reduction(task, ...)` open a task-reduction scope for the worksharing
@@ -4432,6 +4561,10 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
   std::optional<omp::ScheduleModifier> scheduleMod = wsloopOp.getScheduleMod();
   bool isSimd = wsloopOp.getScheduleSimd();
   bool loopNeedsBarrier = !wsloopOp.getNowait();
+  // TODO: Linear clause support needs to be enabled for scan reduction.
+  if (isInScanRegion)
+    assert(wsloopOp.getLinearVars().empty() &&
+           "Linear clause support is not enabled with scan reduction");
 
   // The only legal way for the direct parent to be omp.distribute is that this
   // represents 'distribute parallel do'. Otherwise, this is a regular
@@ -4469,21 +4602,93 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
   if (failed(handleError(regionBlock, opInst)))
     return failure();
 
+  // Generates the loop body for a worksharing loop, including linear-variable
+  // handling and the call into the OpenMPIRBuilder's worksharing-loop helper.
+  // For scan reductions this lambda is invoked twice: once for the input loop
+  // and once for the scan loop.
+  const auto &&wsloopCodeGen = [&](llvm::CanonicalLoopInfo *loopInfo,
+                                   bool noLoopMode,
+                                   bool inputScanLoop) -> LogicalResult {
+    // Emit Initialization and Update IR for linear variables
+    if (!wsloopOp.getLinearVars().empty()) {
+      linearClauseProcessor.initLinearVar(builder, moduleTranslation,
+                                          loopInfo->getPreheader());
+      llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP =
+          moduleTranslation.getOpenMPBuilder()->createBarrier(
+              builder.saveIP(), llvm::omp::OMPD_barrier);
+      if (failed(handleError(afterBarrierIP, *loopOp)))
+        return failure();
+      builder.restoreIP(*afterBarrierIP);
+      linearClauseProcessor.updateLinearVar(builder, loopInfo->getBody(),
+                                            loopInfo->getIndVar());
+      linearClauseProcessor.splitLinearFiniBB(builder, loopInfo->getExit());
+    }
+
+    builder.SetInsertPoint(*regionBlock, (*regionBlock)->begin());
+
+    for (size_t index = 0; index < wsloopOp.getLinearVars().size(); index++)
+      linearClauseProcessor.rewriteInPlace(builder, loopInfo->getBody(),
+                                           loopInfo->getLatch(), index);
+
+    llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP =
+        ompBuilder->applyWorkshareLoop(
+            ompLoc.DL, loopInfo, allocaIP, loopNeedsBarrier,
+            convertToScheduleKind(schedule), chunk, isSimd,
+            scheduleMod == omp::ScheduleModifier::monotonic,
+            scheduleMod == omp::ScheduleModifier::nonmonotonic, isOrdered,
+            workshareLoopType, noLoopMode, hasDistSchedule, distScheduleChunk);
+
+    if (failed(handleError(wsloopIP, opInst)))
+      return failure();
+
+    // Emit finalization for linear vars.
+    if (!wsloopOp.getLinearVars().empty()) {
+      llvm::OpenMPIRBuilder::InsertPointTy oldIP = builder.saveIP();
+      assert(loopInfo->getLastIter() &&
+             "`lastiter` in CanonicalLoopInfo is nullptr");
+      llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP =
+          linearClauseProcessor.finalizeLinearVar(builder, moduleTranslation,
+                                                  loopInfo->getLastIter());
+      if (failed(handleError(afterBarrierIP, *loopOp)))
+        return failure();
+
+      builder.restoreIP(oldIP);
+    }
+
+    // For scan reductions, the cancellation finalization callback is popped
+    // only after the scan (second) loop has been generated.
+    if (!inputScanLoop || !isInScanRegion)
+      popCancelFinalizationCB(cancelTerminators, *ompBuilder, wsloopIP.get());
+
+    return success();
+  };
+
   llvm::CanonicalLoopInfo *loopInfo = findCurrentLoopInfo(moduleTranslation);
 
-  // Emit Initialization and Update IR for linear variables
-  if (!wsloopOp.getLinearVars().empty()) {
-    linearClauseProcessor.initLinearVar(builder, moduleTranslation,
-                                        loopInfo->getPreheader());
-    llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP =
-        moduleTranslation.getOpenMPBuilder()->createBarrier(
-            builder.saveIP(), llvm::omp::OMPD_barrier);
-    if (failed(handleError(afterBarrierIP, *loopOp)))
+  if (isInScanRegion) {
+    // Emit the scan reduction combiner between the input loop and the scan
+    // loop, operating on the per-iteration partial values stored in the shared
+    // buffer.
+    auto inputLoopFinishIp = loopInfo->getAfterIP();
+    builder.restoreIP(inputLoopFinishIp);
+    SmallVector<OwningReductionGen> owningReductionGens;
+    SmallVector<OwningAtomicReductionGen> owningAtomicReductionGens;
+    SmallVector<llvm::OpenMPIRBuilder::ReductionInfo, 2> reductionInfos;
+    SmallVector<OwningDataPtrPtrReductionGen> owningReductionGenRefDataPtrGens;
+    collectReductionInfo(wsloopOp, builder, moduleTranslation, reductionDecls,
+                         owningReductionGens, owningAtomicReductionGens,
+                         owningReductionGenRefDataPtrGens,
+                         privateReductionVariables, reductionInfos, isByRef);
+    llvm::BasicBlock *cont = splitBB(builder, false, "omp.scan.loop.cont");
+    llvm::ScanInfo *scanInfo = findScanInfo(moduleTranslation);
+    llvm::OpenMPIRBuilder::InsertPointOrErrorTy redIP =
+        ompBuilder->emitScanReduction(builder.saveIP(), reductionInfos,
+                                      scanInfo);
+    if (failed(handleError(redIP, opInst)))
       return failure();
-    builder.restoreIP(*afterBarrierIP);
-    linearClauseProcessor.updateLinearVar(builder, loopInfo->getBody(),
-                                          loopInfo->getIndVar());
-    linearClauseProcessor.splitLinearFiniBB(builder, loopInfo->getExit());
+
+    builder.restoreIP(*redIP);
+    builder.CreateBr(cont);
   }
 
   builder.SetInsertPoint(*regionBlock, (*regionBlock)->begin());
@@ -4502,50 +4707,41 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
       noLoopMode = true;
   }
 
-  for (size_t index = 0; index < wsloopOp.getLinearVars().size(); index++)
-    linearClauseProcessor.rewriteInPlace(builder, loopInfo->getBody(),
-                                         loopInfo->getLatch(), index);
-
-  llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP =
-      ompBuilder->applyWorkshareLoop(
-          ompLoc.DL, loopInfo, allocaIP, loopNeedsBarrier,
-          convertToScheduleKind(schedule), chunk, isSimd,
-          scheduleMod == omp::ScheduleModifier::monotonic,
-          scheduleMod == omp::ScheduleModifier::nonmonotonic, isOrdered,
-          workshareLoopType, noLoopMode, hasDistSchedule, distScheduleChunk);
-
-  if (failed(handleError(wsloopIP, opInst)))
+  // For scan loops, the input loop does not pop the cancellation finalization
+  // callback; that happens after the scan loop is generated below.
+  bool inputScanLoop = isInScanRegion;
+  if (failed(wsloopCodeGen(loopInfo, noLoopMode, inputScanLoop)))
     return failure();
+  inputScanLoop = false;
 
-  // Emit finalization and in-place rewrites for linear vars.
-  if (!wsloopOp.getLinearVars().empty()) {
-    llvm::OpenMPIRBuilder::InsertPointTy oldIP = builder.saveIP();
-    assert(loopInfo->getLastIter() &&
-           "`lastiter` in CanonicalLoopInfo is nullptr");
-    llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP =
-        linearClauseProcessor.finalizeLinearVar(builder, moduleTranslation,
-                                                loopInfo->getLastIter());
-    if (failed(handleError(afterBarrierIP, *loopOp)))
+  if (isInScanRegion) {
+    llvm::CanonicalLoopInfo *scanLoopInfo =
+        findCurrentScanLoopInfo(moduleTranslation);
+    if (failed(wsloopCodeGen(scanLoopInfo, noLoopMode, inputScanLoop)))
+      return failure();
+    SmallVector<Region *> reductionRegions;
+    llvm::transform(reductionDecls, std::back_inserter(reductionRegions),
+                    [](omp::DeclareReductionOp reductionDecl) {
+                      return &reductionDecl.getCleanupRegion();
+                    });
+    if (failed(inlineOmpRegionCleanup(
+            reductionRegions, privateReductionVariables, moduleTranslation,
+            builder, "omp.reduction.cleanup")))
       return failure();
+  } else {
+    // Close the task-reduction scope before the worksharing reduction combine.
+    if (isTaskReductionMod)
+      emitTaskReductionModifierFini(/*isWorksharing=*/true, builder,
+                                    moduleTranslation);
 
-    builder.restoreIP(oldIP);
+    // Process the reductions if required.
+    if (failed(createReductionsAndCleanup(
+            wsloopOp, builder, moduleTranslation, allocaIP, reductionDecls,
+            privateReductionVariables, isByRef, wsloopOp.getNowait(),
+            /*isTeamsReduction=*/false)))
+      return failure();
   }
 
-  // Set the correct branch target for task cancellation
-  popCancelFinalizationCB(cancelTerminators, *ompBuilder, wsloopIP.get());
-
-  // Close the task-reduction scope before the worksharing reduction combine.
-  if (isTaskReductionMod)
-    emitTaskReductionModifierFini(/*isWorksharing=*/true, builder,
-                                  moduleTranslation);
-
-  // Process the reductions if required.
-  if (failed(createReductionsAndCleanup(
-          wsloopOp, builder, moduleTranslation, allocaIP, reductionDecls,
-          privateReductionVariables, isByRef, wsloopOp.getNowait(),
-          /*isTeamsReduction=*/false)))
-    return failure();
-
   return cleanupPrivateVars(wsloopOp, builder, moduleTranslation,
                             wsloopOp.getLoc(), privateVarsInfo);
 }
@@ -4579,6 +4775,20 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
       opInst.getReductionMod() == omp::ReductionModifier::task &&
       opInst.getNumReductionVars() > 0;
 
+  // Mark the enclosing alloca stack frame as containing a parallel op. Scan
+  // reductions use the alloca insertion point of the function enclosing the
+  // parallel region to allocate their shared temporary buffer.
+  bool foundParallelOp = false;
+  moduleTranslation.stackWalk<OpenMPAllocStackFrame>(
+      [&](OpenMPAllocStackFrame &frame) {
+        if (foundParallelOp) {
+          frame.containsParallelOp = true;
+          return WalkResult::interrupt();
+        }
+        foundParallelOp = true;
+        return WalkResult::skip();
+      });
+
   auto bodyGenCB =
       [&](InsertPointTy allocaIP, InsertPointTy codeGenIP,
           llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) -> llvm::Error {
@@ -4972,6 +5182,99 @@ convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder,
                             privateVarsInfo);
 }
 
+static LogicalResult
+convertOmpScan(Operation &opInst, llvm::IRBuilderBase &builder,
+               LLVM::ModuleTranslation &moduleTranslation) {
+  if (failed(checkImplementationStatus(opInst)))
+    return failure();
+  auto scanOp = cast<omp::ScanOp>(opInst);
+  bool isInclusive = scanOp.hasInclusiveVars();
+  SmallVector<llvm::Value *> llvmScanVars;
+  SmallVector<llvm::Type *> llvmScanVarsType;
+  mlir::OperandRange mlirScanVars = scanOp.getInclusiveVars();
+  if (!isInclusive)
+    mlirScanVars = scanOp.getExclusiveVars();
+
+  llvm::DenseMap<llvm::Value *, llvm::Type *> *reductionVarToType =
+      findReductionVarTypes(moduleTranslation);
+  for (auto val : mlirScanVars) {
+    llvm::Value *llvmVal = moduleTranslation.lookupValue(val);
+    llvmScanVars.push_back(llvmVal);
+    llvmScanVarsType.push_back((*reductionVarToType)[llvmVal]);
+  }
+  llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
+      findParallelAllocaIP(builder, moduleTranslation);
+  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
+  llvm::ScanInfo *scanInfo = findScanInfo(moduleTranslation);
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
+      moduleTranslation.getOpenMPBuilder()->createScan(
+          ompLoc, allocaIP, llvmScanVars, llvmScanVarsType, isInclusive,
+          scanInfo);
+  if (failed(handleError(afterIP, opInst)))
+    return failure();
+  builder.restoreIP(*afterIP);
+  return success();
+}
+
+/// Re-initialize the scan reduction variables to their identity at the start of
+/// each input-loop iteration of a scan reduction. This is required because the
+/// OpenMPIRBuilder stores the per-iteration reduction value into the scan
+/// buffer (`buffer[i] = red`) and `emitScanReduction` then computes the prefix
+/// sum across the buffer. Without resetting `red` to the reduction identity
+/// each iteration, the buffer would hold a running sum and the prefix sum would
+/// be computed twice. This reset only applies to the input loop; in the scan
+/// loop the reduction variable is loaded back from the buffer.
+static LogicalResult
+initScanReductionVars(omp::LoopNestOp loopOp, llvm::IRBuilderBase &builder,
+                      LLVM::ModuleTranslation &moduleTranslation) {
+  auto wsloopOp = loopOp->getParentOfType<omp::WsloopOp>();
+  if (!wsloopOp || !wsloopOp.getReductionMod() ||
+      wsloopOp.getReductionMod().value() != omp::ReductionModifier::inscan)
+    return success();
+
+  // Only reset in the input loop; in the scan loop the reduction variable holds
+  // the value loaded from the buffer and must not be overwritten.
+  llvm::ScanInfo *scanInfo = findScanInfo(moduleTranslation);
+  if (!scanInfo || !scanInfo->OMPFirstScanLoop)
+    return success();
+
+  unsigned numReductions = wsloopOp.getNumReductionVars();
+  if (numReductions == 0)
+    return success();
+
+  SmallVector<omp::DeclareReductionOp> reductionDecls;
+  collectReductionDecls(wsloopOp, reductionDecls);
+  ArrayRef<bool> isByRef = getIsByRef(wsloopOp.getReductionByref());
+  MutableArrayRef<BlockArgument> reductionArgs =
+      cast<omp::BlockArgOpenMPOpInterface>(wsloopOp.getOperation())
+          .getReductionBlockArgs();
+
+  DenseMap<Value, llvm::Value *> reductionVariableMap;
+  for (unsigned i = 0; i < numReductions; ++i)
+    reductionVariableMap.try_emplace(
+        wsloopOp.getReductionVars()[i],
+        moduleTranslation.lookupValue(reductionArgs[i]));
+
+  for (unsigned i = 0; i < numReductions; ++i) {
+    // Scan reductions only support by-value (scalar) reductions.
+    if (isByRef[i])
+      continue;
+    SmallVector<llvm::Value *, 1> phis;
+    mapInitializationArgs(wsloopOp, moduleTranslation, builder, reductionDecls,
+                          reductionVariableMap, i);
+    if (failed(inlineConvertOmpRegions(
+            reductionDecls[i].getInitializerRegion(),
+            "omp.scan.reduction.init", builder, moduleTranslation, &phis)))
+      return failure();
+    assert(phis.size() == 1 &&
+           "expected one value to be yielded from the reduction init region");
+    setInsertPointForPossiblyEmptyBlock(builder);
+    builder.CreateStore(phis[0],
+                        moduleTranslation.lookupValue(reductionArgs[i]));
+  }
+  return success();
+}
+
 /// Converts an OpenMP loop nest into LLVM IR using OpenMPIRBuilder.
 static LogicalResult
 convertOmpLoopNest(Operation &opInst, llvm::IRBuilderBase &builder,
@@ -5004,6 +5307,10 @@ convertOmpLoopNest(Operation &opInst, llvm::IRBuilderBase &builder,
 
     // Convert the body of the loop.
     builder.restoreIP(ip);
+    // For scan reductions, reset the reduction variables to their identity at
+    // the start of each input-loop iteration, before the input phase runs.
+    if (failed(initScanReductionVars(loopOp, builder, moduleTranslation)))
+      return llvm::make_error<PreviouslyReportedError>();
     llvm::Expected<llvm::BasicBlock *> regionBlock = convertOmpOpRegions(
         loopOp.getRegion(), "omp.loop_nest.region", builder, moduleTranslation);
     if (!regionBlock)
@@ -5036,6 +5343,46 @@ convertOmpLoopNest(Operation &opInst, llvm::IRBuilderBase &builder,
       computeIP = loopInfos.front()->getPreheaderIP();
     }
 
+    // If this loop is the worksharing loop of a scan reduction, generate a
+    // pair of canonical loops (an input loop and a scan loop) instead of a
+    // single loop. The `scan` directive in the body is translated against the
+    // `ScanInfo` recorded here.
+    bool isInScanRegion = false;
+    if (auto wsloopOp = loopOp->getParentOfType<omp::WsloopOp>())
+      isInScanRegion =
+          wsloopOp.getReductionMod() && (wsloopOp.getReductionMod().value() ==
+                                         mlir::omp::ReductionModifier::inscan);
+    if (isInScanRegion) {
+      llvm::Expected<llvm::ScanInfo *> res = ompBuilder->scanInfoInitialize();
+      if (failed(handleError(res, *loopOp)))
+        return failure();
+      llvm::ScanInfo *scanInfo = res.get();
+      moduleTranslation.stackWalk<OpenMPLoopInfoStackFrame>(
+          [&](OpenMPLoopInfoStackFrame &frame) {
+            frame.scanInfo = scanInfo;
+            return WalkResult::interrupt();
+          });
+      llvm::Expected<llvm::SmallVector<llvm::CanonicalLoopInfo *>> loopResults =
+          ompBuilder->createCanonicalScanLoops(
+              loc, bodyGen, lowerBound, upperBound, step,
+              /*IsSigned=*/true, loopOp.getLoopInclusive(), computeIP, "loop",
+              scanInfo);
+
+      if (failed(handleError(loopResults, *loopOp)))
+        return failure();
+      llvm::CanonicalLoopInfo *inputLoop = loopResults.get().front();
+      llvm::CanonicalLoopInfo *scanLoop = loopResults.get().back();
+      moduleTranslation.stackWalk<OpenMPLoopInfoStackFrame>(
+          [&](OpenMPLoopInfoStackFrame &frame) {
+            frame.loopInfo = inputLoop;
+            frame.scanloopInfo = scanLoop;
+            return WalkResult::interrupt();
+          });
+      builder.restoreIP(scanLoop->getAfterIP());
+      // TODO: tiling and collapse are not yet implemented for scan reduction.
+      return success();
+    }
+
     llvm::Expected<llvm::CanonicalLoopInfo *> loopResult =
         ompBuilder->createCanonicalLoop(
             loc, bodyGen, lowerBound, upperBound, step,
@@ -9454,6 +9801,9 @@ LogicalResult OpenMPDialectLLVMIRTranslationInterface::convertOperation(
           .Case([&](omp::WsloopOp) {
             return convertOmpWsloop(*op, builder, moduleTranslation);
           })
+          .Case([&](omp::ScanOp) {
+            return convertOmpScan(*op, builder, moduleTranslation);
+          })
           .Case([&](omp::SimdOp) {
             return convertOmpSimd(*op, builder, moduleTranslation);
           })
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
new file mode 100644
index 0000000000000..c6b8daccad313
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
@@ -0,0 +1,123 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+omp.declare_reduction @add_reduction_i32 : i32 init {
+^bb0(%arg0: i32):
+  %0 = llvm.mlir.constant(0 : i32) : i32
+  omp.yield(%0 : i32)
+} combiner {
+^bb0(%arg0: i32, %arg1: i32):
+  %0 = llvm.add %arg0, %arg1 : i32
+  omp.yield(%0 : i32)
+}
+// CHECK-LABEL: @scan_reduction
+llvm.func @scan_reduction() {
+  %0 = llvm.mlir.constant(1 : i64) : i64
+  %1 = llvm.alloca %0 x i32 {bindc_name = "z"} : (i64) -> !llvm.ptr
+  %3 = llvm.alloca %0 x i32 {bindc_name = "y"} : (i64) -> !llvm.ptr
+  %5 = llvm.alloca %0 x i32 {bindc_name = "x"} : (i64) -> !llvm.ptr
+  %7 = llvm.alloca %0 x i32 {bindc_name = "k"} : (i64) -> !llvm.ptr
+  %10 = llvm.mlir.constant(100 : i32) : i32
+  %11 = llvm.mlir.constant(1 : i32) : i32
+  %12 = llvm.mlir.constant(0 : i32) : i32
+  %13 = llvm.mlir.constant(100 : index) : i64
+  %14 = llvm.mlir.addressof @_QFEa : !llvm.ptr
+  %15 = llvm.mlir.addressof @_QFEb : !llvm.ptr
+  omp.parallel {
+    %37 = llvm.mlir.constant(1 : i64) : i64
+    %38 = llvm.alloca %37 x i32 {bindc_name = "k", pinned} : (i64) -> !llvm.ptr
+    %39 = llvm.mlir.constant(1 : i64) : i64
+    omp.wsloop reduction(mod: inscan, @add_reduction_i32 %5 -> %arg0 : !llvm.ptr) {
+      omp.loop_nest (%arg1) : i32 = (%11) to (%10) inclusive step (%11) {
+        llvm.store %arg1, %38 : i32, !llvm.ptr
+        %40 = llvm.load %arg0 : !llvm.ptr -> i32
+        %41 = llvm.load %38 : !llvm.ptr -> i32
+        %42 = llvm.sext %41 : i32 to i64
+        %50 = llvm.getelementptr %14[%42] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+        %51 = llvm.load %50 : !llvm.ptr -> i32
+        %52 = llvm.add %40, %51 : i32
+        llvm.store %52, %arg0 : i32, !llvm.ptr
+        omp.scan inclusive(%arg0 : !llvm.ptr)
+        llvm.store %arg1, %38 : i32, !llvm.ptr
+        %53 = llvm.load %arg0 : !llvm.ptr -> i32
+        %54 = llvm.load %38 : !llvm.ptr -> i32
+        %55 = llvm.sext %54 : i32 to i64
+        %63 = llvm.getelementptr %15[%55] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+        llvm.store %53, %63 : i32, !llvm.ptr
+        omp.yield
+      }
+    }
+    omp.terminator
+  }
+  llvm.return
+}
+llvm.mlir.global internal @_QFEa() {addr_space = 0 : i32} : !llvm.array<100 x i32> {
+  %0 = llvm.mlir.zero : !llvm.array<100 x i32>
+  llvm.return %0 : !llvm.array<100 x i32>
+}
+llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i32> {
+  %0 = llvm.mlir.zero : !llvm.array<100 x i32>
+  llvm.return %0 : !llvm.array<100 x i32>
+}
+//CHECK: %vla = alloca ptr, align 8
+//CHECK: omp_parallel
+//CHECK: store ptr %vla, ptr %gep_vla, align 8
+//CHECK: @__kmpc_fork_call
+//CHECK: void @scan_reduction..omp_par
+//CHECK: %[[BUFF_PTR:.+]] = load ptr, ptr %gep_vla
+//CHECK: @__kmpc_masked
+//CHECK: @__kmpc_barrier
+//CHECK: @__kmpc_masked
+//CHECK: @__kmpc_barrier
+//CHECK: omp.scan.loop.cont:
+//CHECK: @__kmpc_masked
+//CHECK: @__kmpc_barrier
+//CHECK: %[[FREE_VAR:.+]] = load ptr, ptr %[[BUFF_PTR]], align 8
+//CHECK:  %[[ARRLAST:.+]] = getelementptr inbounds i32, ptr %[[FREE_VAR]], i32 100
+//CHECK:  %[[RES:.+]] = load i32, ptr %[[ARRLAST]], align 4
+//CHECK:  store i32 %[[RES]], ptr %loadgep{{.*}}, align 4
+//CHECK: tail call void @free(ptr %[[FREE_VAR]])
+//CHECK: @__kmpc_end_masked
+//CHECK: omp.inscan.dispatch{{.*}}:                            ; preds = %omp_loop.body{{.*}}
+//CHECK:   %[[BUFFVAR:.+]] = load ptr, ptr %[[BUFF_PTR]], align 8
+//CHECK:   %[[arrayOffset1:.+]] = getelementptr inbounds i32, ptr %[[BUFFVAR]], i32 %{{.*}}
+//CHECK:   %[[BUFFVAL1:.+]] = load i32, ptr %[[arrayOffset1]], align 4
+//CHECK:   store i32 %[[BUFFVAL1]], ptr %{{.*}}, align 4
+//CHECK:   %[[LOG:.+]] = call double @llvm.log2.f64(double 1.000000e+02) #0
+//CHECK:   %[[CEIL:.+]] = call double @llvm.ceil.f64(double %[[LOG]]) #0
+//CHECK:   %[[UB:.+]] = fptoui double %[[CEIL]] to i32
+//CHECK:   br label %omp.outer.log.scan.body
+//CHECK: omp.outer.log.scan.body:
+//CHECK:   %[[K:.+]] = phi i32 [ 0, %{{.*}} ], [ %[[NEXTK:.+]], %omp.inner.log.scan.exit ]
+//CHECK:   %[[I:.+]] = phi i32 [ 1, %{{.*}} ], [ %[[NEXTI:.+]], %omp.inner.log.scan.exit ]
+//CHECK:   %[[CMP1:.+]] = icmp uge i32 99, %[[I]]
+//CHECK:   br i1 %[[CMP1]], label %omp.inner.log.scan.body, label %omp.inner.log.scan.exit
+//CHECK: omp.inner.log.scan.exit:                          ; preds = %omp.inner.log.scan.body, %omp.outer.log.scan.body
+//CHECK:   %[[NEXTK]] = add nuw i32 %[[K]], 1
+//CHECK:   %[[NEXTI]] = shl nuw i32 %[[I]], 1
+//CHECK:   %[[CMP2:.+]] = icmp ne i32 %[[NEXTK]], %[[UB]]
+//CHECK:   br i1 %[[CMP2]], label %omp.outer.log.scan.body, label %omp.outer.log.scan.exit
+//CHECK: omp.outer.log.scan.exit:                          ; preds = %omp.inner.log.scan.exit
+//CHECK: @__kmpc_end_masked
+//CHECK: omp.inner.log.scan.body:                          ; preds = %omp.inner.log.scan.body, %omp.outer.log.scan.body
+//CHECK:   %[[CNT:.+]] = phi i32 [ 99, %omp.outer.log.scan.body ], [ %[[CNTNXT:.+]], %omp.inner.log.scan.body ]
+//CHECK:   %[[BUFF:.+]] = load ptr, ptr %[[BUFF_PTR]]
+//CHECK:   %[[IND1:.+]] = add i32 %[[CNT]], 1
+//CHECK:   %[[IND1PTR:.+]] = getelementptr inbounds i32, ptr %[[BUFF]], i32 %[[IND1]]
+//CHECK:   %[[IND2:.+]] = sub nuw i32 %[[IND1]], %[[I]]
+//CHECK:   %[[IND2PTR:.+]] = getelementptr inbounds i32, ptr %[[BUFF]], i32 %[[IND2]]
+//CHECK:   %[[IND1VAL:.+]] = load i32, ptr %[[IND1PTR]], align 4
+//CHECK:   %[[IND2VAL:.+]] = load i32, ptr %[[IND2PTR]], align 4
+//CHECK:   %[[REDVAL:.+]] = add i32 %[[IND1VAL]], %[[IND2VAL]]
+//CHECK:   store i32 %[[REDVAL]], ptr %[[IND1PTR]], align 4
+//CHECK:   %[[CNTNXT]] = sub nuw i32 %[[CNT]], 1
+//CHECK:   %[[CMP3:.+]] = icmp uge i32 %[[CNTNXT]], %[[I]]
+//CHECK:   br i1 %[[CMP3]], label %omp.inner.log.scan.body, label %omp.inner.log.scan.exit
+//CHECK: omp.inscan.dispatch:                              ; preds = %omp_loop.body
+//CHECK:   br i1 true, label %omp.before.scan.bb, label %omp.after.scan.bb
+//CHECK: omp.before.scan.bb:
+//CHECK:   store i32 0, ptr %[[REDPRIV:.+]], align 4
+//CHECK: omp.loop_nest.region:                             ; preds = %omp.before.scan.bb
+//CHECK:   %[[BUFFER:.+]] = load ptr, ptr %loadgep_vla, align 8
+//CHECK:   %[[ARRAYOFFSET2:.+]] = getelementptr inbounds i32, ptr %[[BUFFER]], i32 %{{.*}}
+//CHECK-NEXT:   %[[REDPRIVVAL:.+]] = load i32, ptr %{{.*}}, align 4
+//CHECK:   store i32 %[[REDPRIVVAL]], ptr %[[ARRAYOFFSET2]], align 4
+//CHECK:   br label %omp.scan.loop.exit
diff --git a/mlir/test/Target/LLVMIR/openmp-todo.mlir b/mlir/test/Target/LLVMIR/openmp-todo.mlir
index 71e8628595aab..7edf88b5ea559 100644
--- a/mlir/test/Target/LLVMIR/openmp-todo.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-todo.mlir
@@ -120,10 +120,10 @@ atomic {
   llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32
   omp.yield
 }
-llvm.func @scan_reduction(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
-  // expected-error at below {{not yet implemented: Unhandled clause reduction with modifier in omp.wsloop operation}}
-  // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
-  omp.wsloop reduction(mod:inscan, @add_f32 %x -> %prv : !llvm.ptr) {
+llvm.func @simd_reduction(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
+  // expected-error at below {{not yet implemented: Unhandled clause reduction with modifier in omp.simd operation}}
+  // expected-error at below {{LLVM Translation failed for operation: omp.simd}}
+  omp.simd reduction(mod:inscan, @add_f32 %x -> %prv : !llvm.ptr) {
     omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
       omp.scan inclusive(%prv : !llvm.ptr)
       omp.yield
@@ -162,6 +162,39 @@ llvm.func @parallel_task_reduction_modifier_byref(%x : !llvm.ptr) {
 
 // -----
 
+omp.declare_reduction @add_f32 : f32
+init {
+^bb0(%arg: f32):
+  %0 = llvm.mlir.constant(0.0 : f32) : f32
+  omp.yield (%0 : f32)
+}
+combiner {
+^bb1(%arg0: f32, %arg1: f32):
+  %1 = llvm.fadd %arg0, %arg1 : f32
+  omp.yield (%1 : f32)
+}
+atomic {
+^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr):
+  %2 = llvm.load %arg3 : !llvm.ptr -> f32
+  llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32
+  omp.yield
+}
+llvm.func @scan_reduction(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
+  // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
+  omp.wsloop reduction(mod:inscan, @add_f32 %x -> %prv : !llvm.ptr) {
+    // expected-error at below {{LLVM Translation failed for operation: omp.loop_nest}}
+    omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
+      // expected-error at below {{not yet implemented: Unhandled clause exclusive in omp.scan operation}}
+      // expected-error at below {{LLVM Translation failed for operation: omp.scan}}
+      omp.scan exclusive(%prv : !llvm.ptr)
+      omp.yield
+    }
+  }
+  llvm.return
+}
+
+// -----
+
 llvm.func @single_allocate(%x : !llvm.ptr) {
   // expected-error at below {{not yet implemented: Unhandled clause allocate in omp.single operation}}
   // expected-error at below {{LLVM Translation failed for operation: omp.single}}
diff --git a/openmp/runtime/test/scan/scan.f90 b/openmp/runtime/test/scan/scan.f90
new file mode 100644
index 0000000000000..76263508105d0
--- /dev/null
+++ b/openmp/runtime/test/scan/scan.f90
@@ -0,0 +1,38 @@
+! RUN: %flang %flags %openmp_flags -fopenmp-version=51 %s -o %t.exe
+! RUN: %t.exe | FileCheck %s --match-full-lines
+program inclusive_scan
+ implicit none
+ integer, parameter :: n = 100
+ integer a(n), b(n)
+ integer x, k, y, z
+
+ ! initialization
+ x = 0
+ do k = 1, n
+  a(k) = k
+ end do
+
+ ! a(k) is included in the computation of producing results in b(k)
+ !$omp parallel do reduction(inscan, +: x)
+ do k = 1, n
+   x = x + a(k)
+   !$omp scan inclusive(x)
+   b(k) = x
+ end do
+
+ print *,'x =', x
+ do k = 1, 10
+  print *, 'b(', k, ') =', b(k)
+ end do
+end program
+!CHECK: x = 5050
+!CHECK: b( 1 ) = 1
+!CHECK: b( 2 ) = 3
+!CHECK: b( 3 ) = 6
+!CHECK: b( 4 ) = 10
+!CHECK: b( 5 ) = 15
+!CHECK: b( 6 ) = 21
+!CHECK: b( 7 ) = 28
+!CHECK: b( 8 ) = 36
+!CHECK: b( 9 ) = 45
+!CHECK: b( 10 ) = 55

>From 4011dd537d9830ee4a0379cba80413d1e4cdf9c3 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Wed, 1 Jul 2026 02:44:02 -0500
Subject: [PATCH 2/9] formatting correction

---
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp        | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 297ebac51de88..e18cc0df79860 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -4534,12 +4534,11 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
   bool isInScanRegion =
       wsloopOp.getReductionMod() && (wsloopOp.getReductionMod().value() ==
                                      mlir::omp::ReductionModifier::inscan);
-  if (failed(initReductionVars(wsloopOp, reductionArgs, builder,
-                               moduleTranslation,
-                               afterAllocas.get()->getSinglePredecessor(),
-                               reductionDecls, privateReductionVariables,
-                               reductionVariableMap, isByRef, deferredStores,
-                               isInScanRegion)))
+  if (failed(initReductionVars(
+          wsloopOp, reductionArgs, builder, moduleTranslation,
+          afterAllocas.get()->getSinglePredecessor(), reductionDecls,
+          privateReductionVariables, reductionVariableMap, isByRef,
+          deferredStores, isInScanRegion)))
     return failure();
 
   // For `reduction(task, ...)` open a task-reduction scope for the worksharing
@@ -5262,9 +5261,9 @@ initScanReductionVars(omp::LoopNestOp loopOp, llvm::IRBuilderBase &builder,
     SmallVector<llvm::Value *, 1> phis;
     mapInitializationArgs(wsloopOp, moduleTranslation, builder, reductionDecls,
                           reductionVariableMap, i);
-    if (failed(inlineConvertOmpRegions(
-            reductionDecls[i].getInitializerRegion(),
-            "omp.scan.reduction.init", builder, moduleTranslation, &phis)))
+    if (failed(inlineConvertOmpRegions(reductionDecls[i].getInitializerRegion(),
+                                       "omp.scan.reduction.init", builder,
+                                       moduleTranslation, &phis)))
       return failure();
     assert(phis.size() == 1 &&
            "expected one value to be yielded from the reduction init region");

>From 64b530d3f37e0cd08ca622b538d9bec1eb348b50 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Wed, 22 Jul 2026 05:09:34 -0500
Subject: [PATCH 3/9] Address review comments

---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     | 46 +++++++++++---
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      | 26 +++++++-
 .../Target/LLVMIR/openmp-reduction-scan.mlir  | 14 +++--
 mlir/test/Target/LLVMIR/openmp-todo.mlir      | 31 ++++++++++
 openmp/runtime/test/scan/scan-bounds.f90      | 61 +++++++++++++++++++
 openmp/runtime/test/scan/scan-edge.f90        | 61 +++++++++++++++++++
 openmp/runtime/test/scan/scan-nonidentity.f90 | 28 +++++++++
 7 files changed, 251 insertions(+), 16 deletions(-)
 create mode 100644 openmp/runtime/test/scan/scan-bounds.f90
 create mode 100644 openmp/runtime/test/scan/scan-edge.f90
 create mode 100644 openmp/runtime/test/scan/scan-nonidentity.f90

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index f8c1999fe1b89..f62ea4cf2f9da 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5307,7 +5307,17 @@ Error OpenMPIRBuilder::emitScanBasedDirectiveFinalsIR(
                                              "arrayOffset");
       Value *Src = Builder.CreateLoad(SrcTy, Val);
 
-      Builder.CreateStore(Src, OrigVar);
+      // For a zero-trip loop the buffer slot is uninitialized; keep the
+      // original variable's incoming value instead of storing garbage. The
+      // load above stays in bounds because the buffer was allocated with
+      // `Span + 1` elements, so index `Span` is always valid.
+      Value *Cur = Builder.CreateLoad(SrcTy, OrigVar);
+      Value *HasIters = Builder.CreateICmpUGT(
+          ScanRedInfo->Span,
+          llvm::ConstantInt::get(ScanRedInfo->Span->getType(), 0));
+      Value *Final = Builder.CreateSelect(HasIters, Src, Cur);
+
+      Builder.CreateStore(Final, OrigVar);
       Builder.CreateFree(Buff);
     }
     return Error::success();
@@ -5370,7 +5380,15 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
         ScanRedInfo->Span,
         llvm::ConstantInt::get(ScanRedInfo->Span->getType(), 1));
     Builder.SetInsertPoint(InputBB);
-    Builder.CreateBr(LoopBB);
+    // The log-scan prefix computation is only well-defined when there are at
+    // least two elements: `log2(Span)` is invalid for Span == 0 and the loop
+    // trip computation underflows/loops forever for Span <= 1. For Span <= 1
+    // the buffer already holds the final value(s), so skip straight to the
+    // exit.
+    llvm::Value *SpanGuard = Builder.CreateICmpUGT(
+        ScanRedInfo->Span,
+        llvm::ConstantInt::get(ScanRedInfo->Span->getType(), 1));
+    Builder.CreateCondBr(SpanGuard, LoopBB, ExitBB);
     emitBlock(LoopBB, CurFn);
     Builder.SetInsertPoint(LoopBB);
 
@@ -5617,7 +5635,6 @@ OpenMPIRBuilder::createCanonicalScanLoops(
 
   auto BodyGen = [=](InsertPointTy CodeGenIP, Value *IV) {
     Builder.restoreIP(CodeGenIP);
-    ScanRedInfo->IV = IV;
     createScanBBs(ScanRedInfo);
     BasicBlock *InputBlock = Builder.GetInsertBlock();
     Instruction *Terminator = InputBlock->getTerminator();
@@ -5636,9 +5653,13 @@ OpenMPIRBuilder::createCanonicalScanLoops(
   };
 
   const auto &&InputLoopGen = [&]() -> Error {
+    // Pass an empty ComputeIP: the trip count must be (re)computed at the
+    // loop's own location. Reusing the outer ComputeIP is invalid here because
+    // it was invalidated by the `scan.init` split above and would emit the
+    // recomputed trip count after a block terminator for runtime bounds.
     Expected<CanonicalLoopInfo *> LoopInfo = createCanonicalLoop(
         Builder.saveIP(), BodyGen, Start, Stop, Step, IsSigned, InclusiveStop,
-        ComputeIP, Name, true, ScanRedInfo);
+        /*ComputeIP=*/InsertPointTy(), Name, true, ScanRedInfo);
     if (!LoopInfo)
       return LoopInfo.takeError();
     Result.push_back(*LoopInfo);
@@ -5646,9 +5667,9 @@ OpenMPIRBuilder::createCanonicalScanLoops(
     return Error::success();
   };
   const auto &&ScanLoopGen = [&](LocationDescription Loc) -> Error {
-    Expected<CanonicalLoopInfo *> LoopInfo =
-        createCanonicalLoop(Loc, BodyGen, Start, Stop, Step, IsSigned,
-                            InclusiveStop, ComputeIP, Name, true, ScanRedInfo);
+    Expected<CanonicalLoopInfo *> LoopInfo = createCanonicalLoop(
+        Loc, BodyGen, Start, Stop, Step, IsSigned, InclusiveStop,
+        /*ComputeIP=*/InsertPointTy(), Name, true, ScanRedInfo);
     if (!LoopInfo)
       return LoopInfo.takeError();
     Result.push_back(*LoopInfo);
@@ -5737,8 +5758,15 @@ Expected<CanonicalLoopInfo *> OpenMPIRBuilder::createCanonicalLoop(
     Builder.restoreIP(CodeGenIP);
     Value *Span = Builder.CreateMul(IV, Step);
     Value *IndVar = Builder.CreateAdd(Span, Start);
-    if (InScan)
-      ScanRedInfo->IV = IndVar;
+    if (InScan) {
+      // The scan buffer is 1-indexed by iteration number (buffer[1..TripCount])
+      // rather than by the source induction variable. Using the source value
+      // would index out of bounds for loops that do not start at 1 or that use
+      // a non-unit stride. `IV` here is the 0-based canonical loop counter, so
+      // the 1-based buffer index is `IV + 1`.
+      ScanRedInfo->IV =
+          Builder.CreateAdd(IV, ConstantInt::get(IV->getType(), 1));
+    }
     return BodyGenCB(Builder.saveIP(), IndVar);
   };
   LocationDescription LoopLoc =
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index e18cc0df79860..5af1a384d3edb 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -430,6 +430,16 @@ static LogicalResult checkImplementationStatus(Operation &op) {
               break;
             }
         }
+      } else if (inscanModifierSupported) {
+        if (auto byref = op.getReductionByref()) {
+          // Scan reductions (inscan modifier) only support by-value (scalar)
+          // reductions for now.
+          for (bool isByRef : *byref)
+            if (isByRef) {
+              result = todo("inscan reduction modifier with by-ref reduction");
+              break;
+            }
+        }
       }
     }
   };
@@ -5268,8 +5278,20 @@ initScanReductionVars(omp::LoopNestOp loopOp, llvm::IRBuilderBase &builder,
     assert(phis.size() == 1 &&
            "expected one value to be yielded from the reduction init region");
     setInsertPointForPossiblyEmptyBlock(builder);
-    builder.CreateStore(phis[0],
-                        moduleTranslation.lookupValue(reductionArgs[i]));
+
+    // On the first iteration, seed the reduction variable with the original
+    // variable's incoming value so that a non-identity initial value
+    // participates in the scan (e.g. `x = 10` before an inclusive-scan loop of
+    // `+1` must yield 11, 12, ...). On subsequent iterations, reset to the
+    // reduction identity. `scanInfo->IV` is the 1-based iteration index.
+    llvm::Value *identity = phis[0];
+    llvm::Value *origVar =
+        moduleTranslation.lookupValue(wsloopOp.getReductionVars()[i]);
+    llvm::Value *origVal = builder.CreateLoad(identity->getType(), origVar);
+    llvm::Value *isFirstIter = builder.CreateICmpEQ(
+        scanInfo->IV, llvm::ConstantInt::get(scanInfo->IV->getType(), 1));
+    llvm::Value *seed = builder.CreateSelect(isFirstIter, origVal, identity);
+    builder.CreateStore(seed, moduleTranslation.lookupValue(reductionArgs[i]));
   }
   return success();
 }
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
index c6b8daccad313..6ef58b5013e08 100644
--- a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
@@ -73,7 +73,9 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK: %[[FREE_VAR:.+]] = load ptr, ptr %[[BUFF_PTR]], align 8
 //CHECK:  %[[ARRLAST:.+]] = getelementptr inbounds i32, ptr %[[FREE_VAR]], i32 100
 //CHECK:  %[[RES:.+]] = load i32, ptr %[[ARRLAST]], align 4
-//CHECK:  store i32 %[[RES]], ptr %loadgep{{.*}}, align 4
+//CHECK:  %[[CUR:.+]] = load i32, ptr %loadgep{{.*}}, align 4
+//CHECK:  %[[FIN:.+]] = select i1 {{.*}}, i32 %[[RES]], i32 %[[CUR]]
+//CHECK:  store i32 %[[FIN]], ptr %loadgep{{.*}}, align 4
 //CHECK: tail call void @free(ptr %[[FREE_VAR]])
 //CHECK: @__kmpc_end_masked
 //CHECK: omp.inscan.dispatch{{.*}}:                            ; preds = %omp_loop.body{{.*}}
@@ -84,7 +86,9 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK:   %[[LOG:.+]] = call double @llvm.log2.f64(double 1.000000e+02) #0
 //CHECK:   %[[CEIL:.+]] = call double @llvm.ceil.f64(double %[[LOG]]) #0
 //CHECK:   %[[UB:.+]] = fptoui double %[[CEIL]] to i32
-//CHECK:   br label %omp.outer.log.scan.body
+//CHECK:   br i1 {{.*}}, label %omp.outer.log.scan.body, label %omp.outer.log.scan.exit
+//CHECK: omp.outer.log.scan.exit:                          ; preds = %omp.inner.log.scan.exit, %omp_region.body{{.*}}
+//CHECK: @__kmpc_end_masked
 //CHECK: omp.outer.log.scan.body:
 //CHECK:   %[[K:.+]] = phi i32 [ 0, %{{.*}} ], [ %[[NEXTK:.+]], %omp.inner.log.scan.exit ]
 //CHECK:   %[[I:.+]] = phi i32 [ 1, %{{.*}} ], [ %[[NEXTI:.+]], %omp.inner.log.scan.exit ]
@@ -95,8 +99,6 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK:   %[[NEXTI]] = shl nuw i32 %[[I]], 1
 //CHECK:   %[[CMP2:.+]] = icmp ne i32 %[[NEXTK]], %[[UB]]
 //CHECK:   br i1 %[[CMP2]], label %omp.outer.log.scan.body, label %omp.outer.log.scan.exit
-//CHECK: omp.outer.log.scan.exit:                          ; preds = %omp.inner.log.scan.exit
-//CHECK: @__kmpc_end_masked
 //CHECK: omp.inner.log.scan.body:                          ; preds = %omp.inner.log.scan.body, %omp.outer.log.scan.body
 //CHECK:   %[[CNT:.+]] = phi i32 [ 99, %omp.outer.log.scan.body ], [ %[[CNTNXT:.+]], %omp.inner.log.scan.body ]
 //CHECK:   %[[BUFF:.+]] = load ptr, ptr %[[BUFF_PTR]]
@@ -114,7 +116,9 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK: omp.inscan.dispatch:                              ; preds = %omp_loop.body
 //CHECK:   br i1 true, label %omp.before.scan.bb, label %omp.after.scan.bb
 //CHECK: omp.before.scan.bb:
-//CHECK:   store i32 0, ptr %[[REDPRIV:.+]], align 4
+//CHECK:   %[[ISFIRST:.+]] = icmp eq i32 %{{.*}}, 1
+//CHECK:   %[[SEED:.+]] = select i1 %[[ISFIRST]], i32 %{{.*}}, i32 0
+//CHECK:   store i32 %[[SEED]], ptr %[[REDPRIV:.+]], align 4
 //CHECK: omp.loop_nest.region:                             ; preds = %omp.before.scan.bb
 //CHECK:   %[[BUFFER:.+]] = load ptr, ptr %loadgep_vla, align 8
 //CHECK:   %[[ARRAYOFFSET2:.+]] = getelementptr inbounds i32, ptr %[[BUFFER]], i32 %{{.*}}
diff --git a/mlir/test/Target/LLVMIR/openmp-todo.mlir b/mlir/test/Target/LLVMIR/openmp-todo.mlir
index 7edf88b5ea559..fe3c4259e75e0 100644
--- a/mlir/test/Target/LLVMIR/openmp-todo.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-todo.mlir
@@ -195,6 +195,37 @@ llvm.func @scan_reduction(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
 
 // -----
 
+omp.declare_reduction @add_f32 : f32
+init {
+^bb0(%arg: f32):
+  %0 = llvm.mlir.constant(0.0 : f32) : f32
+  omp.yield (%0 : f32)
+}
+combiner {
+^bb1(%arg0: f32, %arg1: f32):
+  %1 = llvm.fadd %arg0, %arg1 : f32
+  omp.yield (%1 : f32)
+}
+atomic {
+^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr):
+  %2 = llvm.load %arg3 : !llvm.ptr -> f32
+  llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32
+  omp.yield
+}
+llvm.func @scan_reduction_byref(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
+  // expected-error at below {{not yet implemented: Unhandled clause inscan reduction modifier with by-ref reduction in omp.wsloop operation}}
+  // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
+  omp.wsloop reduction(mod:inscan, byref @add_f32 %x -> %prv : !llvm.ptr) {
+    omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
+      omp.scan inclusive(%prv : !llvm.ptr)
+      omp.yield
+    }
+  }
+  llvm.return
+}
+
+// -----
+
 llvm.func @single_allocate(%x : !llvm.ptr) {
   // expected-error at below {{not yet implemented: Unhandled clause allocate in omp.single operation}}
   // expected-error at below {{LLVM Translation failed for operation: omp.single}}
diff --git a/openmp/runtime/test/scan/scan-bounds.f90 b/openmp/runtime/test/scan/scan-bounds.f90
new file mode 100644
index 0000000000000..9d624954568ff
--- /dev/null
+++ b/openmp/runtime/test/scan/scan-bounds.f90
@@ -0,0 +1,61 @@
+! RUN: %flang %flags %openmp_flags -fopenmp-version=51 %s -o %t.exe
+! RUN: %t.exe | FileCheck %s --match-full-lines
+
+! Arbitrary loop bounds and a non-unit stride must index the scan buffer by the
+! logical iteration number, not by the induction-variable value.
+program scan_bounds
+ implicit none
+ integer, parameter :: n = 30
+ integer :: a(n), b(n)
+ integer :: x, k
+
+ do k = 1, n
+   a(k) = k
+ end do
+
+ ! Arbitrary bounds: iterate from 10 to 20.
+ b = -1
+ x = 0
+ !$omp parallel do reduction(inscan, +: x)
+ do k = 10, 20
+   x = x + a(k)
+   !$omp scan inclusive(x)
+   b(k) = x
+ end do
+ print *, 'bounds x =', x
+ do k = 10, 20
+   print *, 'bb(', k, ') =', b(k)
+ end do
+
+ ! Non-unit stride: iterate from 1 to 10 step 2.
+ b = -1
+ x = 0
+ !$omp parallel do reduction(inscan, +: x)
+ do k = 1, 10, 2
+   x = x + a(k)
+   !$omp scan inclusive(x)
+   b(k) = x
+ end do
+ print *, 'stride x =', x
+ do k = 1, 10, 2
+   print *, 'sb(', k, ') =', b(k)
+ end do
+end program
+!CHECK: bounds x = 165
+!CHECK: bb( 10 ) = 10
+!CHECK: bb( 11 ) = 21
+!CHECK: bb( 12 ) = 33
+!CHECK: bb( 13 ) = 46
+!CHECK: bb( 14 ) = 60
+!CHECK: bb( 15 ) = 75
+!CHECK: bb( 16 ) = 91
+!CHECK: bb( 17 ) = 108
+!CHECK: bb( 18 ) = 126
+!CHECK: bb( 19 ) = 145
+!CHECK: bb( 20 ) = 165
+!CHECK: stride x = 25
+!CHECK: sb( 1 ) = 1
+!CHECK: sb( 3 ) = 4
+!CHECK: sb( 5 ) = 9
+!CHECK: sb( 7 ) = 16
+!CHECK: sb( 9 ) = 25
diff --git a/openmp/runtime/test/scan/scan-edge.f90 b/openmp/runtime/test/scan/scan-edge.f90
new file mode 100644
index 0000000000000..7cc00da958e6e
--- /dev/null
+++ b/openmp/runtime/test/scan/scan-edge.f90
@@ -0,0 +1,61 @@
+! RUN: %flang %flags %openmp_flags -fopenmp-version=51 %s -o %t.exe
+! RUN: %t.exe | FileCheck %s --match-full-lines
+
+! Zero-trip loops must not crash and must leave the reduction variable
+! unchanged, and runtime (non-constant) trip counts must work correctly.
+module scan_edge_mod
+contains
+ subroutine run_scan(n, a, b)
+   implicit none
+   integer :: n, k
+   integer :: a(n), b(n)
+   integer :: x
+   x = 0
+   !$omp parallel do reduction(inscan, +: x)
+   do k = 1, n
+     x = x + a(k)
+     !$omp scan inclusive(x)
+     b(k) = x
+   end do
+ end subroutine
+end module
+
+program scan_edge
+ use scan_edge_mod
+ implicit none
+ integer :: a(6), b(6), k
+ integer :: z, zk
+ integer :: zb(1)
+
+ ! Runtime bounds: the trip count of run_scan is not a compile-time constant.
+ do k = 1, 6
+   a(k) = k
+ end do
+ b = -1
+ call run_scan(6, a, b)
+ do k = 1, 6
+   print *, 'rb(', k, ') =', b(k)
+ end do
+
+ ! Runtime zero-trip: n = 0 must not crash.
+ call run_scan(0, a, b)
+ print *, 'runtime zero-trip ok'
+
+ ! Constant zero-trip with a non-identity start: z must be unchanged.
+ z = 42
+ !$omp parallel do reduction(inscan, +: z)
+ do zk = 1, 0
+   z = z + 1
+   !$omp scan inclusive(z)
+   zb(zk) = z
+ end do
+ print *, 'z =', z
+end program
+!CHECK: rb( 1 ) = 1
+!CHECK: rb( 2 ) = 3
+!CHECK: rb( 3 ) = 6
+!CHECK: rb( 4 ) = 10
+!CHECK: rb( 5 ) = 15
+!CHECK: rb( 6 ) = 21
+!CHECK: runtime zero-trip ok
+!CHECK: z = 42
diff --git a/openmp/runtime/test/scan/scan-nonidentity.f90 b/openmp/runtime/test/scan/scan-nonidentity.f90
new file mode 100644
index 0000000000000..064b0d757af6e
--- /dev/null
+++ b/openmp/runtime/test/scan/scan-nonidentity.f90
@@ -0,0 +1,28 @@
+! RUN: %flang %flags %openmp_flags -fopenmp-version=51 %s -o %t.exe
+! RUN: %t.exe | FileCheck %s --match-full-lines
+
+! The reduction variable's non-identity incoming value must be included in the
+! first partial result of an inclusive scan.
+program scan_nonidentity
+ implicit none
+ integer :: x, k
+ integer :: b(4)
+
+ x = 10
+ !$omp parallel do reduction(inscan, +: x)
+ do k = 1, 4
+   x = x + 1
+   !$omp scan inclusive(x)
+   b(k) = x
+ end do
+
+ print *, 'x =', x
+ do k = 1, 4
+   print *, 'b(', k, ') =', b(k)
+ end do
+end program
+!CHECK: x = 14
+!CHECK: b( 1 ) = 11
+!CHECK: b( 2 ) = 12
+!CHECK: b( 3 ) = 13
+!CHECK: b( 4 ) = 14

>From 5506275c5cec52b0991c9f85bddeb1c8ebabf451 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Mon, 27 Jul 2026 10:07:57 -0500
Subject: [PATCH 4/9] updated with review comments fixes

---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     | 23 +++--
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      | 25 ++++-
 mlir/test/Target/LLVMIR/openmp-todo.mlir      | 94 +++++++++++++++++--
 openmp/runtime/test/scan/scan-i64.f90         | 37 ++++++++
 openmp/runtime/test/scan/scan-nowait.f90      | 38 ++++++++
 5 files changed, 198 insertions(+), 19 deletions(-)
 create mode 100644 openmp/runtime/test/scan/scan-i64.f90
 create mode 100644 openmp/runtime/test/scan/scan-nowait.f90

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 90db03da7380c..0ed215b3e2ea6 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5350,10 +5350,13 @@ Error OpenMPIRBuilder::emitScanBasedDirectiveDeclsIR(
   auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
                        ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
     Builder.restoreIP(CodeGenIP);
-    Value *AllocSpan =
-        Builder.CreateAdd(ScanRedInfo->Span, Builder.getInt32(1));
+    // Use the loop's own index type (matching `Span`) so the buffer allocation
+    // works for both i32 and i64 (e.g. `integer(kind=8)`) induction variables.
+    Type *IndexTy = ScanRedInfo->Span->getType();
+    Value *AllocSpan = Builder.CreateAdd(ScanRedInfo->Span,
+                                         ConstantInt::get(IndexTy, 1));
     for (size_t i = 0; i < ScanVars.size(); i++) {
-      Type *IntPtrTy = Builder.getInt32Ty();
+      Type *IntPtrTy = IndexTy;
       Constant *Allocsize = ConstantExpr::getSizeOf(ScanVarsType[i]);
       Allocsize = ConstantExpr::getTruncOrBitCast(Allocsize, IntPtrTy);
       Value *Buff = Builder.CreateMalloc(IntPtrTy, ScanVarsType[i], Allocsize,
@@ -5453,6 +5456,10 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
                        ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
     Builder.restoreIP(CodeGenIP);
     Function *CurFn = Builder.GetInsertBlock()->getParent();
+    // Index/counter values that participate in buffer-offset arithmetic must
+    // use the loop's own index type (`Span`'s type) so scan works for both i32
+    // and i64 (e.g. `integer(kind=8)`) induction variables.
+    Type *IndexTy = ScanRedInfo->Span->getType();
     // for (int k = 0; k <= ceil(log2(n)); ++k)
     llvm::BasicBlock *LoopBB =
         BasicBlock::Create(CurFn->getContext(), "omp.outer.log.scan.body");
@@ -5488,10 +5495,10 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
 
     PHINode *Counter = Builder.CreatePHI(Builder.getInt32Ty(), 2);
     // size pow2k = 1;
-    PHINode *Pow2K = Builder.CreatePHI(Builder.getInt32Ty(), 2);
+    PHINode *Pow2K = Builder.CreatePHI(IndexTy, 2);
     Counter->addIncoming(llvm::ConstantInt::get(Builder.getInt32Ty(), 0),
                          InputBB);
-    Pow2K->addIncoming(llvm::ConstantInt::get(Builder.getInt32Ty(), 1),
+    Pow2K->addIncoming(llvm::ConstantInt::get(IndexTy, 1),
                        InputBB);
     // for (size i = n - 1; i >= 2 ^ k; --i)
     //   tmp[i] op= tmp[i-pow2k];
@@ -5503,14 +5510,14 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
     Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
     emitBlock(InnerLoopBB, CurFn);
     Builder.SetInsertPoint(InnerLoopBB);
-    PHINode *IVal = Builder.CreatePHI(Builder.getInt32Ty(), 2);
+    PHINode *IVal = Builder.CreatePHI(IndexTy, 2);
     IVal->addIncoming(NMin1, LoopBB);
     for (ReductionInfo RedInfo : ReductionInfos) {
       Value *ReductionVal = RedInfo.PrivateVariable;
       Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ReductionVal];
       Value *Buff = Builder.CreateLoad(Builder.getPtrTy(), BuffPtr);
       Type *DestTy = RedInfo.ElementType;
-      Value *IV = Builder.CreateAdd(IVal, Builder.getInt32(1));
+      Value *IV = Builder.CreateAdd(IVal, ConstantInt::get(IndexTy, 1));
       Value *LHSPtr =
           Builder.CreateInBoundsGEP(DestTy, Buff, IV, "arrayOffset");
       Value *OffsetIval = Builder.CreateNUWSub(IV, Pow2K);
@@ -5526,7 +5533,7 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
       Builder.CreateStore(Result, LHSPtr);
     }
     llvm::Value *NextIVal = Builder.CreateNUWSub(
-        IVal, llvm::ConstantInt::get(Builder.getInt32Ty(), 1));
+        IVal, llvm::ConstantInt::get(IndexTy, 1));
     IVal->addIncoming(NextIVal, Builder.GetInsertBlock());
     CmpI = Builder.CreateICmpUGE(NextIVal, Pow2K);
     Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 7b8be030da4dc..b7381244a50b0 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -471,6 +471,22 @@ static LogicalResult checkImplementationStatus(Operation &op) {
               break;
             }
         }
+        // Scan reductions allocate a shared temporary buffer in the alloca
+        // frame of the enclosing parallel region. An orphaned worksharing loop
+        // (e.g. an `omp do` in a subroutine that is called from within a
+        // parallel region) has no such frame in its own function, so the buffer
+        // would be allocated per-thread and produce wrong results.
+        if (!op->template getParentOfType<omp::ParallelOp>())
+          result = todo("inscan reduction on an orphaned worksharing loop "
+                        "(no enclosing parallel region)");
+        // Scan reductions are only implemented for a single loop; a collapsed
+        // (multi-dimensional) loop nest is not yet supported.
+        if (auto wsloopOp = dyn_cast<omp::WsloopOp>(op.getOperation()))
+          if (auto loopNest = dyn_cast_or_null<omp::LoopNestOp>(
+                  wsloopOp.getWrappedLoop()))
+            if (loopNest.getNumLoops() > 1)
+              result = todo("inscan reduction on a collapsed "
+                            "(multi-dimensional) loop nest");
       }
     }
   };
@@ -4699,9 +4715,16 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
       linearClauseProcessor.rewriteInPlace(builder, loopInfo->getBody(),
                                            loopInfo->getLatch(), index);
 
+    // Scan reductions need a barrier at the end of the input (first) loop so
+    // that every thread has finished writing the temporary buffer before the
+    // masked prefix-sum reads it. A source-level `nowait` only elides the final
+    // barrier after the scan (second) loop, so force the barrier for the input
+    // loop regardless of `nowait` to avoid a data race.
+    bool needsBarrier =
+        loopNeedsBarrier || (isInScanRegion && inputScanLoop);
     llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP =
         ompBuilder->applyWorkshareLoop(
-            ompLoc.DL, loopInfo, allocaIP, loopNeedsBarrier,
+            ompLoc.DL, loopInfo, allocaIP, needsBarrier,
             convertToScheduleKind(schedule), chunk, isSimd,
             scheduleMod == omp::ScheduleModifier::monotonic,
             scheduleMod == omp::ScheduleModifier::nonmonotonic, isOrdered,
diff --git a/mlir/test/Target/LLVMIR/openmp-todo.mlir b/mlir/test/Target/LLVMIR/openmp-todo.mlir
index 44817110d8b20..befa9bba2680e 100644
--- a/mlir/test/Target/LLVMIR/openmp-todo.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-todo.mlir
@@ -180,15 +180,19 @@ atomic {
   omp.yield
 }
 llvm.func @scan_reduction(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
-  // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
-  omp.wsloop reduction(mod:inscan, @add_f32 %x -> %prv : !llvm.ptr) {
-    // expected-error at below {{LLVM Translation failed for operation: omp.loop_nest}}
-    omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
-      // expected-error at below {{not yet implemented: Unhandled clause exclusive in omp.scan operation}}
-      // expected-error at below {{LLVM Translation failed for operation: omp.scan}}
-      omp.scan exclusive(%prv : !llvm.ptr)
-      omp.yield
+  // expected-error at below {{LLVM Translation failed for operation: omp.parallel}}
+  omp.parallel {
+    // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
+    omp.wsloop reduction(mod:inscan, @add_f32 %x -> %prv : !llvm.ptr) {
+      // expected-error at below {{LLVM Translation failed for operation: omp.loop_nest}}
+      omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
+        // expected-error at below {{not yet implemented: Unhandled clause exclusive in omp.scan operation}}
+        // expected-error at below {{LLVM Translation failed for operation: omp.scan}}
+        omp.scan exclusive(%prv : !llvm.ptr)
+        omp.yield
+      }
     }
+    omp.terminator
   }
   llvm.return
 }
@@ -213,9 +217,44 @@ atomic {
   omp.yield
 }
 llvm.func @scan_reduction_byref(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
-  // expected-error at below {{not yet implemented: Unhandled clause inscan reduction modifier with by-ref reduction in omp.wsloop operation}}
+  // expected-error at below {{LLVM Translation failed for operation: omp.parallel}}
+  omp.parallel {
+    // expected-error at below {{not yet implemented: Unhandled clause inscan reduction modifier with by-ref reduction in omp.wsloop operation}}
+    // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
+    omp.wsloop reduction(mod:inscan, byref @add_f32 %x -> %prv : !llvm.ptr) {
+      omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
+        omp.scan inclusive(%prv : !llvm.ptr)
+        omp.yield
+      }
+    }
+    omp.terminator
+  }
+  llvm.return
+}
+
+// -----
+
+omp.declare_reduction @add_f32 : f32
+init {
+^bb0(%arg: f32):
+  %0 = llvm.mlir.constant(0.0 : f32) : f32
+  omp.yield (%0 : f32)
+}
+combiner {
+^bb1(%arg0: f32, %arg1: f32):
+  %1 = llvm.fadd %arg0, %arg1 : f32
+  omp.yield (%1 : f32)
+}
+atomic {
+^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr):
+  %2 = llvm.load %arg3 : !llvm.ptr -> f32
+  llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32
+  omp.yield
+}
+llvm.func @scan_reduction_orphaned(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
+  // expected-error at below {{not yet implemented: Unhandled clause inscan reduction on an orphaned worksharing loop (no enclosing parallel region) in omp.wsloop operation}}
   // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
-  omp.wsloop reduction(mod:inscan, byref @add_f32 %x -> %prv : !llvm.ptr) {
+  omp.wsloop reduction(mod:inscan, @add_f32 %x -> %prv : !llvm.ptr) {
     omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
       omp.scan inclusive(%prv : !llvm.ptr)
       omp.yield
@@ -226,6 +265,41 @@ llvm.func @scan_reduction_byref(%lb : i32, %ub : i32, %step : i32, %x : !llvm.pt
 
 // -----
 
+omp.declare_reduction @add_f32 : f32
+init {
+^bb0(%arg: f32):
+  %0 = llvm.mlir.constant(0.0 : f32) : f32
+  omp.yield (%0 : f32)
+}
+combiner {
+^bb1(%arg0: f32, %arg1: f32):
+  %1 = llvm.fadd %arg0, %arg1 : f32
+  omp.yield (%1 : f32)
+}
+atomic {
+^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr):
+  %2 = llvm.load %arg3 : !llvm.ptr -> f32
+  llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32
+  omp.yield
+}
+llvm.func @scan_reduction_multidim(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
+  // expected-error at below {{LLVM Translation failed for operation: omp.parallel}}
+  omp.parallel {
+    // expected-error at below {{not yet implemented: Unhandled clause inscan reduction on a collapsed (multi-dimensional) loop nest in omp.wsloop operation}}
+    // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
+    omp.wsloop reduction(mod:inscan, @add_f32 %x -> %prv : !llvm.ptr) {
+      omp.loop_nest (%iv, %iv2) : i32 = (%lb, %lb) to (%ub, %ub) step (%step, %step) {
+        omp.scan inclusive(%prv : !llvm.ptr)
+        omp.yield
+      }
+    }
+    omp.terminator
+  }
+  llvm.return
+}
+
+// -----
+
 llvm.func @single_allocate(%x : !llvm.ptr) {
   // expected-error at below {{not yet implemented: Unhandled clause allocate in omp.single operation}}
   // expected-error at below {{LLVM Translation failed for operation: omp.single}}
diff --git a/openmp/runtime/test/scan/scan-i64.f90 b/openmp/runtime/test/scan/scan-i64.f90
new file mode 100644
index 0000000000000..c2ef3e6e0ac76
--- /dev/null
+++ b/openmp/runtime/test/scan/scan-i64.f90
@@ -0,0 +1,37 @@
+! RUN: %flang %flags %openmp_flags -fopenmp-version=51 %s -o %t.exe
+! RUN: %t.exe | FileCheck %s --match-full-lines
+
+! An `integer(kind=8)` (i64) loop induction variable must work with an inscan
+! reduction. The scan temporary buffer is indexed using the loop's own index
+! type, so an i64 induction variable must not trip the i32/i64 type mismatch in
+! the scan buffer allocation or the prefix-sum computation.
+program scan_i64
+ implicit none
+ integer(kind=8) :: i, n
+ integer :: k
+ integer :: x
+ integer :: b(8)
+
+ n = 8
+ x = 0
+ !$omp parallel do reduction(inscan, +: x)
+ do i = 1_8, n
+   x = x + 1
+   !$omp scan inclusive(x)
+   b(i) = x
+ end do
+
+ print *, 'x =', x
+ do k = 1, 8
+  print *, 'b(', k, ') =', b(k)
+ end do
+end program
+!CHECK: x = 8
+!CHECK: b( 1 ) = 1
+!CHECK: b( 2 ) = 2
+!CHECK: b( 3 ) = 3
+!CHECK: b( 4 ) = 4
+!CHECK: b( 5 ) = 5
+!CHECK: b( 6 ) = 6
+!CHECK: b( 7 ) = 7
+!CHECK: b( 8 ) = 8
diff --git a/openmp/runtime/test/scan/scan-nowait.f90 b/openmp/runtime/test/scan/scan-nowait.f90
new file mode 100644
index 0000000000000..bb17aeeedf9d9
--- /dev/null
+++ b/openmp/runtime/test/scan/scan-nowait.f90
@@ -0,0 +1,38 @@
+! RUN: %flang %flags %openmp_flags -fopenmp-version=51 %s -o %t.exe
+! RUN: env OMP_NUM_THREADS=4 %t.exe | FileCheck %s --match-full-lines
+
+! A source-level `nowait` on a scan worksharing loop must only remove the final
+! barrier after the scan (second) loop. The barrier at the end of the input
+! (first) loop is required so that every thread finishes writing the temporary
+! buffer before the masked prefix-sum reads it. Forcing multiple threads
+! exercises the data race that occurred when `nowait` incorrectly removed that
+! internal barrier.
+program scan_nowait
+ implicit none
+ integer :: x, k
+ integer :: b(8)
+
+ b = -1
+ x = 0
+ !$omp parallel
+ !$omp do reduction(inscan, +: x)
+ do k = 1, 8
+   x = x + 1
+   !$omp scan inclusive(x)
+   b(k) = x
+ end do
+ !$omp end do nowait
+ !$omp end parallel
+
+ do k = 1, 8
+  print *, 'b(', k, ') =', b(k)
+ end do
+end program
+!CHECK: b( 1 ) = 1
+!CHECK: b( 2 ) = 2
+!CHECK: b( 3 ) = 3
+!CHECK: b( 4 ) = 4
+!CHECK: b( 5 ) = 5
+!CHECK: b( 6 ) = 6
+!CHECK: b( 7 ) = 7
+!CHECK: b( 8 ) = 8

>From c5283fcf901ea9d20f64a59ac15a2795c19abac0 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Mon, 27 Jul 2026 11:37:51 -0500
Subject: [PATCH 5/9] git formatting

---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp             | 11 +++++------
 .../Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp      |  7 +++----
 2 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 9f82ba18ded81..ff5d325075b93 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5378,8 +5378,8 @@ Error OpenMPIRBuilder::emitScanBasedDirectiveDeclsIR(
     // Use the loop's own index type (matching `Span`) so the buffer allocation
     // works for both i32 and i64 (e.g. `integer(kind=8)`) induction variables.
     Type *IndexTy = ScanRedInfo->Span->getType();
-    Value *AllocSpan = Builder.CreateAdd(ScanRedInfo->Span,
-                                         ConstantInt::get(IndexTy, 1));
+    Value *AllocSpan =
+        Builder.CreateAdd(ScanRedInfo->Span, ConstantInt::get(IndexTy, 1));
     for (size_t i = 0; i < ScanVars.size(); i++) {
       Type *IntPtrTy = IndexTy;
       Constant *Allocsize = ConstantExpr::getSizeOf(ScanVarsType[i]);
@@ -5523,8 +5523,7 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
     PHINode *Pow2K = Builder.CreatePHI(IndexTy, 2);
     Counter->addIncoming(llvm::ConstantInt::get(Builder.getInt32Ty(), 0),
                          InputBB);
-    Pow2K->addIncoming(llvm::ConstantInt::get(IndexTy, 1),
-                       InputBB);
+    Pow2K->addIncoming(llvm::ConstantInt::get(IndexTy, 1), InputBB);
     // for (size i = n - 1; i >= 2 ^ k; --i)
     //   tmp[i] op= tmp[i-pow2k];
     llvm::BasicBlock *InnerLoopBB =
@@ -5557,8 +5556,8 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
         return AfterIP.takeError();
       Builder.CreateStore(Result, LHSPtr);
     }
-    llvm::Value *NextIVal = Builder.CreateNUWSub(
-        IVal, llvm::ConstantInt::get(IndexTy, 1));
+    llvm::Value *NextIVal =
+        Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(IndexTy, 1));
     IVal->addIncoming(NextIVal, Builder.GetInsertBlock());
     CmpI = Builder.CreateICmpUGE(NextIVal, Pow2K);
     Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index d1a31a6870ced..7c27a18d285fb 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -540,8 +540,8 @@ static LogicalResult checkImplementationStatus(Operation &op) {
         // Scan reductions are only implemented for a single loop; a collapsed
         // (multi-dimensional) loop nest is not yet supported.
         if (auto wsloopOp = dyn_cast<omp::WsloopOp>(op.getOperation()))
-          if (auto loopNest = dyn_cast_or_null<omp::LoopNestOp>(
-                  wsloopOp.getWrappedLoop()))
+          if (auto loopNest =
+                  dyn_cast_or_null<omp::LoopNestOp>(wsloopOp.getWrappedLoop()))
             if (loopNest.getNumLoops() > 1)
               result = todo("inscan reduction on a collapsed "
                             "(multi-dimensional) loop nest");
@@ -4876,8 +4876,7 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
     // masked prefix-sum reads it. A source-level `nowait` only elides the final
     // barrier after the scan (second) loop, so force the barrier for the input
     // loop regardless of `nowait` to avoid a data race.
-    bool needsBarrier =
-        loopNeedsBarrier || (isInScanRegion && inputScanLoop);
+    bool needsBarrier = loopNeedsBarrier || (isInScanRegion && inputScanLoop);
     llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP =
         ompBuilder->applyWorkshareLoop(
             ompLoc.DL, loopInfo, allocaIP, needsBarrier,

>From 42d781704befcc65923ff419c6ba461d4796afbb Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Tue, 28 Jul 2026 11:01:16 -0500
Subject: [PATCH 6/9] rework on second race cond

---
 .../llvm/Frontend/OpenMP/OMPIRBuilder.h       | 14 +++-
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     | 40 +++++++---
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      | 19 +++--
 .../LLVMIR/openmp-reduction-scan-nowait.mlir  | 73 +++++++++++++++++++
 4 files changed, 127 insertions(+), 19 deletions(-)
 create mode 100644 mlir/test/Target/LLVMIR/openmp-reduction-scan-nowait.mlir

diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index 05307414c94d0..87dc1f067463e 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -1940,11 +1940,17 @@ class OpenMPIRBuilder {
   /// \param ReductionInfos Array type containing the ReductionOps.
   /// \param ScanRedInfo    Pointer to the ScanInfo objected created using
   ///                       `ScanInfoInitialize`.
+  /// \param NoWait         Whether the enclosing worksharing construct has a
+  ///                       `nowait` clause. The barrier that keeps the shared
+  ///                       scan buffer alive until every thread has finished
+  ///                       reading it is always emitted; only the
+  ///                       end-of-construct barrier after the masked region is
+  ///                       skipped when \p NoWait is set.
   ///
   /// \return error if any produced, else return success.
   Error emitScanBasedDirectiveFinalsIR(
       ArrayRef<llvm::OpenMPIRBuilder::ReductionInfo> ReductionInfos,
-      ScanInfo *ScanInfo);
+      ScanInfo *ScanInfo, bool NoWait = false);
 
   /// This function emits a helper that gathers Reduce lists from the first
   /// lane of every active warp to lanes in the first warp.
@@ -3173,12 +3179,16 @@ class OpenMPIRBuilder {
   /// \param ReductionInfos Array type containing the ReductionOps.
   /// \param ScanRedInfo    Pointer to the ScanInfo objected created using
   ///                       `ScanInfoInitialize`.
+  /// \param NoWait         Whether the enclosing worksharing construct has a
+  ///                       `nowait` clause. Only the end-of-construct barrier
+  ///                       emitted after freeing the shared scan buffer is
+  ///                       elided; barriers required for correctness are kept.
   ///
   /// \returns The insertion position *after* the masked.
   LLVM_ABI InsertPointOrErrorTy emitScanReduction(
       const LocationDescription &Loc,
       ArrayRef<llvm::OpenMPIRBuilder::ReductionInfo> ReductionInfos,
-      ScanInfo *ScanRedInfo);
+      ScanInfo *ScanRedInfo, bool NoWait = false);
 
   /// This directive split and directs the control flow to input phase
   ///  blocks or scan phase blocks based on 1. whether input loop or scan loop
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index ff5d325075b93..e81768e884922 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5414,7 +5414,8 @@ Error OpenMPIRBuilder::emitScanBasedDirectiveDeclsIR(
 }
 
 Error OpenMPIRBuilder::emitScanBasedDirectiveFinalsIR(
-    ArrayRef<ReductionInfo> ReductionInfos, ScanInfo *ScanRedInfo) {
+    ArrayRef<ReductionInfo> ReductionInfos, ScanInfo *ScanRedInfo,
+    bool NoWait) {
   auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
                        ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
     Builder.restoreIP(CodeGenIP);
@@ -5453,27 +5454,45 @@ Error OpenMPIRBuilder::emitScanBasedDirectiveFinalsIR(
   else
     Builder.SetInsertPoint(ScanRedInfo->OMPScanFinish);
 
-  llvm::Value *FilterVal = Builder.getInt32(0);
+  // Synchronize before the masked region frees the shared scan buffer. The
+  // scan (second) loop loads each thread's prefix value from the buffer, so
+  // every thread must have finished that loop before the single masked thread
+  // frees it; otherwise a thread still executing the loop could load from
+  // freed memory. A `masked` region is not a barrier, so this synchronization
+  // is required for correctness and is emitted even under `nowait`.
   llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
-      createMasked(Builder.saveIP(), BodyGenCB, FiniCB, FilterVal);
-
+      createBarrier(Builder.saveIP(), llvm::omp::OMPD_barrier);
   if (!AfterIP)
     return AfterIP.takeError();
   Builder.restoreIP(*AfterIP);
-  BasicBlock *InputBB = Builder.GetInsertBlock();
-  if (InputBB->hasTerminator())
-    Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
-  AfterIP = createBarrier(Builder.saveIP(), llvm::omp::OMPD_barrier);
+
+  llvm::Value *FilterVal = Builder.getInt32(0);
+  AfterIP = createMasked(Builder.saveIP(), BodyGenCB, FiniCB, FilterVal);
+
   if (!AfterIP)
     return AfterIP.takeError();
   Builder.restoreIP(*AfterIP);
+
+  // The barrier after the masked region provides the construct's
+  // end-of-construct synchronization and publishes the final reduction value
+  // stored into the original variable inside the masked region. Skip it when
+  // the construct has `nowait`, matching a normal worksharing loop.
+  if (!NoWait) {
+    BasicBlock *InputBB = Builder.GetInsertBlock();
+    if (InputBB->hasTerminator())
+      Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
+    AfterIP = createBarrier(Builder.saveIP(), llvm::omp::OMPD_barrier);
+    if (!AfterIP)
+      return AfterIP.takeError();
+    Builder.restoreIP(*AfterIP);
+  }
   return Error::success();
 }
 
 OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
     const LocationDescription &Loc,
     ArrayRef<llvm::OpenMPIRBuilder::ReductionInfo> ReductionInfos,
-    ScanInfo *ScanRedInfo) {
+    ScanInfo *ScanRedInfo, bool NoWait) {
 
   if (!updateToLocation(Loc))
     return Loc.IP;
@@ -5590,7 +5609,8 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
   if (!AfterIP)
     return AfterIP.takeError();
   Builder.restoreIP(*AfterIP);
-  Error Err = emitScanBasedDirectiveFinalsIR(ReductionInfos, ScanRedInfo);
+  Error Err =
+      emitScanBasedDirectiveFinalsIR(ReductionInfos, ScanRedInfo, NoWait);
   if (Err)
     return Err;
 
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 7c27a18d285fb..e5e0acf008116 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -4871,12 +4871,17 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
       linearClauseProcessor.rewriteInPlace(builder, loopInfo->getBody(),
                                            loopInfo->getLatch(), index);
 
-    // Scan reductions need a barrier at the end of the input (first) loop so
-    // that every thread has finished writing the temporary buffer before the
-    // masked prefix-sum reads it. A source-level `nowait` only elides the final
-    // barrier after the scan (second) loop, so force the barrier for the input
-    // loop regardless of `nowait` to avoid a data race.
-    bool needsBarrier = loopNeedsBarrier || (isInScanRegion && inputScanLoop);
+    // Scan reductions manage the barriers around the shared temporary buffer
+    // explicitly:
+    //  * The input (first) loop always needs its trailing barrier so that every
+    //    thread has finished writing the buffer before the masked prefix-sum
+    //    reads it; this is required even under `nowait`.
+    //  * The scan (second) loop must not emit its own trailing barrier here.
+    //    `emitScanReduction` emits a barrier *before* the masked region frees
+    //    the buffer (so no thread frees it while another is still reading it)
+    //    and, unless `nowait` is present, a barrier *after* it for
+    //    end-of-construct synchronization.
+    bool needsBarrier = isInScanRegion ? inputScanLoop : loopNeedsBarrier;
     llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP =
         ompBuilder->applyWorkshareLoop(
             ompLoc.DL, loopInfo, allocaIP, needsBarrier,
@@ -4930,7 +4935,7 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
     llvm::ScanInfo *scanInfo = findScanInfo(moduleTranslation);
     llvm::OpenMPIRBuilder::InsertPointOrErrorTy redIP =
         ompBuilder->emitScanReduction(builder.saveIP(), reductionInfos,
-                                      scanInfo);
+                                      scanInfo, wsloopOp.getNowait());
     if (failed(handleError(redIP, opInst)))
       return failure();
 
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan-nowait.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan-nowait.mlir
new file mode 100644
index 0000000000000..95306535c92cd
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan-nowait.mlir
@@ -0,0 +1,73 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// Verify buffer-lifetime handling for an inclusive scan reduction on a
+// worksharing loop with `nowait`. The scan (second) loop reads each thread's
+// prefix value from the shared scan buffer, and a masked region then frees
+// that buffer. A barrier must be emitted *before* the masked region so the
+// buffer cannot be freed while another thread is still reading it (a masked
+// region is not a barrier). Because of `nowait`, no end-of-construct barrier
+// is emitted *after* the masked region.
+
+omp.declare_reduction @add_reduction_i32 : i32 init {
+^bb0(%arg0: i32):
+  %0 = llvm.mlir.constant(0 : i32) : i32
+  omp.yield(%0 : i32)
+} combiner {
+^bb0(%arg0: i32, %arg1: i32):
+  %0 = llvm.add %arg0, %arg1 : i32
+  omp.yield(%0 : i32)
+}
+llvm.func @scan_reduction_nowait() {
+  %0 = llvm.mlir.constant(1 : i64) : i64
+  %5 = llvm.alloca %0 x i32 {bindc_name = "x"} : (i64) -> !llvm.ptr
+  %10 = llvm.mlir.constant(100 : i32) : i32
+  %11 = llvm.mlir.constant(1 : i32) : i32
+  %14 = llvm.mlir.addressof @_QFEa : !llvm.ptr
+  %15 = llvm.mlir.addressof @_QFEb : !llvm.ptr
+  omp.parallel {
+    %37 = llvm.mlir.constant(1 : i64) : i64
+    %38 = llvm.alloca %37 x i32 {bindc_name = "k", pinned} : (i64) -> !llvm.ptr
+    omp.wsloop nowait reduction(mod: inscan, @add_reduction_i32 %5 -> %arg0 : !llvm.ptr) {
+      omp.loop_nest (%arg1) : i32 = (%11) to (%10) inclusive step (%11) {
+        llvm.store %arg1, %38 : i32, !llvm.ptr
+        %40 = llvm.load %arg0 : !llvm.ptr -> i32
+        %41 = llvm.load %38 : !llvm.ptr -> i32
+        %42 = llvm.sext %41 : i32 to i64
+        %50 = llvm.getelementptr %14[%42] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+        %51 = llvm.load %50 : !llvm.ptr -> i32
+        %52 = llvm.add %40, %51 : i32
+        llvm.store %52, %arg0 : i32, !llvm.ptr
+        omp.scan inclusive(%arg0 : !llvm.ptr)
+        llvm.store %arg1, %38 : i32, !llvm.ptr
+        %53 = llvm.load %arg0 : !llvm.ptr -> i32
+        %54 = llvm.load %38 : !llvm.ptr -> i32
+        %55 = llvm.sext %54 : i32 to i64
+        %63 = llvm.getelementptr %15[%55] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+        llvm.store %53, %63 : i32, !llvm.ptr
+        omp.yield
+      }
+    }
+    omp.terminator
+  }
+  llvm.return
+}
+llvm.mlir.global internal @_QFEa() {addr_space = 0 : i32} : !llvm.array<100 x i32> {
+  %0 = llvm.mlir.zero : !llvm.array<100 x i32>
+  llvm.return %0 : !llvm.array<100 x i32>
+}
+llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i32> {
+  %0 = llvm.mlir.zero : !llvm.array<100 x i32>
+  llvm.return %0 : !llvm.array<100 x i32>
+}
+
+// CHECK-LABEL: define internal void @scan_reduction_nowait..omp_par
+// After the scan (second) loop, the mandatory buffer-lifetime barrier is
+// emitted before the masked region that frees the shared scan buffer.
+// CHECK: omp.scan.loop.cont:
+// CHECK: call void @__kmpc_barrier
+// CHECK: call i32 @__kmpc_masked
+// CHECK: call void @free(ptr
+// CHECK: call void @__kmpc_end_masked
+// With `nowait`, there is no end-of-construct barrier after the buffer is freed.
+// CHECK-NOT: call void @__kmpc_barrier
+// CHECK: ret void

>From 46cbc9122d8fa036d02c61ea3d740030701eb824 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Tue, 28 Jul 2026 12:18:10 -0500
Subject: [PATCH 7/9] fixex iteration input now  starts from reduction identity

---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     | 32 +++++++++++++++++++
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      | 21 ++++++------
 .../Target/LLVMIR/openmp-reduction-scan.mlir  | 16 ++++++++--
 3 files changed, 54 insertions(+), 15 deletions(-)

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index e81768e884922..a75c39be196c1 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5525,6 +5525,38 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
         ScanRedInfo->Span,
         llvm::ConstantInt::get(ScanRedInfo->Span->getType(), 1));
     Builder.SetInsertPoint(InputBB);
+    // Combine the original variable's incoming value (orig-val) into the first
+    // buffer element before computing the prefix sum. Per the OpenMP scan
+    // semantics orig-val is a single prefix element that must be reflected in
+    // every inclusive/exclusive scan result and in the final reduction value.
+    // Folding it in here lets the log-scan prefix computation propagate it into
+    // every element (and into the finals' read of `buffer[Span]`), regardless
+    // of whether an iteration's input phase accumulates into or overwrites the
+    // reduction variable. `select(Span > 0, 1, 0)` keeps the access in bounds
+    // when Span == 0: buffer[0] is otherwise unused and the finals' zero-trip
+    // guard discards it.
+    llvm::Value *HasElem = Builder.CreateICmpUGT(
+        ScanRedInfo->Span, llvm::ConstantInt::get(IndexTy, 0));
+    llvm::Value *FirstIdx =
+        Builder.CreateSelect(HasElem, llvm::ConstantInt::get(IndexTy, 1),
+                             llvm::ConstantInt::get(IndexTy, 0));
+    for (ReductionInfo RedInfo : ReductionInfos) {
+      Value *ReductionVal = RedInfo.PrivateVariable;
+      Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ReductionVal];
+      Value *Buff = Builder.CreateLoad(Builder.getPtrTy(), BuffPtr);
+      Type *DestTy = RedInfo.ElementType;
+      Value *ElemPtr =
+          Builder.CreateInBoundsGEP(DestTy, Buff, FirstIdx, "arrayOffset");
+      Value *OrigVal = Builder.CreateLoad(DestTy, RedInfo.Variable);
+      Value *Elem = Builder.CreateLoad(DestTy, ElemPtr);
+      llvm::Value *Combined;
+      InsertPointOrErrorTy AfterIP =
+          RedInfo.ReductionGen(Builder.saveIP(), Elem, OrigVal, Combined);
+      if (!AfterIP)
+        return AfterIP.takeError();
+      Builder.restoreIP(*AfterIP);
+      Builder.CreateStore(Combined, ElemPtr);
+    }
     // The log-scan prefix computation is only well-defined when there are at
     // least two elements: `log2(Span)` is invalid for Span == 0 and the loop
     // trip computation underflows/loops forever for Span <= 1. For Span <= 1
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index e5e0acf008116..23f2d06fdabe2 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -5526,19 +5526,16 @@ initScanReductionVars(omp::LoopNestOp loopOp, llvm::IRBuilderBase &builder,
            "expected one value to be yielded from the reduction init region");
     setInsertPointForPossiblyEmptyBlock(builder);
 
-    // On the first iteration, seed the reduction variable with the original
-    // variable's incoming value so that a non-identity initial value
-    // participates in the scan (e.g. `x = 10` before an inclusive-scan loop of
-    // `+1` must yield 11, 12, ...). On subsequent iterations, reset to the
-    // reduction identity. `scanInfo->IV` is the 1-based iteration index.
+    // Reset the private reduction variable to the reduction identity at the
+    // start of every input-loop iteration. The original variable's incoming
+    // value (orig-val) must not be folded into any single iteration here: per
+    // the OpenMP scan semantics it is a separate prefix element that is
+    // combined once into the scan buffer (see `emitScanReduction`). Seeding a
+    // particular iteration would be lost whenever the input phase overwrites
+    // the reduction variable (e.g. `x = i`), producing incorrect results.
     llvm::Value *identity = phis[0];
-    llvm::Value *origVar =
-        moduleTranslation.lookupValue(wsloopOp.getReductionVars()[i]);
-    llvm::Value *origVal = builder.CreateLoad(identity->getType(), origVar);
-    llvm::Value *isFirstIter = builder.CreateICmpEQ(
-        scanInfo->IV, llvm::ConstantInt::get(scanInfo->IV->getType(), 1));
-    llvm::Value *seed = builder.CreateSelect(isFirstIter, origVal, identity);
-    builder.CreateStore(seed, moduleTranslation.lookupValue(reductionArgs[i]));
+    builder.CreateStore(identity,
+                        moduleTranslation.lookupValue(reductionArgs[i]));
   }
   return success();
 }
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
index 6ef58b5013e08..2aa100d686bc1 100644
--- a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
@@ -86,6 +86,15 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK:   %[[LOG:.+]] = call double @llvm.log2.f64(double 1.000000e+02) #0
 //CHECK:   %[[CEIL:.+]] = call double @llvm.ceil.f64(double %[[LOG]]) #0
 //CHECK:   %[[UB:.+]] = fptoui double %[[CEIL]] to i32
+// Combine the original variable's incoming value (orig-val) into the first
+// buffer element once, before the prefix sum, so it is reflected in every
+// scan result and in the final reduction value.
+//CHECK:   %[[OBUFF:.+]] = load ptr, ptr %{{.*}}, align 8
+//CHECK:   %[[OELEMPTR:.+]] = getelementptr inbounds i32, ptr %[[OBUFF]], i32 1
+//CHECK:   %[[OORIG:.+]] = load i32, ptr %{{.*}}, align 4
+//CHECK:   %[[OELEM:.+]] = load i32, ptr %[[OELEMPTR]], align 4
+//CHECK:   %[[OCOMB:.+]] = add i32 %[[OELEM]], %[[OORIG]]
+//CHECK:   store i32 %[[OCOMB]], ptr %[[OELEMPTR]], align 4
 //CHECK:   br i1 {{.*}}, label %omp.outer.log.scan.body, label %omp.outer.log.scan.exit
 //CHECK: omp.outer.log.scan.exit:                          ; preds = %omp.inner.log.scan.exit, %omp_region.body{{.*}}
 //CHECK: @__kmpc_end_masked
@@ -116,9 +125,10 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK: omp.inscan.dispatch:                              ; preds = %omp_loop.body
 //CHECK:   br i1 true, label %omp.before.scan.bb, label %omp.after.scan.bb
 //CHECK: omp.before.scan.bb:
-//CHECK:   %[[ISFIRST:.+]] = icmp eq i32 %{{.*}}, 1
-//CHECK:   %[[SEED:.+]] = select i1 %[[ISFIRST]], i32 %{{.*}}, i32 0
-//CHECK:   store i32 %[[SEED]], ptr %[[REDPRIV:.+]], align 4
+// Each input-loop iteration resets the private reduction variable to the
+// reduction identity (0 for `+`). The original variable's incoming value
+// (orig-val) is NOT seeded here; it is combined once in the masked region.
+//CHECK:   store i32 0, ptr %{{.*}}, align 4
 //CHECK: omp.loop_nest.region:                             ; preds = %omp.before.scan.bb
 //CHECK:   %[[BUFFER:.+]] = load ptr, ptr %loadgep_vla, align 8
 //CHECK:   %[[ARRAYOFFSET2:.+]] = getelementptr inbounds i32, ptr %[[BUFFER]], i32 %{{.*}}

>From cbfd4440aa06d37f99c1bcd8e0e8649199b3d9f0 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Wed, 29 Jul 2026 04:52:51 -0500
Subject: [PATCH 8/9] inscan reduction orig-val handling and guard the scan
 seed for zero-trip loops

---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     | 37 +++++++++++--------
 .../Target/LLVMIR/openmp-reduction-scan.mlir  | 12 ++++--
 2 files changed, 30 insertions(+), 19 deletions(-)

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index a75c39be196c1..fd9b857d88b9b 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5525,6 +5525,18 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
         ScanRedInfo->Span,
         llvm::ConstantInt::get(ScanRedInfo->Span->getType(), 1));
     Builder.SetInsertPoint(InputBB);
+    // Branch around the orig-val seed load and combine when the loop has zero
+    // logical iterations. When Span == 0 no scan-buffer entry is populated, so
+    // loading buffer[0] and feeding it to a (possibly input-sensitive
+    // user-defined) reduction combiner would be undefined behavior. In that
+    // case skip straight to the exit; the finals' zero-trip guard preserves
+    // orig-val as the result.
+    llvm::Value *HasElem = Builder.CreateICmpUGT(
+        ScanRedInfo->Span, llvm::ConstantInt::get(IndexTy, 0));
+    llvm::BasicBlock *SeedBB =
+        BasicBlock::Create(CurFn->getContext(), "omp.scan.seed", CurFn, ExitBB);
+    Builder.CreateCondBr(HasElem, SeedBB, ExitBB);
+    Builder.SetInsertPoint(SeedBB);
     // Combine the original variable's incoming value (orig-val) into the first
     // buffer element before computing the prefix sum. Per the OpenMP scan
     // semantics orig-val is a single prefix element that must be reflected in
@@ -5532,21 +5544,14 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
     // Folding it in here lets the log-scan prefix computation propagate it into
     // every element (and into the finals' read of `buffer[Span]`), regardless
     // of whether an iteration's input phase accumulates into or overwrites the
-    // reduction variable. `select(Span > 0, 1, 0)` keeps the access in bounds
-    // when Span == 0: buffer[0] is otherwise unused and the finals' zero-trip
-    // guard discards it.
-    llvm::Value *HasElem = Builder.CreateICmpUGT(
-        ScanRedInfo->Span, llvm::ConstantInt::get(IndexTy, 0));
-    llvm::Value *FirstIdx =
-        Builder.CreateSelect(HasElem, llvm::ConstantInt::get(IndexTy, 1),
-                             llvm::ConstantInt::get(IndexTy, 0));
+    // reduction variable. Span > 0 is guaranteed here, so buffer[1] is valid.
     for (ReductionInfo RedInfo : ReductionInfos) {
       Value *ReductionVal = RedInfo.PrivateVariable;
       Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ReductionVal];
       Value *Buff = Builder.CreateLoad(Builder.getPtrTy(), BuffPtr);
       Type *DestTy = RedInfo.ElementType;
-      Value *ElemPtr =
-          Builder.CreateInBoundsGEP(DestTy, Buff, FirstIdx, "arrayOffset");
+      Value *ElemPtr = Builder.CreateInBoundsGEP(
+          DestTy, Buff, llvm::ConstantInt::get(IndexTy, 1), "arrayOffset");
       Value *OrigVal = Builder.CreateLoad(DestTy, RedInfo.Variable);
       Value *Elem = Builder.CreateLoad(DestTy, ElemPtr);
       llvm::Value *Combined;
@@ -5559,9 +5564,9 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
     }
     // The log-scan prefix computation is only well-defined when there are at
     // least two elements: `log2(Span)` is invalid for Span == 0 and the loop
-    // trip computation underflows/loops forever for Span <= 1. For Span <= 1
-    // the buffer already holds the final value(s), so skip straight to the
-    // exit.
+    // trip computation underflows/loops forever for Span <= 1. For Span == 1
+    // the buffer already holds the final (seeded) value, so skip straight to
+    // the exit.
     llvm::Value *SpanGuard = Builder.CreateICmpUGT(
         ScanRedInfo->Span,
         llvm::ConstantInt::get(ScanRedInfo->Span->getType(), 1));
@@ -5572,9 +5577,11 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
     PHINode *Counter = Builder.CreatePHI(Builder.getInt32Ty(), 2);
     // size pow2k = 1;
     PHINode *Pow2K = Builder.CreatePHI(IndexTy, 2);
+    // The loop is now entered from the seed block (guarded by Span > 0), not
+    // directly from InputBB.
     Counter->addIncoming(llvm::ConstantInt::get(Builder.getInt32Ty(), 0),
-                         InputBB);
-    Pow2K->addIncoming(llvm::ConstantInt::get(IndexTy, 1), InputBB);
+                         SeedBB);
+    Pow2K->addIncoming(llvm::ConstantInt::get(IndexTy, 1), SeedBB);
     // for (size i = n - 1; i >= 2 ^ k; --i)
     //   tmp[i] op= tmp[i-pow2k];
     llvm::BasicBlock *InnerLoopBB =
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
index 2aa100d686bc1..38e413a504315 100644
--- a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
@@ -86,9 +86,15 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK:   %[[LOG:.+]] = call double @llvm.log2.f64(double 1.000000e+02) #0
 //CHECK:   %[[CEIL:.+]] = call double @llvm.ceil.f64(double %[[LOG]]) #0
 //CHECK:   %[[UB:.+]] = fptoui double %[[CEIL]] to i32
+// Zero-trip guard: skip the orig-val seed load/combine when Span == 0 so
+// buffer[0] is never loaded or passed to the reduction combiner.
+//CHECK:   br i1 {{.*}}, label %omp.scan.seed, label %omp.outer.log.scan.exit
+//CHECK: omp.outer.log.scan.exit:
+//CHECK: @__kmpc_end_masked
+//CHECK: omp.scan.seed:
 // Combine the original variable's incoming value (orig-val) into the first
-// buffer element once, before the prefix sum, so it is reflected in every
-// scan result and in the final reduction value.
+// buffer element once (Span > 0 is guaranteed here), before the prefix sum, so
+// it is reflected in every scan result and in the final reduction value.
 //CHECK:   %[[OBUFF:.+]] = load ptr, ptr %{{.*}}, align 8
 //CHECK:   %[[OELEMPTR:.+]] = getelementptr inbounds i32, ptr %[[OBUFF]], i32 1
 //CHECK:   %[[OORIG:.+]] = load i32, ptr %{{.*}}, align 4
@@ -96,8 +102,6 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK:   %[[OCOMB:.+]] = add i32 %[[OELEM]], %[[OORIG]]
 //CHECK:   store i32 %[[OCOMB]], ptr %[[OELEMPTR]], align 4
 //CHECK:   br i1 {{.*}}, label %omp.outer.log.scan.body, label %omp.outer.log.scan.exit
-//CHECK: omp.outer.log.scan.exit:                          ; preds = %omp.inner.log.scan.exit, %omp_region.body{{.*}}
-//CHECK: @__kmpc_end_masked
 //CHECK: omp.outer.log.scan.body:
 //CHECK:   %[[K:.+]] = phi i32 [ 0, %{{.*}} ], [ %[[NEXTK:.+]], %omp.inner.log.scan.exit ]
 //CHECK:   %[[I:.+]] = phi i32 [ 1, %{{.*}} ], [ %[[NEXTI:.+]], %omp.inner.log.scan.exit ]

>From a51ea62ab9586e3219388f07e8df525af46733d9 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Mon, 3 Aug 2026 04:37:41 -0500
Subject: [PATCH 9/9] fix operand order in reduction seeding

---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     | 14 ++-
 .../openmp-reduction-scan-noncommutative.mlir | 95 +++++++++++++++++++
 .../Target/LLVMIR/openmp-reduction-scan.mlir  |  4 +-
 3 files changed, 109 insertions(+), 4 deletions(-)
 create mode 100644 mlir/test/Target/LLVMIR/openmp-reduction-scan-noncommutative.mlir

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index fd9b857d88b9b..961ec39af4efc 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5555,8 +5555,12 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
       Value *OrigVal = Builder.CreateLoad(DestTy, RedInfo.Variable);
       Value *Elem = Builder.CreateLoad(DestTy, ElemPtr);
       llvm::Value *Combined;
+      // orig-val is the leftmost/earliest element of the prefix, so it must be
+      // the accumulator (`omp_out`, the combiner's first/lhs operand) while the
+      // first buffer element is the incoming value (`omp_in`). The combiner is
+      // associative but not necessarily commutative, so this order matters.
       InsertPointOrErrorTy AfterIP =
-          RedInfo.ReductionGen(Builder.saveIP(), Elem, OrigVal, Combined);
+          RedInfo.ReductionGen(Builder.saveIP(), OrigVal, Elem, Combined);
       if (!AfterIP)
         return AfterIP.takeError();
       Builder.restoreIP(*AfterIP);
@@ -5608,8 +5612,14 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
       Value *LHS = Builder.CreateLoad(DestTy, LHSPtr);
       Value *RHS = Builder.CreateLoad(DestTy, RHSPtr);
       llvm::Value *Result;
+      // buffer[IV-pow2k] holds the earlier partial prefix and buffer[IV] the
+      // later one. The earlier element must be the accumulator (`omp_out`, the
+      // combiner's first/lhs operand) and the later element the incoming value
+      // (`omp_in`); the combiner is associative but not necessarily
+      // commutative, so passing them in prefix order is required. The result
+      // is written back into the later slot (LHSPtr = buffer[IV]).
       InsertPointOrErrorTy AfterIP =
-          RedInfo.ReductionGen(Builder.saveIP(), LHS, RHS, Result);
+          RedInfo.ReductionGen(Builder.saveIP(), RHS, LHS, Result);
       if (!AfterIP)
         return AfterIP.takeError();
       Builder.CreateStore(Result, LHSPtr);
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan-noncommutative.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan-noncommutative.mlir
new file mode 100644
index 0000000000000..0c595de9e80c1
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan-noncommutative.mlir
@@ -0,0 +1,95 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// Regression test for inscan reduction combiner operand ordering. The combiner
+// here is intentionally NONcommutative (subtraction) so that reversing the two
+// operands passed to the reduction combiner changes the emitted IR. This pins
+// the operand order at the two combiner call sites of the log-scan lowering:
+// the orig-val seed and the prefix computation. A commutative combiner such as
+// `+` produces identical IR regardless of operand order and would not catch a
+// reversal, so this complements openmp-reduction-scan.mlir. The scan combines
+// elements left-to-right, so the earlier prefix element must be the
+// accumulator (`omp_out`, the combiner's first operand) and the later element
+// the incoming value (`omp_in`, the second operand).
+
+omp.declare_reduction @sub_reduction_i32 : i32 init {
+^bb0(%arg0: i32):
+  %0 = llvm.mlir.constant(0 : i32) : i32
+  omp.yield(%0 : i32)
+} combiner {
+^bb0(%arg0: i32, %arg1: i32):
+  %0 = llvm.sub %arg0, %arg1 : i32
+  omp.yield(%0 : i32)
+}
+// CHECK-LABEL: @scan_reduction_noncommutative
+llvm.func @scan_reduction_noncommutative() {
+  %0 = llvm.mlir.constant(1 : i64) : i64
+  %1 = llvm.alloca %0 x i32 {bindc_name = "z"} : (i64) -> !llvm.ptr
+  %3 = llvm.alloca %0 x i32 {bindc_name = "y"} : (i64) -> !llvm.ptr
+  %5 = llvm.alloca %0 x i32 {bindc_name = "x"} : (i64) -> !llvm.ptr
+  %7 = llvm.alloca %0 x i32 {bindc_name = "k"} : (i64) -> !llvm.ptr
+  %10 = llvm.mlir.constant(100 : i32) : i32
+  %11 = llvm.mlir.constant(1 : i32) : i32
+  %12 = llvm.mlir.constant(0 : i32) : i32
+  %13 = llvm.mlir.constant(100 : index) : i64
+  %14 = llvm.mlir.addressof @_QFEa : !llvm.ptr
+  %15 = llvm.mlir.addressof @_QFEb : !llvm.ptr
+  omp.parallel {
+    %37 = llvm.mlir.constant(1 : i64) : i64
+    %38 = llvm.alloca %37 x i32 {bindc_name = "k", pinned} : (i64) -> !llvm.ptr
+    %39 = llvm.mlir.constant(1 : i64) : i64
+    omp.wsloop reduction(mod: inscan, @sub_reduction_i32 %5 -> %arg0 : !llvm.ptr) {
+      omp.loop_nest (%arg1) : i32 = (%11) to (%10) inclusive step (%11) {
+        llvm.store %arg1, %38 : i32, !llvm.ptr
+        %40 = llvm.load %arg0 : !llvm.ptr -> i32
+        %41 = llvm.load %38 : !llvm.ptr -> i32
+        %42 = llvm.sext %41 : i32 to i64
+        %50 = llvm.getelementptr %14[%42] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+        %51 = llvm.load %50 : !llvm.ptr -> i32
+        %52 = llvm.add %40, %51 : i32
+        llvm.store %52, %arg0 : i32, !llvm.ptr
+        omp.scan inclusive(%arg0 : !llvm.ptr)
+        llvm.store %arg1, %38 : i32, !llvm.ptr
+        %53 = llvm.load %arg0 : !llvm.ptr -> i32
+        %54 = llvm.load %38 : !llvm.ptr -> i32
+        %55 = llvm.sext %54 : i32 to i64
+        %63 = llvm.getelementptr %15[%55] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+        llvm.store %53, %63 : i32, !llvm.ptr
+        omp.yield
+      }
+    }
+    omp.terminator
+  }
+  llvm.return
+}
+llvm.mlir.global internal @_QFEa() {addr_space = 0 : i32} : !llvm.array<100 x i32> {
+  %0 = llvm.mlir.zero : !llvm.array<100 x i32>
+  llvm.return %0 : !llvm.array<100 x i32>
+}
+llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i32> {
+  %0 = llvm.mlir.zero : !llvm.array<100 x i32>
+  llvm.return %0 : !llvm.array<100 x i32>
+}
+
+// The orig-val seed must combine as sub(orig-val, buffer[1]): the original
+// variable's incoming value (the leftmost prefix element) is the accumulator
+// (first operand) and buffer[1] is the incoming value (second operand).
+//CHECK: omp.scan.seed:
+//CHECK:   %[[OBUFF:.+]] = load ptr, ptr %{{.*}}, align 8
+//CHECK:   %[[OELEMPTR:.+]] = getelementptr inbounds i32, ptr %[[OBUFF]], i32 1
+//CHECK:   %[[OORIG:.+]] = load i32, ptr %{{.*}}, align 4
+//CHECK:   %[[OELEM:.+]] = load i32, ptr %[[OELEMPTR]], align 4
+//CHECK:   %[[OCOMB:.+]] = sub i32 %[[OORIG]], %[[OELEM]]
+//CHECK:   store i32 %[[OCOMB]], ptr %[[OELEMPTR]], align 4
+
+// The prefix computation must combine as sub(buffer[i-pow2k], buffer[i]): the
+// earlier partial prefix is the accumulator (first operand) and the later one
+// the incoming value (second operand). buffer[i] (the later slot) is updated.
+//CHECK: omp.inner.log.scan.body:
+//CHECK:   %[[IND1:.+]] = add i32 %{{.*}}, 1
+//CHECK:   %[[IND1PTR:.+]] = getelementptr inbounds i32, ptr %{{.*}}, i32 %[[IND1]]
+//CHECK:   %[[IND2:.+]] = sub nuw i32 %[[IND1]], %{{.*}}
+//CHECK:   %[[IND2PTR:.+]] = getelementptr inbounds i32, ptr %{{.*}}, i32 %[[IND2]]
+//CHECK:   %[[IND1VAL:.+]] = load i32, ptr %[[IND1PTR]], align 4
+//CHECK:   %[[IND2VAL:.+]] = load i32, ptr %[[IND2PTR]], align 4
+//CHECK:   %[[REDVAL:.+]] = sub i32 %[[IND2VAL]], %[[IND1VAL]]
+//CHECK:   store i32 %[[REDVAL]], ptr %[[IND1PTR]], align 4
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
index 38e413a504315..0006434167c7c 100644
--- a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
@@ -99,7 +99,7 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK:   %[[OELEMPTR:.+]] = getelementptr inbounds i32, ptr %[[OBUFF]], i32 1
 //CHECK:   %[[OORIG:.+]] = load i32, ptr %{{.*}}, align 4
 //CHECK:   %[[OELEM:.+]] = load i32, ptr %[[OELEMPTR]], align 4
-//CHECK:   %[[OCOMB:.+]] = add i32 %[[OELEM]], %[[OORIG]]
+//CHECK:   %[[OCOMB:.+]] = add i32 %[[OORIG]], %[[OELEM]]
 //CHECK:   store i32 %[[OCOMB]], ptr %[[OELEMPTR]], align 4
 //CHECK:   br i1 {{.*}}, label %omp.outer.log.scan.body, label %omp.outer.log.scan.exit
 //CHECK: omp.outer.log.scan.body:
@@ -121,7 +121,7 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK:   %[[IND2PTR:.+]] = getelementptr inbounds i32, ptr %[[BUFF]], i32 %[[IND2]]
 //CHECK:   %[[IND1VAL:.+]] = load i32, ptr %[[IND1PTR]], align 4
 //CHECK:   %[[IND2VAL:.+]] = load i32, ptr %[[IND2PTR]], align 4
-//CHECK:   %[[REDVAL:.+]] = add i32 %[[IND1VAL]], %[[IND2VAL]]
+//CHECK:   %[[REDVAL:.+]] = add i32 %[[IND2VAL]], %[[IND1VAL]]
 //CHECK:   store i32 %[[REDVAL]], ptr %[[IND1PTR]], align 4
 //CHECK:   %[[CNTNXT]] = sub nuw i32 %[[CNT]], 1
 //CHECK:   %[[CMP3:.+]] = icmp uge i32 %[[CNTNXT]], %[[I]]


From openmp-commits at lists.llvm.org  Mon Aug 10 11:50:28 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Mon, 10 Aug 2026 11:50:28 -0700 (PDT)
Subject: [Openmp-commits] [flang] [llvm] [mlir] [openmp] [Flang][OpenMP]
 Lower scan directive and inscan reduction modifier (PR #206747)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206747@github.com>
Message-ID: <6a7a1d74.af6ba3ed.fcb70.7cb5@mx.google.com>


================
@@ -3616,12 +3616,59 @@ genParallelOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
 
 static mlir::omp::ScanOp
 genScanOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
-          semantics::SemanticsContext &semaCtx, mlir::Location loc,
-          const ConstructQueue &queue, ConstructQueue::const_iterator item) {
+          semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+          mlir::Location loc, const ConstructQueue &queue,
+          ConstructQueue::const_iterator item) {
   mlir::omp::ScanOperands clauseOps;
   genScanClauses(converter, semaCtx, item->clauses, loc, clauseOps);
-  return mlir::omp::ScanOp::create(converter.getFirOpBuilder(),
-                                   converter.getCurrentLocation(), clauseOps);
+  mlir::omp::ScanOp scanOp = mlir::omp::ScanOp::create(
+      converter.getFirOpBuilder(), converter.getCurrentLocation(), clauseOps);
+
+  /// Scan reduction is not implemented with nested workshare loops, linear
+  /// clause, tiling
+  mlir::omp::LoopNestOp loopNestOp =
+      scanOp->getParentOfType<mlir::omp::LoopNestOp>();
+  llvm::SmallVector<mlir::omp::LoopWrapperInterface> loopWrappers;
+  loopNestOp.gatherWrappers(loopWrappers);
+  mlir::Operation *loopWrapperOp = loopWrappers.front().getOperation();
+  if (llvm::isa<mlir::omp::SimdOp>(loopWrapperOp))
+    TODO(loc, "unsupported simd");
+  if (loopWrappers.size() > 1)
+    TODO(loc, "unsupported composite");
+  mlir::omp::WsloopOp wsLoopOp = llvm::cast<mlir::omp::WsloopOp>(loopWrapperOp);
+  bool isNested =
+      (loopNestOp.getNumLoops() > 1) ||
+      (wsLoopOp && (wsLoopOp->getParentOfType<mlir::omp::WsloopOp>()));
+  if (isNested)
+    TODO(loc, "Scan directive inside nested workshare loops");
+  if (wsLoopOp && !wsLoopOp.getLinearVars().empty())
+    TODO(loc, "Scan directive with linear clause");
+  if (loopNestOp.getTileSizes())
+    TODO(loc, "Scan directive with loop tiling");
+
+  // All loop indices should be loaded after the scan construct as otherwise,
+  // it would result in using the index variable across scan directive.
+  // (`Intra-iteration dependences from a statement in the structured
+  // block sequence that precede a scan directive to a statement in the
+  // structured block sequence that follows a scan directive must not exist,
+  // except for dependences for the list items specified in an inclusive or
+  // exclusive clause.`).
+  // TODO: Nested loops are not handled.
+  mlir::Region &region = loopNestOp->getRegion(0);
+  mlir::Value indexVal = fir::getBase(region.getArgument(0));
+  lower::pft::Evaluation *doConstructEval = eval.parentConstruct;
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+  lower::pft::Evaluation *doLoop = &doConstructEval->getFirstNestedEvaluation();
+  auto *doStmt = doLoop->getIf<parser::NonLabelDoStmt>();
+  assert(doStmt && "Expected do loop to be in the nested evaluation");
+  const auto &loopControl =
+      std::get<std::optional<parser::LoopControl>>(doStmt->t);
----------------
chichunchen wrote:

Should we use `getIterationVariableSymbol(*doConstructEval)` instead? Since right now bounds assume loopControl is non-null and there's also no guarantee that bounds is non-null.

https://github.com/llvm/llvm-project/pull/206747

From openmp-commits at lists.llvm.org  Mon Aug 10 18:06:52 2026
From: openmp-commits at lists.llvm.org (Johannes Doerfert via Openmp-commits)
Date: Mon, 10 Aug 2026 18:06:52 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [offload] Pool host and shared
	allocations (PR #214754)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/214754@github.com>
Message-ID: <6a7a75ac.27e38385.2dd06b.5dda@mx.google.com>


jdoerfert wrote:

> IIRC host allocations are a little strange so I'll need to double check. HSA does its own pooling, but allocations that pin memory use full page granularity so they're not as easily split. E.g. if you want to pin 4 bytes you round down to the page size and pin the whole page. If you then unpin that and free it you could unpin in-flight memory that shares the page, so I think HSA just gives you an entire page.

We could keep track of users of an unpinned side and co-locate ourselves anyway. Just a thought.

https://github.com/llvm/llvm-project/pull/214754

From openmp-commits at lists.llvm.org  Mon Aug 10 18:08:36 2026
From: openmp-commits at lists.llvm.org (Johannes Doerfert via Openmp-commits)
Date: Mon, 10 Aug 2026 18:08:36 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [offload] Pool host and shared
	allocations (PR #214754)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/214754@github.com>
Message-ID: <6a7a7614.d1496011.24792e.cb63@mx.google.com>


================
@@ -4529,6 +4535,12 @@ Expected<void *> AMDGPUDeviceTy::allocate(size_t Size, void *,
     // Enable all valid kernel agents to access the buffer.
     if (auto Err = MemoryPool->enableAccess(Alloc, Size, Agents))
       return std::move(Err);
+
+    // Register host / shared memory as pinned memory, so that transfers reading
+    // from it can take a device-accessible path.
+    if (Kind == TARGET_ALLOC_HOST || Kind == TARGET_ALLOC_SHARED)
+      if (auto Err = PinnedAllocs.registerHostBuffer(Alloc, Alloc, Size))
+        return std::move(Err);
----------------
jdoerfert wrote:

Can we have a quick call about this PR?
The 4 of us, for example. I am confused by different things and I think talking might be easier to figure this out.

https://github.com/llvm/llvm-project/pull/214754

From openmp-commits at lists.llvm.org  Mon Aug 10 18:10:41 2026
From: openmp-commits at lists.llvm.org (Joseph Huber via Openmp-commits)
Date: Mon, 10 Aug 2026 18:10:41 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [offload] Pool host and shared
	allocations (PR #214754)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/214754@github.com>
Message-ID: <6a7a7691.4a342cbc.f12fe.de3f@mx.google.com>


================
@@ -4529,6 +4535,12 @@ Expected<void *> AMDGPUDeviceTy::allocate(size_t Size, void *,
     // Enable all valid kernel agents to access the buffer.
     if (auto Err = MemoryPool->enableAccess(Alloc, Size, Agents))
       return std::move(Err);
+
+    // Register host / shared memory as pinned memory, so that transfers reading
+    // from it can take a device-accessible path.
+    if (Kind == TARGET_ALLOC_HOST || Kind == TARGET_ALLOC_SHARED)
+      if (auto Err = PinnedAllocs.registerHostBuffer(Alloc, Alloc, Size))
+        return std::move(Err);
----------------
jhuber6 wrote:

The Wednesday meeting is too early for you, right?

https://github.com/llvm/llvm-project/pull/214754

From openmp-commits at lists.llvm.org  Tue Aug 11 06:41:21 2026
From: openmp-commits at lists.llvm.org (CHANDRA GHALE via Openmp-commits)
Date: Tue, 11 Aug 2026 06:41:21 -0700 (PDT)
Subject: [Openmp-commits] [flang] [llvm] [mlir] [openmp] [Flang][OpenMP]
 Lower scan directive and inscan reduction modifier (PR #206747)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206747@github.com>
Message-ID: <6a7b2681.2915925c.3a79d1.a778@mx.google.com>


https://github.com/chandraghale updated https://github.com/llvm/llvm-project/pull/206747

>From 82416bc775c4befdbcce1f999a5f5b97ef954cd8 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Tue, 30 Jun 2026 10:00:56 -0500
Subject: [PATCH 01/10] Lower scan directive and inscan reduction modifier

---
 flang/lib/Lower/OpenMP/OpenMP.cpp             |  57 ++-
 .../Lower/OpenMP/Todo/nested-wsloop-scan.f90  |  34 ++
 .../OpenMP/Todo/wsloop-scan-collapse.f90      |  29 ++
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      | 476 +++++++++++++++---
 .../Target/LLVMIR/openmp-reduction-scan.mlir  | 123 +++++
 mlir/test/Target/LLVMIR/openmp-todo.mlir      |  41 +-
 openmp/runtime/test/scan/scan.f90             |  38 ++
 7 files changed, 726 insertions(+), 72 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/Todo/nested-wsloop-scan.f90
 create mode 100644 flang/test/Lower/OpenMP/Todo/wsloop-scan-collapse.f90
 create mode 100644 mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
 create mode 100644 openmp/runtime/test/scan/scan.f90

diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp
index ea8c279962508..e77584a142eea 100644
--- a/flang/lib/Lower/OpenMP/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP/OpenMP.cpp
@@ -2972,12 +2972,59 @@ genParallelOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
 
 static mlir::omp::ScanOp
 genScanOp(lower::AbstractConverter &converter, lower::SymMap &symTable,
-          semantics::SemanticsContext &semaCtx, mlir::Location loc,
-          const ConstructQueue &queue, ConstructQueue::const_iterator item) {
+          semantics::SemanticsContext &semaCtx, lower::pft::Evaluation &eval,
+          mlir::Location loc, const ConstructQueue &queue,
+          ConstructQueue::const_iterator item) {
   mlir::omp::ScanOperands clauseOps;
   genScanClauses(converter, semaCtx, item->clauses, loc, clauseOps);
-  return mlir::omp::ScanOp::create(converter.getFirOpBuilder(),
-                                   converter.getCurrentLocation(), clauseOps);
+  mlir::omp::ScanOp scanOp = mlir::omp::ScanOp::create(
+      converter.getFirOpBuilder(), converter.getCurrentLocation(), clauseOps);
+
+  /// Scan reduction is not implemented with nested workshare loops, linear
+  /// clause, tiling
+  mlir::omp::LoopNestOp loopNestOp =
+      scanOp->getParentOfType<mlir::omp::LoopNestOp>();
+  llvm::SmallVector<mlir::omp::LoopWrapperInterface> loopWrappers;
+  loopNestOp.gatherWrappers(loopWrappers);
+  mlir::Operation *loopWrapperOp = loopWrappers.front().getOperation();
+  if (llvm::isa<mlir::omp::SimdOp>(loopWrapperOp))
+    TODO(loc, "unsupported simd");
+  if (loopWrappers.size() > 1)
+    TODO(loc, "unsupported composite");
+  mlir::omp::WsloopOp wsLoopOp = llvm::cast<mlir::omp::WsloopOp>(loopWrapperOp);
+  bool isNested =
+      (loopNestOp.getNumLoops() > 1) ||
+      (wsLoopOp && (wsLoopOp->getParentOfType<mlir::omp::WsloopOp>()));
+  if (isNested)
+    TODO(loc, "Scan directive inside nested workshare loops");
+  if (wsLoopOp && !wsLoopOp.getLinearVars().empty())
+    TODO(loc, "Scan directive with linear clause");
+  if (loopNestOp.getTileSizes())
+    TODO(loc, "Scan directive with loop tiling");
+
+  // All loop indices should be loaded after the scan construct as otherwise,
+  // it would result in using the index variable across scan directive.
+  // (`Intra-iteration dependences from a statement in the structured
+  // block sequence that precede a scan directive to a statement in the
+  // structured block sequence that follows a scan directive must not exist,
+  // except for dependences for the list items specified in an inclusive or
+  // exclusive clause.`).
+  // TODO: Nested loops are not handled.
+  mlir::Region &region = loopNestOp->getRegion(0);
+  mlir::Value indexVal = fir::getBase(region.getArgument(0));
+  lower::pft::Evaluation *doConstructEval = eval.parentConstruct;
+  fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
+  lower::pft::Evaluation *doLoop = &doConstructEval->getFirstNestedEvaluation();
+  auto *doStmt = doLoop->getIf<parser::NonLabelDoStmt>();
+  assert(doStmt && "Expected do loop to be in the nested evaluation");
+  const auto &loopControl =
+      std::get<std::optional<parser::LoopControl>>(doStmt->t);
+  const parser::LoopControl::Bounds *bounds =
+      std::get_if<parser::LoopControl::Bounds>(&loopControl->u);
+  mlir::Operation *storeOp =
+      setLoopVar(converter, loc, indexVal, bounds->Name().thing.symbol);
+  firOpBuilder.setInsertionPointAfter(storeOp);
+  return scanOp;
 }
 
 static mlir::omp::SectionsOp
@@ -4305,7 +4352,7 @@ static void genOMPDispatch(lower::AbstractConverter &converter,
                                   loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_scan:
-    newOp = genScanOp(converter, symTable, semaCtx, loc, queue, item);
+    newOp = genScanOp(converter, symTable, semaCtx, eval, loc, queue, item);
     break;
   case llvm::omp::Directive::OMPD_section:
     llvm_unreachable("genOMPDispatch: OMPD_section");
diff --git a/flang/test/Lower/OpenMP/Todo/nested-wsloop-scan.f90 b/flang/test/Lower/OpenMP/Todo/nested-wsloop-scan.f90
new file mode 100644
index 0000000000000..414e2ef2d5aed
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/nested-wsloop-scan.f90
@@ -0,0 +1,34 @@
+! Tests scan reduction behavior when used in nested workshare loops
+
+! RUN: %not_todo_cmd %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+program nested_scan_example
+  implicit none
+  integer, parameter :: n = 4, m = 5
+  integer :: a(n, m), b(n, m)
+  integer :: i, j
+  integer :: row_sum, col_sum
+
+  do i = 1, n
+     do j = 1, m
+        a(i, j) = i + j
+     end do
+  end do
+
+  !$omp parallel do reduction(inscan, +: row_sum) private(col_sum, j)
+  do i = 1, n
+     row_sum = row_sum + i
+     !$omp scan inclusive(row_sum)
+
+     col_sum = 0
+     !$omp parallel do reduction(inscan, +: col_sum)
+     do j = 1, m
+        col_sum = col_sum + a(i, j)
+        !CHECK: not yet implemented: Scan directive inside nested workshare loops
+        !$omp scan inclusive(col_sum)
+        b(i, j) = col_sum + row_sum 
+     end do
+     !$omp end parallel do
+  end do
+  !$omp end parallel do
+end program nested_scan_example
diff --git a/flang/test/Lower/OpenMP/Todo/wsloop-scan-collapse.f90 b/flang/test/Lower/OpenMP/Todo/wsloop-scan-collapse.f90
new file mode 100644
index 0000000000000..b8e6e831884ab
--- /dev/null
+++ b/flang/test/Lower/OpenMP/Todo/wsloop-scan-collapse.f90
@@ -0,0 +1,29 @@
+! Tests scan reduction behavior when used in nested workshare loops
+
+! RUN: %not_todo_cmd %flang_fc1 -emit-hlfir -fopenmp -o - %s 2>&1 | FileCheck %s
+
+program nested_loop_example
+  implicit none
+  integer :: i, j, x
+  integer, parameter :: N = 100, M = 200
+  real :: A(N, M), B(N, M)
+  x = 0
+
+  do i = 1, N
+    do j = 1, M
+      A(i, j) = i * j
+    end do
+  end do
+  
+  !$omp parallel do collapse(2) reduction(inscan, +:x)
+  do i = 1, N
+    do j = 1, M
+      x = x + A(i,j)
+      !CHECK: not yet implemented: Scan directive inside nested workshare loops
+      !$omp scan inclusive(x)
+      B(i, j) = x
+    end do
+  end do
+  !$omp end parallel do
+
+end program nested_loop_example
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index d860c408e0fd2..297ebac51de88 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -41,6 +41,8 @@
 #include "llvm/TargetParser/Triple.h"
 #include "llvm/Transforms/Utils/ModuleUtils.h"
 
+#include <cassert>
+#include <cstddef>
 #include <cstdint>
 #include <iterator>
 #include <numeric>
@@ -84,6 +86,10 @@ class OpenMPAllocStackFrame
       : allocInsertPoint(allocaIP), deallocBlocks(deallocBlocks) {}
   llvm::OpenMPIRBuilder::InsertPointTy allocInsertPoint;
   llvm::SmallVector<llvm::BasicBlock *> deallocBlocks;
+  /// Set to true when this alloca frame encloses an omp.parallel operation.
+  /// The alloca insertion point of a function in which a parallel op is
+  /// defined may be used to allocate the temporary buffer for scan reductions.
+  bool containsParallelOp = false;
 };
 
 /// Stack frame to hold a \see llvm::CanonicalLoopInfo representing the
@@ -93,7 +99,17 @@ class OpenMPLoopInfoStackFrame
     : public StateStackFrameBase<OpenMPLoopInfoStackFrame> {
 public:
   MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(OpenMPLoopInfoStackFrame)
+  /// For constructs like `scan`, a single `omp.loop_nest` is split into an
+  /// input loop and a scan loop. In that case `loopInfo` holds the input loop
+  /// info and `scanloopInfo` holds the scan loop info.
   llvm::CanonicalLoopInfo *loopInfo = nullptr;
+  llvm::CanonicalLoopInfo *scanloopInfo = nullptr;
+  llvm::ScanInfo *scanInfo = nullptr;
+  /// Map reduction variables to their LLVM types. Populated when the reduction
+  /// clause is processed and used when a `scan` directive is encountered in the
+  /// loop body.
+  std::unique_ptr<llvm::DenseMap<llvm::Value *, llvm::Type *>>
+      reductionVarToType = nullptr;
 };
 
 /// Custom error class to signal translation errors that don't need reporting,
@@ -359,6 +375,10 @@ static LogicalResult checkImplementationStatus(Operation &op) {
     if (!op.getDependVars().empty() || op.getDependKinds())
       result = todo("depend");
   };
+  auto checkExclusive = [&todo](auto op, LogicalResult &result) {
+    if (!op.getExclusiveVars().empty())
+      result = todo("exclusive");
+  };
   auto checkHint = [](auto op, LogicalResult &) {
     if (op.getHint())
       op.emitWarning("hint clause discarded");
@@ -389,21 +409,27 @@ static LogicalResult checkImplementationStatus(Operation &op) {
         op.getReductionMod().value() != omp::ReductionModifier::defaultmod) {
       omp::ReductionModifier mod = op.getReductionMod().value();
       // The `task` reduction modifier is supported on the parallel and
-      // worksharing (do/for and sections) constructs. Other modifiers, and the
-      // `task` modifier on other constructs, are not yet implemented.
+      // worksharing (do/for and sections) constructs. The `inscan` modifier is
+      // supported on the worksharing-loop construct (it is translated as a scan
+      // reduction). Other modifiers, and these modifiers on other constructs,
+      // are not yet implemented.
       bool taskModifierSupported =
           mod == omp::ReductionModifier::task &&
           isa<omp::ParallelOp, omp::WsloopOp, omp::SectionsOp>(op);
-      if (!taskModifierSupported) {
+      bool inscanModifierSupported =
+          mod == omp::ReductionModifier::inscan && isa<omp::WsloopOp>(op);
+      if (!taskModifierSupported && !inscanModifierSupported) {
         result = todo("reduction with modifier");
-      } else if (auto byref = op.getReductionByref()) {
-        // The task reduction modifier lowering only handles non-byref
-        // reductions for now.
-        for (bool isByRef : *byref)
-          if (isByRef) {
-            result = todo("task reduction modifier with by-ref reduction");
-            break;
-          }
+      } else if (taskModifierSupported) {
+        if (auto byref = op.getReductionByref()) {
+          // The task reduction modifier lowering only handles non-byref
+          // reductions for now.
+          for (bool isByRef : *byref)
+            if (isByRef) {
+              result = todo("task reduction modifier with by-ref reduction");
+              break;
+            }
+        }
       }
     }
   };
@@ -460,6 +486,7 @@ static LogicalResult checkImplementationStatus(Operation &op) {
         checkAllocate(op, result);
         checkOrder(op, result);
       })
+      .Case([&](omp::ScanOp op) { checkExclusive(op, result); })
       .Case([&](omp::SectionsOp op) {
         checkAllocate(op, result);
         checkPrivate(op, result);
@@ -650,6 +677,97 @@ findCurrentLoopInfo(LLVM::ModuleTranslation &moduleTranslation) {
   return loopInfo;
 }
 
+/// Find the scan loop information structure for the scan loop nest being
+/// translated. It will return a `null` value unless called from the
+/// translation function for a loop wrapper operation after successfully
+/// translating its body.
+static llvm::CanonicalLoopInfo *
+findCurrentScanLoopInfo(LLVM::ModuleTranslation &moduleTranslation) {
+  llvm::CanonicalLoopInfo *scanLoopInfo = nullptr;
+  moduleTranslation.stackWalk<OpenMPLoopInfoStackFrame>(
+      [&](OpenMPLoopInfoStackFrame &frame) {
+        scanLoopInfo = frame.scanloopInfo;
+        return WalkResult::interrupt();
+      });
+  return scanLoopInfo;
+}
+
+/// Find the `ScanInfo` stored on the loop stack frame. Upon encountering an
+/// `inscan` reduction modifier, `scanInfoInitialize` initializes the
+/// `ScanInfo`, which is then used when a `scan` directive is encountered in the
+/// body of the loop nest.
+static llvm::ScanInfo *
+findScanInfo(LLVM::ModuleTranslation &moduleTranslation) {
+  llvm::ScanInfo *scanInfo = nullptr;
+  moduleTranslation.stackWalk<OpenMPLoopInfoStackFrame>(
+      [&](OpenMPLoopInfoStackFrame &frame) {
+        scanInfo = frame.scanInfo;
+        return WalkResult::interrupt();
+      });
+  return scanInfo;
+}
+
+/// The types of reduction variables are used for lowering a `scan` directive
+/// that appears in the body of the loop. The types are stored in the loop frame
+/// when the reduction clause is encountered and used when the `scan` directive
+/// is encountered.
+static llvm::DenseMap<llvm::Value *, llvm::Type *> *
+findReductionVarTypes(LLVM::ModuleTranslation &moduleTranslation) {
+  llvm::DenseMap<llvm::Value *, llvm::Type *> *reductionVarToType = nullptr;
+  moduleTranslation.stackWalk<OpenMPLoopInfoStackFrame>(
+      [&](OpenMPLoopInfoStackFrame &frame) {
+        if (!frame.reductionVarToType)
+          frame.reductionVarToType =
+              std::make_unique<llvm::DenseMap<llvm::Value *, llvm::Type *>>();
+        reductionVarToType = frame.reductionVarToType.get();
+        return WalkResult::interrupt();
+      });
+  return reductionVarToType;
+}
+
+/// Scan reduction requires a shared buffer to be allocated to perform the
+/// reduction. The allocation needs to be done outside the parallel region in
+/// which the scan operation is used.
+static llvm::OpenMPIRBuilder::InsertPointTy
+findParallelAllocaIP(llvm::IRBuilderBase &builder,
+                     LLVM::ModuleTranslation &moduleTranslation) {
+  // If there is an alloca insertion point on the stack belonging to a frame
+  // that encloses a parallel op, use it.
+  llvm::OpenMPIRBuilder::InsertPointTy allocaInsertPoint;
+  WalkResult walkResult = moduleTranslation.stackWalk<OpenMPAllocStackFrame>(
+      [&](OpenMPAllocStackFrame &frame) {
+        if (frame.containsParallelOp) {
+          allocaInsertPoint = frame.allocInsertPoint;
+          return WalkResult::interrupt();
+        }
+        return WalkResult::skip();
+      });
+  if (walkResult.wasInterrupted())
+    return allocaInsertPoint;
+  // Otherwise, insert into the entry block of the surrounding function.
+  // If the current IRBuilder InsertPoint is the function's entry, it cannot
+  // also be used for alloca insertion which would result in insertion order
+  // confusion. Create a new BasicBlock for the Builder and use the entry block
+  // for the allocs.
+  // TODO: Create a dedicated alloca BasicBlock at function creation such that
+  // we do not need to move the current InsertPoint here.
+  if (builder.GetInsertBlock() ==
+      &builder.GetInsertBlock()->getParent()->getEntryBlock()) {
+    assert(builder.GetInsertPoint() == builder.GetInsertBlock()->end() &&
+           "Assuming end of basic block");
+    llvm::BasicBlock *entryBB = llvm::BasicBlock::Create(
+        builder.getContext(), "entry", builder.GetInsertBlock()->getParent(),
+        builder.GetInsertBlock()->getNextNode());
+    builder.CreateBr(entryBB);
+    builder.SetInsertPoint(entryBB);
+  }
+
+  llvm::BasicBlock &funcEntryBlock =
+      builder.GetInsertBlock()->getParent()->getEntryBlock();
+  return llvm::OpenMPIRBuilder::InsertPointTy(
+      &funcEntryBlock, funcEntryBlock.getFirstInsertionPt());
+}
+
 /// Converts the given region that appears within an OpenMP dialect operation to
 /// LLVM IR, creating a branch from the `sourceBlock` to the entry block of the
 /// region, and a branch from any block with an successor-less OpenMP terminator
@@ -1414,7 +1532,8 @@ initReductionVars(OP op, ArrayRef<BlockArgument> reductionArgs,
                   SmallVectorImpl<llvm::Value *> &privateReductionVariables,
                   DenseMap<Value, llvm::Value *> &reductionVariableMap,
                   llvm::ArrayRef<bool> isByRef,
-                  SmallVectorImpl<DeferredStore> &deferredStores) {
+                  SmallVectorImpl<DeferredStore> &deferredStores,
+                  bool isInScanRegion = false) {
   if (op.getNumReductionVars() == 0)
     return success();
 
@@ -1451,11 +1570,17 @@ initReductionVars(OP op, ArrayRef<BlockArgument> reductionArgs,
   for (auto [data, addr] : deferredStores)
     builder.CreateStore(data, addr);
 
+  llvm::DenseMap<llvm::Value *, llvm::Type *> *reductionVarToType =
+      findReductionVarTypes(moduleTranslation);
   // Before the loop, store the initial values of reductions into reduction
   // variables. Although this could be done after allocas, we don't want to mess
   // up with the alloca insertion point.
   for (unsigned i = 0; i < op.getNumReductionVars(); ++i) {
     SmallVector<llvm::Value *, 1> phis;
+    llvm::Type *reductionType =
+        moduleTranslation.convertType(reductionDecls[i].getType());
+    if (isInScanRegion && reductionVarToType != nullptr)
+      (*reductionVarToType)[privateReductionVariables[i]] = reductionType;
 
     // map block argument to initializer region
     mapInitializationArgs(op, moduleTranslation, builder, reductionDecls,
@@ -4406,11 +4531,15 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
     return failure();
 
   assert(afterAllocas.get()->getSinglePredecessor());
+  bool isInScanRegion =
+      wsloopOp.getReductionMod() && (wsloopOp.getReductionMod().value() ==
+                                     mlir::omp::ReductionModifier::inscan);
   if (failed(initReductionVars(wsloopOp, reductionArgs, builder,
                                moduleTranslation,
                                afterAllocas.get()->getSinglePredecessor(),
                                reductionDecls, privateReductionVariables,
-                               reductionVariableMap, isByRef, deferredStores)))
+                               reductionVariableMap, isByRef, deferredStores,
+                               isInScanRegion)))
     return failure();
 
   // For `reduction(task, ...)` open a task-reduction scope for the worksharing
@@ -4432,6 +4561,10 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
   std::optional<omp::ScheduleModifier> scheduleMod = wsloopOp.getScheduleMod();
   bool isSimd = wsloopOp.getScheduleSimd();
   bool loopNeedsBarrier = !wsloopOp.getNowait();
+  // TODO: Linear clause support needs to be enabled for scan reduction.
+  if (isInScanRegion)
+    assert(wsloopOp.getLinearVars().empty() &&
+           "Linear clause support is not enabled with scan reduction");
 
   // The only legal way for the direct parent to be omp.distribute is that this
   // represents 'distribute parallel do'. Otherwise, this is a regular
@@ -4469,21 +4602,93 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
   if (failed(handleError(regionBlock, opInst)))
     return failure();
 
+  // Generates the loop body for a worksharing loop, including linear-variable
+  // handling and the call into the OpenMPIRBuilder's worksharing-loop helper.
+  // For scan reductions this lambda is invoked twice: once for the input loop
+  // and once for the scan loop.
+  const auto &&wsloopCodeGen = [&](llvm::CanonicalLoopInfo *loopInfo,
+                                   bool noLoopMode,
+                                   bool inputScanLoop) -> LogicalResult {
+    // Emit Initialization and Update IR for linear variables
+    if (!wsloopOp.getLinearVars().empty()) {
+      linearClauseProcessor.initLinearVar(builder, moduleTranslation,
+                                          loopInfo->getPreheader());
+      llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP =
+          moduleTranslation.getOpenMPBuilder()->createBarrier(
+              builder.saveIP(), llvm::omp::OMPD_barrier);
+      if (failed(handleError(afterBarrierIP, *loopOp)))
+        return failure();
+      builder.restoreIP(*afterBarrierIP);
+      linearClauseProcessor.updateLinearVar(builder, loopInfo->getBody(),
+                                            loopInfo->getIndVar());
+      linearClauseProcessor.splitLinearFiniBB(builder, loopInfo->getExit());
+    }
+
+    builder.SetInsertPoint(*regionBlock, (*regionBlock)->begin());
+
+    for (size_t index = 0; index < wsloopOp.getLinearVars().size(); index++)
+      linearClauseProcessor.rewriteInPlace(builder, loopInfo->getBody(),
+                                           loopInfo->getLatch(), index);
+
+    llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP =
+        ompBuilder->applyWorkshareLoop(
+            ompLoc.DL, loopInfo, allocaIP, loopNeedsBarrier,
+            convertToScheduleKind(schedule), chunk, isSimd,
+            scheduleMod == omp::ScheduleModifier::monotonic,
+            scheduleMod == omp::ScheduleModifier::nonmonotonic, isOrdered,
+            workshareLoopType, noLoopMode, hasDistSchedule, distScheduleChunk);
+
+    if (failed(handleError(wsloopIP, opInst)))
+      return failure();
+
+    // Emit finalization for linear vars.
+    if (!wsloopOp.getLinearVars().empty()) {
+      llvm::OpenMPIRBuilder::InsertPointTy oldIP = builder.saveIP();
+      assert(loopInfo->getLastIter() &&
+             "`lastiter` in CanonicalLoopInfo is nullptr");
+      llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP =
+          linearClauseProcessor.finalizeLinearVar(builder, moduleTranslation,
+                                                  loopInfo->getLastIter());
+      if (failed(handleError(afterBarrierIP, *loopOp)))
+        return failure();
+
+      builder.restoreIP(oldIP);
+    }
+
+    // For scan reductions, the cancellation finalization callback is popped
+    // only after the scan (second) loop has been generated.
+    if (!inputScanLoop || !isInScanRegion)
+      popCancelFinalizationCB(cancelTerminators, *ompBuilder, wsloopIP.get());
+
+    return success();
+  };
+
   llvm::CanonicalLoopInfo *loopInfo = findCurrentLoopInfo(moduleTranslation);
 
-  // Emit Initialization and Update IR for linear variables
-  if (!wsloopOp.getLinearVars().empty()) {
-    linearClauseProcessor.initLinearVar(builder, moduleTranslation,
-                                        loopInfo->getPreheader());
-    llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP =
-        moduleTranslation.getOpenMPBuilder()->createBarrier(
-            builder.saveIP(), llvm::omp::OMPD_barrier);
-    if (failed(handleError(afterBarrierIP, *loopOp)))
+  if (isInScanRegion) {
+    // Emit the scan reduction combiner between the input loop and the scan
+    // loop, operating on the per-iteration partial values stored in the shared
+    // buffer.
+    auto inputLoopFinishIp = loopInfo->getAfterIP();
+    builder.restoreIP(inputLoopFinishIp);
+    SmallVector<OwningReductionGen> owningReductionGens;
+    SmallVector<OwningAtomicReductionGen> owningAtomicReductionGens;
+    SmallVector<llvm::OpenMPIRBuilder::ReductionInfo, 2> reductionInfos;
+    SmallVector<OwningDataPtrPtrReductionGen> owningReductionGenRefDataPtrGens;
+    collectReductionInfo(wsloopOp, builder, moduleTranslation, reductionDecls,
+                         owningReductionGens, owningAtomicReductionGens,
+                         owningReductionGenRefDataPtrGens,
+                         privateReductionVariables, reductionInfos, isByRef);
+    llvm::BasicBlock *cont = splitBB(builder, false, "omp.scan.loop.cont");
+    llvm::ScanInfo *scanInfo = findScanInfo(moduleTranslation);
+    llvm::OpenMPIRBuilder::InsertPointOrErrorTy redIP =
+        ompBuilder->emitScanReduction(builder.saveIP(), reductionInfos,
+                                      scanInfo);
+    if (failed(handleError(redIP, opInst)))
       return failure();
-    builder.restoreIP(*afterBarrierIP);
-    linearClauseProcessor.updateLinearVar(builder, loopInfo->getBody(),
-                                          loopInfo->getIndVar());
-    linearClauseProcessor.splitLinearFiniBB(builder, loopInfo->getExit());
+
+    builder.restoreIP(*redIP);
+    builder.CreateBr(cont);
   }
 
   builder.SetInsertPoint(*regionBlock, (*regionBlock)->begin());
@@ -4502,50 +4707,41 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
       noLoopMode = true;
   }
 
-  for (size_t index = 0; index < wsloopOp.getLinearVars().size(); index++)
-    linearClauseProcessor.rewriteInPlace(builder, loopInfo->getBody(),
-                                         loopInfo->getLatch(), index);
-
-  llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP =
-      ompBuilder->applyWorkshareLoop(
-          ompLoc.DL, loopInfo, allocaIP, loopNeedsBarrier,
-          convertToScheduleKind(schedule), chunk, isSimd,
-          scheduleMod == omp::ScheduleModifier::monotonic,
-          scheduleMod == omp::ScheduleModifier::nonmonotonic, isOrdered,
-          workshareLoopType, noLoopMode, hasDistSchedule, distScheduleChunk);
-
-  if (failed(handleError(wsloopIP, opInst)))
+  // For scan loops, the input loop does not pop the cancellation finalization
+  // callback; that happens after the scan loop is generated below.
+  bool inputScanLoop = isInScanRegion;
+  if (failed(wsloopCodeGen(loopInfo, noLoopMode, inputScanLoop)))
     return failure();
+  inputScanLoop = false;
 
-  // Emit finalization and in-place rewrites for linear vars.
-  if (!wsloopOp.getLinearVars().empty()) {
-    llvm::OpenMPIRBuilder::InsertPointTy oldIP = builder.saveIP();
-    assert(loopInfo->getLastIter() &&
-           "`lastiter` in CanonicalLoopInfo is nullptr");
-    llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterBarrierIP =
-        linearClauseProcessor.finalizeLinearVar(builder, moduleTranslation,
-                                                loopInfo->getLastIter());
-    if (failed(handleError(afterBarrierIP, *loopOp)))
+  if (isInScanRegion) {
+    llvm::CanonicalLoopInfo *scanLoopInfo =
+        findCurrentScanLoopInfo(moduleTranslation);
+    if (failed(wsloopCodeGen(scanLoopInfo, noLoopMode, inputScanLoop)))
+      return failure();
+    SmallVector<Region *> reductionRegions;
+    llvm::transform(reductionDecls, std::back_inserter(reductionRegions),
+                    [](omp::DeclareReductionOp reductionDecl) {
+                      return &reductionDecl.getCleanupRegion();
+                    });
+    if (failed(inlineOmpRegionCleanup(
+            reductionRegions, privateReductionVariables, moduleTranslation,
+            builder, "omp.reduction.cleanup")))
       return failure();
+  } else {
+    // Close the task-reduction scope before the worksharing reduction combine.
+    if (isTaskReductionMod)
+      emitTaskReductionModifierFini(/*isWorksharing=*/true, builder,
+                                    moduleTranslation);
 
-    builder.restoreIP(oldIP);
+    // Process the reductions if required.
+    if (failed(createReductionsAndCleanup(
+            wsloopOp, builder, moduleTranslation, allocaIP, reductionDecls,
+            privateReductionVariables, isByRef, wsloopOp.getNowait(),
+            /*isTeamsReduction=*/false)))
+      return failure();
   }
 
-  // Set the correct branch target for task cancellation
-  popCancelFinalizationCB(cancelTerminators, *ompBuilder, wsloopIP.get());
-
-  // Close the task-reduction scope before the worksharing reduction combine.
-  if (isTaskReductionMod)
-    emitTaskReductionModifierFini(/*isWorksharing=*/true, builder,
-                                  moduleTranslation);
-
-  // Process the reductions if required.
-  if (failed(createReductionsAndCleanup(
-          wsloopOp, builder, moduleTranslation, allocaIP, reductionDecls,
-          privateReductionVariables, isByRef, wsloopOp.getNowait(),
-          /*isTeamsReduction=*/false)))
-    return failure();
-
   return cleanupPrivateVars(wsloopOp, builder, moduleTranslation,
                             wsloopOp.getLoc(), privateVarsInfo);
 }
@@ -4579,6 +4775,20 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
       opInst.getReductionMod() == omp::ReductionModifier::task &&
       opInst.getNumReductionVars() > 0;
 
+  // Mark the enclosing alloca stack frame as containing a parallel op. Scan
+  // reductions use the alloca insertion point of the function enclosing the
+  // parallel region to allocate their shared temporary buffer.
+  bool foundParallelOp = false;
+  moduleTranslation.stackWalk<OpenMPAllocStackFrame>(
+      [&](OpenMPAllocStackFrame &frame) {
+        if (foundParallelOp) {
+          frame.containsParallelOp = true;
+          return WalkResult::interrupt();
+        }
+        foundParallelOp = true;
+        return WalkResult::skip();
+      });
+
   auto bodyGenCB =
       [&](InsertPointTy allocaIP, InsertPointTy codeGenIP,
           llvm::ArrayRef<llvm::BasicBlock *> deallocBlocks) -> llvm::Error {
@@ -4972,6 +5182,99 @@ convertOmpSimd(Operation &opInst, llvm::IRBuilderBase &builder,
                             privateVarsInfo);
 }
 
+static LogicalResult
+convertOmpScan(Operation &opInst, llvm::IRBuilderBase &builder,
+               LLVM::ModuleTranslation &moduleTranslation) {
+  if (failed(checkImplementationStatus(opInst)))
+    return failure();
+  auto scanOp = cast<omp::ScanOp>(opInst);
+  bool isInclusive = scanOp.hasInclusiveVars();
+  SmallVector<llvm::Value *> llvmScanVars;
+  SmallVector<llvm::Type *> llvmScanVarsType;
+  mlir::OperandRange mlirScanVars = scanOp.getInclusiveVars();
+  if (!isInclusive)
+    mlirScanVars = scanOp.getExclusiveVars();
+
+  llvm::DenseMap<llvm::Value *, llvm::Type *> *reductionVarToType =
+      findReductionVarTypes(moduleTranslation);
+  for (auto val : mlirScanVars) {
+    llvm::Value *llvmVal = moduleTranslation.lookupValue(val);
+    llvmScanVars.push_back(llvmVal);
+    llvmScanVarsType.push_back((*reductionVarToType)[llvmVal]);
+  }
+  llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
+      findParallelAllocaIP(builder, moduleTranslation);
+  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
+  llvm::ScanInfo *scanInfo = findScanInfo(moduleTranslation);
+  llvm::OpenMPIRBuilder::InsertPointOrErrorTy afterIP =
+      moduleTranslation.getOpenMPBuilder()->createScan(
+          ompLoc, allocaIP, llvmScanVars, llvmScanVarsType, isInclusive,
+          scanInfo);
+  if (failed(handleError(afterIP, opInst)))
+    return failure();
+  builder.restoreIP(*afterIP);
+  return success();
+}
+
+/// Re-initialize the scan reduction variables to their identity at the start of
+/// each input-loop iteration of a scan reduction. This is required because the
+/// OpenMPIRBuilder stores the per-iteration reduction value into the scan
+/// buffer (`buffer[i] = red`) and `emitScanReduction` then computes the prefix
+/// sum across the buffer. Without resetting `red` to the reduction identity
+/// each iteration, the buffer would hold a running sum and the prefix sum would
+/// be computed twice. This reset only applies to the input loop; in the scan
+/// loop the reduction variable is loaded back from the buffer.
+static LogicalResult
+initScanReductionVars(omp::LoopNestOp loopOp, llvm::IRBuilderBase &builder,
+                      LLVM::ModuleTranslation &moduleTranslation) {
+  auto wsloopOp = loopOp->getParentOfType<omp::WsloopOp>();
+  if (!wsloopOp || !wsloopOp.getReductionMod() ||
+      wsloopOp.getReductionMod().value() != omp::ReductionModifier::inscan)
+    return success();
+
+  // Only reset in the input loop; in the scan loop the reduction variable holds
+  // the value loaded from the buffer and must not be overwritten.
+  llvm::ScanInfo *scanInfo = findScanInfo(moduleTranslation);
+  if (!scanInfo || !scanInfo->OMPFirstScanLoop)
+    return success();
+
+  unsigned numReductions = wsloopOp.getNumReductionVars();
+  if (numReductions == 0)
+    return success();
+
+  SmallVector<omp::DeclareReductionOp> reductionDecls;
+  collectReductionDecls(wsloopOp, reductionDecls);
+  ArrayRef<bool> isByRef = getIsByRef(wsloopOp.getReductionByref());
+  MutableArrayRef<BlockArgument> reductionArgs =
+      cast<omp::BlockArgOpenMPOpInterface>(wsloopOp.getOperation())
+          .getReductionBlockArgs();
+
+  DenseMap<Value, llvm::Value *> reductionVariableMap;
+  for (unsigned i = 0; i < numReductions; ++i)
+    reductionVariableMap.try_emplace(
+        wsloopOp.getReductionVars()[i],
+        moduleTranslation.lookupValue(reductionArgs[i]));
+
+  for (unsigned i = 0; i < numReductions; ++i) {
+    // Scan reductions only support by-value (scalar) reductions.
+    if (isByRef[i])
+      continue;
+    SmallVector<llvm::Value *, 1> phis;
+    mapInitializationArgs(wsloopOp, moduleTranslation, builder, reductionDecls,
+                          reductionVariableMap, i);
+    if (failed(inlineConvertOmpRegions(
+            reductionDecls[i].getInitializerRegion(),
+            "omp.scan.reduction.init", builder, moduleTranslation, &phis)))
+      return failure();
+    assert(phis.size() == 1 &&
+           "expected one value to be yielded from the reduction init region");
+    setInsertPointForPossiblyEmptyBlock(builder);
+    builder.CreateStore(phis[0],
+                        moduleTranslation.lookupValue(reductionArgs[i]));
+  }
+  return success();
+}
+
 /// Converts an OpenMP loop nest into LLVM IR using OpenMPIRBuilder.
 static LogicalResult
 convertOmpLoopNest(Operation &opInst, llvm::IRBuilderBase &builder,
@@ -5004,6 +5307,10 @@ convertOmpLoopNest(Operation &opInst, llvm::IRBuilderBase &builder,
 
     // Convert the body of the loop.
     builder.restoreIP(ip);
+    // For scan reductions, reset the reduction variables to their identity at
+    // the start of each input-loop iteration, before the input phase runs.
+    if (failed(initScanReductionVars(loopOp, builder, moduleTranslation)))
+      return llvm::make_error<PreviouslyReportedError>();
     llvm::Expected<llvm::BasicBlock *> regionBlock = convertOmpOpRegions(
         loopOp.getRegion(), "omp.loop_nest.region", builder, moduleTranslation);
     if (!regionBlock)
@@ -5036,6 +5343,46 @@ convertOmpLoopNest(Operation &opInst, llvm::IRBuilderBase &builder,
       computeIP = loopInfos.front()->getPreheaderIP();
     }
 
+    // If this loop is the worksharing loop of a scan reduction, generate a
+    // pair of canonical loops (an input loop and a scan loop) instead of a
+    // single loop. The `scan` directive in the body is translated against the
+    // `ScanInfo` recorded here.
+    bool isInScanRegion = false;
+    if (auto wsloopOp = loopOp->getParentOfType<omp::WsloopOp>())
+      isInScanRegion =
+          wsloopOp.getReductionMod() && (wsloopOp.getReductionMod().value() ==
+                                         mlir::omp::ReductionModifier::inscan);
+    if (isInScanRegion) {
+      llvm::Expected<llvm::ScanInfo *> res = ompBuilder->scanInfoInitialize();
+      if (failed(handleError(res, *loopOp)))
+        return failure();
+      llvm::ScanInfo *scanInfo = res.get();
+      moduleTranslation.stackWalk<OpenMPLoopInfoStackFrame>(
+          [&](OpenMPLoopInfoStackFrame &frame) {
+            frame.scanInfo = scanInfo;
+            return WalkResult::interrupt();
+          });
+      llvm::Expected<llvm::SmallVector<llvm::CanonicalLoopInfo *>> loopResults =
+          ompBuilder->createCanonicalScanLoops(
+              loc, bodyGen, lowerBound, upperBound, step,
+              /*IsSigned=*/true, loopOp.getLoopInclusive(), computeIP, "loop",
+              scanInfo);
+
+      if (failed(handleError(loopResults, *loopOp)))
+        return failure();
+      llvm::CanonicalLoopInfo *inputLoop = loopResults.get().front();
+      llvm::CanonicalLoopInfo *scanLoop = loopResults.get().back();
+      moduleTranslation.stackWalk<OpenMPLoopInfoStackFrame>(
+          [&](OpenMPLoopInfoStackFrame &frame) {
+            frame.loopInfo = inputLoop;
+            frame.scanloopInfo = scanLoop;
+            return WalkResult::interrupt();
+          });
+      builder.restoreIP(scanLoop->getAfterIP());
+      // TODO: tiling and collapse are not yet implemented for scan reduction.
+      return success();
+    }
+
     llvm::Expected<llvm::CanonicalLoopInfo *> loopResult =
         ompBuilder->createCanonicalLoop(
             loc, bodyGen, lowerBound, upperBound, step,
@@ -9454,6 +9801,9 @@ LogicalResult OpenMPDialectLLVMIRTranslationInterface::convertOperation(
           .Case([&](omp::WsloopOp) {
             return convertOmpWsloop(*op, builder, moduleTranslation);
           })
+          .Case([&](omp::ScanOp) {
+            return convertOmpScan(*op, builder, moduleTranslation);
+          })
           .Case([&](omp::SimdOp) {
             return convertOmpSimd(*op, builder, moduleTranslation);
           })
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
new file mode 100644
index 0000000000000..c6b8daccad313
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
@@ -0,0 +1,123 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+omp.declare_reduction @add_reduction_i32 : i32 init {
+^bb0(%arg0: i32):
+  %0 = llvm.mlir.constant(0 : i32) : i32
+  omp.yield(%0 : i32)
+} combiner {
+^bb0(%arg0: i32, %arg1: i32):
+  %0 = llvm.add %arg0, %arg1 : i32
+  omp.yield(%0 : i32)
+}
+// CHECK-LABEL: @scan_reduction
+llvm.func @scan_reduction() {
+  %0 = llvm.mlir.constant(1 : i64) : i64
+  %1 = llvm.alloca %0 x i32 {bindc_name = "z"} : (i64) -> !llvm.ptr
+  %3 = llvm.alloca %0 x i32 {bindc_name = "y"} : (i64) -> !llvm.ptr
+  %5 = llvm.alloca %0 x i32 {bindc_name = "x"} : (i64) -> !llvm.ptr
+  %7 = llvm.alloca %0 x i32 {bindc_name = "k"} : (i64) -> !llvm.ptr
+  %10 = llvm.mlir.constant(100 : i32) : i32
+  %11 = llvm.mlir.constant(1 : i32) : i32
+  %12 = llvm.mlir.constant(0 : i32) : i32
+  %13 = llvm.mlir.constant(100 : index) : i64
+  %14 = llvm.mlir.addressof @_QFEa : !llvm.ptr
+  %15 = llvm.mlir.addressof @_QFEb : !llvm.ptr
+  omp.parallel {
+    %37 = llvm.mlir.constant(1 : i64) : i64
+    %38 = llvm.alloca %37 x i32 {bindc_name = "k", pinned} : (i64) -> !llvm.ptr
+    %39 = llvm.mlir.constant(1 : i64) : i64
+    omp.wsloop reduction(mod: inscan, @add_reduction_i32 %5 -> %arg0 : !llvm.ptr) {
+      omp.loop_nest (%arg1) : i32 = (%11) to (%10) inclusive step (%11) {
+        llvm.store %arg1, %38 : i32, !llvm.ptr
+        %40 = llvm.load %arg0 : !llvm.ptr -> i32
+        %41 = llvm.load %38 : !llvm.ptr -> i32
+        %42 = llvm.sext %41 : i32 to i64
+        %50 = llvm.getelementptr %14[%42] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+        %51 = llvm.load %50 : !llvm.ptr -> i32
+        %52 = llvm.add %40, %51 : i32
+        llvm.store %52, %arg0 : i32, !llvm.ptr
+        omp.scan inclusive(%arg0 : !llvm.ptr)
+        llvm.store %arg1, %38 : i32, !llvm.ptr
+        %53 = llvm.load %arg0 : !llvm.ptr -> i32
+        %54 = llvm.load %38 : !llvm.ptr -> i32
+        %55 = llvm.sext %54 : i32 to i64
+        %63 = llvm.getelementptr %15[%55] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+        llvm.store %53, %63 : i32, !llvm.ptr
+        omp.yield
+      }
+    }
+    omp.terminator
+  }
+  llvm.return
+}
+llvm.mlir.global internal @_QFEa() {addr_space = 0 : i32} : !llvm.array<100 x i32> {
+  %0 = llvm.mlir.zero : !llvm.array<100 x i32>
+  llvm.return %0 : !llvm.array<100 x i32>
+}
+llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i32> {
+  %0 = llvm.mlir.zero : !llvm.array<100 x i32>
+  llvm.return %0 : !llvm.array<100 x i32>
+}
+//CHECK: %vla = alloca ptr, align 8
+//CHECK: omp_parallel
+//CHECK: store ptr %vla, ptr %gep_vla, align 8
+//CHECK: @__kmpc_fork_call
+//CHECK: void @scan_reduction..omp_par
+//CHECK: %[[BUFF_PTR:.+]] = load ptr, ptr %gep_vla
+//CHECK: @__kmpc_masked
+//CHECK: @__kmpc_barrier
+//CHECK: @__kmpc_masked
+//CHECK: @__kmpc_barrier
+//CHECK: omp.scan.loop.cont:
+//CHECK: @__kmpc_masked
+//CHECK: @__kmpc_barrier
+//CHECK: %[[FREE_VAR:.+]] = load ptr, ptr %[[BUFF_PTR]], align 8
+//CHECK:  %[[ARRLAST:.+]] = getelementptr inbounds i32, ptr %[[FREE_VAR]], i32 100
+//CHECK:  %[[RES:.+]] = load i32, ptr %[[ARRLAST]], align 4
+//CHECK:  store i32 %[[RES]], ptr %loadgep{{.*}}, align 4
+//CHECK: tail call void @free(ptr %[[FREE_VAR]])
+//CHECK: @__kmpc_end_masked
+//CHECK: omp.inscan.dispatch{{.*}}:                            ; preds = %omp_loop.body{{.*}}
+//CHECK:   %[[BUFFVAR:.+]] = load ptr, ptr %[[BUFF_PTR]], align 8
+//CHECK:   %[[arrayOffset1:.+]] = getelementptr inbounds i32, ptr %[[BUFFVAR]], i32 %{{.*}}
+//CHECK:   %[[BUFFVAL1:.+]] = load i32, ptr %[[arrayOffset1]], align 4
+//CHECK:   store i32 %[[BUFFVAL1]], ptr %{{.*}}, align 4
+//CHECK:   %[[LOG:.+]] = call double @llvm.log2.f64(double 1.000000e+02) #0
+//CHECK:   %[[CEIL:.+]] = call double @llvm.ceil.f64(double %[[LOG]]) #0
+//CHECK:   %[[UB:.+]] = fptoui double %[[CEIL]] to i32
+//CHECK:   br label %omp.outer.log.scan.body
+//CHECK: omp.outer.log.scan.body:
+//CHECK:   %[[K:.+]] = phi i32 [ 0, %{{.*}} ], [ %[[NEXTK:.+]], %omp.inner.log.scan.exit ]
+//CHECK:   %[[I:.+]] = phi i32 [ 1, %{{.*}} ], [ %[[NEXTI:.+]], %omp.inner.log.scan.exit ]
+//CHECK:   %[[CMP1:.+]] = icmp uge i32 99, %[[I]]
+//CHECK:   br i1 %[[CMP1]], label %omp.inner.log.scan.body, label %omp.inner.log.scan.exit
+//CHECK: omp.inner.log.scan.exit:                          ; preds = %omp.inner.log.scan.body, %omp.outer.log.scan.body
+//CHECK:   %[[NEXTK]] = add nuw i32 %[[K]], 1
+//CHECK:   %[[NEXTI]] = shl nuw i32 %[[I]], 1
+//CHECK:   %[[CMP2:.+]] = icmp ne i32 %[[NEXTK]], %[[UB]]
+//CHECK:   br i1 %[[CMP2]], label %omp.outer.log.scan.body, label %omp.outer.log.scan.exit
+//CHECK: omp.outer.log.scan.exit:                          ; preds = %omp.inner.log.scan.exit
+//CHECK: @__kmpc_end_masked
+//CHECK: omp.inner.log.scan.body:                          ; preds = %omp.inner.log.scan.body, %omp.outer.log.scan.body
+//CHECK:   %[[CNT:.+]] = phi i32 [ 99, %omp.outer.log.scan.body ], [ %[[CNTNXT:.+]], %omp.inner.log.scan.body ]
+//CHECK:   %[[BUFF:.+]] = load ptr, ptr %[[BUFF_PTR]]
+//CHECK:   %[[IND1:.+]] = add i32 %[[CNT]], 1
+//CHECK:   %[[IND1PTR:.+]] = getelementptr inbounds i32, ptr %[[BUFF]], i32 %[[IND1]]
+//CHECK:   %[[IND2:.+]] = sub nuw i32 %[[IND1]], %[[I]]
+//CHECK:   %[[IND2PTR:.+]] = getelementptr inbounds i32, ptr %[[BUFF]], i32 %[[IND2]]
+//CHECK:   %[[IND1VAL:.+]] = load i32, ptr %[[IND1PTR]], align 4
+//CHECK:   %[[IND2VAL:.+]] = load i32, ptr %[[IND2PTR]], align 4
+//CHECK:   %[[REDVAL:.+]] = add i32 %[[IND1VAL]], %[[IND2VAL]]
+//CHECK:   store i32 %[[REDVAL]], ptr %[[IND1PTR]], align 4
+//CHECK:   %[[CNTNXT]] = sub nuw i32 %[[CNT]], 1
+//CHECK:   %[[CMP3:.+]] = icmp uge i32 %[[CNTNXT]], %[[I]]
+//CHECK:   br i1 %[[CMP3]], label %omp.inner.log.scan.body, label %omp.inner.log.scan.exit
+//CHECK: omp.inscan.dispatch:                              ; preds = %omp_loop.body
+//CHECK:   br i1 true, label %omp.before.scan.bb, label %omp.after.scan.bb
+//CHECK: omp.before.scan.bb:
+//CHECK:   store i32 0, ptr %[[REDPRIV:.+]], align 4
+//CHECK: omp.loop_nest.region:                             ; preds = %omp.before.scan.bb
+//CHECK:   %[[BUFFER:.+]] = load ptr, ptr %loadgep_vla, align 8
+//CHECK:   %[[ARRAYOFFSET2:.+]] = getelementptr inbounds i32, ptr %[[BUFFER]], i32 %{{.*}}
+//CHECK-NEXT:   %[[REDPRIVVAL:.+]] = load i32, ptr %{{.*}}, align 4
+//CHECK:   store i32 %[[REDPRIVVAL]], ptr %[[ARRAYOFFSET2]], align 4
+//CHECK:   br label %omp.scan.loop.exit
diff --git a/mlir/test/Target/LLVMIR/openmp-todo.mlir b/mlir/test/Target/LLVMIR/openmp-todo.mlir
index 71e8628595aab..7edf88b5ea559 100644
--- a/mlir/test/Target/LLVMIR/openmp-todo.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-todo.mlir
@@ -120,10 +120,10 @@ atomic {
   llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32
   omp.yield
 }
-llvm.func @scan_reduction(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
-  // expected-error at below {{not yet implemented: Unhandled clause reduction with modifier in omp.wsloop operation}}
-  // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
-  omp.wsloop reduction(mod:inscan, @add_f32 %x -> %prv : !llvm.ptr) {
+llvm.func @simd_reduction(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
+  // expected-error at below {{not yet implemented: Unhandled clause reduction with modifier in omp.simd operation}}
+  // expected-error at below {{LLVM Translation failed for operation: omp.simd}}
+  omp.simd reduction(mod:inscan, @add_f32 %x -> %prv : !llvm.ptr) {
     omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
       omp.scan inclusive(%prv : !llvm.ptr)
       omp.yield
@@ -162,6 +162,39 @@ llvm.func @parallel_task_reduction_modifier_byref(%x : !llvm.ptr) {
 
 // -----
 
+omp.declare_reduction @add_f32 : f32
+init {
+^bb0(%arg: f32):
+  %0 = llvm.mlir.constant(0.0 : f32) : f32
+  omp.yield (%0 : f32)
+}
+combiner {
+^bb1(%arg0: f32, %arg1: f32):
+  %1 = llvm.fadd %arg0, %arg1 : f32
+  omp.yield (%1 : f32)
+}
+atomic {
+^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr):
+  %2 = llvm.load %arg3 : !llvm.ptr -> f32
+  llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32
+  omp.yield
+}
+llvm.func @scan_reduction(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
+  // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
+  omp.wsloop reduction(mod:inscan, @add_f32 %x -> %prv : !llvm.ptr) {
+    // expected-error at below {{LLVM Translation failed for operation: omp.loop_nest}}
+    omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
+      // expected-error at below {{not yet implemented: Unhandled clause exclusive in omp.scan operation}}
+      // expected-error at below {{LLVM Translation failed for operation: omp.scan}}
+      omp.scan exclusive(%prv : !llvm.ptr)
+      omp.yield
+    }
+  }
+  llvm.return
+}
+
+// -----
+
 llvm.func @single_allocate(%x : !llvm.ptr) {
   // expected-error at below {{not yet implemented: Unhandled clause allocate in omp.single operation}}
   // expected-error at below {{LLVM Translation failed for operation: omp.single}}
diff --git a/openmp/runtime/test/scan/scan.f90 b/openmp/runtime/test/scan/scan.f90
new file mode 100644
index 0000000000000..76263508105d0
--- /dev/null
+++ b/openmp/runtime/test/scan/scan.f90
@@ -0,0 +1,38 @@
+! RUN: %flang %flags %openmp_flags -fopenmp-version=51 %s -o %t.exe
+! RUN: %t.exe | FileCheck %s --match-full-lines
+program inclusive_scan
+ implicit none
+ integer, parameter :: n = 100
+ integer a(n), b(n)
+ integer x, k, y, z
+
+ ! initialization
+ x = 0
+ do k = 1, n
+  a(k) = k
+ end do
+
+ ! a(k) is included in the computation of producing results in b(k)
+ !$omp parallel do reduction(inscan, +: x)
+ do k = 1, n
+   x = x + a(k)
+   !$omp scan inclusive(x)
+   b(k) = x
+ end do
+
+ print *,'x =', x
+ do k = 1, 10
+  print *, 'b(', k, ') =', b(k)
+ end do
+end program
+!CHECK: x = 5050
+!CHECK: b( 1 ) = 1
+!CHECK: b( 2 ) = 3
+!CHECK: b( 3 ) = 6
+!CHECK: b( 4 ) = 10
+!CHECK: b( 5 ) = 15
+!CHECK: b( 6 ) = 21
+!CHECK: b( 7 ) = 28
+!CHECK: b( 8 ) = 36
+!CHECK: b( 9 ) = 45
+!CHECK: b( 10 ) = 55

>From 4011dd537d9830ee4a0379cba80413d1e4cdf9c3 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Wed, 1 Jul 2026 02:44:02 -0500
Subject: [PATCH 02/10] formatting correction

---
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp        | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 297ebac51de88..e18cc0df79860 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -4534,12 +4534,11 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
   bool isInScanRegion =
       wsloopOp.getReductionMod() && (wsloopOp.getReductionMod().value() ==
                                      mlir::omp::ReductionModifier::inscan);
-  if (failed(initReductionVars(wsloopOp, reductionArgs, builder,
-                               moduleTranslation,
-                               afterAllocas.get()->getSinglePredecessor(),
-                               reductionDecls, privateReductionVariables,
-                               reductionVariableMap, isByRef, deferredStores,
-                               isInScanRegion)))
+  if (failed(initReductionVars(
+          wsloopOp, reductionArgs, builder, moduleTranslation,
+          afterAllocas.get()->getSinglePredecessor(), reductionDecls,
+          privateReductionVariables, reductionVariableMap, isByRef,
+          deferredStores, isInScanRegion)))
     return failure();
 
   // For `reduction(task, ...)` open a task-reduction scope for the worksharing
@@ -5262,9 +5261,9 @@ initScanReductionVars(omp::LoopNestOp loopOp, llvm::IRBuilderBase &builder,
     SmallVector<llvm::Value *, 1> phis;
     mapInitializationArgs(wsloopOp, moduleTranslation, builder, reductionDecls,
                           reductionVariableMap, i);
-    if (failed(inlineConvertOmpRegions(
-            reductionDecls[i].getInitializerRegion(),
-            "omp.scan.reduction.init", builder, moduleTranslation, &phis)))
+    if (failed(inlineConvertOmpRegions(reductionDecls[i].getInitializerRegion(),
+                                       "omp.scan.reduction.init", builder,
+                                       moduleTranslation, &phis)))
       return failure();
     assert(phis.size() == 1 &&
            "expected one value to be yielded from the reduction init region");

>From 64b530d3f37e0cd08ca622b538d9bec1eb348b50 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Wed, 22 Jul 2026 05:09:34 -0500
Subject: [PATCH 03/10] Address review comments

---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     | 46 +++++++++++---
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      | 26 +++++++-
 .../Target/LLVMIR/openmp-reduction-scan.mlir  | 14 +++--
 mlir/test/Target/LLVMIR/openmp-todo.mlir      | 31 ++++++++++
 openmp/runtime/test/scan/scan-bounds.f90      | 61 +++++++++++++++++++
 openmp/runtime/test/scan/scan-edge.f90        | 61 +++++++++++++++++++
 openmp/runtime/test/scan/scan-nonidentity.f90 | 28 +++++++++
 7 files changed, 251 insertions(+), 16 deletions(-)
 create mode 100644 openmp/runtime/test/scan/scan-bounds.f90
 create mode 100644 openmp/runtime/test/scan/scan-edge.f90
 create mode 100644 openmp/runtime/test/scan/scan-nonidentity.f90

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index f8c1999fe1b89..f62ea4cf2f9da 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5307,7 +5307,17 @@ Error OpenMPIRBuilder::emitScanBasedDirectiveFinalsIR(
                                              "arrayOffset");
       Value *Src = Builder.CreateLoad(SrcTy, Val);
 
-      Builder.CreateStore(Src, OrigVar);
+      // For a zero-trip loop the buffer slot is uninitialized; keep the
+      // original variable's incoming value instead of storing garbage. The
+      // load above stays in bounds because the buffer was allocated with
+      // `Span + 1` elements, so index `Span` is always valid.
+      Value *Cur = Builder.CreateLoad(SrcTy, OrigVar);
+      Value *HasIters = Builder.CreateICmpUGT(
+          ScanRedInfo->Span,
+          llvm::ConstantInt::get(ScanRedInfo->Span->getType(), 0));
+      Value *Final = Builder.CreateSelect(HasIters, Src, Cur);
+
+      Builder.CreateStore(Final, OrigVar);
       Builder.CreateFree(Buff);
     }
     return Error::success();
@@ -5370,7 +5380,15 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
         ScanRedInfo->Span,
         llvm::ConstantInt::get(ScanRedInfo->Span->getType(), 1));
     Builder.SetInsertPoint(InputBB);
-    Builder.CreateBr(LoopBB);
+    // The log-scan prefix computation is only well-defined when there are at
+    // least two elements: `log2(Span)` is invalid for Span == 0 and the loop
+    // trip computation underflows/loops forever for Span <= 1. For Span <= 1
+    // the buffer already holds the final value(s), so skip straight to the
+    // exit.
+    llvm::Value *SpanGuard = Builder.CreateICmpUGT(
+        ScanRedInfo->Span,
+        llvm::ConstantInt::get(ScanRedInfo->Span->getType(), 1));
+    Builder.CreateCondBr(SpanGuard, LoopBB, ExitBB);
     emitBlock(LoopBB, CurFn);
     Builder.SetInsertPoint(LoopBB);
 
@@ -5617,7 +5635,6 @@ OpenMPIRBuilder::createCanonicalScanLoops(
 
   auto BodyGen = [=](InsertPointTy CodeGenIP, Value *IV) {
     Builder.restoreIP(CodeGenIP);
-    ScanRedInfo->IV = IV;
     createScanBBs(ScanRedInfo);
     BasicBlock *InputBlock = Builder.GetInsertBlock();
     Instruction *Terminator = InputBlock->getTerminator();
@@ -5636,9 +5653,13 @@ OpenMPIRBuilder::createCanonicalScanLoops(
   };
 
   const auto &&InputLoopGen = [&]() -> Error {
+    // Pass an empty ComputeIP: the trip count must be (re)computed at the
+    // loop's own location. Reusing the outer ComputeIP is invalid here because
+    // it was invalidated by the `scan.init` split above and would emit the
+    // recomputed trip count after a block terminator for runtime bounds.
     Expected<CanonicalLoopInfo *> LoopInfo = createCanonicalLoop(
         Builder.saveIP(), BodyGen, Start, Stop, Step, IsSigned, InclusiveStop,
-        ComputeIP, Name, true, ScanRedInfo);
+        /*ComputeIP=*/InsertPointTy(), Name, true, ScanRedInfo);
     if (!LoopInfo)
       return LoopInfo.takeError();
     Result.push_back(*LoopInfo);
@@ -5646,9 +5667,9 @@ OpenMPIRBuilder::createCanonicalScanLoops(
     return Error::success();
   };
   const auto &&ScanLoopGen = [&](LocationDescription Loc) -> Error {
-    Expected<CanonicalLoopInfo *> LoopInfo =
-        createCanonicalLoop(Loc, BodyGen, Start, Stop, Step, IsSigned,
-                            InclusiveStop, ComputeIP, Name, true, ScanRedInfo);
+    Expected<CanonicalLoopInfo *> LoopInfo = createCanonicalLoop(
+        Loc, BodyGen, Start, Stop, Step, IsSigned, InclusiveStop,
+        /*ComputeIP=*/InsertPointTy(), Name, true, ScanRedInfo);
     if (!LoopInfo)
       return LoopInfo.takeError();
     Result.push_back(*LoopInfo);
@@ -5737,8 +5758,15 @@ Expected<CanonicalLoopInfo *> OpenMPIRBuilder::createCanonicalLoop(
     Builder.restoreIP(CodeGenIP);
     Value *Span = Builder.CreateMul(IV, Step);
     Value *IndVar = Builder.CreateAdd(Span, Start);
-    if (InScan)
-      ScanRedInfo->IV = IndVar;
+    if (InScan) {
+      // The scan buffer is 1-indexed by iteration number (buffer[1..TripCount])
+      // rather than by the source induction variable. Using the source value
+      // would index out of bounds for loops that do not start at 1 or that use
+      // a non-unit stride. `IV` here is the 0-based canonical loop counter, so
+      // the 1-based buffer index is `IV + 1`.
+      ScanRedInfo->IV =
+          Builder.CreateAdd(IV, ConstantInt::get(IV->getType(), 1));
+    }
     return BodyGenCB(Builder.saveIP(), IndVar);
   };
   LocationDescription LoopLoc =
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index e18cc0df79860..5af1a384d3edb 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -430,6 +430,16 @@ static LogicalResult checkImplementationStatus(Operation &op) {
               break;
             }
         }
+      } else if (inscanModifierSupported) {
+        if (auto byref = op.getReductionByref()) {
+          // Scan reductions (inscan modifier) only support by-value (scalar)
+          // reductions for now.
+          for (bool isByRef : *byref)
+            if (isByRef) {
+              result = todo("inscan reduction modifier with by-ref reduction");
+              break;
+            }
+        }
       }
     }
   };
@@ -5268,8 +5278,20 @@ initScanReductionVars(omp::LoopNestOp loopOp, llvm::IRBuilderBase &builder,
     assert(phis.size() == 1 &&
            "expected one value to be yielded from the reduction init region");
     setInsertPointForPossiblyEmptyBlock(builder);
-    builder.CreateStore(phis[0],
-                        moduleTranslation.lookupValue(reductionArgs[i]));
+
+    // On the first iteration, seed the reduction variable with the original
+    // variable's incoming value so that a non-identity initial value
+    // participates in the scan (e.g. `x = 10` before an inclusive-scan loop of
+    // `+1` must yield 11, 12, ...). On subsequent iterations, reset to the
+    // reduction identity. `scanInfo->IV` is the 1-based iteration index.
+    llvm::Value *identity = phis[0];
+    llvm::Value *origVar =
+        moduleTranslation.lookupValue(wsloopOp.getReductionVars()[i]);
+    llvm::Value *origVal = builder.CreateLoad(identity->getType(), origVar);
+    llvm::Value *isFirstIter = builder.CreateICmpEQ(
+        scanInfo->IV, llvm::ConstantInt::get(scanInfo->IV->getType(), 1));
+    llvm::Value *seed = builder.CreateSelect(isFirstIter, origVal, identity);
+    builder.CreateStore(seed, moduleTranslation.lookupValue(reductionArgs[i]));
   }
   return success();
 }
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
index c6b8daccad313..6ef58b5013e08 100644
--- a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
@@ -73,7 +73,9 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK: %[[FREE_VAR:.+]] = load ptr, ptr %[[BUFF_PTR]], align 8
 //CHECK:  %[[ARRLAST:.+]] = getelementptr inbounds i32, ptr %[[FREE_VAR]], i32 100
 //CHECK:  %[[RES:.+]] = load i32, ptr %[[ARRLAST]], align 4
-//CHECK:  store i32 %[[RES]], ptr %loadgep{{.*}}, align 4
+//CHECK:  %[[CUR:.+]] = load i32, ptr %loadgep{{.*}}, align 4
+//CHECK:  %[[FIN:.+]] = select i1 {{.*}}, i32 %[[RES]], i32 %[[CUR]]
+//CHECK:  store i32 %[[FIN]], ptr %loadgep{{.*}}, align 4
 //CHECK: tail call void @free(ptr %[[FREE_VAR]])
 //CHECK: @__kmpc_end_masked
 //CHECK: omp.inscan.dispatch{{.*}}:                            ; preds = %omp_loop.body{{.*}}
@@ -84,7 +86,9 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK:   %[[LOG:.+]] = call double @llvm.log2.f64(double 1.000000e+02) #0
 //CHECK:   %[[CEIL:.+]] = call double @llvm.ceil.f64(double %[[LOG]]) #0
 //CHECK:   %[[UB:.+]] = fptoui double %[[CEIL]] to i32
-//CHECK:   br label %omp.outer.log.scan.body
+//CHECK:   br i1 {{.*}}, label %omp.outer.log.scan.body, label %omp.outer.log.scan.exit
+//CHECK: omp.outer.log.scan.exit:                          ; preds = %omp.inner.log.scan.exit, %omp_region.body{{.*}}
+//CHECK: @__kmpc_end_masked
 //CHECK: omp.outer.log.scan.body:
 //CHECK:   %[[K:.+]] = phi i32 [ 0, %{{.*}} ], [ %[[NEXTK:.+]], %omp.inner.log.scan.exit ]
 //CHECK:   %[[I:.+]] = phi i32 [ 1, %{{.*}} ], [ %[[NEXTI:.+]], %omp.inner.log.scan.exit ]
@@ -95,8 +99,6 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK:   %[[NEXTI]] = shl nuw i32 %[[I]], 1
 //CHECK:   %[[CMP2:.+]] = icmp ne i32 %[[NEXTK]], %[[UB]]
 //CHECK:   br i1 %[[CMP2]], label %omp.outer.log.scan.body, label %omp.outer.log.scan.exit
-//CHECK: omp.outer.log.scan.exit:                          ; preds = %omp.inner.log.scan.exit
-//CHECK: @__kmpc_end_masked
 //CHECK: omp.inner.log.scan.body:                          ; preds = %omp.inner.log.scan.body, %omp.outer.log.scan.body
 //CHECK:   %[[CNT:.+]] = phi i32 [ 99, %omp.outer.log.scan.body ], [ %[[CNTNXT:.+]], %omp.inner.log.scan.body ]
 //CHECK:   %[[BUFF:.+]] = load ptr, ptr %[[BUFF_PTR]]
@@ -114,7 +116,9 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK: omp.inscan.dispatch:                              ; preds = %omp_loop.body
 //CHECK:   br i1 true, label %omp.before.scan.bb, label %omp.after.scan.bb
 //CHECK: omp.before.scan.bb:
-//CHECK:   store i32 0, ptr %[[REDPRIV:.+]], align 4
+//CHECK:   %[[ISFIRST:.+]] = icmp eq i32 %{{.*}}, 1
+//CHECK:   %[[SEED:.+]] = select i1 %[[ISFIRST]], i32 %{{.*}}, i32 0
+//CHECK:   store i32 %[[SEED]], ptr %[[REDPRIV:.+]], align 4
 //CHECK: omp.loop_nest.region:                             ; preds = %omp.before.scan.bb
 //CHECK:   %[[BUFFER:.+]] = load ptr, ptr %loadgep_vla, align 8
 //CHECK:   %[[ARRAYOFFSET2:.+]] = getelementptr inbounds i32, ptr %[[BUFFER]], i32 %{{.*}}
diff --git a/mlir/test/Target/LLVMIR/openmp-todo.mlir b/mlir/test/Target/LLVMIR/openmp-todo.mlir
index 7edf88b5ea559..fe3c4259e75e0 100644
--- a/mlir/test/Target/LLVMIR/openmp-todo.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-todo.mlir
@@ -195,6 +195,37 @@ llvm.func @scan_reduction(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
 
 // -----
 
+omp.declare_reduction @add_f32 : f32
+init {
+^bb0(%arg: f32):
+  %0 = llvm.mlir.constant(0.0 : f32) : f32
+  omp.yield (%0 : f32)
+}
+combiner {
+^bb1(%arg0: f32, %arg1: f32):
+  %1 = llvm.fadd %arg0, %arg1 : f32
+  omp.yield (%1 : f32)
+}
+atomic {
+^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr):
+  %2 = llvm.load %arg3 : !llvm.ptr -> f32
+  llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32
+  omp.yield
+}
+llvm.func @scan_reduction_byref(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
+  // expected-error at below {{not yet implemented: Unhandled clause inscan reduction modifier with by-ref reduction in omp.wsloop operation}}
+  // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
+  omp.wsloop reduction(mod:inscan, byref @add_f32 %x -> %prv : !llvm.ptr) {
+    omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
+      omp.scan inclusive(%prv : !llvm.ptr)
+      omp.yield
+    }
+  }
+  llvm.return
+}
+
+// -----
+
 llvm.func @single_allocate(%x : !llvm.ptr) {
   // expected-error at below {{not yet implemented: Unhandled clause allocate in omp.single operation}}
   // expected-error at below {{LLVM Translation failed for operation: omp.single}}
diff --git a/openmp/runtime/test/scan/scan-bounds.f90 b/openmp/runtime/test/scan/scan-bounds.f90
new file mode 100644
index 0000000000000..9d624954568ff
--- /dev/null
+++ b/openmp/runtime/test/scan/scan-bounds.f90
@@ -0,0 +1,61 @@
+! RUN: %flang %flags %openmp_flags -fopenmp-version=51 %s -o %t.exe
+! RUN: %t.exe | FileCheck %s --match-full-lines
+
+! Arbitrary loop bounds and a non-unit stride must index the scan buffer by the
+! logical iteration number, not by the induction-variable value.
+program scan_bounds
+ implicit none
+ integer, parameter :: n = 30
+ integer :: a(n), b(n)
+ integer :: x, k
+
+ do k = 1, n
+   a(k) = k
+ end do
+
+ ! Arbitrary bounds: iterate from 10 to 20.
+ b = -1
+ x = 0
+ !$omp parallel do reduction(inscan, +: x)
+ do k = 10, 20
+   x = x + a(k)
+   !$omp scan inclusive(x)
+   b(k) = x
+ end do
+ print *, 'bounds x =', x
+ do k = 10, 20
+   print *, 'bb(', k, ') =', b(k)
+ end do
+
+ ! Non-unit stride: iterate from 1 to 10 step 2.
+ b = -1
+ x = 0
+ !$omp parallel do reduction(inscan, +: x)
+ do k = 1, 10, 2
+   x = x + a(k)
+   !$omp scan inclusive(x)
+   b(k) = x
+ end do
+ print *, 'stride x =', x
+ do k = 1, 10, 2
+   print *, 'sb(', k, ') =', b(k)
+ end do
+end program
+!CHECK: bounds x = 165
+!CHECK: bb( 10 ) = 10
+!CHECK: bb( 11 ) = 21
+!CHECK: bb( 12 ) = 33
+!CHECK: bb( 13 ) = 46
+!CHECK: bb( 14 ) = 60
+!CHECK: bb( 15 ) = 75
+!CHECK: bb( 16 ) = 91
+!CHECK: bb( 17 ) = 108
+!CHECK: bb( 18 ) = 126
+!CHECK: bb( 19 ) = 145
+!CHECK: bb( 20 ) = 165
+!CHECK: stride x = 25
+!CHECK: sb( 1 ) = 1
+!CHECK: sb( 3 ) = 4
+!CHECK: sb( 5 ) = 9
+!CHECK: sb( 7 ) = 16
+!CHECK: sb( 9 ) = 25
diff --git a/openmp/runtime/test/scan/scan-edge.f90 b/openmp/runtime/test/scan/scan-edge.f90
new file mode 100644
index 0000000000000..7cc00da958e6e
--- /dev/null
+++ b/openmp/runtime/test/scan/scan-edge.f90
@@ -0,0 +1,61 @@
+! RUN: %flang %flags %openmp_flags -fopenmp-version=51 %s -o %t.exe
+! RUN: %t.exe | FileCheck %s --match-full-lines
+
+! Zero-trip loops must not crash and must leave the reduction variable
+! unchanged, and runtime (non-constant) trip counts must work correctly.
+module scan_edge_mod
+contains
+ subroutine run_scan(n, a, b)
+   implicit none
+   integer :: n, k
+   integer :: a(n), b(n)
+   integer :: x
+   x = 0
+   !$omp parallel do reduction(inscan, +: x)
+   do k = 1, n
+     x = x + a(k)
+     !$omp scan inclusive(x)
+     b(k) = x
+   end do
+ end subroutine
+end module
+
+program scan_edge
+ use scan_edge_mod
+ implicit none
+ integer :: a(6), b(6), k
+ integer :: z, zk
+ integer :: zb(1)
+
+ ! Runtime bounds: the trip count of run_scan is not a compile-time constant.
+ do k = 1, 6
+   a(k) = k
+ end do
+ b = -1
+ call run_scan(6, a, b)
+ do k = 1, 6
+   print *, 'rb(', k, ') =', b(k)
+ end do
+
+ ! Runtime zero-trip: n = 0 must not crash.
+ call run_scan(0, a, b)
+ print *, 'runtime zero-trip ok'
+
+ ! Constant zero-trip with a non-identity start: z must be unchanged.
+ z = 42
+ !$omp parallel do reduction(inscan, +: z)
+ do zk = 1, 0
+   z = z + 1
+   !$omp scan inclusive(z)
+   zb(zk) = z
+ end do
+ print *, 'z =', z
+end program
+!CHECK: rb( 1 ) = 1
+!CHECK: rb( 2 ) = 3
+!CHECK: rb( 3 ) = 6
+!CHECK: rb( 4 ) = 10
+!CHECK: rb( 5 ) = 15
+!CHECK: rb( 6 ) = 21
+!CHECK: runtime zero-trip ok
+!CHECK: z = 42
diff --git a/openmp/runtime/test/scan/scan-nonidentity.f90 b/openmp/runtime/test/scan/scan-nonidentity.f90
new file mode 100644
index 0000000000000..064b0d757af6e
--- /dev/null
+++ b/openmp/runtime/test/scan/scan-nonidentity.f90
@@ -0,0 +1,28 @@
+! RUN: %flang %flags %openmp_flags -fopenmp-version=51 %s -o %t.exe
+! RUN: %t.exe | FileCheck %s --match-full-lines
+
+! The reduction variable's non-identity incoming value must be included in the
+! first partial result of an inclusive scan.
+program scan_nonidentity
+ implicit none
+ integer :: x, k
+ integer :: b(4)
+
+ x = 10
+ !$omp parallel do reduction(inscan, +: x)
+ do k = 1, 4
+   x = x + 1
+   !$omp scan inclusive(x)
+   b(k) = x
+ end do
+
+ print *, 'x =', x
+ do k = 1, 4
+   print *, 'b(', k, ') =', b(k)
+ end do
+end program
+!CHECK: x = 14
+!CHECK: b( 1 ) = 11
+!CHECK: b( 2 ) = 12
+!CHECK: b( 3 ) = 13
+!CHECK: b( 4 ) = 14

>From 5506275c5cec52b0991c9f85bddeb1c8ebabf451 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Mon, 27 Jul 2026 10:07:57 -0500
Subject: [PATCH 04/10] updated with review comments fixes

---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     | 23 +++--
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      | 25 ++++-
 mlir/test/Target/LLVMIR/openmp-todo.mlir      | 94 +++++++++++++++++--
 openmp/runtime/test/scan/scan-i64.f90         | 37 ++++++++
 openmp/runtime/test/scan/scan-nowait.f90      | 38 ++++++++
 5 files changed, 198 insertions(+), 19 deletions(-)
 create mode 100644 openmp/runtime/test/scan/scan-i64.f90
 create mode 100644 openmp/runtime/test/scan/scan-nowait.f90

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 90db03da7380c..0ed215b3e2ea6 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5350,10 +5350,13 @@ Error OpenMPIRBuilder::emitScanBasedDirectiveDeclsIR(
   auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
                        ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
     Builder.restoreIP(CodeGenIP);
-    Value *AllocSpan =
-        Builder.CreateAdd(ScanRedInfo->Span, Builder.getInt32(1));
+    // Use the loop's own index type (matching `Span`) so the buffer allocation
+    // works for both i32 and i64 (e.g. `integer(kind=8)`) induction variables.
+    Type *IndexTy = ScanRedInfo->Span->getType();
+    Value *AllocSpan = Builder.CreateAdd(ScanRedInfo->Span,
+                                         ConstantInt::get(IndexTy, 1));
     for (size_t i = 0; i < ScanVars.size(); i++) {
-      Type *IntPtrTy = Builder.getInt32Ty();
+      Type *IntPtrTy = IndexTy;
       Constant *Allocsize = ConstantExpr::getSizeOf(ScanVarsType[i]);
       Allocsize = ConstantExpr::getTruncOrBitCast(Allocsize, IntPtrTy);
       Value *Buff = Builder.CreateMalloc(IntPtrTy, ScanVarsType[i], Allocsize,
@@ -5453,6 +5456,10 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
                        ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
     Builder.restoreIP(CodeGenIP);
     Function *CurFn = Builder.GetInsertBlock()->getParent();
+    // Index/counter values that participate in buffer-offset arithmetic must
+    // use the loop's own index type (`Span`'s type) so scan works for both i32
+    // and i64 (e.g. `integer(kind=8)`) induction variables.
+    Type *IndexTy = ScanRedInfo->Span->getType();
     // for (int k = 0; k <= ceil(log2(n)); ++k)
     llvm::BasicBlock *LoopBB =
         BasicBlock::Create(CurFn->getContext(), "omp.outer.log.scan.body");
@@ -5488,10 +5495,10 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
 
     PHINode *Counter = Builder.CreatePHI(Builder.getInt32Ty(), 2);
     // size pow2k = 1;
-    PHINode *Pow2K = Builder.CreatePHI(Builder.getInt32Ty(), 2);
+    PHINode *Pow2K = Builder.CreatePHI(IndexTy, 2);
     Counter->addIncoming(llvm::ConstantInt::get(Builder.getInt32Ty(), 0),
                          InputBB);
-    Pow2K->addIncoming(llvm::ConstantInt::get(Builder.getInt32Ty(), 1),
+    Pow2K->addIncoming(llvm::ConstantInt::get(IndexTy, 1),
                        InputBB);
     // for (size i = n - 1; i >= 2 ^ k; --i)
     //   tmp[i] op= tmp[i-pow2k];
@@ -5503,14 +5510,14 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
     Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
     emitBlock(InnerLoopBB, CurFn);
     Builder.SetInsertPoint(InnerLoopBB);
-    PHINode *IVal = Builder.CreatePHI(Builder.getInt32Ty(), 2);
+    PHINode *IVal = Builder.CreatePHI(IndexTy, 2);
     IVal->addIncoming(NMin1, LoopBB);
     for (ReductionInfo RedInfo : ReductionInfos) {
       Value *ReductionVal = RedInfo.PrivateVariable;
       Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ReductionVal];
       Value *Buff = Builder.CreateLoad(Builder.getPtrTy(), BuffPtr);
       Type *DestTy = RedInfo.ElementType;
-      Value *IV = Builder.CreateAdd(IVal, Builder.getInt32(1));
+      Value *IV = Builder.CreateAdd(IVal, ConstantInt::get(IndexTy, 1));
       Value *LHSPtr =
           Builder.CreateInBoundsGEP(DestTy, Buff, IV, "arrayOffset");
       Value *OffsetIval = Builder.CreateNUWSub(IV, Pow2K);
@@ -5526,7 +5533,7 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
       Builder.CreateStore(Result, LHSPtr);
     }
     llvm::Value *NextIVal = Builder.CreateNUWSub(
-        IVal, llvm::ConstantInt::get(Builder.getInt32Ty(), 1));
+        IVal, llvm::ConstantInt::get(IndexTy, 1));
     IVal->addIncoming(NextIVal, Builder.GetInsertBlock());
     CmpI = Builder.CreateICmpUGE(NextIVal, Pow2K);
     Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 7b8be030da4dc..b7381244a50b0 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -471,6 +471,22 @@ static LogicalResult checkImplementationStatus(Operation &op) {
               break;
             }
         }
+        // Scan reductions allocate a shared temporary buffer in the alloca
+        // frame of the enclosing parallel region. An orphaned worksharing loop
+        // (e.g. an `omp do` in a subroutine that is called from within a
+        // parallel region) has no such frame in its own function, so the buffer
+        // would be allocated per-thread and produce wrong results.
+        if (!op->template getParentOfType<omp::ParallelOp>())
+          result = todo("inscan reduction on an orphaned worksharing loop "
+                        "(no enclosing parallel region)");
+        // Scan reductions are only implemented for a single loop; a collapsed
+        // (multi-dimensional) loop nest is not yet supported.
+        if (auto wsloopOp = dyn_cast<omp::WsloopOp>(op.getOperation()))
+          if (auto loopNest = dyn_cast_or_null<omp::LoopNestOp>(
+                  wsloopOp.getWrappedLoop()))
+            if (loopNest.getNumLoops() > 1)
+              result = todo("inscan reduction on a collapsed "
+                            "(multi-dimensional) loop nest");
       }
     }
   };
@@ -4699,9 +4715,16 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
       linearClauseProcessor.rewriteInPlace(builder, loopInfo->getBody(),
                                            loopInfo->getLatch(), index);
 
+    // Scan reductions need a barrier at the end of the input (first) loop so
+    // that every thread has finished writing the temporary buffer before the
+    // masked prefix-sum reads it. A source-level `nowait` only elides the final
+    // barrier after the scan (second) loop, so force the barrier for the input
+    // loop regardless of `nowait` to avoid a data race.
+    bool needsBarrier =
+        loopNeedsBarrier || (isInScanRegion && inputScanLoop);
     llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP =
         ompBuilder->applyWorkshareLoop(
-            ompLoc.DL, loopInfo, allocaIP, loopNeedsBarrier,
+            ompLoc.DL, loopInfo, allocaIP, needsBarrier,
             convertToScheduleKind(schedule), chunk, isSimd,
             scheduleMod == omp::ScheduleModifier::monotonic,
             scheduleMod == omp::ScheduleModifier::nonmonotonic, isOrdered,
diff --git a/mlir/test/Target/LLVMIR/openmp-todo.mlir b/mlir/test/Target/LLVMIR/openmp-todo.mlir
index 44817110d8b20..befa9bba2680e 100644
--- a/mlir/test/Target/LLVMIR/openmp-todo.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-todo.mlir
@@ -180,15 +180,19 @@ atomic {
   omp.yield
 }
 llvm.func @scan_reduction(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
-  // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
-  omp.wsloop reduction(mod:inscan, @add_f32 %x -> %prv : !llvm.ptr) {
-    // expected-error at below {{LLVM Translation failed for operation: omp.loop_nest}}
-    omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
-      // expected-error at below {{not yet implemented: Unhandled clause exclusive in omp.scan operation}}
-      // expected-error at below {{LLVM Translation failed for operation: omp.scan}}
-      omp.scan exclusive(%prv : !llvm.ptr)
-      omp.yield
+  // expected-error at below {{LLVM Translation failed for operation: omp.parallel}}
+  omp.parallel {
+    // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
+    omp.wsloop reduction(mod:inscan, @add_f32 %x -> %prv : !llvm.ptr) {
+      // expected-error at below {{LLVM Translation failed for operation: omp.loop_nest}}
+      omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
+        // expected-error at below {{not yet implemented: Unhandled clause exclusive in omp.scan operation}}
+        // expected-error at below {{LLVM Translation failed for operation: omp.scan}}
+        omp.scan exclusive(%prv : !llvm.ptr)
+        omp.yield
+      }
     }
+    omp.terminator
   }
   llvm.return
 }
@@ -213,9 +217,44 @@ atomic {
   omp.yield
 }
 llvm.func @scan_reduction_byref(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
-  // expected-error at below {{not yet implemented: Unhandled clause inscan reduction modifier with by-ref reduction in omp.wsloop operation}}
+  // expected-error at below {{LLVM Translation failed for operation: omp.parallel}}
+  omp.parallel {
+    // expected-error at below {{not yet implemented: Unhandled clause inscan reduction modifier with by-ref reduction in omp.wsloop operation}}
+    // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
+    omp.wsloop reduction(mod:inscan, byref @add_f32 %x -> %prv : !llvm.ptr) {
+      omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
+        omp.scan inclusive(%prv : !llvm.ptr)
+        omp.yield
+      }
+    }
+    omp.terminator
+  }
+  llvm.return
+}
+
+// -----
+
+omp.declare_reduction @add_f32 : f32
+init {
+^bb0(%arg: f32):
+  %0 = llvm.mlir.constant(0.0 : f32) : f32
+  omp.yield (%0 : f32)
+}
+combiner {
+^bb1(%arg0: f32, %arg1: f32):
+  %1 = llvm.fadd %arg0, %arg1 : f32
+  omp.yield (%1 : f32)
+}
+atomic {
+^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr):
+  %2 = llvm.load %arg3 : !llvm.ptr -> f32
+  llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32
+  omp.yield
+}
+llvm.func @scan_reduction_orphaned(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
+  // expected-error at below {{not yet implemented: Unhandled clause inscan reduction on an orphaned worksharing loop (no enclosing parallel region) in omp.wsloop operation}}
   // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
-  omp.wsloop reduction(mod:inscan, byref @add_f32 %x -> %prv : !llvm.ptr) {
+  omp.wsloop reduction(mod:inscan, @add_f32 %x -> %prv : !llvm.ptr) {
     omp.loop_nest (%iv) : i32 = (%lb) to (%ub) step (%step) {
       omp.scan inclusive(%prv : !llvm.ptr)
       omp.yield
@@ -226,6 +265,41 @@ llvm.func @scan_reduction_byref(%lb : i32, %ub : i32, %step : i32, %x : !llvm.pt
 
 // -----
 
+omp.declare_reduction @add_f32 : f32
+init {
+^bb0(%arg: f32):
+  %0 = llvm.mlir.constant(0.0 : f32) : f32
+  omp.yield (%0 : f32)
+}
+combiner {
+^bb1(%arg0: f32, %arg1: f32):
+  %1 = llvm.fadd %arg0, %arg1 : f32
+  omp.yield (%1 : f32)
+}
+atomic {
+^bb2(%arg2: !llvm.ptr, %arg3: !llvm.ptr):
+  %2 = llvm.load %arg3 : !llvm.ptr -> f32
+  llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32
+  omp.yield
+}
+llvm.func @scan_reduction_multidim(%lb : i32, %ub : i32, %step : i32, %x : !llvm.ptr) {
+  // expected-error at below {{LLVM Translation failed for operation: omp.parallel}}
+  omp.parallel {
+    // expected-error at below {{not yet implemented: Unhandled clause inscan reduction on a collapsed (multi-dimensional) loop nest in omp.wsloop operation}}
+    // expected-error at below {{LLVM Translation failed for operation: omp.wsloop}}
+    omp.wsloop reduction(mod:inscan, @add_f32 %x -> %prv : !llvm.ptr) {
+      omp.loop_nest (%iv, %iv2) : i32 = (%lb, %lb) to (%ub, %ub) step (%step, %step) {
+        omp.scan inclusive(%prv : !llvm.ptr)
+        omp.yield
+      }
+    }
+    omp.terminator
+  }
+  llvm.return
+}
+
+// -----
+
 llvm.func @single_allocate(%x : !llvm.ptr) {
   // expected-error at below {{not yet implemented: Unhandled clause allocate in omp.single operation}}
   // expected-error at below {{LLVM Translation failed for operation: omp.single}}
diff --git a/openmp/runtime/test/scan/scan-i64.f90 b/openmp/runtime/test/scan/scan-i64.f90
new file mode 100644
index 0000000000000..c2ef3e6e0ac76
--- /dev/null
+++ b/openmp/runtime/test/scan/scan-i64.f90
@@ -0,0 +1,37 @@
+! RUN: %flang %flags %openmp_flags -fopenmp-version=51 %s -o %t.exe
+! RUN: %t.exe | FileCheck %s --match-full-lines
+
+! An `integer(kind=8)` (i64) loop induction variable must work with an inscan
+! reduction. The scan temporary buffer is indexed using the loop's own index
+! type, so an i64 induction variable must not trip the i32/i64 type mismatch in
+! the scan buffer allocation or the prefix-sum computation.
+program scan_i64
+ implicit none
+ integer(kind=8) :: i, n
+ integer :: k
+ integer :: x
+ integer :: b(8)
+
+ n = 8
+ x = 0
+ !$omp parallel do reduction(inscan, +: x)
+ do i = 1_8, n
+   x = x + 1
+   !$omp scan inclusive(x)
+   b(i) = x
+ end do
+
+ print *, 'x =', x
+ do k = 1, 8
+  print *, 'b(', k, ') =', b(k)
+ end do
+end program
+!CHECK: x = 8
+!CHECK: b( 1 ) = 1
+!CHECK: b( 2 ) = 2
+!CHECK: b( 3 ) = 3
+!CHECK: b( 4 ) = 4
+!CHECK: b( 5 ) = 5
+!CHECK: b( 6 ) = 6
+!CHECK: b( 7 ) = 7
+!CHECK: b( 8 ) = 8
diff --git a/openmp/runtime/test/scan/scan-nowait.f90 b/openmp/runtime/test/scan/scan-nowait.f90
new file mode 100644
index 0000000000000..bb17aeeedf9d9
--- /dev/null
+++ b/openmp/runtime/test/scan/scan-nowait.f90
@@ -0,0 +1,38 @@
+! RUN: %flang %flags %openmp_flags -fopenmp-version=51 %s -o %t.exe
+! RUN: env OMP_NUM_THREADS=4 %t.exe | FileCheck %s --match-full-lines
+
+! A source-level `nowait` on a scan worksharing loop must only remove the final
+! barrier after the scan (second) loop. The barrier at the end of the input
+! (first) loop is required so that every thread finishes writing the temporary
+! buffer before the masked prefix-sum reads it. Forcing multiple threads
+! exercises the data race that occurred when `nowait` incorrectly removed that
+! internal barrier.
+program scan_nowait
+ implicit none
+ integer :: x, k
+ integer :: b(8)
+
+ b = -1
+ x = 0
+ !$omp parallel
+ !$omp do reduction(inscan, +: x)
+ do k = 1, 8
+   x = x + 1
+   !$omp scan inclusive(x)
+   b(k) = x
+ end do
+ !$omp end do nowait
+ !$omp end parallel
+
+ do k = 1, 8
+  print *, 'b(', k, ') =', b(k)
+ end do
+end program
+!CHECK: b( 1 ) = 1
+!CHECK: b( 2 ) = 2
+!CHECK: b( 3 ) = 3
+!CHECK: b( 4 ) = 4
+!CHECK: b( 5 ) = 5
+!CHECK: b( 6 ) = 6
+!CHECK: b( 7 ) = 7
+!CHECK: b( 8 ) = 8

>From c5283fcf901ea9d20f64a59ac15a2795c19abac0 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Mon, 27 Jul 2026 11:37:51 -0500
Subject: [PATCH 05/10] git formatting

---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp             | 11 +++++------
 .../Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp      |  7 +++----
 2 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 9f82ba18ded81..ff5d325075b93 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5378,8 +5378,8 @@ Error OpenMPIRBuilder::emitScanBasedDirectiveDeclsIR(
     // Use the loop's own index type (matching `Span`) so the buffer allocation
     // works for both i32 and i64 (e.g. `integer(kind=8)`) induction variables.
     Type *IndexTy = ScanRedInfo->Span->getType();
-    Value *AllocSpan = Builder.CreateAdd(ScanRedInfo->Span,
-                                         ConstantInt::get(IndexTy, 1));
+    Value *AllocSpan =
+        Builder.CreateAdd(ScanRedInfo->Span, ConstantInt::get(IndexTy, 1));
     for (size_t i = 0; i < ScanVars.size(); i++) {
       Type *IntPtrTy = IndexTy;
       Constant *Allocsize = ConstantExpr::getSizeOf(ScanVarsType[i]);
@@ -5523,8 +5523,7 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
     PHINode *Pow2K = Builder.CreatePHI(IndexTy, 2);
     Counter->addIncoming(llvm::ConstantInt::get(Builder.getInt32Ty(), 0),
                          InputBB);
-    Pow2K->addIncoming(llvm::ConstantInt::get(IndexTy, 1),
-                       InputBB);
+    Pow2K->addIncoming(llvm::ConstantInt::get(IndexTy, 1), InputBB);
     // for (size i = n - 1; i >= 2 ^ k; --i)
     //   tmp[i] op= tmp[i-pow2k];
     llvm::BasicBlock *InnerLoopBB =
@@ -5557,8 +5556,8 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
         return AfterIP.takeError();
       Builder.CreateStore(Result, LHSPtr);
     }
-    llvm::Value *NextIVal = Builder.CreateNUWSub(
-        IVal, llvm::ConstantInt::get(IndexTy, 1));
+    llvm::Value *NextIVal =
+        Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(IndexTy, 1));
     IVal->addIncoming(NextIVal, Builder.GetInsertBlock());
     CmpI = Builder.CreateICmpUGE(NextIVal, Pow2K);
     Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index d1a31a6870ced..7c27a18d285fb 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -540,8 +540,8 @@ static LogicalResult checkImplementationStatus(Operation &op) {
         // Scan reductions are only implemented for a single loop; a collapsed
         // (multi-dimensional) loop nest is not yet supported.
         if (auto wsloopOp = dyn_cast<omp::WsloopOp>(op.getOperation()))
-          if (auto loopNest = dyn_cast_or_null<omp::LoopNestOp>(
-                  wsloopOp.getWrappedLoop()))
+          if (auto loopNest =
+                  dyn_cast_or_null<omp::LoopNestOp>(wsloopOp.getWrappedLoop()))
             if (loopNest.getNumLoops() > 1)
               result = todo("inscan reduction on a collapsed "
                             "(multi-dimensional) loop nest");
@@ -4876,8 +4876,7 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
     // masked prefix-sum reads it. A source-level `nowait` only elides the final
     // barrier after the scan (second) loop, so force the barrier for the input
     // loop regardless of `nowait` to avoid a data race.
-    bool needsBarrier =
-        loopNeedsBarrier || (isInScanRegion && inputScanLoop);
+    bool needsBarrier = loopNeedsBarrier || (isInScanRegion && inputScanLoop);
     llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP =
         ompBuilder->applyWorkshareLoop(
             ompLoc.DL, loopInfo, allocaIP, needsBarrier,

>From 42d781704befcc65923ff419c6ba461d4796afbb Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Tue, 28 Jul 2026 11:01:16 -0500
Subject: [PATCH 06/10] rework on second race cond

---
 .../llvm/Frontend/OpenMP/OMPIRBuilder.h       | 14 +++-
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     | 40 +++++++---
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      | 19 +++--
 .../LLVMIR/openmp-reduction-scan-nowait.mlir  | 73 +++++++++++++++++++
 4 files changed, 127 insertions(+), 19 deletions(-)
 create mode 100644 mlir/test/Target/LLVMIR/openmp-reduction-scan-nowait.mlir

diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index 05307414c94d0..87dc1f067463e 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -1940,11 +1940,17 @@ class OpenMPIRBuilder {
   /// \param ReductionInfos Array type containing the ReductionOps.
   /// \param ScanRedInfo    Pointer to the ScanInfo objected created using
   ///                       `ScanInfoInitialize`.
+  /// \param NoWait         Whether the enclosing worksharing construct has a
+  ///                       `nowait` clause. The barrier that keeps the shared
+  ///                       scan buffer alive until every thread has finished
+  ///                       reading it is always emitted; only the
+  ///                       end-of-construct barrier after the masked region is
+  ///                       skipped when \p NoWait is set.
   ///
   /// \return error if any produced, else return success.
   Error emitScanBasedDirectiveFinalsIR(
       ArrayRef<llvm::OpenMPIRBuilder::ReductionInfo> ReductionInfos,
-      ScanInfo *ScanInfo);
+      ScanInfo *ScanInfo, bool NoWait = false);
 
   /// This function emits a helper that gathers Reduce lists from the first
   /// lane of every active warp to lanes in the first warp.
@@ -3173,12 +3179,16 @@ class OpenMPIRBuilder {
   /// \param ReductionInfos Array type containing the ReductionOps.
   /// \param ScanRedInfo    Pointer to the ScanInfo objected created using
   ///                       `ScanInfoInitialize`.
+  /// \param NoWait         Whether the enclosing worksharing construct has a
+  ///                       `nowait` clause. Only the end-of-construct barrier
+  ///                       emitted after freeing the shared scan buffer is
+  ///                       elided; barriers required for correctness are kept.
   ///
   /// \returns The insertion position *after* the masked.
   LLVM_ABI InsertPointOrErrorTy emitScanReduction(
       const LocationDescription &Loc,
       ArrayRef<llvm::OpenMPIRBuilder::ReductionInfo> ReductionInfos,
-      ScanInfo *ScanRedInfo);
+      ScanInfo *ScanRedInfo, bool NoWait = false);
 
   /// This directive split and directs the control flow to input phase
   ///  blocks or scan phase blocks based on 1. whether input loop or scan loop
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index ff5d325075b93..e81768e884922 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5414,7 +5414,8 @@ Error OpenMPIRBuilder::emitScanBasedDirectiveDeclsIR(
 }
 
 Error OpenMPIRBuilder::emitScanBasedDirectiveFinalsIR(
-    ArrayRef<ReductionInfo> ReductionInfos, ScanInfo *ScanRedInfo) {
+    ArrayRef<ReductionInfo> ReductionInfos, ScanInfo *ScanRedInfo,
+    bool NoWait) {
   auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
                        ArrayRef<BasicBlock *> DeallocBlocks) -> Error {
     Builder.restoreIP(CodeGenIP);
@@ -5453,27 +5454,45 @@ Error OpenMPIRBuilder::emitScanBasedDirectiveFinalsIR(
   else
     Builder.SetInsertPoint(ScanRedInfo->OMPScanFinish);
 
-  llvm::Value *FilterVal = Builder.getInt32(0);
+  // Synchronize before the masked region frees the shared scan buffer. The
+  // scan (second) loop loads each thread's prefix value from the buffer, so
+  // every thread must have finished that loop before the single masked thread
+  // frees it; otherwise a thread still executing the loop could load from
+  // freed memory. A `masked` region is not a barrier, so this synchronization
+  // is required for correctness and is emitted even under `nowait`.
   llvm::OpenMPIRBuilder::InsertPointOrErrorTy AfterIP =
-      createMasked(Builder.saveIP(), BodyGenCB, FiniCB, FilterVal);
-
+      createBarrier(Builder.saveIP(), llvm::omp::OMPD_barrier);
   if (!AfterIP)
     return AfterIP.takeError();
   Builder.restoreIP(*AfterIP);
-  BasicBlock *InputBB = Builder.GetInsertBlock();
-  if (InputBB->hasTerminator())
-    Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
-  AfterIP = createBarrier(Builder.saveIP(), llvm::omp::OMPD_barrier);
+
+  llvm::Value *FilterVal = Builder.getInt32(0);
+  AfterIP = createMasked(Builder.saveIP(), BodyGenCB, FiniCB, FilterVal);
+
   if (!AfterIP)
     return AfterIP.takeError();
   Builder.restoreIP(*AfterIP);
+
+  // The barrier after the masked region provides the construct's
+  // end-of-construct synchronization and publishes the final reduction value
+  // stored into the original variable inside the masked region. Skip it when
+  // the construct has `nowait`, matching a normal worksharing loop.
+  if (!NoWait) {
+    BasicBlock *InputBB = Builder.GetInsertBlock();
+    if (InputBB->hasTerminator())
+      Builder.SetInsertPoint(Builder.GetInsertBlock()->getTerminator());
+    AfterIP = createBarrier(Builder.saveIP(), llvm::omp::OMPD_barrier);
+    if (!AfterIP)
+      return AfterIP.takeError();
+    Builder.restoreIP(*AfterIP);
+  }
   return Error::success();
 }
 
 OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
     const LocationDescription &Loc,
     ArrayRef<llvm::OpenMPIRBuilder::ReductionInfo> ReductionInfos,
-    ScanInfo *ScanRedInfo) {
+    ScanInfo *ScanRedInfo, bool NoWait) {
 
   if (!updateToLocation(Loc))
     return Loc.IP;
@@ -5590,7 +5609,8 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
   if (!AfterIP)
     return AfterIP.takeError();
   Builder.restoreIP(*AfterIP);
-  Error Err = emitScanBasedDirectiveFinalsIR(ReductionInfos, ScanRedInfo);
+  Error Err =
+      emitScanBasedDirectiveFinalsIR(ReductionInfos, ScanRedInfo, NoWait);
   if (Err)
     return Err;
 
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 7c27a18d285fb..e5e0acf008116 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -4871,12 +4871,17 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
       linearClauseProcessor.rewriteInPlace(builder, loopInfo->getBody(),
                                            loopInfo->getLatch(), index);
 
-    // Scan reductions need a barrier at the end of the input (first) loop so
-    // that every thread has finished writing the temporary buffer before the
-    // masked prefix-sum reads it. A source-level `nowait` only elides the final
-    // barrier after the scan (second) loop, so force the barrier for the input
-    // loop regardless of `nowait` to avoid a data race.
-    bool needsBarrier = loopNeedsBarrier || (isInScanRegion && inputScanLoop);
+    // Scan reductions manage the barriers around the shared temporary buffer
+    // explicitly:
+    //  * The input (first) loop always needs its trailing barrier so that every
+    //    thread has finished writing the buffer before the masked prefix-sum
+    //    reads it; this is required even under `nowait`.
+    //  * The scan (second) loop must not emit its own trailing barrier here.
+    //    `emitScanReduction` emits a barrier *before* the masked region frees
+    //    the buffer (so no thread frees it while another is still reading it)
+    //    and, unless `nowait` is present, a barrier *after* it for
+    //    end-of-construct synchronization.
+    bool needsBarrier = isInScanRegion ? inputScanLoop : loopNeedsBarrier;
     llvm::OpenMPIRBuilder::InsertPointOrErrorTy wsloopIP =
         ompBuilder->applyWorkshareLoop(
             ompLoc.DL, loopInfo, allocaIP, needsBarrier,
@@ -4930,7 +4935,7 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder,
     llvm::ScanInfo *scanInfo = findScanInfo(moduleTranslation);
     llvm::OpenMPIRBuilder::InsertPointOrErrorTy redIP =
         ompBuilder->emitScanReduction(builder.saveIP(), reductionInfos,
-                                      scanInfo);
+                                      scanInfo, wsloopOp.getNowait());
     if (failed(handleError(redIP, opInst)))
       return failure();
 
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan-nowait.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan-nowait.mlir
new file mode 100644
index 0000000000000..95306535c92cd
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan-nowait.mlir
@@ -0,0 +1,73 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// Verify buffer-lifetime handling for an inclusive scan reduction on a
+// worksharing loop with `nowait`. The scan (second) loop reads each thread's
+// prefix value from the shared scan buffer, and a masked region then frees
+// that buffer. A barrier must be emitted *before* the masked region so the
+// buffer cannot be freed while another thread is still reading it (a masked
+// region is not a barrier). Because of `nowait`, no end-of-construct barrier
+// is emitted *after* the masked region.
+
+omp.declare_reduction @add_reduction_i32 : i32 init {
+^bb0(%arg0: i32):
+  %0 = llvm.mlir.constant(0 : i32) : i32
+  omp.yield(%0 : i32)
+} combiner {
+^bb0(%arg0: i32, %arg1: i32):
+  %0 = llvm.add %arg0, %arg1 : i32
+  omp.yield(%0 : i32)
+}
+llvm.func @scan_reduction_nowait() {
+  %0 = llvm.mlir.constant(1 : i64) : i64
+  %5 = llvm.alloca %0 x i32 {bindc_name = "x"} : (i64) -> !llvm.ptr
+  %10 = llvm.mlir.constant(100 : i32) : i32
+  %11 = llvm.mlir.constant(1 : i32) : i32
+  %14 = llvm.mlir.addressof @_QFEa : !llvm.ptr
+  %15 = llvm.mlir.addressof @_QFEb : !llvm.ptr
+  omp.parallel {
+    %37 = llvm.mlir.constant(1 : i64) : i64
+    %38 = llvm.alloca %37 x i32 {bindc_name = "k", pinned} : (i64) -> !llvm.ptr
+    omp.wsloop nowait reduction(mod: inscan, @add_reduction_i32 %5 -> %arg0 : !llvm.ptr) {
+      omp.loop_nest (%arg1) : i32 = (%11) to (%10) inclusive step (%11) {
+        llvm.store %arg1, %38 : i32, !llvm.ptr
+        %40 = llvm.load %arg0 : !llvm.ptr -> i32
+        %41 = llvm.load %38 : !llvm.ptr -> i32
+        %42 = llvm.sext %41 : i32 to i64
+        %50 = llvm.getelementptr %14[%42] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+        %51 = llvm.load %50 : !llvm.ptr -> i32
+        %52 = llvm.add %40, %51 : i32
+        llvm.store %52, %arg0 : i32, !llvm.ptr
+        omp.scan inclusive(%arg0 : !llvm.ptr)
+        llvm.store %arg1, %38 : i32, !llvm.ptr
+        %53 = llvm.load %arg0 : !llvm.ptr -> i32
+        %54 = llvm.load %38 : !llvm.ptr -> i32
+        %55 = llvm.sext %54 : i32 to i64
+        %63 = llvm.getelementptr %15[%55] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+        llvm.store %53, %63 : i32, !llvm.ptr
+        omp.yield
+      }
+    }
+    omp.terminator
+  }
+  llvm.return
+}
+llvm.mlir.global internal @_QFEa() {addr_space = 0 : i32} : !llvm.array<100 x i32> {
+  %0 = llvm.mlir.zero : !llvm.array<100 x i32>
+  llvm.return %0 : !llvm.array<100 x i32>
+}
+llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i32> {
+  %0 = llvm.mlir.zero : !llvm.array<100 x i32>
+  llvm.return %0 : !llvm.array<100 x i32>
+}
+
+// CHECK-LABEL: define internal void @scan_reduction_nowait..omp_par
+// After the scan (second) loop, the mandatory buffer-lifetime barrier is
+// emitted before the masked region that frees the shared scan buffer.
+// CHECK: omp.scan.loop.cont:
+// CHECK: call void @__kmpc_barrier
+// CHECK: call i32 @__kmpc_masked
+// CHECK: call void @free(ptr
+// CHECK: call void @__kmpc_end_masked
+// With `nowait`, there is no end-of-construct barrier after the buffer is freed.
+// CHECK-NOT: call void @__kmpc_barrier
+// CHECK: ret void

>From 46cbc9122d8fa036d02c61ea3d740030701eb824 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Tue, 28 Jul 2026 12:18:10 -0500
Subject: [PATCH 07/10] fixex iteration input now  starts from reduction
 identity

---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     | 32 +++++++++++++++++++
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      | 21 ++++++------
 .../Target/LLVMIR/openmp-reduction-scan.mlir  | 16 ++++++++--
 3 files changed, 54 insertions(+), 15 deletions(-)

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index e81768e884922..a75c39be196c1 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5525,6 +5525,38 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
         ScanRedInfo->Span,
         llvm::ConstantInt::get(ScanRedInfo->Span->getType(), 1));
     Builder.SetInsertPoint(InputBB);
+    // Combine the original variable's incoming value (orig-val) into the first
+    // buffer element before computing the prefix sum. Per the OpenMP scan
+    // semantics orig-val is a single prefix element that must be reflected in
+    // every inclusive/exclusive scan result and in the final reduction value.
+    // Folding it in here lets the log-scan prefix computation propagate it into
+    // every element (and into the finals' read of `buffer[Span]`), regardless
+    // of whether an iteration's input phase accumulates into or overwrites the
+    // reduction variable. `select(Span > 0, 1, 0)` keeps the access in bounds
+    // when Span == 0: buffer[0] is otherwise unused and the finals' zero-trip
+    // guard discards it.
+    llvm::Value *HasElem = Builder.CreateICmpUGT(
+        ScanRedInfo->Span, llvm::ConstantInt::get(IndexTy, 0));
+    llvm::Value *FirstIdx =
+        Builder.CreateSelect(HasElem, llvm::ConstantInt::get(IndexTy, 1),
+                             llvm::ConstantInt::get(IndexTy, 0));
+    for (ReductionInfo RedInfo : ReductionInfos) {
+      Value *ReductionVal = RedInfo.PrivateVariable;
+      Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ReductionVal];
+      Value *Buff = Builder.CreateLoad(Builder.getPtrTy(), BuffPtr);
+      Type *DestTy = RedInfo.ElementType;
+      Value *ElemPtr =
+          Builder.CreateInBoundsGEP(DestTy, Buff, FirstIdx, "arrayOffset");
+      Value *OrigVal = Builder.CreateLoad(DestTy, RedInfo.Variable);
+      Value *Elem = Builder.CreateLoad(DestTy, ElemPtr);
+      llvm::Value *Combined;
+      InsertPointOrErrorTy AfterIP =
+          RedInfo.ReductionGen(Builder.saveIP(), Elem, OrigVal, Combined);
+      if (!AfterIP)
+        return AfterIP.takeError();
+      Builder.restoreIP(*AfterIP);
+      Builder.CreateStore(Combined, ElemPtr);
+    }
     // The log-scan prefix computation is only well-defined when there are at
     // least two elements: `log2(Span)` is invalid for Span == 0 and the loop
     // trip computation underflows/loops forever for Span <= 1. For Span <= 1
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index e5e0acf008116..23f2d06fdabe2 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -5526,19 +5526,16 @@ initScanReductionVars(omp::LoopNestOp loopOp, llvm::IRBuilderBase &builder,
            "expected one value to be yielded from the reduction init region");
     setInsertPointForPossiblyEmptyBlock(builder);
 
-    // On the first iteration, seed the reduction variable with the original
-    // variable's incoming value so that a non-identity initial value
-    // participates in the scan (e.g. `x = 10` before an inclusive-scan loop of
-    // `+1` must yield 11, 12, ...). On subsequent iterations, reset to the
-    // reduction identity. `scanInfo->IV` is the 1-based iteration index.
+    // Reset the private reduction variable to the reduction identity at the
+    // start of every input-loop iteration. The original variable's incoming
+    // value (orig-val) must not be folded into any single iteration here: per
+    // the OpenMP scan semantics it is a separate prefix element that is
+    // combined once into the scan buffer (see `emitScanReduction`). Seeding a
+    // particular iteration would be lost whenever the input phase overwrites
+    // the reduction variable (e.g. `x = i`), producing incorrect results.
     llvm::Value *identity = phis[0];
-    llvm::Value *origVar =
-        moduleTranslation.lookupValue(wsloopOp.getReductionVars()[i]);
-    llvm::Value *origVal = builder.CreateLoad(identity->getType(), origVar);
-    llvm::Value *isFirstIter = builder.CreateICmpEQ(
-        scanInfo->IV, llvm::ConstantInt::get(scanInfo->IV->getType(), 1));
-    llvm::Value *seed = builder.CreateSelect(isFirstIter, origVal, identity);
-    builder.CreateStore(seed, moduleTranslation.lookupValue(reductionArgs[i]));
+    builder.CreateStore(identity,
+                        moduleTranslation.lookupValue(reductionArgs[i]));
   }
   return success();
 }
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
index 6ef58b5013e08..2aa100d686bc1 100644
--- a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
@@ -86,6 +86,15 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK:   %[[LOG:.+]] = call double @llvm.log2.f64(double 1.000000e+02) #0
 //CHECK:   %[[CEIL:.+]] = call double @llvm.ceil.f64(double %[[LOG]]) #0
 //CHECK:   %[[UB:.+]] = fptoui double %[[CEIL]] to i32
+// Combine the original variable's incoming value (orig-val) into the first
+// buffer element once, before the prefix sum, so it is reflected in every
+// scan result and in the final reduction value.
+//CHECK:   %[[OBUFF:.+]] = load ptr, ptr %{{.*}}, align 8
+//CHECK:   %[[OELEMPTR:.+]] = getelementptr inbounds i32, ptr %[[OBUFF]], i32 1
+//CHECK:   %[[OORIG:.+]] = load i32, ptr %{{.*}}, align 4
+//CHECK:   %[[OELEM:.+]] = load i32, ptr %[[OELEMPTR]], align 4
+//CHECK:   %[[OCOMB:.+]] = add i32 %[[OELEM]], %[[OORIG]]
+//CHECK:   store i32 %[[OCOMB]], ptr %[[OELEMPTR]], align 4
 //CHECK:   br i1 {{.*}}, label %omp.outer.log.scan.body, label %omp.outer.log.scan.exit
 //CHECK: omp.outer.log.scan.exit:                          ; preds = %omp.inner.log.scan.exit, %omp_region.body{{.*}}
 //CHECK: @__kmpc_end_masked
@@ -116,9 +125,10 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK: omp.inscan.dispatch:                              ; preds = %omp_loop.body
 //CHECK:   br i1 true, label %omp.before.scan.bb, label %omp.after.scan.bb
 //CHECK: omp.before.scan.bb:
-//CHECK:   %[[ISFIRST:.+]] = icmp eq i32 %{{.*}}, 1
-//CHECK:   %[[SEED:.+]] = select i1 %[[ISFIRST]], i32 %{{.*}}, i32 0
-//CHECK:   store i32 %[[SEED]], ptr %[[REDPRIV:.+]], align 4
+// Each input-loop iteration resets the private reduction variable to the
+// reduction identity (0 for `+`). The original variable's incoming value
+// (orig-val) is NOT seeded here; it is combined once in the masked region.
+//CHECK:   store i32 0, ptr %{{.*}}, align 4
 //CHECK: omp.loop_nest.region:                             ; preds = %omp.before.scan.bb
 //CHECK:   %[[BUFFER:.+]] = load ptr, ptr %loadgep_vla, align 8
 //CHECK:   %[[ARRAYOFFSET2:.+]] = getelementptr inbounds i32, ptr %[[BUFFER]], i32 %{{.*}}

>From cbfd4440aa06d37f99c1bcd8e0e8649199b3d9f0 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Wed, 29 Jul 2026 04:52:51 -0500
Subject: [PATCH 08/10] inscan reduction orig-val handling and guard the scan
 seed for zero-trip loops

---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     | 37 +++++++++++--------
 .../Target/LLVMIR/openmp-reduction-scan.mlir  | 12 ++++--
 2 files changed, 30 insertions(+), 19 deletions(-)

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index a75c39be196c1..fd9b857d88b9b 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5525,6 +5525,18 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
         ScanRedInfo->Span,
         llvm::ConstantInt::get(ScanRedInfo->Span->getType(), 1));
     Builder.SetInsertPoint(InputBB);
+    // Branch around the orig-val seed load and combine when the loop has zero
+    // logical iterations. When Span == 0 no scan-buffer entry is populated, so
+    // loading buffer[0] and feeding it to a (possibly input-sensitive
+    // user-defined) reduction combiner would be undefined behavior. In that
+    // case skip straight to the exit; the finals' zero-trip guard preserves
+    // orig-val as the result.
+    llvm::Value *HasElem = Builder.CreateICmpUGT(
+        ScanRedInfo->Span, llvm::ConstantInt::get(IndexTy, 0));
+    llvm::BasicBlock *SeedBB =
+        BasicBlock::Create(CurFn->getContext(), "omp.scan.seed", CurFn, ExitBB);
+    Builder.CreateCondBr(HasElem, SeedBB, ExitBB);
+    Builder.SetInsertPoint(SeedBB);
     // Combine the original variable's incoming value (orig-val) into the first
     // buffer element before computing the prefix sum. Per the OpenMP scan
     // semantics orig-val is a single prefix element that must be reflected in
@@ -5532,21 +5544,14 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
     // Folding it in here lets the log-scan prefix computation propagate it into
     // every element (and into the finals' read of `buffer[Span]`), regardless
     // of whether an iteration's input phase accumulates into or overwrites the
-    // reduction variable. `select(Span > 0, 1, 0)` keeps the access in bounds
-    // when Span == 0: buffer[0] is otherwise unused and the finals' zero-trip
-    // guard discards it.
-    llvm::Value *HasElem = Builder.CreateICmpUGT(
-        ScanRedInfo->Span, llvm::ConstantInt::get(IndexTy, 0));
-    llvm::Value *FirstIdx =
-        Builder.CreateSelect(HasElem, llvm::ConstantInt::get(IndexTy, 1),
-                             llvm::ConstantInt::get(IndexTy, 0));
+    // reduction variable. Span > 0 is guaranteed here, so buffer[1] is valid.
     for (ReductionInfo RedInfo : ReductionInfos) {
       Value *ReductionVal = RedInfo.PrivateVariable;
       Value *BuffPtr = (*(ScanRedInfo->ScanBuffPtrs))[ReductionVal];
       Value *Buff = Builder.CreateLoad(Builder.getPtrTy(), BuffPtr);
       Type *DestTy = RedInfo.ElementType;
-      Value *ElemPtr =
-          Builder.CreateInBoundsGEP(DestTy, Buff, FirstIdx, "arrayOffset");
+      Value *ElemPtr = Builder.CreateInBoundsGEP(
+          DestTy, Buff, llvm::ConstantInt::get(IndexTy, 1), "arrayOffset");
       Value *OrigVal = Builder.CreateLoad(DestTy, RedInfo.Variable);
       Value *Elem = Builder.CreateLoad(DestTy, ElemPtr);
       llvm::Value *Combined;
@@ -5559,9 +5564,9 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
     }
     // The log-scan prefix computation is only well-defined when there are at
     // least two elements: `log2(Span)` is invalid for Span == 0 and the loop
-    // trip computation underflows/loops forever for Span <= 1. For Span <= 1
-    // the buffer already holds the final value(s), so skip straight to the
-    // exit.
+    // trip computation underflows/loops forever for Span <= 1. For Span == 1
+    // the buffer already holds the final (seeded) value, so skip straight to
+    // the exit.
     llvm::Value *SpanGuard = Builder.CreateICmpUGT(
         ScanRedInfo->Span,
         llvm::ConstantInt::get(ScanRedInfo->Span->getType(), 1));
@@ -5572,9 +5577,11 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
     PHINode *Counter = Builder.CreatePHI(Builder.getInt32Ty(), 2);
     // size pow2k = 1;
     PHINode *Pow2K = Builder.CreatePHI(IndexTy, 2);
+    // The loop is now entered from the seed block (guarded by Span > 0), not
+    // directly from InputBB.
     Counter->addIncoming(llvm::ConstantInt::get(Builder.getInt32Ty(), 0),
-                         InputBB);
-    Pow2K->addIncoming(llvm::ConstantInt::get(IndexTy, 1), InputBB);
+                         SeedBB);
+    Pow2K->addIncoming(llvm::ConstantInt::get(IndexTy, 1), SeedBB);
     // for (size i = n - 1; i >= 2 ^ k; --i)
     //   tmp[i] op= tmp[i-pow2k];
     llvm::BasicBlock *InnerLoopBB =
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
index 2aa100d686bc1..38e413a504315 100644
--- a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
@@ -86,9 +86,15 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK:   %[[LOG:.+]] = call double @llvm.log2.f64(double 1.000000e+02) #0
 //CHECK:   %[[CEIL:.+]] = call double @llvm.ceil.f64(double %[[LOG]]) #0
 //CHECK:   %[[UB:.+]] = fptoui double %[[CEIL]] to i32
+// Zero-trip guard: skip the orig-val seed load/combine when Span == 0 so
+// buffer[0] is never loaded or passed to the reduction combiner.
+//CHECK:   br i1 {{.*}}, label %omp.scan.seed, label %omp.outer.log.scan.exit
+//CHECK: omp.outer.log.scan.exit:
+//CHECK: @__kmpc_end_masked
+//CHECK: omp.scan.seed:
 // Combine the original variable's incoming value (orig-val) into the first
-// buffer element once, before the prefix sum, so it is reflected in every
-// scan result and in the final reduction value.
+// buffer element once (Span > 0 is guaranteed here), before the prefix sum, so
+// it is reflected in every scan result and in the final reduction value.
 //CHECK:   %[[OBUFF:.+]] = load ptr, ptr %{{.*}}, align 8
 //CHECK:   %[[OELEMPTR:.+]] = getelementptr inbounds i32, ptr %[[OBUFF]], i32 1
 //CHECK:   %[[OORIG:.+]] = load i32, ptr %{{.*}}, align 4
@@ -96,8 +102,6 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK:   %[[OCOMB:.+]] = add i32 %[[OELEM]], %[[OORIG]]
 //CHECK:   store i32 %[[OCOMB]], ptr %[[OELEMPTR]], align 4
 //CHECK:   br i1 {{.*}}, label %omp.outer.log.scan.body, label %omp.outer.log.scan.exit
-//CHECK: omp.outer.log.scan.exit:                          ; preds = %omp.inner.log.scan.exit, %omp_region.body{{.*}}
-//CHECK: @__kmpc_end_masked
 //CHECK: omp.outer.log.scan.body:
 //CHECK:   %[[K:.+]] = phi i32 [ 0, %{{.*}} ], [ %[[NEXTK:.+]], %omp.inner.log.scan.exit ]
 //CHECK:   %[[I:.+]] = phi i32 [ 1, %{{.*}} ], [ %[[NEXTI:.+]], %omp.inner.log.scan.exit ]

>From a51ea62ab9586e3219388f07e8df525af46733d9 Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Mon, 3 Aug 2026 04:37:41 -0500
Subject: [PATCH 09/10] fix operand order in reduction seeding

---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     | 14 ++-
 .../openmp-reduction-scan-noncommutative.mlir | 95 +++++++++++++++++++
 .../Target/LLVMIR/openmp-reduction-scan.mlir  |  4 +-
 3 files changed, 109 insertions(+), 4 deletions(-)
 create mode 100644 mlir/test/Target/LLVMIR/openmp-reduction-scan-noncommutative.mlir

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index fd9b857d88b9b..961ec39af4efc 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5555,8 +5555,12 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
       Value *OrigVal = Builder.CreateLoad(DestTy, RedInfo.Variable);
       Value *Elem = Builder.CreateLoad(DestTy, ElemPtr);
       llvm::Value *Combined;
+      // orig-val is the leftmost/earliest element of the prefix, so it must be
+      // the accumulator (`omp_out`, the combiner's first/lhs operand) while the
+      // first buffer element is the incoming value (`omp_in`). The combiner is
+      // associative but not necessarily commutative, so this order matters.
       InsertPointOrErrorTy AfterIP =
-          RedInfo.ReductionGen(Builder.saveIP(), Elem, OrigVal, Combined);
+          RedInfo.ReductionGen(Builder.saveIP(), OrigVal, Elem, Combined);
       if (!AfterIP)
         return AfterIP.takeError();
       Builder.restoreIP(*AfterIP);
@@ -5608,8 +5612,14 @@ OpenMPIRBuilder::InsertPointOrErrorTy OpenMPIRBuilder::emitScanReduction(
       Value *LHS = Builder.CreateLoad(DestTy, LHSPtr);
       Value *RHS = Builder.CreateLoad(DestTy, RHSPtr);
       llvm::Value *Result;
+      // buffer[IV-pow2k] holds the earlier partial prefix and buffer[IV] the
+      // later one. The earlier element must be the accumulator (`omp_out`, the
+      // combiner's first/lhs operand) and the later element the incoming value
+      // (`omp_in`); the combiner is associative but not necessarily
+      // commutative, so passing them in prefix order is required. The result
+      // is written back into the later slot (LHSPtr = buffer[IV]).
       InsertPointOrErrorTy AfterIP =
-          RedInfo.ReductionGen(Builder.saveIP(), LHS, RHS, Result);
+          RedInfo.ReductionGen(Builder.saveIP(), RHS, LHS, Result);
       if (!AfterIP)
         return AfterIP.takeError();
       Builder.CreateStore(Result, LHSPtr);
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan-noncommutative.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan-noncommutative.mlir
new file mode 100644
index 0000000000000..0c595de9e80c1
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan-noncommutative.mlir
@@ -0,0 +1,95 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// Regression test for inscan reduction combiner operand ordering. The combiner
+// here is intentionally NONcommutative (subtraction) so that reversing the two
+// operands passed to the reduction combiner changes the emitted IR. This pins
+// the operand order at the two combiner call sites of the log-scan lowering:
+// the orig-val seed and the prefix computation. A commutative combiner such as
+// `+` produces identical IR regardless of operand order and would not catch a
+// reversal, so this complements openmp-reduction-scan.mlir. The scan combines
+// elements left-to-right, so the earlier prefix element must be the
+// accumulator (`omp_out`, the combiner's first operand) and the later element
+// the incoming value (`omp_in`, the second operand).
+
+omp.declare_reduction @sub_reduction_i32 : i32 init {
+^bb0(%arg0: i32):
+  %0 = llvm.mlir.constant(0 : i32) : i32
+  omp.yield(%0 : i32)
+} combiner {
+^bb0(%arg0: i32, %arg1: i32):
+  %0 = llvm.sub %arg0, %arg1 : i32
+  omp.yield(%0 : i32)
+}
+// CHECK-LABEL: @scan_reduction_noncommutative
+llvm.func @scan_reduction_noncommutative() {
+  %0 = llvm.mlir.constant(1 : i64) : i64
+  %1 = llvm.alloca %0 x i32 {bindc_name = "z"} : (i64) -> !llvm.ptr
+  %3 = llvm.alloca %0 x i32 {bindc_name = "y"} : (i64) -> !llvm.ptr
+  %5 = llvm.alloca %0 x i32 {bindc_name = "x"} : (i64) -> !llvm.ptr
+  %7 = llvm.alloca %0 x i32 {bindc_name = "k"} : (i64) -> !llvm.ptr
+  %10 = llvm.mlir.constant(100 : i32) : i32
+  %11 = llvm.mlir.constant(1 : i32) : i32
+  %12 = llvm.mlir.constant(0 : i32) : i32
+  %13 = llvm.mlir.constant(100 : index) : i64
+  %14 = llvm.mlir.addressof @_QFEa : !llvm.ptr
+  %15 = llvm.mlir.addressof @_QFEb : !llvm.ptr
+  omp.parallel {
+    %37 = llvm.mlir.constant(1 : i64) : i64
+    %38 = llvm.alloca %37 x i32 {bindc_name = "k", pinned} : (i64) -> !llvm.ptr
+    %39 = llvm.mlir.constant(1 : i64) : i64
+    omp.wsloop reduction(mod: inscan, @sub_reduction_i32 %5 -> %arg0 : !llvm.ptr) {
+      omp.loop_nest (%arg1) : i32 = (%11) to (%10) inclusive step (%11) {
+        llvm.store %arg1, %38 : i32, !llvm.ptr
+        %40 = llvm.load %arg0 : !llvm.ptr -> i32
+        %41 = llvm.load %38 : !llvm.ptr -> i32
+        %42 = llvm.sext %41 : i32 to i64
+        %50 = llvm.getelementptr %14[%42] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+        %51 = llvm.load %50 : !llvm.ptr -> i32
+        %52 = llvm.add %40, %51 : i32
+        llvm.store %52, %arg0 : i32, !llvm.ptr
+        omp.scan inclusive(%arg0 : !llvm.ptr)
+        llvm.store %arg1, %38 : i32, !llvm.ptr
+        %53 = llvm.load %arg0 : !llvm.ptr -> i32
+        %54 = llvm.load %38 : !llvm.ptr -> i32
+        %55 = llvm.sext %54 : i32 to i64
+        %63 = llvm.getelementptr %15[%55] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+        llvm.store %53, %63 : i32, !llvm.ptr
+        omp.yield
+      }
+    }
+    omp.terminator
+  }
+  llvm.return
+}
+llvm.mlir.global internal @_QFEa() {addr_space = 0 : i32} : !llvm.array<100 x i32> {
+  %0 = llvm.mlir.zero : !llvm.array<100 x i32>
+  llvm.return %0 : !llvm.array<100 x i32>
+}
+llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i32> {
+  %0 = llvm.mlir.zero : !llvm.array<100 x i32>
+  llvm.return %0 : !llvm.array<100 x i32>
+}
+
+// The orig-val seed must combine as sub(orig-val, buffer[1]): the original
+// variable's incoming value (the leftmost prefix element) is the accumulator
+// (first operand) and buffer[1] is the incoming value (second operand).
+//CHECK: omp.scan.seed:
+//CHECK:   %[[OBUFF:.+]] = load ptr, ptr %{{.*}}, align 8
+//CHECK:   %[[OELEMPTR:.+]] = getelementptr inbounds i32, ptr %[[OBUFF]], i32 1
+//CHECK:   %[[OORIG:.+]] = load i32, ptr %{{.*}}, align 4
+//CHECK:   %[[OELEM:.+]] = load i32, ptr %[[OELEMPTR]], align 4
+//CHECK:   %[[OCOMB:.+]] = sub i32 %[[OORIG]], %[[OELEM]]
+//CHECK:   store i32 %[[OCOMB]], ptr %[[OELEMPTR]], align 4
+
+// The prefix computation must combine as sub(buffer[i-pow2k], buffer[i]): the
+// earlier partial prefix is the accumulator (first operand) and the later one
+// the incoming value (second operand). buffer[i] (the later slot) is updated.
+//CHECK: omp.inner.log.scan.body:
+//CHECK:   %[[IND1:.+]] = add i32 %{{.*}}, 1
+//CHECK:   %[[IND1PTR:.+]] = getelementptr inbounds i32, ptr %{{.*}}, i32 %[[IND1]]
+//CHECK:   %[[IND2:.+]] = sub nuw i32 %[[IND1]], %{{.*}}
+//CHECK:   %[[IND2PTR:.+]] = getelementptr inbounds i32, ptr %{{.*}}, i32 %[[IND2]]
+//CHECK:   %[[IND1VAL:.+]] = load i32, ptr %[[IND1PTR]], align 4
+//CHECK:   %[[IND2VAL:.+]] = load i32, ptr %[[IND2PTR]], align 4
+//CHECK:   %[[REDVAL:.+]] = sub i32 %[[IND2VAL]], %[[IND1VAL]]
+//CHECK:   store i32 %[[REDVAL]], ptr %[[IND1PTR]], align 4
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
index 38e413a504315..0006434167c7c 100644
--- a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
@@ -99,7 +99,7 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK:   %[[OELEMPTR:.+]] = getelementptr inbounds i32, ptr %[[OBUFF]], i32 1
 //CHECK:   %[[OORIG:.+]] = load i32, ptr %{{.*}}, align 4
 //CHECK:   %[[OELEM:.+]] = load i32, ptr %[[OELEMPTR]], align 4
-//CHECK:   %[[OCOMB:.+]] = add i32 %[[OELEM]], %[[OORIG]]
+//CHECK:   %[[OCOMB:.+]] = add i32 %[[OORIG]], %[[OELEM]]
 //CHECK:   store i32 %[[OCOMB]], ptr %[[OELEMPTR]], align 4
 //CHECK:   br i1 {{.*}}, label %omp.outer.log.scan.body, label %omp.outer.log.scan.exit
 //CHECK: omp.outer.log.scan.body:
@@ -121,7 +121,7 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK:   %[[IND2PTR:.+]] = getelementptr inbounds i32, ptr %[[BUFF]], i32 %[[IND2]]
 //CHECK:   %[[IND1VAL:.+]] = load i32, ptr %[[IND1PTR]], align 4
 //CHECK:   %[[IND2VAL:.+]] = load i32, ptr %[[IND2PTR]], align 4
-//CHECK:   %[[REDVAL:.+]] = add i32 %[[IND1VAL]], %[[IND2VAL]]
+//CHECK:   %[[REDVAL:.+]] = add i32 %[[IND2VAL]], %[[IND1VAL]]
 //CHECK:   store i32 %[[REDVAL]], ptr %[[IND1PTR]], align 4
 //CHECK:   %[[CNTNXT]] = sub nuw i32 %[[CNT]], 1
 //CHECK:   %[[CMP3:.+]] = icmp uge i32 %[[CNTNXT]], %[[I]]

>From f2d45f68ecb92b63e1d2161301c8bb8f8756828c Mon Sep 17 00:00:00 2001
From: Chandra Ghale <ghale at pe34genoa.hpc.amslabs.hpecorp.net>
Date: Tue, 11 Aug 2026 08:40:34 -0500
Subject: [PATCH 10/10] fix nested-parallel-segfault

---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     |  8 +-
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      | 23 +++---
 .../LLVMIR/openmp-reduction-scan-nested.mlir  | 74 +++++++++++++++++++
 .../Target/LLVMIR/openmp-reduction-scan.mlir  |  8 ++
 4 files changed, 102 insertions(+), 11 deletions(-)
 create mode 100644 mlir/test/Target/LLVMIR/openmp-reduction-scan-nested.mlir

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index 11dc6ca976938..0de85d48702ee 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -5530,8 +5530,14 @@ Error OpenMPIRBuilder::emitScanBasedDirectiveDeclsIR(
     Type *IndexTy = ScanRedInfo->Span->getType();
     Value *AllocSpan =
         Builder.CreateAdd(ScanRedInfo->Span, ConstantInt::get(IndexTy, 1));
+    // Compute the allocation size in the target's pointer-width integer type
+    // (size_t) rather than the loop index type. `CreateMalloc` multiplies the
+    // element size by the element count in this type, and performing that
+    // multiplication in a narrow index type (e.g. i32) could overflow before
+    // the call to malloc for large scans; the element count is instead
+    // zero-extended to this wider type.
+    Type *IntPtrTy = M.getDataLayout().getIntPtrType(M.getContext());
     for (size_t i = 0; i < ScanVars.size(); i++) {
-      Type *IntPtrTy = IndexTy;
       Constant *Allocsize = ConstantExpr::getSizeOf(ScanVarsType[i]);
       Allocsize = ConstantExpr::getTruncOrBitCast(Allocsize, IntPtrTy);
       Value *Buff = Builder.CreateMalloc(IntPtrTy, ScanVarsType[i], Allocsize,
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index 3098f567442db..22cd617472f8a 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -5128,18 +5128,21 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder,
       opInst.getReductionMod() == omp::ReductionModifier::task &&
       opInst.getNumReductionVars() > 0;
 
-  // Mark the enclosing alloca stack frame as containing a parallel op. Scan
-  // reductions use the alloca insertion point of the function enclosing the
-  // parallel region to allocate their shared temporary buffer.
-  bool foundParallelOp = false;
+  // Mark the alloca stack frame of the region immediately enclosing this
+  // parallel op as containing a parallel op. Scan reductions allocate a shared
+  // temporary buffer at that frame's alloca insertion point: the buffer must be
+  // shared across this parallel's team (one buffer per team) yet private to each
+  // instance of the enclosing region, so that independent nested parallel teams
+  // do not race on a single buffer. `stackWalk` visits the innermost frame
+  // first, and this parallel op's own frame has not been pushed yet (that
+  // happens later inside `bodyGenCB`), so the first frame visited is exactly the
+  // immediately enclosing region. If there is no enclosing frame (a top-level
+  // parallel), nothing is marked and `findParallelAllocaIP` falls back to the
+  // enclosing function's entry block.
   moduleTranslation.stackWalk<OpenMPAllocStackFrame>(
       [&](OpenMPAllocStackFrame &frame) {
-        if (foundParallelOp) {
-          frame.containsParallelOp = true;
-          return WalkResult::interrupt();
-        }
-        foundParallelOp = true;
-        return WalkResult::skip();
+        frame.containsParallelOp = true;
+        return WalkResult::interrupt();
       });
 
   auto bodyGenCB =
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan-nested.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan-nested.mlir
new file mode 100644
index 0000000000000..6626d9e6e06e5
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan-nested.mlir
@@ -0,0 +1,74 @@
+// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s
+
+// Verify that an inscan reduction on a worksharing loop nested inside two
+// parallel regions allocates its shared temporary buffer inside the *outer*
+// parallel's outlined region, not in the enclosing function. If the buffer
+// (pointer) were allocated in the top-level function it would be shared by
+// every nested inner-parallel team, so independent teams would race on and
+// double-free the same buffer.
+
+omp.declare_reduction @add_reduction_i32 : i32 init {
+^bb0(%arg0: i32):
+  %0 = llvm.mlir.constant(0 : i32) : i32
+  omp.yield(%0 : i32)
+} combiner {
+^bb0(%arg0: i32, %arg1: i32):
+  %0 = llvm.add %arg0, %arg1 : i32
+  omp.yield(%0 : i32)
+}
+llvm.func @nested_scan_reduction() {
+  %0 = llvm.mlir.constant(1 : i64) : i64
+  %5 = llvm.alloca %0 x i32 {bindc_name = "x"} : (i64) -> !llvm.ptr
+  %10 = llvm.mlir.constant(100 : i32) : i32
+  %11 = llvm.mlir.constant(1 : i32) : i32
+  %14 = llvm.mlir.addressof @_QFEa : !llvm.ptr
+  %15 = llvm.mlir.addressof @_QFEb : !llvm.ptr
+  omp.parallel {
+    omp.parallel {
+      %38 = llvm.alloca %0 x i32 {bindc_name = "k", pinned} : (i64) -> !llvm.ptr
+      omp.wsloop reduction(mod: inscan, @add_reduction_i32 %5 -> %arg0 : !llvm.ptr) {
+        omp.loop_nest (%arg1) : i32 = (%11) to (%10) inclusive step (%11) {
+          llvm.store %arg1, %38 : i32, !llvm.ptr
+          %40 = llvm.load %arg0 : !llvm.ptr -> i32
+          %41 = llvm.load %38 : !llvm.ptr -> i32
+          %42 = llvm.sext %41 : i32 to i64
+          %50 = llvm.getelementptr %14[%42] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+          %51 = llvm.load %50 : !llvm.ptr -> i32
+          %52 = llvm.add %40, %51 : i32
+          llvm.store %52, %arg0 : i32, !llvm.ptr
+          omp.scan inclusive(%arg0 : !llvm.ptr)
+          llvm.store %arg1, %38 : i32, !llvm.ptr
+          %53 = llvm.load %arg0 : !llvm.ptr -> i32
+          %54 = llvm.load %38 : !llvm.ptr -> i32
+          %55 = llvm.sext %54 : i32 to i64
+          %63 = llvm.getelementptr %15[%55] : (!llvm.ptr, i64) -> !llvm.ptr, i32
+          llvm.store %53, %63 : i32, !llvm.ptr
+          omp.yield
+        }
+      }
+      omp.terminator
+    }
+    omp.terminator
+  }
+  llvm.return
+}
+llvm.mlir.global internal @_QFEa() {addr_space = 0 : i32} : !llvm.array<100 x i32> {
+  %0 = llvm.mlir.zero : !llvm.array<100 x i32>
+  llvm.return %0 : !llvm.array<100 x i32>
+}
+llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i32> {
+  %0 = llvm.mlir.zero : !llvm.array<100 x i32>
+  llvm.return %0 : !llvm.array<100 x i32>
+}
+
+// The enclosing function only forks the outer parallel; the scan buffer pointer
+// must NOT be allocated here (that would be shared across all nested teams).
+// CHECK-LABEL: define void @nested_scan_reduction()
+// CHECK-NOT: alloca ptr
+// CHECK: call void {{.*}}@__kmpc_fork_call({{.*}} @nested_scan_reduction..omp_par.1
+
+// The buffer pointer is allocated inside the outer parallel's outlined region,
+// so each outer-parallel thread has its own buffer for its nested inner team.
+// CHECK: define internal void @nested_scan_reduction..omp_par.1
+// CHECK: %vla = alloca ptr
+// CHECK: call void {{.*}}@__kmpc_fork_call({{.*}} @nested_scan_reduction..omp_par,
diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
index 0006434167c7c..ec956b9395423 100644
--- a/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-reduction-scan.mlir
@@ -139,3 +139,11 @@ llvm.mlir.global internal @_QFEb() {addr_space = 0 : i32} : !llvm.array<100 x i3
 //CHECK-NEXT:   %[[REDPRIVVAL:.+]] = load i32, ptr %{{.*}}, align 4
 //CHECK:   store i32 %[[REDPRIVVAL]], ptr %[[ARRAYOFFSET2]], align 4
 //CHECK:   br label %omp.scan.loop.exit
+
+// The shared scan buffer is allocated with malloc in the masked region. The
+// element count is multiplied by the element size in the target's pointer-width
+// integer type (size_t), not the loop index type, so the size computation
+// cannot overflow a narrow (e.g. i32) index before the call to malloc.
+//CHECK: %[[MALLOCSZ:.+]] = mul i64 {{.*}}ptrtoint
+//CHECK: %[[ARR:.+]] = tail call ptr @malloc(i64 %[[MALLOCSZ]])
+//CHECK: store ptr %[[ARR]], ptr %{{.*}}, align 8


From openmp-commits at lists.llvm.org  Tue Aug 11 07:46:32 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Tue, 11 Aug 2026 07:46:32 -0700 (PDT)
Subject: [Openmp-commits] [openmp] 9c317bb - [OpenMP][DeviceRTL] Use the
 actual block size in the SPMD no-loop distribute path (#214073)
Message-ID: <6a7b35c8.3ff6034f.eb95e.02ef@mx.google.com>


Author: Spencer Bryngelson
Date: 2026-08-11T10:46:23-04:00
New Revision: 9c317bbbdbeaaf2b2e039ece201baba8f21f9d2a

URL: https://github.com/llvm/llvm-project/commit/9c317bbbdbeaaf2b2e039ece201baba8f21f9d2a
DIFF: https://github.com/llvm/llvm-project/commit/9c317bbbdbeaaf2b2e039ece201baba8f21f9d2a.diff

LOG: [OpenMP][DeviceRTL] Use the actual block size in the SPMD no-loop distribute path (#214073)

In the SPMD no-loop path the iteration index is `BId * NumThreads +
TId`, so `NumThreads` has to
be the size of the block the threads are actually in. `DistributeFor`
uses the caller-supplied
value as-is. If that value is larger than the block size, the index
strides past the end of each
block and the iterations in between are never run.

Fixes #198621.

Reproduced on gfx90a, `num_teams(4)`, 128 iterations, varying the value
passed as `NumThreads`
against the threads the block actually has:

| NumThreads passed | threads in block | iterations not run |
|---|---|---|
| 256 | 32 | 96 of 128 |
| 256 | 64 | 64 of 128 |
| 128 | 32 | 96 of 128 |
| 64 | 64 | 0 |

Exactly as many iterations run as there are threads in the block, and
the rest are dropped. With
this patch all four cases run all 128.

AMD carries the same fix downstream as ROCm/llvm-project#3058.

### Testing

The new test calls `__kmpc_distribute_for_static_loop_4u` directly. That
is unusual, and the reason
is that no C or C++ construct reaches it: clang lowers worksharing loops
to
`__kmpc_for_static_init_4`, and only flang emits
`__kmpc_distribute_for_static_loop_*`. Verified on
device IR from both frontends. Calling the entry directly keeps the test
independent of the Fortran
runtime, which upstream does not ship for the device.

Verified the test fails without the patch (`unwritten: 96`) and passes
with it (`unwritten: 0`).

Also ran 66 Fortran offload probes covering collapse, reductions,
private and firstprivate arrays,
nested parallel, simd, allocatable, atomic and complex arithmetic, in
no-loop mode with
`exec_mode` confirmed to be 6 rather than assumed: all pass before and
after, so the clamp does not
disturb the cases where the caller's value was already correct.

---

Parts of this change were written or audited with Claude Code. I have
reviewed all of it and take
full responsibility for the contribution. See
`llvm/docs/AIToolPolicy.md`.

Added: 
    offload/test/offloading/distribute_for_no_loop_num_threads.c

Modified: 
    openmp/device/src/Workshare.cpp

Removed: 
    


################################################################################
diff  --git a/offload/test/offloading/distribute_for_no_loop_num_threads.c b/offload/test/offloading/distribute_for_no_loop_num_threads.c
new file mode 100644
index 0000000000000..4ad4012efb81d
--- /dev/null
+++ b/offload/test/offloading/distribute_for_no_loop_num_threads.c
@@ -0,0 +1,71 @@
+// SPMD no-loop distribute: the index is BId * NumThreads + TId, so a NumThreads
+// larger than the real block size drops iterations.
+//
+// The entry is called directly because no C/C++ construct reaches it: clang
+// emits __kmpc_for_static_init_4, only flang emits this one.
+//
+// RUN: %libomptarget-compile-run-and-check-generic
+// REQUIRES: gpu
+
+#include <omp.h>
+#include <stdio.h>
+
+#define N 128
+#define NUM_THREADS 256
+#define THREAD_LIMIT 32
+
+struct Args {
+  double *Out;
+};
+
+#pragma omp begin declare target
+extern void __kmpc_distribute_for_static_loop_4u(
+    void *Loc, void (*Fn)(unsigned, void *), void *Arg, unsigned NumIters,
+    unsigned NumThreads, unsigned BlockChunk, unsigned ThreadChunk,
+    unsigned char OneIterationPerThread);
+
+__attribute__((noinline)) static void body(unsigned I, void *A) {
+  ((struct Args *)A)->Out[I] = (double)(I + 1);
+}
+#pragma omp end declare target
+
+// For the host fallback copy only; the device uses the runtime's definition.
+#ifndef __AMDGCN__
+void __kmpc_distribute_for_static_loop_4u(
+    void *Loc, void (*Fn)(unsigned, void *), void *Arg, unsigned NumIters,
+    unsigned NumThreads, unsigned BlockChunk, unsigned ThreadChunk,
+    unsigned char OneIterationPerThread) {
+  for (unsigned I = 0; I < NumIters; ++I)
+    Fn(I, Arg);
+}
+#endif
+
+int main(void) {
+  static double Out[N];
+  for (int I = 0; I < N; ++I)
+    Out[I] = -1.0;
+
+  // NUM_THREADS deliberately exceeds the block's actual thread count.
+#pragma omp target teams map(tofrom : Out[0 : N]) num_teams(4)                 \
+    thread_limit(THREAD_LIMIT)
+  {
+#pragma omp parallel
+    {
+      struct Args A;
+      A.Out = Out;
+      __kmpc_distribute_for_static_loop_4u(0, body, &A, N, NUM_THREADS,
+                                           /*BlockChunk=*/0,
+                                           /*ThreadChunk=*/0,
+                                           /*OneIterationPerThread=*/1);
+    }
+  }
+
+  int Unwritten = 0;
+  for (int I = 0; I < N; ++I)
+    if (Out[I] == -1.0)
+      ++Unwritten;
+
+  // CHECK: unwritten: 0
+  printf("unwritten: %d\n", Unwritten);
+  return 0;
+}

diff  --git a/openmp/device/src/Workshare.cpp b/openmp/device/src/Workshare.cpp
index 6e6440b690db0..a16ef93707989 100644
--- a/openmp/device/src/Workshare.cpp
+++ b/openmp/device/src/Workshare.cpp
@@ -895,6 +895,11 @@ template <typename Ty> class StaticLoopChunker {
     Ty NumBlocks = mapping::getNumberOfBlocksInKernel();
     Ty BId = mapping::getBlockIdInKernel();
 
+    // The index is BId * NumThreads + TId, so NumThreads must be the real
+    // block size; a larger value strides past each block and drops iterations.
+    if (OneIterationPerThread)
+      NumThreads = static_cast<Ty>(mapping::getMaxTeamThreads());
+
     // If the block chunk is not specified we pick a default now.
     if (BlockChunk == 0)
       BlockChunk = NumThreads;


        

From openmp-commits at lists.llvm.org  Tue Aug 11 07:46:33 2026
From: openmp-commits at lists.llvm.org (Shilei Tian via Openmp-commits)
Date: Tue, 11 Aug 2026 07:46:33 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMP][DeviceRTL] Use the actual
 block size in the SPMD no-loop distribute path (PR #214073)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/214073@github.com>
Message-ID: <6a7b35c9.d1496011.98923.f8ac@mx.google.com>


https://github.com/shiltian closed https://github.com/llvm/llvm-project/pull/214073

From openmp-commits at lists.llvm.org  Tue Aug 11 13:24:09 2026
From: openmp-commits at lists.llvm.org (Amit Tiwari via Openmp-commits)
Date: Tue, 11 Aug 2026 13:24:09 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [mlir] [openmp] [Clang][OpenMP]
 Canonicalize Intra-tiles in Loop Tiling (PR #191114)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/191114@github.com>
Message-ID: <6a7b84e9.a1accbb4.2cf7.70c8@mx.google.com>


https://github.com/loopacino updated https://github.com/llvm/llvm-project/pull/191114

>From cc69f5d3c05ae97a99cad8fa4f87aaffb6201063 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Apr 2026 07:07:16 -0400
Subject: [PATCH 01/18] nested-predicate

---
 clang/lib/Sema/SemaOpenMP.cpp | 167 ++++++++++++++++++++++++----------
 1 file changed, 121 insertions(+), 46 deletions(-)

diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 76bb0d38d428f..c8d75a0b333b4 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -15081,8 +15081,10 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
   // Create iteration variables for the generated loops.
   SmallVector<VarDecl *, 4> FloorIndVars;
   SmallVector<VarDecl *, 4> TileIndVars;
+  SmallVector<VarDecl *, 4> TileCntVars;
   FloorIndVars.resize(NumLoops);
   TileIndVars.resize(NumLoops);
+  TileCntVars.resize(NumLoops);
   for (unsigned I = 0; I < NumLoops; ++I) {
     OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
 
@@ -15102,27 +15104,101 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
       FloorIndVars[I] = FloorCntDecl;
     }
 
-    // Iteration variable for the tile (i.e. inner) loop.
+    // Logical iteration variable for the tile loop. Retains the meaning of
+    // the original logical iteration number (floor_iv + tile_cnt) so that
+    // LoopHelper.Updates can derive the original loop variable unchanged.
     {
-      std::string TileCntName =
+      std::string TileIVName =
           (Twine(".tile_") + llvm::utostr(I) + ".iv." + OrigVarName).str();
 
-      // Reuse the iteration variable created by checkOpenMPLoop. It is also
-      // used by the expressions to derive the original iteration variable's
-      // value from the logical iteration number.
-      auto *TileCntDecl = cast<VarDecl>(IterVarRef->getDecl());
-      TileCntDecl->setDeclName(
-          &SemaRef.PP.getIdentifierTable().get(TileCntName));
-      TileIndVars[I] = TileCntDecl;
+      auto *TileIVDecl = cast<VarDecl>(IterVarRef->getDecl());
+      TileIVDecl->setDeclName(&SemaRef.PP.getIdentifierTable().get(TileIVName));
+      TileIndVars[I] = TileIVDecl;
+    }
+
+    // Loop counter for the rectangular tile loop [0, TileSize).
+    {
+      std::string TileCntName =
+          (Twine(".tile.cnt.") + llvm::utostr(I) + ".iv." + OrigVarName).str();
+      VarDecl *TileCntDecl =
+          buildVarDecl(SemaRef, {}, CntTy, TileCntName, nullptr, OrigCntVar);
+      TileCntVars[I] = TileCntDecl;
     }
 
     addLoopPreInits(Context, LoopHelper, LoopStmts[I], OriginalInits[I],
                     PreInits);
+
+    // Declare the logical tile IV in PreInits so it is in scope for the
+    // entire loop nest (it will be assigned in each tile loop body).
+    Decl *TileIVDeclPtr = TileIndVars[I];
+    PreInits.push_back(new (Context) DeclStmt(
+        DeclGroupRef::Create(Context, &TileIVDeclPtr, 1), {}, {}));
   }
 
   // Once the original iteration values are set, append the innermost body.
   Stmt *Inner = Body;
 
+  // Build a combined validity predicate that guards the innermost body.
+  // For each tiled dimension, check that the logical iteration number
+  // (.tile.iv) is within the original trip count. This is required because the
+  // tile loop now has rectangular (constant) bounds and may overshoot on the
+  // remainder tile. The predicate is: .tile.iv.0 < N0 && .tile.iv.1 < N1 ...
+  //
+  // Optimization: if every dimension's trip count is a compile-time constant
+  // that is evenly divisible by the corresponding tile size (also a constant),
+  // then the remainder tile is empty and the predicate is trivially true.
+  {
+    bool PredicateNeeded = false;
+    for (unsigned I = 0; I < NumLoops; ++I) {
+      Expr *TSExpr = SizesClause->getSizesRefs()[I];
+      Expr *NExpr = LoopHelpers[I].NumIterations;
+      llvm::APSInt TileVal, TripVal;
+      bool TSConst =
+          !TSExpr->containsErrors() && TSExpr->isIntegerConstantExpr(Context);
+      bool NConst = NExpr->isIntegerConstantExpr(Context);
+      if (TSConst && NConst) {
+        Expr::EvalResult TSResult;
+        TSExpr->EvaluateAsInt(TSResult, Context);
+        TileVal = TSResult.Val.getInt();
+        Expr::EvalResult NResult;
+        NExpr->EvaluateAsInt(NResult, Context);
+        TripVal = NResult.Val.getInt();
+        if (TileVal.isStrictlyPositive() && (TripVal.srem(TileVal)).isZero())
+          continue;
+      }
+      PredicateNeeded = true;
+      break;
+    }
+
+    if (PredicateNeeded) {
+      Expr *CombinedPred = nullptr;
+      for (unsigned I = 0; I < NumLoops; ++I) {
+        auto *OrigCntVar = cast<DeclRefExpr>(LoopHelpers[I].Counters[0]);
+        QualType IVTy = LoopHelpers[I].NumIterations->getType();
+        Expr *TileIVRef = buildDeclRefExpr(SemaRef, TileIndVars[I], IVTy,
+                                           OrigCntVar->getExprLoc());
+        ExprResult DimPred =
+            SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(), BO_LT,
+                               TileIVRef, LoopHelpers[I].NumIterations);
+        if (!DimPred.isUsable())
+          return StmtError();
+        if (CombinedPred) {
+          ExprResult Combined =
+              SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(), BO_LAnd,
+                                 CombinedPred, DimPred.get());
+          if (!Combined.isUsable())
+            return StmtError();
+          CombinedPred = Combined.get();
+        } else {
+          CombinedPred = DimPred.get();
+        }
+      }
+      Inner = IfStmt::Create(
+          Context, SourceLocation(), IfStatementKind::Ordinary, nullptr,
+          nullptr, CombinedPred, SourceLocation(), SourceLocation(), Inner);
+    }
+  }
+
   auto MakeDimTileSize = [&SemaRef = this->SemaRef, &CopyTransformer, &Context,
                           SizesClause, CurScope](int I) -> Expr * {
     Expr *DimTileSizeExpr = SizesClause->getSizesRefs()[I];
@@ -15130,7 +15206,7 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     if (DimTileSizeExpr->containsErrors())
       return nullptr;
 
-    if (isa<ConstantExpr>(DimTileSizeExpr))
+    if (DimTileSizeExpr->isIntegerConstantExpr(Context))
       return AssertSuccess(CopyTransformer.TransformExpr(DimTileSizeExpr));
 
     // When the tile size is not a constant but a variable, it is possible to
@@ -15166,6 +15242,9 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
   };
 
   // Create tile loops from the inside to the outside.
+  // Each tile loop uses .tile.cnt as its counter with rectangular bounds
+  // [0, TileSize), and computes .tile.iv = .floor.iv + .tile.cnt to set
+  // the logical iteration number for LoopHelper.Updates.
   for (int I = NumLoops - 1; I >= 0; --I) {
     OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
     Expr *NumIterations = LoopHelper.NumIterations;
@@ -15176,70 +15255,65 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     // Commonly used variables. One of the constraints of an AST is that every
     // node object must appear at most once, hence we define a lambda that
     // creates a new AST node at every use.
+    auto MakeTileCntRef = [&SemaRef = this->SemaRef, &TileCntVars, I, IVTy,
+                           OrigCntVar]() {
+      return buildDeclRefExpr(SemaRef, TileCntVars[I], IVTy,
+                              OrigCntVar->getExprLoc());
+    };
     auto MakeTileIVRef = [&SemaRef = this->SemaRef, &TileIndVars, I, IVTy,
                           OrigCntVar]() {
       return buildDeclRefExpr(SemaRef, TileIndVars[I], IVTy,
                               OrigCntVar->getExprLoc());
     };
 
-    // For init-statement: auto .tile.iv = .floor.iv
+    // For init-statement: auto .tile.cnt = 0
     SemaRef.AddInitializerToDecl(
-        TileIndVars[I],
-        SemaRef
-            .DefaultLvalueConversion(
-                makeFloorIVRef(SemaRef, FloorIndVars, I, IVTy, OrigCntVar))
-            .get(),
+        TileCntVars[I],
+        SemaRef.ActOnIntegerConstant(LoopHelper.Init->getExprLoc(), 0).get(),
         /*DirectInit=*/false);
-    Decl *CounterDecl = TileIndVars[I];
+    Decl *CounterDecl = TileCntVars[I];
     StmtResult InitStmt = new (Context)
         DeclStmt(DeclGroupRef::Create(Context, &CounterDecl, 1),
                  OrigCntVar->getBeginLoc(), OrigCntVar->getEndLoc());
     if (!InitStmt.isUsable())
       return StmtError();
 
-    // For cond-expression:
-    //   .tile.iv < min(.floor.iv + DimTileSize, NumIterations)
+    // For cond-expression: .tile.cnt < DimTileSize  (rectangular bound)
     Expr *DimTileSize = MakeDimTileSize(I);
     if (!DimTileSize)
       return StmtError();
-    ExprResult EndOfTile = SemaRef.BuildBinOp(
-        CurScope, LoopHelper.Cond->getExprLoc(), BO_Add,
-        makeFloorIVRef(SemaRef, FloorIndVars, I, IVTy, OrigCntVar),
-        DimTileSize);
-    if (!EndOfTile.isUsable())
-      return StmtError();
-    ExprResult IsPartialTile =
-        SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
-                           NumIterations, EndOfTile.get());
-    if (!IsPartialTile.isUsable())
-      return StmtError();
-    ExprResult MinTileAndIterSpace = SemaRef.ActOnConditionalOp(
-        LoopHelper.Cond->getBeginLoc(), LoopHelper.Cond->getEndLoc(),
-        IsPartialTile.get(), NumIterations, EndOfTile.get());
-    if (!MinTileAndIterSpace.isUsable())
-      return StmtError();
     ExprResult CondExpr =
         SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
-                           MakeTileIVRef(), MinTileAndIterSpace.get());
+                           MakeTileCntRef(), DimTileSize);
     if (!CondExpr.isUsable())
       return StmtError();
 
-    // For incr-statement: ++.tile.iv
+    // For incr-statement: ++.tile.cnt
     ExprResult IncrStmt = SemaRef.BuildUnaryOp(
-        CurScope, LoopHelper.Inc->getExprLoc(), UO_PreInc, MakeTileIVRef());
+        CurScope, LoopHelper.Inc->getExprLoc(), UO_PreInc, MakeTileCntRef());
     if (!IncrStmt.isUsable())
       return StmtError();
 
-    // Statements to set the original iteration variable's value from the
-    // logical iteration number.
+    // Compute the logical iteration number:
+    //   .tile.iv = .floor.iv + .tile.cnt
+    ExprResult FloorPlusCnt = SemaRef.BuildBinOp(
+        CurScope, OrigCntVar->getExprLoc(), BO_Add,
+        makeFloorIVRef(SemaRef, FloorIndVars, I, IVTy, OrigCntVar),
+        MakeTileCntRef());
+    if (!FloorPlusCnt.isUsable())
+      return StmtError();
+    ExprResult TileIVAssign =
+        SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(), BO_Assign,
+                           MakeTileIVRef(), FloorPlusCnt.get());
+    if (!TileIVAssign.isUsable())
+      return StmtError();
+
     // Generated for loop is:
     // \code
-    // Original_for_init;
-    // for (auto .tile.iv = .floor.iv;
-    //      .tile.iv < min(.floor.iv + DimTileSize, NumIterations);
-    //      ++.tile.iv) {
-    //   Original_Body;
-    //   Original_counter_update;
+    // for (auto .tile.cnt = 0; .tile.cnt < DimTileSize; ++.tile.cnt) {
+    //   .tile.iv = .floor.iv + .tile.cnt;
+    //   Original_counter_update;  // derives orig var from .tile.iv
+    //   Inner;                    // predicated body or inner tile loops
     // }
     // \endcode
     // FIXME: If the innermost body is an loop itself, inserting these
@@ -15247,6 +15321,7 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     // for applying tiling again). If this is the case, sink the expressions
     // further into the inner loop.
     SmallVector<Stmt *, 4> BodyParts;
+    BodyParts.push_back(TileIVAssign.get());
     BodyParts.append(LoopHelper.Updates.begin(), LoopHelper.Updates.end());
     if (auto *SourceCXXFor = dyn_cast<CXXForRangeStmt>(LoopStmt))
       BodyParts.push_back(SourceCXXFor->getLoopVarStmt());

>From fbf1b6503562dedf0f6073263c2a9bc4c885b597 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Apr 2026 07:13:48 -0400
Subject: [PATCH 02/18] ir-builder

---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 64 ++++++++++++++---------
 1 file changed, 39 insertions(+), 25 deletions(-)

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index fc3812502fb2e..dba68179567e3 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -6984,7 +6984,7 @@ OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
   // Compute the trip counts of the floor loops.
   Builder.SetCurrentDebugLocation(DL);
   Builder.restoreIP(OutermostLoop->getPreheaderIP());
-  SmallVector<Value *, 4> FloorCompleteCount, FloorCount, FloorRems;
+  SmallVector<Value *, 4> FloorCount;
   for (int i = 0; i < NumLoops; ++i) {
     Value *TileSize = TileSizes[i];
     Value *OrigTripCount = OrigTripCounts[i];
@@ -7008,10 +7008,7 @@ OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
         Builder.CreateAdd(FloorCompleteTripCount, FloorTripOverflow,
                           "omp_floor" + Twine(i) + ".tripcount", true);
 
-    // Remember some values for later use.
-    FloorCompleteCount.push_back(FloorCompleteTripCount);
     FloorCount.push_back(FloorTripCount);
-    FloorRems.push_back(FloorTripRem);
   }
 
   // Generate the new loop nest, from the outermost to the innermost.
@@ -7055,23 +7052,10 @@ OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
 
   EmbeddNewLoops(FloorCount, "floor");
 
-  // Within the innermost floor loop, emit the code that computes the tile
-  // sizes.
-  Builder.SetInsertPoint(Enter->getTerminator());
-  SmallVector<Value *, 4> TileCounts;
-  for (int i = 0; i < NumLoops; ++i) {
-    CanonicalLoopInfo *FloorLoop = Result[i];
-    Value *TileSize = TileSizes[i];
-
-    Value *FloorIsEpilogue =
-        Builder.CreateICmpEQ(FloorLoop->getIndVar(), FloorCompleteCount[i]);
-    Value *TileTripCount =
-        Builder.CreateSelect(FloorIsEpilogue, FloorRems[i], TileSize);
-
-    TileCounts.push_back(TileTripCount);
-  }
-
-  // Create the tile loops.
+  // Create the tile loops with rectangular (constant) trip counts equal to
+  // the tile sizes. A validity predicate in the body guards against
+  // out-of-bounds iterations in the remainder tile.
+  SmallVector<Value *, 4> TileCounts(TileSizes.begin(), TileSizes.end());
   EmbeddNewLoops(TileCounts, "tile");
 
   // Insert the inbetween code into the body.
@@ -7100,6 +7084,7 @@ OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
   // Replace the original induction variable with an induction variable computed
   // from the tile and floor induction variables.
   Builder.restoreIP(Result.back()->getBodyIP());
+  SmallVector<Value *, 4> ShiftValues;
   for (int i = 0; i < NumLoops; ++i) {
     CanonicalLoopInfo *FloorLoop = Result[i];
     CanonicalLoopInfo *TileLoop = Result[NumLoops + i];
@@ -7111,18 +7096,47 @@ OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
     Value *Shift =
         Builder.CreateAdd(Scale, TileLoop->getIndVar(), {}, /*HasNUW=*/true);
     OrigIndVar->replaceAllUsesWith(Shift);
+    ShiftValues.push_back(Shift);
+  }
+
+  // Build a validity predicate: for each tiled dimension, check that the
+  // computed original index is within the original trip count. This guards
+  // against executing out-of-bounds iterations in the remainder tile.
+  Value *Pred = nullptr;
+  for (int i = 0; i < NumLoops; ++i) {
+    Value *DimPred = Builder.CreateICmpULT(ShiftValues[i], OrigTripCounts[i],
+                                           "omp_tile" + Twine(i) + ".inbounds");
+    Pred = Pred ? Builder.CreateAnd(Pred, DimPred) : DimPred;
   }
 
+  // Insert the conditional guard: split the body flow so that out-of-bounds
+  // iterations skip directly to a merge block before the tile latch.
+  BasicBlock *TileBodyBB = Builder.GetInsertBlock();
+  Instruction *BodyTerm = TileBodyBB->getTerminator();
+  BasicBlock *BodySucc = cast<BranchInst>(BodyTerm)->getSuccessor(0);
+  BasicBlock *TileLatch = Result.back()->getLatch();
+
+  BasicBlock *MergeBB =
+      BasicBlock::Create(F->getContext(), "omp_tile.body.merge", F, TileLatch);
+  Builder.SetInsertPoint(MergeBB);
+  Builder.CreateBr(TileLatch);
+
+  // Redirect the body chain's exit from the tile latch to the merge block.
+  for (BasicBlock *PredBB : llvm::make_early_inc_range(predecessors(TileLatch)))
+    if (PredBB != MergeBB)
+      PredBB->getTerminator()->replaceUsesOfWith(TileLatch, MergeBB);
+
+  // Replace the tile body's unconditional branch with a conditional one.
+  BodyTerm->eraseFromParent();
+  Builder.SetInsertPoint(TileBodyBB);
+  Builder.CreateCondBr(Pred, BodySucc, MergeBB);
+
   // Remove unused parts of the original loops.
   removeUnusedBlocksFromParent(OldControlBBs);
 
   for (CanonicalLoopInfo *L : Loops)
     L->invalidate();
 
-#ifndef NDEBUG
-  for (CanonicalLoopInfo *GenL : Result)
-    GenL->assertOK();
-#endif
   return Result;
 }
 

>From 28270e10df2128b1ccd2ad673ebe85c436c41b55 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Thu, 9 Apr 2026 02:27:14 -0400
Subject: [PATCH 03/18] UncondBrInst

---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index dba68179567e3..a38ecede63376 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -7113,7 +7113,7 @@ OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
   // iterations skip directly to a merge block before the tile latch.
   BasicBlock *TileBodyBB = Builder.GetInsertBlock();
   Instruction *BodyTerm = TileBodyBB->getTerminator();
-  BasicBlock *BodySucc = cast<BranchInst>(BodyTerm)->getSuccessor(0);
+  BasicBlock *BodySucc = cast<UncondBrInst>(BodyTerm)->getSuccessor(0);
   BasicBlock *TileLatch = Result.back()->getLatch();
 
   BasicBlock *MergeBB =

>From 08fa07fe6c0a451f05fd36ec9a019518b22e2a0d Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Sat, 11 Apr 2026 11:05:18 -0400
Subject: [PATCH 04/18] mlir-tests-rect-update

---
 .../test/Target/LLVMIR/openmp-cli-tile01.mlir |  18 +--
 .../test/Target/LLVMIR/openmp-cli-tile02.mlir |  30 +++--
 .../test/Target/LLVMIR/openmp-cli-tile03.mlir | 120 +++++++++---------
 3 files changed, 89 insertions(+), 79 deletions(-)

diff --git a/mlir/test/Target/LLVMIR/openmp-cli-tile01.mlir b/mlir/test/Target/LLVMIR/openmp-cli-tile01.mlir
index 0d559b69a3ad1..948a9815851a5 100644
--- a/mlir/test/Target/LLVMIR/openmp-cli-tile01.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-cli-tile01.mlir
@@ -38,8 +38,6 @@ llvm.func @tile_trivial_loop(%baseptr: !llvm.ptr, %tc: i32, %ts: i32) -> () {
 // CHECK-NEXT:    br i1 %[[OMP_FLOOR0_CMP:.+]], label %[[OMP_FLOOR0_BODY:.+]], label %[[OMP_FLOOR0_EXIT:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_FLOOR0_BODY]]:
-// CHECK-NEXT:    %[[TMP8:.+]] = icmp eq i32 %[[OMP_FLOOR0_IV:.+]], %[[TMP4:.+]]
-// CHECK-NEXT:    %[[TMP9:.+]] = select i1 %[[TMP8:.+]], i32 %[[TMP5:.+]], i32 %[[TMP2:.+]]
 // CHECK-NEXT:    br label %[[OMP_TILE0_PREHEADER:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_PREHEADER]]:
@@ -50,23 +48,27 @@ llvm.func @tile_trivial_loop(%baseptr: !llvm.ptr, %tc: i32, %ts: i32) -> () {
 // CHECK-NEXT:    br label %[[OMP_TILE0_COND:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_COND]]:
-// CHECK-NEXT:    %[[OMP_TILE0_CMP:.+]] = icmp ult i32 %[[OMP_TILE0_IV:.+]], %[[TMP9:.+]]
+// CHECK-NEXT:    %[[OMP_TILE0_CMP:.+]] = icmp ult i32 %[[OMP_TILE0_IV:.+]], %[[TMP2:.+]]
 // CHECK-NEXT:    br i1 %[[OMP_TILE0_CMP:.+]], label %[[OMP_TILE0_BODY:.+]], label %[[OMP_TILE0_EXIT:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_BODY]]:
-// CHECK-NEXT:    %[[TMP10:.+]] = mul nuw i32 %[[TMP2:.+]], %[[OMP_FLOOR0_IV:.+]]
-// CHECK-NEXT:    %[[TMP11:.+]] = add nuw i32 %[[TMP10:.+]], %[[OMP_TILE0_IV:.+]]
-// CHECK-NEXT:    br label %[[OMP_OMP_LOOP_BODY:.+]]
+// CHECK-NEXT:    %[[TMP8:.+]] = mul nuw i32 %[[TMP2:.+]], %[[OMP_FLOOR0_IV:.+]]
+// CHECK-NEXT:    %[[TMP9:.+]] = add nuw i32 %[[TMP8:.+]], %[[OMP_TILE0_IV:.+]]
+// CHECK-NEXT:    %[[OMP_TILE0_INBOUNDS:.+]] = icmp ult i32 %[[TMP9:.+]], %[[TMP1:.+]]
+// CHECK-NEXT:    br i1 %[[OMP_TILE0_INBOUNDS:.+]], label %[[OMP_OMP_LOOP_BODY:.+]], label %[[OMP_TILE_BODY_MERGE:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_OMP_LOOP_BODY]]:
 // CHECK-NEXT:    br label %[[OMP_LOOP_REGION:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_LOOP_REGION]]:
-// CHECK-NEXT:    %[[TMP12:.+]] = getelementptr inbounds float, ptr %[[TMP0:.+]], i32 %[[TMP11:.+]]
-// CHECK-NEXT:    store float 4.200000e+01, ptr %[[TMP12:.+]], align 4
+// CHECK-NEXT:    %[[TMP10:.+]] = getelementptr inbounds float, ptr %[[TMP0:.+]], i32 %[[TMP9:.+]]
+// CHECK-NEXT:    store float 4.200000e+01, ptr %[[TMP10:.+]], align 4
 // CHECK-NEXT:    br label %[[OMP_REGION_CONT:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_REGION_CONT]]:
+// CHECK-NEXT:    br label %[[OMP_TILE_BODY_MERGE:.+]]
+// CHECK-EMPTY:
+// CHECK-NEXT:  [[OMP_TILE_BODY_MERGE]]:
 // CHECK-NEXT:    br label %[[OMP_TILE0_INC:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_INC]]:
diff --git a/mlir/test/Target/LLVMIR/openmp-cli-tile02.mlir b/mlir/test/Target/LLVMIR/openmp-cli-tile02.mlir
index 22c2973164159..59ffa50037f4b 100644
--- a/mlir/test/Target/LLVMIR/openmp-cli-tile02.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-cli-tile02.mlir
@@ -79,10 +79,6 @@ llvm.func @tile_2d_loop(%baseptr: !llvm.ptr, %tc1: i32, %tc2: i32, %ts1: i32, %t
 // CHECK-NEXT:    br i1 %[[OMP_FLOOR1_CMP:.+]], label %[[OMP_FLOOR1_BODY:.+]], label %[[OMP_FLOOR1_EXIT:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_FLOOR1_BODY]]:
-// CHECK-NEXT:    %[[TMP14:.+]] = icmp eq i32 %[[OMP_FLOOR0_IV:.+]], %[[TMP6:.+]]
-// CHECK-NEXT:    %[[TMP15:.+]] = select i1 %[[TMP14:.+]], i32 %[[TMP7:.+]], i32 %[[TMP3:.+]]
-// CHECK-NEXT:    %[[TMP16:.+]] = icmp eq i32 %[[OMP_FLOOR1_IV:.+]], %[[TMP10:.+]]
-// CHECK-NEXT:    %[[TMP17:.+]] = select i1 %[[TMP16:.+]], i32 %[[TMP11:.+]], i32 %[[TMP4:.+]]
 // CHECK-NEXT:    br label %[[OMP_TILE0_PREHEADER:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_PREHEADER]]:
@@ -93,7 +89,7 @@ llvm.func @tile_2d_loop(%baseptr: !llvm.ptr, %tc1: i32, %tc2: i32, %ts1: i32, %t
 // CHECK-NEXT:    br label %[[OMP_TILE0_COND:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_COND]]:
-// CHECK-NEXT:    %[[OMP_TILE0_CMP:.+]] = icmp ult i32 %[[OMP_TILE0_IV:.+]], %[[TMP15:.+]]
+// CHECK-NEXT:    %[[OMP_TILE0_CMP:.+]] = icmp ult i32 %[[OMP_TILE0_IV:.+]], %[[TMP3:.+]]
 // CHECK-NEXT:    br i1 %[[OMP_TILE0_CMP:.+]], label %[[OMP_TILE0_BODY:.+]], label %[[OMP_TILE0_EXIT:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_BODY]]:
@@ -107,26 +103,32 @@ llvm.func @tile_2d_loop(%baseptr: !llvm.ptr, %tc1: i32, %tc2: i32, %ts1: i32, %t
 // CHECK-NEXT:    br label %[[OMP_TILE1_COND:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE1_COND]]:
-// CHECK-NEXT:    %[[OMP_TILE1_CMP:.+]] = icmp ult i32 %[[OMP_TILE1_IV:.+]], %[[TMP17:.+]]
+// CHECK-NEXT:    %[[OMP_TILE1_CMP:.+]] = icmp ult i32 %[[OMP_TILE1_IV:.+]], %[[TMP4:.+]]
 // CHECK-NEXT:    br i1 %[[OMP_TILE1_CMP:.+]], label %[[OMP_TILE1_BODY:.+]], label %[[OMP_TILE1_EXIT:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE1_BODY]]:
-// CHECK-NEXT:    %[[TMP18:.+]] = mul nuw i32 %[[TMP3:.+]], %[[OMP_FLOOR0_IV:.+]]
-// CHECK-NEXT:    %[[TMP19:.+]] = add nuw i32 %[[TMP18:.+]], %[[OMP_TILE0_IV:.+]]
-// CHECK-NEXT:    %[[TMP20:.+]] = mul nuw i32 %[[TMP4:.+]], %[[OMP_FLOOR1_IV:.+]]
-// CHECK-NEXT:    %[[TMP21:.+]] = add nuw i32 %[[TMP20:.+]], %[[OMP_TILE1_IV:.+]]
-// CHECK-NEXT:    br label %[[OMP_OMP_LOOP_BODY:.+]]
+// CHECK-NEXT:    %[[TMP14:.+]] = mul nuw i32 %[[TMP3:.+]], %[[OMP_FLOOR0_IV:.+]]
+// CHECK-NEXT:    %[[TMP15:.+]] = add nuw i32 %[[TMP14:.+]], %[[OMP_TILE0_IV:.+]]
+// CHECK-NEXT:    %[[TMP16:.+]] = mul nuw i32 %[[TMP4:.+]], %[[OMP_FLOOR1_IV:.+]]
+// CHECK-NEXT:    %[[TMP17:.+]] = add nuw i32 %[[TMP16:.+]], %[[OMP_TILE1_IV:.+]]
+// CHECK-NEXT:    %[[OMP_TILE0_INBOUNDS:.+]] = icmp ult i32 %[[TMP15:.+]], %[[TMP1:.+]]
+// CHECK-NEXT:    %[[OMP_TILE1_INBOUNDS:.+]] = icmp ult i32 %[[TMP17:.+]], %[[TMP2:.+]]
+// CHECK-NEXT:    %[[OMP_TILE_PRED:.+]] = and i1 %[[OMP_TILE0_INBOUNDS:.+]], %[[OMP_TILE1_INBOUNDS:.+]]
+// CHECK-NEXT:    br i1 %[[OMP_TILE_PRED:.+]], label %[[OMP_OMP_LOOP_BODY:.+]], label %[[OMP_TILE_BODY_MERGE:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_OMP_LOOP_BODY4]]:
 // CHECK-NEXT:    br label %[[OMP_LOOP_REGION12:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_LOOP_REGION12]]:
-// CHECK-NEXT:    %[[TMP22:.+]] = add i32 %[[TMP19:.+]], %[[TMP21:.+]]
-// CHECK-NEXT:    %[[TMP23:.+]] = getelementptr inbounds float, ptr %[[TMP0:.+]], i32 %[[TMP22:.+]]
-// CHECK-NEXT:    store float 4.200000e+01, ptr %[[TMP23:.+]], align 4
+// CHECK-NEXT:    %[[TMP18:.+]] = add i32 %[[TMP15:.+]], %[[TMP17:.+]]
+// CHECK-NEXT:    %[[TMP19:.+]] = getelementptr inbounds float, ptr %[[TMP0:.+]], i32 %[[TMP18:.+]]
+// CHECK-NEXT:    store float 4.200000e+01, ptr %[[TMP19:.+]], align 4
 // CHECK-NEXT:    br label %[[OMP_REGION_CONT11:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_REGION_CONT11]]:
+// CHECK-NEXT:    br label %[[OMP_TILE_BODY_MERGE:.+]]
+// CHECK-EMPTY:
+// CHECK-NEXT:  [[OMP_TILE_BODY_MERGE]]:
 // CHECK-NEXT:    br label %[[OMP_TILE1_INC:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE1_INC]]:
diff --git a/mlir/test/Target/LLVMIR/openmp-cli-tile03.mlir b/mlir/test/Target/LLVMIR/openmp-cli-tile03.mlir
index 0747ff8909362..fcc37af2ca8f3 100644
--- a/mlir/test/Target/LLVMIR/openmp-cli-tile03.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-cli-tile03.mlir
@@ -56,8 +56,6 @@ llvm.func @tile_composition(%baseptr: !llvm.ptr, %tc: i32, %ts: i32, %grid_ts: i
 // CHECK-NEXT:    br i1 %[[OMP_FLOOR0_CMP10:.+]], label %[[OMP_FLOOR0_BODY5:.+]], label %[[OMP_FLOOR0_EXIT7:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_FLOOR0_BODY5]]:
-// CHECK-NEXT:    %[[TMP14:.+]] = icmp eq i32 %[[OMP_FLOOR0_IV9:.+]], %[[TMP10:.+]]
-// CHECK-NEXT:    %[[TMP15:.+]] = select i1 %[[TMP14:.+]], i32 %[[TMP11:.+]], i32 %[[TMP3:.+]]
 // CHECK-NEXT:    br label %[[OMP_TILE0_PREHEADER12:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_PREHEADER12]]:
@@ -68,83 +66,88 @@ llvm.func @tile_composition(%baseptr: !llvm.ptr, %tc: i32, %ts: i32, %grid_ts: i
 // CHECK-NEXT:    br label %[[OMP_TILE0_COND14:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_COND14]]:
-// CHECK-NEXT:    %[[OMP_TILE0_CMP20:.+]] = icmp ult i32 %[[OMP_TILE0_IV19:.+]], %[[TMP15:.+]]
+// CHECK-NEXT:    %[[OMP_TILE0_CMP20:.+]] = icmp ult i32 %[[OMP_TILE0_IV19:.+]], %[[TMP3:.+]]
 // CHECK-NEXT:    br i1 %[[OMP_TILE0_CMP20:.+]], label %[[OMP_TILE0_BODY15:.+]], label %[[OMP_TILE0_EXIT17:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_BODY15]]:
-// CHECK-NEXT:    %[[TMP16:.+]] = mul nuw i32 %[[TMP3:.+]], %[[OMP_FLOOR0_IV9:.+]]
-// CHECK-NEXT:    %[[TMP17:.+]] = add nuw i32 %[[TMP16:.+]], %[[OMP_TILE0_IV19:.+]]
-// CHECK-NEXT:    br label %[[OMP_FLOOR0_BODY:.+]]
-// CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_FLOOR0_BODY]]:
-// CHECK-NEXT:    %[[TMP18:.+]] = icmp eq i32 %[[TMP17:.+]], %[[TMP6:.+]]
-// CHECK-NEXT:    %[[TMP19:.+]] = select i1 %[[TMP18:.+]], i32 %[[TMP7:.+]], i32 %[[TMP2:.+]]
-// CHECK-NEXT:    br label %[[OMP_TILE0_PREHEADER:.+]]
-// CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_PREHEADER]]:
-// CHECK-NEXT:    %[[TMP20:.+]] = udiv i32 %[[TMP19:.+]], %[[TMP4:.+]]
-// CHECK-NEXT:    %[[TMP21:.+]] = urem i32 %[[TMP19:.+]], %[[TMP4:.+]]
-// CHECK-NEXT:    %[[TMP22:.+]] = icmp ne i32 %[[TMP21:.+]], 0
-// CHECK-NEXT:    %[[TMP23:.+]] = zext i1 %[[TMP22:.+]] to i32
-// CHECK-NEXT:    %[[OMP_FLOOR0_TRIPCOUNT22:.+]] = add nuw i32 %[[TMP20:.+]], %[[TMP23:.+]]
-// CHECK-NEXT:    br label %[[OMP_FLOOR0_PREHEADER23:.+]]
-// CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_FLOOR0_PREHEADER23]]:
+// CHECK-NEXT:    %[[TMP14:.+]] = mul nuw i32 %[[TMP3:.+]], %[[OMP_FLOOR0_IV9:.+]]
+// CHECK-NEXT:    %[[TMP15:.+]] = add nuw i32 %[[TMP14:.+]], %[[OMP_TILE0_IV19:.+]]
+// CHECK-NEXT:    %[[OMP_TILE0_INBOUNDS22:.+]] = icmp ult i32 %[[TMP15:.+]], %[[OMP_FLOOR0_TRIPCOUNT:.+]]
+// CHECK-NEXT:    br i1 %[[OMP_TILE0_INBOUNDS22:.+]], label %[[OMP_FLOOR_INNER_ENTRY:.+]], label %[[OMP_TILE_BODY_MERGE23:.+]]
+// CHECK-EMPTY:
+// CHECK-NEXT:  [[OMP_FLOOR_INNER_ENTRY]]:
+// CHECK-NEXT:    br label %[[OMP_TILE0_PREHEADER_INNER:.+]]
+// CHECK-EMPTY:
+// CHECK-NEXT:  [[OMP_TILE0_PREHEADER_INNER]]:
+// CHECK-NEXT:    %[[TMP16:.+]] = udiv i32 %[[TMP2:.+]], %[[TMP4:.+]]
+// CHECK-NEXT:    %[[TMP17:.+]] = urem i32 %[[TMP2:.+]], %[[TMP4:.+]]
+// CHECK-NEXT:    %[[TMP18:.+]] = icmp ne i32 %[[TMP17:.+]], 0
+// CHECK-NEXT:    %[[TMP19:.+]] = zext i1 %[[TMP18:.+]] to i32
+// CHECK-NEXT:    %[[OMP_FLOOR0_TRIPCOUNT24:.+]] = add nuw i32 %[[TMP16:.+]], %[[TMP19:.+]]
+// CHECK-NEXT:    br label %[[OMP_FLOOR0_PREHEADER25:.+]]
+// CHECK-EMPTY:
+// CHECK-NEXT:  [[OMP_FLOOR0_PREHEADER25]]:
 // CHECK-NEXT:    br label %[[OMP_FLOOR0_HEADER:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_FLOOR0_HEADER]]:
-// CHECK-NEXT:    %[[OMP_FLOOR0_IV:.+]] = phi i32 [ 0, %[[OMP_FLOOR0_PREHEADER23:.+]] ], [ %[[OMP_FLOOR0_NEXT:.+]], %[[OMP_FLOOR0_INC:.+]] ]
+// CHECK-NEXT:    %[[OMP_FLOOR0_IV:.+]] = phi i32 [ 0, %[[OMP_FLOOR0_PREHEADER25:.+]] ], [ %[[OMP_FLOOR0_NEXT:.+]], %[[OMP_FLOOR0_INC:.+]] ]
 // CHECK-NEXT:    br label %[[OMP_FLOOR0_COND:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_FLOOR0_COND]]:
-// CHECK-NEXT:    %[[OMP_FLOOR0_CMP:.+]] = icmp ult i32 %[[OMP_FLOOR0_IV:.+]], %[[OMP_FLOOR0_TRIPCOUNT22:.+]]
-// CHECK-NEXT:    br i1 %[[OMP_FLOOR0_CMP:.+]], label %[[OMP_FLOOR0_BODY24:.+]], label %[[OMP_FLOOR0_EXIT:.+]]
+// CHECK-NEXT:    %[[OMP_FLOOR0_CMP:.+]] = icmp ult i32 %[[OMP_FLOOR0_IV:.+]], %[[OMP_FLOOR0_TRIPCOUNT24:.+]]
+// CHECK-NEXT:    br i1 %[[OMP_FLOOR0_CMP:.+]], label %[[OMP_FLOOR0_BODY26:.+]], label %[[OMP_FLOOR0_EXIT:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_FLOOR0_BODY24]]:
-// CHECK-NEXT:    %[[TMP24:.+]] = icmp eq i32 %[[OMP_FLOOR0_IV:.+]], %[[TMP20:.+]]
-// CHECK-NEXT:    %[[TMP25:.+]] = select i1 %[[TMP24:.+]], i32 %[[TMP21:.+]], i32 %[[TMP4:.+]]
-// CHECK-NEXT:    br label %[[OMP_TILE0_PREHEADER26:.+]]
+// CHECK-NEXT:  [[OMP_FLOOR0_BODY26]]:
+// CHECK-NEXT:    br label %[[OMP_TILE0_PREHEADER28:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_PREHEADER26]]:
-// CHECK-NEXT:    br label %[[OMP_TILE0_HEADER27:.+]]
+// CHECK-NEXT:  [[OMP_TILE0_PREHEADER28]]:
+// CHECK-NEXT:    br label %[[OMP_TILE0_HEADER29:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_HEADER27]]:
-// CHECK-NEXT:    %[[OMP_TILE0_IV33:.+]] = phi i32 [ 0, %[[OMP_TILE0_PREHEADER26:.+]] ], [ %[[OMP_TILE0_NEXT35:.+]], %[[OMP_TILE0_INC30:.+]] ]
-// CHECK-NEXT:    br label %[[OMP_TILE0_COND28:.+]]
+// CHECK-NEXT:  [[OMP_TILE0_HEADER29]]:
+// CHECK-NEXT:    %[[OMP_TILE0_IV35:.+]] = phi i32 [ 0, %[[OMP_TILE0_PREHEADER28:.+]] ], [ %[[OMP_TILE0_NEXT37:.+]], %[[OMP_TILE0_INC32:.+]] ]
+// CHECK-NEXT:    br label %[[OMP_TILE0_COND30:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_COND28]]:
-// CHECK-NEXT:    %[[OMP_TILE0_CMP34:.+]] = icmp ult i32 %[[OMP_TILE0_IV33:.+]], %[[TMP25:.+]]
-// CHECK-NEXT:    br i1 %[[OMP_TILE0_CMP34:.+]], label %[[OMP_TILE0_BODY29:.+]], label %[[OMP_TILE0_EXIT31:.+]]
+// CHECK-NEXT:  [[OMP_TILE0_COND30]]:
+// CHECK-NEXT:    %[[OMP_TILE0_CMP36:.+]] = icmp ult i32 %[[OMP_TILE0_IV35:.+]], %[[TMP4:.+]]
+// CHECK-NEXT:    br i1 %[[OMP_TILE0_CMP36:.+]], label %[[OMP_TILE0_BODY31:.+]], label %[[OMP_TILE0_EXIT33:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_BODY29]]:
-// CHECK-NEXT:    %[[TMP26:.+]] = mul nuw i32 %[[TMP4:.+]], %[[OMP_FLOOR0_IV:.+]]
-// CHECK-NEXT:    %[[TMP27:.+]] = add nuw i32 %[[TMP26:.+]], %[[OMP_TILE0_IV33:.+]]
-// CHECK-NEXT:    br label %[[OMP_TILE0_BODY:.+]]
+// CHECK-NEXT:  [[OMP_TILE0_BODY31]]:
+// CHECK-NEXT:    %[[TMP20:.+]] = mul nuw i32 %[[TMP4:.+]], %[[OMP_FLOOR0_IV:.+]]
+// CHECK-NEXT:    %[[TMP21:.+]] = add nuw i32 %[[TMP20:.+]], %[[OMP_TILE0_IV35:.+]]
+// CHECK-NEXT:    %[[OMP_TILE0_INBOUNDS38:.+]] = icmp ult i32 %[[TMP21:.+]], %[[TMP2:.+]]
+// CHECK-NEXT:    br i1 %[[OMP_TILE0_INBOUNDS38:.+]], label %[[OMP_TILE0_BODY_STORE:.+]], label %[[OMP_TILE_BODY_MERGE39:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_BODY]]:
-// CHECK-NEXT:    %[[TMP28:.+]] = mul nuw i32 %[[TMP2:.+]], %[[TMP17:.+]]
-// CHECK-NEXT:    %[[TMP29:.+]] = add nuw i32 %[[TMP28:.+]], %[[TMP27:.+]]
-// CHECK-NEXT:    br label %[[OMP_OMP_LOOP_BODY:.+]]
+// CHECK-NEXT:  [[OMP_TILE0_BODY_STORE]]:
+// CHECK-NEXT:    %[[TMP22:.+]] = mul nuw i32 %[[TMP2:.+]], %[[TMP15:.+]]
+// CHECK-NEXT:    %[[TMP23:.+]] = add nuw i32 %[[TMP22:.+]], %[[TMP21:.+]]
+// CHECK-NEXT:    %[[OMP_TILE0_INBOUNDS:.+]] = icmp ult i32 %[[TMP23:.+]], %[[TMP1:.+]]
+// CHECK-NEXT:    br i1 %[[OMP_TILE0_INBOUNDS:.+]], label %[[OMP_OMP_LOOP_BODY:.+]], label %[[OMP_TILE_BODY_MERGE:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_OMP_LOOP_BODY]]:
 // CHECK-NEXT:    br label %[[OMP_LOOP_REGION:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_LOOP_REGION]]:
-// CHECK-NEXT:    %[[TMP30:.+]] = getelementptr inbounds float, ptr %[[TMP0:.+]], i32 %[[TMP29:.+]]
-// CHECK-NEXT:    store float 4.200000e+01, ptr %[[TMP30:.+]], align 4
+// CHECK-NEXT:    %[[TMP24:.+]] = getelementptr inbounds float, ptr %[[TMP0:.+]], i32 %[[TMP23:.+]]
+// CHECK-NEXT:    store float 4.200000e+01, ptr %[[TMP24:.+]], align 4
 // CHECK-NEXT:    br label %[[OMP_REGION_CONT:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_REGION_CONT]]:
-// CHECK-NEXT:    br label %[[OMP_TILE0_INC30:.+]]
+// CHECK-NEXT:    br label %[[OMP_TILE_BODY_MERGE:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_INC30]]:
-// CHECK-NEXT:    %[[OMP_TILE0_NEXT35:.+]] = add nuw i32 %[[OMP_TILE0_IV33:.+]], 1
-// CHECK-NEXT:    br label %[[OMP_TILE0_HEADER27:.+]]
+// CHECK-NEXT:  [[OMP_TILE_BODY_MERGE]]:
+// CHECK-NEXT:    br label %[[OMP_TILE_BODY_MERGE39:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_EXIT31]]:
-// CHECK-NEXT:    br label %[[OMP_TILE0_AFTER32:.+]]
+// CHECK-NEXT:  [[OMP_TILE_BODY_MERGE39]]:
+// CHECK-NEXT:    br label %[[OMP_TILE0_INC32:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_AFTER32]]:
+// CHECK-NEXT:  [[OMP_TILE0_INC32]]:
+// CHECK-NEXT:    %[[OMP_TILE0_NEXT37:.+]] = add nuw i32 %[[OMP_TILE0_IV35:.+]], 1
+// CHECK-NEXT:    br label %[[OMP_TILE0_HEADER29:.+]]
+// CHECK-EMPTY:
+// CHECK-NEXT:  [[OMP_TILE0_EXIT33]]:
+// CHECK-NEXT:    br label %[[OMP_TILE0_AFTER34:.+]]
+// CHECK-EMPTY:
+// CHECK-NEXT:  [[OMP_TILE0_AFTER34]]:
 // CHECK-NEXT:    br label %[[OMP_FLOOR0_INC:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_FLOOR0_INC]]:
@@ -152,12 +155,15 @@ llvm.func @tile_composition(%baseptr: !llvm.ptr, %tc: i32, %ts: i32, %grid_ts: i
 // CHECK-NEXT:    br label %[[OMP_FLOOR0_HEADER:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_FLOOR0_EXIT]]:
-// CHECK-NEXT:    br label %[[OMP_FLOOR0_AFTER25:.+]]
+// CHECK-NEXT:    br label %[[OMP_FLOOR0_AFTER27:.+]]
+// CHECK-EMPTY:
+// CHECK-NEXT:  [[OMP_FLOOR0_AFTER27]]:
+// CHECK-NEXT:    br label %[[OMP_TILE0_AFTER_INNER:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_FLOOR0_AFTER25]]:
-// CHECK-NEXT:    br label %[[OMP_TILE0_AFTER:.+]]
+// CHECK-NEXT:  [[OMP_TILE0_AFTER_INNER]]:
+// CHECK-NEXT:    br label %[[OMP_TILE_BODY_MERGE23:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_AFTER]]:
+// CHECK-NEXT:  [[OMP_TILE_BODY_MERGE23]]:
 // CHECK-NEXT:    br label %[[OMP_TILE0_INC16:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_INC16]]:

>From c44c2822377ae53b790998dcb333cdf4cf3640b6 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Sat, 11 Apr 2026 11:12:42 -0400
Subject: [PATCH 05/18] tile-codegen

---
 clang/test/OpenMP/tile_codegen.cpp            | 2504 ++++++++---------
 .../OpenMP/tile_codegen_for_dependent.cpp     |  308 +-
 clang/test/OpenMP/tile_codegen_tile_for.cpp   |  417 ++-
 clang/test/OpenMP/tile_messages.cpp           |    2 +-
 4 files changed, 1439 insertions(+), 1792 deletions(-)

diff --git a/clang/test/OpenMP/tile_codegen.cpp b/clang/test/OpenMP/tile_codegen.cpp
index 85b004dc73f4f..1fd02d74a1209 100644
--- a/clang/test/OpenMP/tile_codegen.cpp
+++ b/clang/test/OpenMP/tile_codegen.cpp
@@ -56,7 +56,7 @@ extern "C" void foo4() {
 
 
 extern "C" void foo5() {
-#pragma omp for collapse(3)
+#pragma omp for collapse(2)
 #pragma omp tile sizes(5)
   for (int i = 7; i < 17; i += 3)
     for (int j = 7; j < 17; j += 3)
@@ -136,8 +136,9 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[I2:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[I:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
@@ -149,50 +150,47 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END12:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END11:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK1:       for.cond4:
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 5
-// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
-// CHECK1-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK1:       cond.true:
-// CHECK1-NEXT:    br label [[COND_END:%.*]]
-// CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP4]], 5
-// CHECK1-NEXT:    br label [[COND_END]]
-// CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
-// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body8:
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
-// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[I2]], align 8
-// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[TMP6]], align 4
-// CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[I2]], align 8
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP1]], 5
+// CHECK1-NEXT:    br i1 [[CMP5]], label [[FOR_BODY6:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body6:
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
+// CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP4]], 3
+// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
+// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[TMP5]], align 4
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP8:%.*]] = icmp slt i32 [[TMP6]], 4
+// CHECK1-NEXT:    br i1 [[CMP8]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK1:       if.then:
+// CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2]], !align [[META3]]
 // CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
 // CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP8]])
+// CHECK1-NEXT:    br label [[IF_END]]
+// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP9]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP3:![0-9]+]]
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP4:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC10:%.*]]
-// CHECK1:       for.inc10:
+// CHECK1-NEXT:    br label [[FOR_INC9:%.*]]
+// CHECK1:       for.inc9:
 // CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP10]], 5
-// CHECK1-NEXT:    store i32 [[ADD11]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP5:![0-9]+]]
-// CHECK1:       for.end12:
+// CHECK1-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP10]], 5
+// CHECK1-NEXT:    store i32 [[ADD10]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
+// CHECK1:       for.end11:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -207,8 +205,9 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -237,54 +236,49 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK1-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP8]], [[ADD5]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END15:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK1:       for.cond6:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[ADD7:%.*]] = add i32 [[TMP12]], 1
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD8:%.*]] = add i32 [[TMP13]], 5
-// CHECK1-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
-// CHECK1-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK1:       cond.true:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[ADD10:%.*]] = add i32 [[TMP14]], 1
-// CHECK1-NEXT:    br label [[COND_END:%.*]]
-// CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD11:%.*]] = add i32 [[TMP15]], 5
-// CHECK1-NEXT:    br label [[COND_END]]
-// CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP11]], [[COND]]
-// CHECK1-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body13:
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP17]], [[TMP18]]
-// CHECK1-NEXT:    [[ADD14:%.*]] = add i32 [[TMP16]], [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP19]])
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP10]], 5
+// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body8:
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD9:%.*]] = add i32 [[TMP11]], [[TMP12]]
+// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP14]], [[TMP15]]
+// CHECK1-NEXT:    [[ADD10:%.*]] = add i32 [[TMP13]], [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD10]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD11:%.*]] = add i32 [[TMP17]], 1
+// CHECK1-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP16]], [[ADD11]]
+// CHECK1-NEXT:    br i1 [[CMP12]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK1:       if.then:
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP18]])
+// CHECK1-NEXT:    br label [[IF_END]]
+// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP20]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP6:![0-9]+]]
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP19]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP7:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC15:%.*]]
-// CHECK1:       for.inc15:
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD16:%.*]] = add i32 [[TMP21]], 5
-// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP7:![0-9]+]]
-// CHECK1:       for.end17:
+// CHECK1-NEXT:    br label [[FOR_INC13:%.*]]
+// CHECK1:       for.inc13:
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD14:%.*]] = add i32 [[TMP20]], 5
+// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
+// CHECK1:       for.end15:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -295,11 +289,13 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -310,94 +306,86 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END30:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
 // CHECK1:       for.body:
 // CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK1:       for.cond1:
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 4
-// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END27:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END21:%.*]]
 // CHECK1:       for.body3:
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 [[TMP2]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK1:       for.cond4:
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], 5
-// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
-// CHECK1-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK1:       cond.true:
-// CHECK1-NEXT:    br label [[COND_END:%.*]]
-// CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP5]], 5
-// CHECK1-NEXT:    br label [[COND_END]]
-// CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP3]], [[COND]]
-// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END24:%.*]]
-// CHECK1:       for.body8:
-// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
-// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP2]], 5
+// CHECK1-NEXT:    br i1 [[CMP5]], label [[FOR_BODY6:%.*]], label [[FOR_END18:%.*]]
+// CHECK1:       for.body6:
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[TMP4]]
+// CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
+// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND8:%.*]]
+// CHECK1:       for.cond8:
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP6]], 5
+// CHECK1-NEXT:    br i1 [[CMP9]], label [[FOR_BODY10:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body10:
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND10:%.*]]
-// CHECK1:       for.cond10:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP9]], 5
-// CHECK1-NEXT:    [[CMP12:%.*]] = icmp slt i32 4, [[ADD11]]
-// CHECK1-NEXT:    br i1 [[CMP12]], label [[COND_TRUE13:%.*]], label [[COND_FALSE14:%.*]]
-// CHECK1:       cond.true13:
-// CHECK1-NEXT:    br label [[COND_END16:%.*]]
-// CHECK1:       cond.false14:
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP10]], 5
-// CHECK1-NEXT:    br label [[COND_END16]]
-// CHECK1:       cond.end16:
-// CHECK1-NEXT:    [[COND17:%.*]] = phi i32 [ 4, [[COND_TRUE13]] ], [ [[ADD15]], [[COND_FALSE14]] ]
-// CHECK1-NEXT:    [[CMP18:%.*]] = icmp slt i32 [[TMP8]], [[COND17]]
-// CHECK1-NEXT:    br i1 [[CMP18]], label [[FOR_BODY19:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body19:
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP7]], [[TMP8]]
+// CHECK1-NEXT:    store i32 [[ADD11]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[MUL12:%.*]] = mul nsw i32 [[TMP9]], 3
+// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i32 7, [[MUL12]]
+// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP10]], 4
+// CHECK1-NEXT:    br i1 [[CMP14]], label [[LAND_LHS_TRUE:%.*]], label [[IF_END:%.*]]
+// CHECK1:       land.lhs.true:
 // CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[TMP11]], 3
-// CHECK1-NEXT:    [[ADD21:%.*]] = add nsw i32 7, [[MUL20]]
-// CHECK1-NEXT:    store i32 [[ADD21]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[CMP15:%.*]] = icmp slt i32 [[TMP11]], 4
+// CHECK1-NEXT:    br i1 [[CMP15]], label [[IF_THEN:%.*]], label [[IF_END]]
+// CHECK1:       if.then:
 // CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
 // CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP12]], i32 noundef [[TMP13]])
+// CHECK1-NEXT:    br label [[IF_END]]
+// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND10]], !llvm.loop [[LOOP8:![0-9]+]]
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP9:![0-9]+]]
 // CHECK1:       for.end:
+// CHECK1-NEXT:    br label [[FOR_INC16:%.*]]
+// CHECK1:       for.inc16:
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC17:%.*]] = add nsw i32 [[TMP15]], 1
+// CHECK1-NEXT:    store i32 [[INC17]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP10:![0-9]+]]
+// CHECK1:       for.end18:
+// CHECK1-NEXT:    br label [[FOR_INC19:%.*]]
+// CHECK1:       for.inc19:
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP16]], 5
+// CHECK1-NEXT:    store i32 [[ADD20]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP11:![0-9]+]]
+// CHECK1:       for.end21:
 // CHECK1-NEXT:    br label [[FOR_INC22:%.*]]
 // CHECK1:       for.inc22:
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC23:%.*]] = add nsw i32 [[TMP15]], 1
-// CHECK1-NEXT:    store i32 [[INC23]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP9:![0-9]+]]
-// CHECK1:       for.end24:
-// CHECK1-NEXT:    br label [[FOR_INC25:%.*]]
-// CHECK1:       for.inc25:
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD26:%.*]] = add nsw i32 [[TMP16]], 5
-// CHECK1-NEXT:    store i32 [[ADD26]], ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP10:![0-9]+]]
-// CHECK1:       for.end27:
-// CHECK1-NEXT:    br label [[FOR_INC28:%.*]]
-// CHECK1:       for.inc28:
 // CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD29:%.*]] = add nsw i32 [[TMP17]], 5
-// CHECK1-NEXT:    store i32 [[ADD29]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP11:![0-9]+]]
-// CHECK1:       for.end30:
+// CHECK1-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP17]], 5
+// CHECK1-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP12:![0-9]+]]
+// CHECK1:       for.end24:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -407,15 +395,17 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
 // CHECK1-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK1-NEXT:    store i32 7, ptr [[J]], align 4
@@ -453,87 +443,79 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP7]], 4
-// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END32:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END22:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 [[TMP8]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND3:%.*]]
 // CHECK1:       for.cond3:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD4:%.*]] = add nsw i32 [[TMP10]], 5
-// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD4]]
-// CHECK1-NEXT:    br i1 [[CMP5]], label [[COND_TRUE6:%.*]], label [[COND_FALSE7:%.*]]
-// CHECK1:       cond.true6:
-// CHECK1-NEXT:    br label [[COND_END9:%.*]]
-// CHECK1:       cond.false7:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP11]], 5
-// CHECK1-NEXT:    br label [[COND_END9]]
-// CHECK1:       cond.end9:
-// CHECK1-NEXT:    [[COND10:%.*]] = phi i32 [ 4, [[COND_TRUE6]] ], [ [[ADD8]], [[COND_FALSE7]] ]
-// CHECK1-NEXT:    [[CMP11:%.*]] = icmp slt i32 [[TMP9]], [[COND10]]
-// CHECK1-NEXT:    br i1 [[CMP11]], label [[FOR_BODY12:%.*]], label [[FOR_END29:%.*]]
-// CHECK1:       for.body12:
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL13:%.*]] = mul nsw i32 [[TMP12]], 3
-// CHECK1-NEXT:    [[ADD14:%.*]] = add nsw i32 7, [[MUL13]]
-// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP8]], 5
+// CHECK1-NEXT:    br i1 [[CMP4]], label [[FOR_BODY5:%.*]], label [[FOR_END19:%.*]]
+// CHECK1:       for.body5:
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
+// CHECK1-NEXT:    store i32 [[ADD6]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL7:%.*]] = mul nsw i32 [[TMP11]], 3
+// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i32 7, [[MUL7]]
+// CHECK1-NEXT:    store i32 [[ADD8]], ptr [[I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND9:%.*]]
+// CHECK1:       for.cond9:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i32 [[TMP12]], 5
+// CHECK1-NEXT:    br i1 [[CMP10]], label [[FOR_BODY11:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body11:
 // CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    store i32 [[TMP13]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND15:%.*]]
-// CHECK1:       for.cond15:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP15]], 5
-// CHECK1-NEXT:    [[CMP17:%.*]] = icmp slt i32 4, [[ADD16]]
-// CHECK1-NEXT:    br i1 [[CMP17]], label [[COND_TRUE18:%.*]], label [[COND_FALSE19:%.*]]
-// CHECK1:       cond.true18:
-// CHECK1-NEXT:    br label [[COND_END21:%.*]]
-// CHECK1:       cond.false19:
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP16]], 5
-// CHECK1-NEXT:    br label [[COND_END21]]
-// CHECK1:       cond.end21:
-// CHECK1-NEXT:    [[COND22:%.*]] = phi i32 [ 4, [[COND_TRUE18]] ], [ [[ADD20]], [[COND_FALSE19]] ]
-// CHECK1-NEXT:    [[CMP23:%.*]] = icmp slt i32 [[TMP14]], [[COND22]]
-// CHECK1-NEXT:    br i1 [[CMP23]], label [[FOR_BODY24:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body24:
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP13]], [[TMP14]]
+// CHECK1-NEXT:    store i32 [[ADD12]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[MUL13:%.*]] = mul nsw i32 [[TMP15]], 3
+// CHECK1-NEXT:    [[ADD14:%.*]] = add nsw i32 7, [[MUL13]]
+// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP15:%.*]] = icmp slt i32 [[TMP16]], 4
+// CHECK1-NEXT:    br i1 [[CMP15]], label [[LAND_LHS_TRUE:%.*]], label [[IF_END:%.*]]
+// CHECK1:       land.lhs.true:
 // CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[MUL25:%.*]] = mul nsw i32 [[TMP17]], 3
-// CHECK1-NEXT:    [[ADD26:%.*]] = add nsw i32 7, [[MUL25]]
-// CHECK1-NEXT:    store i32 [[ADD26]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[CMP16:%.*]] = icmp slt i32 [[TMP17]], 4
+// CHECK1-NEXT:    br i1 [[CMP16]], label [[IF_THEN:%.*]], label [[IF_END]]
+// CHECK1:       if.then:
 // CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[J]], align 4
 // CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP18]], i32 noundef [[TMP19]])
+// CHECK1-NEXT:    br label [[IF_END]]
+// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP20]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND15]], !llvm.loop [[LOOP12:![0-9]+]]
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND9]], !llvm.loop [[LOOP13:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC27:%.*]]
-// CHECK1:       for.inc27:
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC28:%.*]] = add nsw i32 [[TMP21]], 1
-// CHECK1-NEXT:    store i32 [[INC28]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND3]], !llvm.loop [[LOOP13:![0-9]+]]
-// CHECK1:       for.end29:
-// CHECK1-NEXT:    br label [[FOR_INC30:%.*]]
-// CHECK1:       for.inc30:
+// CHECK1-NEXT:    br label [[FOR_INC17:%.*]]
+// CHECK1:       for.inc17:
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC18:%.*]] = add nsw i32 [[TMP21]], 1
+// CHECK1-NEXT:    store i32 [[INC18]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND3]], !llvm.loop [[LOOP14:![0-9]+]]
+// CHECK1:       for.end19:
+// CHECK1-NEXT:    br label [[FOR_INC20:%.*]]
+// CHECK1:       for.inc20:
 // CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD31:%.*]] = add nsw i32 [[TMP22]], 5
-// CHECK1-NEXT:    store i32 [[ADD31]], ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP14:![0-9]+]]
-// CHECK1:       for.end32:
+// CHECK1-NEXT:    [[ADD21:%.*]] = add nsw i32 [[TMP22]], 5
+// CHECK1-NEXT:    store i32 [[ADD21]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP15:![0-9]+]]
+// CHECK1:       for.end22:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
 // CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[ADD33:%.*]] = add nsw i32 [[TMP23]], 1
-// CHECK1-NEXT:    store i32 [[ADD33]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP23]], 1
+// CHECK1-NEXT:    store i32 [[ADD23]], ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -550,7 +532,9 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[_TMP1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
@@ -558,8 +542,8 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[K:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK1-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK1-NEXT:    store i32 7, ptr [[J]], align 4
@@ -606,87 +590,79 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP9]], 4
-// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY:%.*]], label [[FOR_END37:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY:%.*]], label [[FOR_END27:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND8:%.*]]
 // CHECK1:       for.cond8:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i32 [[TMP12]], 5
-// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i32 4, [[ADD9]]
-// CHECK1-NEXT:    br i1 [[CMP10]], label [[COND_TRUE11:%.*]], label [[COND_FALSE12:%.*]]
-// CHECK1:       cond.true11:
-// CHECK1-NEXT:    br label [[COND_END14:%.*]]
-// CHECK1:       cond.false12:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP13]], 5
-// CHECK1-NEXT:    br label [[COND_END14]]
-// CHECK1:       cond.end14:
-// CHECK1-NEXT:    [[COND15:%.*]] = phi i32 [ 4, [[COND_TRUE11]] ], [ [[ADD13]], [[COND_FALSE12]] ]
-// CHECK1-NEXT:    [[CMP16:%.*]] = icmp slt i32 [[TMP11]], [[COND15]]
-// CHECK1-NEXT:    br i1 [[CMP16]], label [[FOR_BODY17:%.*]], label [[FOR_END34:%.*]]
-// CHECK1:       for.body17:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL18:%.*]] = mul nsw i32 [[TMP14]], 3
-// CHECK1-NEXT:    [[ADD19:%.*]] = add nsw i32 7, [[MUL18]]
-// CHECK1-NEXT:    store i32 [[ADD19]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP10]], 5
+// CHECK1-NEXT:    br i1 [[CMP9]], label [[FOR_BODY10:%.*]], label [[FOR_END24:%.*]]
+// CHECK1:       for.body10:
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP11]], [[TMP12]]
+// CHECK1-NEXT:    store i32 [[ADD11]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL12:%.*]] = mul nsw i32 [[TMP13]], 3
+// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i32 7, [[MUL12]]
+// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND14:%.*]]
+// CHECK1:       for.cond14:
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[CMP15:%.*]] = icmp slt i32 [[TMP14]], 5
+// CHECK1-NEXT:    br i1 [[CMP15]], label [[FOR_BODY16:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body16:
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    store i32 [[TMP15]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND20:%.*]]
-// CHECK1:       for.cond20:
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD21:%.*]] = add nsw i32 [[TMP17]], 5
-// CHECK1-NEXT:    [[CMP22:%.*]] = icmp slt i32 4, [[ADD21]]
-// CHECK1-NEXT:    br i1 [[CMP22]], label [[COND_TRUE23:%.*]], label [[COND_FALSE24:%.*]]
-// CHECK1:       cond.true23:
-// CHECK1-NEXT:    br label [[COND_END26:%.*]]
-// CHECK1:       cond.false24:
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD25:%.*]] = add nsw i32 [[TMP18]], 5
-// CHECK1-NEXT:    br label [[COND_END26]]
-// CHECK1:       cond.end26:
-// CHECK1-NEXT:    [[COND27:%.*]] = phi i32 [ 4, [[COND_TRUE23]] ], [ [[ADD25]], [[COND_FALSE24]] ]
-// CHECK1-NEXT:    [[CMP28:%.*]] = icmp slt i32 [[TMP16]], [[COND27]]
-// CHECK1-NEXT:    br i1 [[CMP28]], label [[FOR_BODY29:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body29:
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD17:%.*]] = add nsw i32 [[TMP15]], [[TMP16]]
+// CHECK1-NEXT:    store i32 [[ADD17]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[MUL18:%.*]] = mul nsw i32 [[TMP17]], 3
+// CHECK1-NEXT:    [[ADD19:%.*]] = add nsw i32 7, [[MUL18]]
+// CHECK1-NEXT:    store i32 [[ADD19]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP20:%.*]] = icmp slt i32 [[TMP18]], 4
+// CHECK1-NEXT:    br i1 [[CMP20]], label [[LAND_LHS_TRUE:%.*]], label [[IF_END:%.*]]
+// CHECK1:       land.lhs.true:
 // CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[MUL30:%.*]] = mul nsw i32 [[TMP19]], 3
-// CHECK1-NEXT:    [[ADD31:%.*]] = add nsw i32 7, [[MUL30]]
-// CHECK1-NEXT:    store i32 [[ADD31]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[CMP21:%.*]] = icmp slt i32 [[TMP19]], 4
+// CHECK1-NEXT:    br i1 [[CMP21]], label [[IF_THEN:%.*]], label [[IF_END]]
+// CHECK1:       if.then:
 // CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[J]], align 4
 // CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP20]], i32 noundef [[TMP21]])
+// CHECK1-NEXT:    br label [[IF_END]]
+// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP22]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND20]], !llvm.loop [[LOOP15:![0-9]+]]
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND14]], !llvm.loop [[LOOP16:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC32:%.*]]
-// CHECK1:       for.inc32:
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC33:%.*]] = add nsw i32 [[TMP23]], 1
-// CHECK1-NEXT:    store i32 [[INC33]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP16:![0-9]+]]
-// CHECK1:       for.end34:
-// CHECK1-NEXT:    br label [[FOR_INC35:%.*]]
-// CHECK1:       for.inc35:
+// CHECK1-NEXT:    br label [[FOR_INC22:%.*]]
+// CHECK1:       for.inc22:
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC23:%.*]] = add nsw i32 [[TMP23]], 1
+// CHECK1-NEXT:    store i32 [[INC23]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP17:![0-9]+]]
+// CHECK1:       for.end24:
+// CHECK1-NEXT:    br label [[FOR_INC25:%.*]]
+// CHECK1:       for.inc25:
 // CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD36:%.*]] = add nsw i32 [[TMP24]], 5
-// CHECK1-NEXT:    store i32 [[ADD36]], ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP17:![0-9]+]]
-// CHECK1:       for.end37:
+// CHECK1-NEXT:    [[ADD26:%.*]] = add nsw i32 [[TMP24]], 5
+// CHECK1-NEXT:    store i32 [[ADD26]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP18:![0-9]+]]
+// CHECK1:       for.end27:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
 // CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[ADD38:%.*]] = add nsw i32 [[TMP25]], 1
-// CHECK1-NEXT:    store i32 [[ADD38]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[ADD28:%.*]] = add nsw i32 [[TMP25]], 1
+// CHECK1-NEXT:    store i32 [[ADD28]], ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -699,208 +675,101 @@ extern "C" void foo10(data_t data) {
 // CHECK1-LABEL: define dso_local void @foo5(
 // CHECK1-SAME: ) #[[ATTR0]] {
 // CHECK1-NEXT:  entry:
-// CHECK1-NEXT:    [[DOTOMP_IV:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[_TMP1:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[_TMP2:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_5:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I11:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I12:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[J13:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK1-NEXT:    store i32 7, ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP]], align 4
-// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP]], align 4
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], 5
-// CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
+// CHECK1-NEXT:    store i32 4, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
+// CHECK1-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], 4
 // CHECK1-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK1:       cond.true:
 // CHECK1-NEXT:    br label [[COND_END:%.*]]
 // CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP]], align 4
-// CHECK1-NEXT:    [[ADD4:%.*]] = add nsw i32 [[TMP3]], 5
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
 // CHECK1-NEXT:    br label [[COND_END]]
 // CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD4]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[SUB:%.*]] = sub i32 [[TMP4]], [[TMP5]]
-// CHECK1-NEXT:    [[SUB6:%.*]] = sub i32 [[SUB]], 1
-// CHECK1-NEXT:    [[ADD7:%.*]] = add i32 [[SUB6]], 1
-// CHECK1-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD7]], 1
-// CHECK1-NEXT:    [[CONV:%.*]] = zext i32 [[DIV]] to i64
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 1, [[CONV]]
-// CHECK1-NEXT:    [[MUL8:%.*]] = mul nsw i64 [[MUL]], 4
-// CHECK1-NEXT:    [[SUB9:%.*]] = sub nsw i64 [[MUL8]], 1
-// CHECK1-NEXT:    store i64 [[SUB9]], ptr [[DOTCAPTURE_EXPR_5]], align 8
-// CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    store i32 [[TMP6]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 7, ptr [[J]], align 4
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i32 [[TMP7]], [[TMP8]]
-// CHECK1-NEXT:    br i1 [[CMP10]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END:%.*]]
-// CHECK1:       omp.precond.then:
-// CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_5]], align 8
-// CHECK1-NEXT:    store i64 [[TMP9]], ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    store i64 1, ptr [[DOTOMP_STRIDE]], align 8
-// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
-// CHECK1-NEXT:    call void @__kmpc_for_static_init_8(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i64 1, i64 1)
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_5]], align 8
-// CHECK1-NEXT:    [[CMP14:%.*]] = icmp sgt i64 [[TMP10]], [[TMP11]]
-// CHECK1-NEXT:    br i1 [[CMP14]], label [[COND_TRUE15:%.*]], label [[COND_FALSE16:%.*]]
-// CHECK1:       cond.true15:
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_5]], align 8
-// CHECK1-NEXT:    br label [[COND_END17:%.*]]
-// CHECK1:       cond.false16:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    br label [[COND_END17]]
-// CHECK1:       cond.end17:
-// CHECK1-NEXT:    [[COND18:%.*]] = phi i64 [ [[TMP12]], [[COND_TRUE15]] ], [ [[TMP13]], [[COND_FALSE16]] ]
-// CHECK1-NEXT:    store i64 [[COND18]], ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
-// CHECK1-NEXT:    store i64 [[TMP14]], ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[TMP2]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
+// CHECK1-NEXT:    store i32 [[TMP3]], ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK1:       omp.inner.for.cond:
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[CMP19:%.*]] = icmp sle i64 [[TMP15]], [[TMP16]]
-// CHECK1-NEXT:    br i1 [[CMP19]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], [[TMP5]]
+// CHECK1-NEXT:    br i1 [[CMP2]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
 // CHECK1:       omp.inner.for.body:
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[SUB20:%.*]] = sub i32 [[TMP18]], [[TMP19]]
-// CHECK1-NEXT:    [[SUB21:%.*]] = sub i32 [[SUB20]], 1
-// CHECK1-NEXT:    [[ADD22:%.*]] = add i32 [[SUB21]], 1
-// CHECK1-NEXT:    [[DIV23:%.*]] = udiv i32 [[ADD22]], 1
-// CHECK1-NEXT:    [[MUL24:%.*]] = mul i32 1, [[DIV23]]
-// CHECK1-NEXT:    [[MUL25:%.*]] = mul i32 [[MUL24]], 4
-// CHECK1-NEXT:    [[CONV26:%.*]] = zext i32 [[MUL25]] to i64
-// CHECK1-NEXT:    [[DIV27:%.*]] = sdiv i64 [[TMP17]], [[CONV26]]
-// CHECK1-NEXT:    [[MUL28:%.*]] = mul nsw i64 [[DIV27]], 5
-// CHECK1-NEXT:    [[ADD29:%.*]] = add nsw i64 0, [[MUL28]]
-// CHECK1-NEXT:    [[CONV30:%.*]] = trunc i64 [[ADD29]] to i32
-// CHECK1-NEXT:    store i32 [[CONV30]], ptr [[DOTFLOOR_0_IV_I11]], align 4
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[CONV31:%.*]] = sext i32 [[TMP20]] to i64
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP22:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[SUB32:%.*]] = sub i32 [[TMP23]], [[TMP24]]
-// CHECK1-NEXT:    [[SUB33:%.*]] = sub i32 [[SUB32]], 1
-// CHECK1-NEXT:    [[ADD34:%.*]] = add i32 [[SUB33]], 1
-// CHECK1-NEXT:    [[DIV35:%.*]] = udiv i32 [[ADD34]], 1
-// CHECK1-NEXT:    [[MUL36:%.*]] = mul i32 1, [[DIV35]]
-// CHECK1-NEXT:    [[MUL37:%.*]] = mul i32 [[MUL36]], 4
-// CHECK1-NEXT:    [[CONV38:%.*]] = zext i32 [[MUL37]] to i64
-// CHECK1-NEXT:    [[DIV39:%.*]] = sdiv i64 [[TMP22]], [[CONV38]]
-// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[SUB40:%.*]] = sub i32 [[TMP25]], [[TMP26]]
-// CHECK1-NEXT:    [[SUB41:%.*]] = sub i32 [[SUB40]], 1
-// CHECK1-NEXT:    [[ADD42:%.*]] = add i32 [[SUB41]], 1
-// CHECK1-NEXT:    [[DIV43:%.*]] = udiv i32 [[ADD42]], 1
-// CHECK1-NEXT:    [[MUL44:%.*]] = mul i32 1, [[DIV43]]
-// CHECK1-NEXT:    [[MUL45:%.*]] = mul i32 [[MUL44]], 4
-// CHECK1-NEXT:    [[CONV46:%.*]] = zext i32 [[MUL45]] to i64
-// CHECK1-NEXT:    [[MUL47:%.*]] = mul nsw i64 [[DIV39]], [[CONV46]]
-// CHECK1-NEXT:    [[SUB48:%.*]] = sub nsw i64 [[TMP21]], [[MUL47]]
-// CHECK1-NEXT:    [[DIV49:%.*]] = sdiv i64 [[SUB48]], 4
-// CHECK1-NEXT:    [[MUL50:%.*]] = mul nsw i64 [[DIV49]], 1
-// CHECK1-NEXT:    [[ADD51:%.*]] = add nsw i64 [[CONV31]], [[MUL50]]
-// CHECK1-NEXT:    [[CONV52:%.*]] = trunc i64 [[ADD51]] to i32
-// CHECK1-NEXT:    store i32 [[CONV52]], ptr [[DOTTILE_0_IV_I12]], align 4
-// CHECK1-NEXT:    [[TMP27:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP28:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[SUB53:%.*]] = sub i32 [[TMP29]], [[TMP30]]
-// CHECK1-NEXT:    [[SUB54:%.*]] = sub i32 [[SUB53]], 1
-// CHECK1-NEXT:    [[ADD55:%.*]] = add i32 [[SUB54]], 1
-// CHECK1-NEXT:    [[DIV56:%.*]] = udiv i32 [[ADD55]], 1
-// CHECK1-NEXT:    [[MUL57:%.*]] = mul i32 1, [[DIV56]]
-// CHECK1-NEXT:    [[MUL58:%.*]] = mul i32 [[MUL57]], 4
-// CHECK1-NEXT:    [[CONV59:%.*]] = zext i32 [[MUL58]] to i64
-// CHECK1-NEXT:    [[DIV60:%.*]] = sdiv i64 [[TMP28]], [[CONV59]]
-// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[SUB61:%.*]] = sub i32 [[TMP31]], [[TMP32]]
-// CHECK1-NEXT:    [[SUB62:%.*]] = sub i32 [[SUB61]], 1
-// CHECK1-NEXT:    [[ADD63:%.*]] = add i32 [[SUB62]], 1
-// CHECK1-NEXT:    [[DIV64:%.*]] = udiv i32 [[ADD63]], 1
-// CHECK1-NEXT:    [[MUL65:%.*]] = mul i32 1, [[DIV64]]
-// CHECK1-NEXT:    [[MUL66:%.*]] = mul i32 [[MUL65]], 4
-// CHECK1-NEXT:    [[CONV67:%.*]] = zext i32 [[MUL66]] to i64
-// CHECK1-NEXT:    [[MUL68:%.*]] = mul nsw i64 [[DIV60]], [[CONV67]]
-// CHECK1-NEXT:    [[SUB69:%.*]] = sub nsw i64 [[TMP27]], [[MUL68]]
-// CHECK1-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK1-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[SUB70:%.*]] = sub i32 [[TMP35]], [[TMP36]]
-// CHECK1-NEXT:    [[SUB71:%.*]] = sub i32 [[SUB70]], 1
-// CHECK1-NEXT:    [[ADD72:%.*]] = add i32 [[SUB71]], 1
-// CHECK1-NEXT:    [[DIV73:%.*]] = udiv i32 [[ADD72]], 1
-// CHECK1-NEXT:    [[MUL74:%.*]] = mul i32 1, [[DIV73]]
-// CHECK1-NEXT:    [[MUL75:%.*]] = mul i32 [[MUL74]], 4
-// CHECK1-NEXT:    [[CONV76:%.*]] = zext i32 [[MUL75]] to i64
-// CHECK1-NEXT:    [[DIV77:%.*]] = sdiv i64 [[TMP34]], [[CONV76]]
-// CHECK1-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK1-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[SUB78:%.*]] = sub i32 [[TMP37]], [[TMP38]]
-// CHECK1-NEXT:    [[SUB79:%.*]] = sub i32 [[SUB78]], 1
-// CHECK1-NEXT:    [[ADD80:%.*]] = add i32 [[SUB79]], 1
-// CHECK1-NEXT:    [[DIV81:%.*]] = udiv i32 [[ADD80]], 1
-// CHECK1-NEXT:    [[MUL82:%.*]] = mul i32 1, [[DIV81]]
-// CHECK1-NEXT:    [[MUL83:%.*]] = mul i32 [[MUL82]], 4
-// CHECK1-NEXT:    [[CONV84:%.*]] = zext i32 [[MUL83]] to i64
-// CHECK1-NEXT:    [[MUL85:%.*]] = mul nsw i64 [[DIV77]], [[CONV84]]
-// CHECK1-NEXT:    [[SUB86:%.*]] = sub nsw i64 [[TMP33]], [[MUL85]]
-// CHECK1-NEXT:    [[DIV87:%.*]] = sdiv i64 [[SUB86]], 4
-// CHECK1-NEXT:    [[MUL88:%.*]] = mul nsw i64 [[DIV87]], 4
-// CHECK1-NEXT:    [[SUB89:%.*]] = sub nsw i64 [[SUB69]], [[MUL88]]
-// CHECK1-NEXT:    [[MUL90:%.*]] = mul nsw i64 [[SUB89]], 3
-// CHECK1-NEXT:    [[ADD91:%.*]] = add nsw i64 7, [[MUL90]]
-// CHECK1-NEXT:    [[CONV92:%.*]] = trunc i64 [[ADD91]] to i32
-// CHECK1-NEXT:    store i32 [[CONV92]], ptr [[J13]], align 4
-// CHECK1-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTTILE_0_IV_I12]], align 4
-// CHECK1-NEXT:    [[MUL93:%.*]] = mul nsw i32 [[TMP39]], 3
-// CHECK1-NEXT:    [[ADD94:%.*]] = add nsw i32 7, [[MUL93]]
-// CHECK1-NEXT:    store i32 [[ADD94]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP41:%.*]] = load i32, ptr [[J13]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP40]], i32 noundef [[TMP41]])
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP6]], 5
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[DIV]], 5
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[DIV3:%.*]] = sdiv i32 [[TMP8]], 5
+// CHECK1-NEXT:    [[MUL4:%.*]] = mul nsw i32 [[DIV3]], 5
+// CHECK1-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP7]], [[MUL4]]
+// CHECK1-NEXT:    [[MUL5:%.*]] = mul nsw i32 [[SUB]], 1
+// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL5]]
+// CHECK1-NEXT:    store i32 [[ADD6]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
+// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL8:%.*]] = mul nsw i32 [[TMP11]], 3
+// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL8]]
+// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i32 [[TMP12]], 4
+// CHECK1-NEXT:    br i1 [[CMP10]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK1:       if.then:
+// CHECK1-NEXT:    store i32 7, ptr [[J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND:%.*]]
+// CHECK1:       for.cond:
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    [[CMP11:%.*]] = icmp slt i32 [[TMP13]], 17
+// CHECK1-NEXT:    br i1 [[CMP11]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body:
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP14]], i32 noundef [[TMP15]])
+// CHECK1-NEXT:    br label [[FOR_INC:%.*]]
+// CHECK1:       for.inc:
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP16]], 3
+// CHECK1-NEXT:    store i32 [[ADD12]], ptr [[J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP19:![0-9]+]]
+// CHECK1:       for.end:
+// CHECK1-NEXT:    br label [[IF_END]]
+// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP42:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[ADD95:%.*]] = add nsw i64 [[TMP42]], 1
-// CHECK1-NEXT:    store i64 [[ADD95]], ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP17]], 1
+// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
 // CHECK1:       omp.loop.exit:
 // CHECK1-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP0]])
-// CHECK1-NEXT:    br label [[OMP_PRECOND_END]]
-// CHECK1:       omp.precond.end:
 // CHECK1-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[TMP0]])
 // CHECK1-NEXT:    ret void
 //
@@ -920,12 +789,13 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
 // CHECK1-NEXT:    store i32 7, ptr [[I]], align 4
@@ -960,46 +830,43 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP7]], 5
 // CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 [[TMP8]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND:%.*]]
 // CHECK1:       for.cond:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD2:%.*]] = add nsw i32 [[TMP10]], 5
-// CHECK1-NEXT:    [[CMP3:%.*]] = icmp slt i32 4, [[ADD2]]
-// CHECK1-NEXT:    br i1 [[CMP3]], label [[COND_TRUE4:%.*]], label [[COND_FALSE5:%.*]]
-// CHECK1:       cond.true4:
-// CHECK1-NEXT:    br label [[COND_END7:%.*]]
-// CHECK1:       cond.false5:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP11]], 5
-// CHECK1-NEXT:    br label [[COND_END7]]
-// CHECK1:       cond.end7:
-// CHECK1-NEXT:    [[COND8:%.*]] = phi i32 [ 4, [[COND_TRUE4]] ], [ [[ADD6]], [[COND_FALSE5]] ]
-// CHECK1-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP9]], [[COND8]]
-// CHECK1-NEXT:    br i1 [[CMP9]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP8]], 5
+// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
 // CHECK1:       for.body:
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
+// CHECK1-NEXT:    store i32 [[ADD3]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL4:%.*]] = mul nsw i32 [[TMP11]], 3
+// CHECK1-NEXT:    [[ADD5:%.*]] = add nsw i32 7, [[MUL4]]
+// CHECK1-NEXT:    store i32 [[ADD5]], ptr [[I]], align 4
 // CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL10:%.*]] = mul nsw i32 [[TMP12]], 3
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 7, [[MUL10]]
-// CHECK1-NEXT:    store i32 [[ADD11]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[CMP6:%.*]] = icmp slt i32 [[TMP12]], 4
+// CHECK1-NEXT:    br i1 [[CMP6]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK1:       if.then:
 // CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP13]])
+// CHECK1-NEXT:    br label [[IF_END]]
+// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP18:![0-9]+]]
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP20:![0-9]+]]
 // CHECK1:       for.end:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP15]], 1
-// CHECK1-NEXT:    store i32 [[ADD12]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 [[TMP15]], 1
+// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -1024,8 +891,9 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[START_ADDR]], align 4
@@ -1049,53 +917,48 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK1-NEXT:    [[ADD5:%.*]] = add i32 [[TMP6]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP5]], [[ADD5]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END15:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK1:       for.cond6:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[ADD7:%.*]] = add i32 [[TMP9]], 1
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD8:%.*]] = add i32 [[TMP10]], 5
-// CHECK1-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
-// CHECK1-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK1:       cond.true:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[ADD10:%.*]] = add i32 [[TMP11]], 1
-// CHECK1-NEXT:    br label [[COND_END:%.*]]
-// CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD11:%.*]] = add i32 [[TMP12]], 5
-// CHECK1-NEXT:    br label [[COND_END]]
-// CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP8]], [[COND]]
-// CHECK1-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body13:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP14]], 3
-// CHECK1-NEXT:    [[ADD14:%.*]] = add i32 [[TMP13]], [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP15]])
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP7]], 5
+// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body8:
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD9:%.*]] = add i32 [[TMP8]], [[TMP9]]
+// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP11]], 3
+// CHECK1-NEXT:    [[ADD10:%.*]] = add i32 [[TMP10]], [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD10]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD11:%.*]] = add i32 [[TMP13]], 1
+// CHECK1-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP12]], [[ADD11]]
+// CHECK1-NEXT:    br i1 [[CMP12]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK1:       if.then:
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP14]])
+// CHECK1-NEXT:    br label [[IF_END]]
+// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP16]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP21:![0-9]+]]
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP15]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP23:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC15:%.*]]
-// CHECK1:       for.inc15:
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD16:%.*]] = add i32 [[TMP17]], 5
-// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP22:![0-9]+]]
-// CHECK1:       for.end17:
+// CHECK1-NEXT:    br label [[FOR_INC13:%.*]]
+// CHECK1:       for.inc13:
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD14:%.*]] = add i32 [[TMP16]], 5
+// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP24:![0-9]+]]
+// CHECK1:       for.end15:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -1104,8 +967,9 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
 // CHECK1-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
@@ -1113,78 +977,65 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END14:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK1:       for.cond1:
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], 0
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP2]], 0
 // CHECK1-NEXT:    br i1 [[CMP2]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK1:       cond.true:
 // CHECK1-NEXT:    br label [[COND_END:%.*]]
 // CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[A_ADDR]], align 4
 // CHECK1-NEXT:    br label [[COND_END]]
 // CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP5]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[COND]]
-// CHECK1-NEXT:    [[CMP3:%.*]] = icmp slt i32 4, [[ADD]]
-// CHECK1-NEXT:    br i1 [[CMP3]], label [[COND_TRUE4:%.*]], label [[COND_FALSE5:%.*]]
-// CHECK1:       cond.true4:
-// CHECK1-NEXT:    br label [[COND_END12:%.*]]
-// CHECK1:       cond.false5:
-// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    [[CMP6:%.*]] = icmp sle i32 [[TMP7]], 0
-// CHECK1-NEXT:    br i1 [[CMP6]], label [[COND_TRUE7:%.*]], label [[COND_FALSE8:%.*]]
-// CHECK1:       cond.true7:
-// CHECK1-NEXT:    br label [[COND_END9:%.*]]
-// CHECK1:       cond.false8:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    br label [[COND_END9]]
-// CHECK1:       cond.end9:
-// CHECK1-NEXT:    [[COND10:%.*]] = phi i32 [ 1, [[COND_TRUE7]] ], [ [[TMP8]], [[COND_FALSE8]] ]
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP6]], [[COND10]]
-// CHECK1-NEXT:    br label [[COND_END12]]
-// CHECK1:       cond.end12:
-// CHECK1-NEXT:    [[COND13:%.*]] = phi i32 [ 4, [[COND_TRUE4]] ], [ [[ADD11]], [[COND_END9]] ]
-// CHECK1-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP2]], [[COND13]]
-// CHECK1-NEXT:    br i1 [[CMP14]], label [[FOR_BODY15:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body15:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP9]], 3
-// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP10]])
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP3]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[TMP1]], [[COND]]
+// CHECK1-NEXT:    br i1 [[CMP3]], label [[FOR_BODY4:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body4:
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
+// CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
+// CHECK1-NEXT:    [[ADD5:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD5]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP6:%.*]] = icmp slt i32 [[TMP7]], 4
+// CHECK1-NEXT:    br i1 [[CMP6]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK1:       if.then:
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP8]])
+// CHECK1-NEXT:    br label [[IF_END]]
+// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP23:![0-9]+]]
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP9]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP25:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC17:%.*]]
-// CHECK1:       for.inc17:
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    [[CMP18:%.*]] = icmp sle i32 [[TMP12]], 0
-// CHECK1-NEXT:    br i1 [[CMP18]], label [[COND_TRUE19:%.*]], label [[COND_FALSE20:%.*]]
-// CHECK1:       cond.true19:
-// CHECK1-NEXT:    br label [[COND_END21:%.*]]
-// CHECK1:       cond.false20:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    br label [[COND_END21]]
-// CHECK1:       cond.end21:
-// CHECK1-NEXT:    [[COND22:%.*]] = phi i32 [ 1, [[COND_TRUE19]] ], [ [[TMP13]], [[COND_FALSE20]] ]
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP14]], [[COND22]]
-// CHECK1-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP24:![0-9]+]]
-// CHECK1:       for.end24:
+// CHECK1-NEXT:    br label [[FOR_INC7:%.*]]
+// CHECK1:       for.inc7:
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    [[CMP8:%.*]] = icmp sle i32 [[TMP10]], 0
+// CHECK1-NEXT:    br i1 [[CMP8]], label [[COND_TRUE9:%.*]], label [[COND_FALSE10:%.*]]
+// CHECK1:       cond.true9:
+// CHECK1-NEXT:    br label [[COND_END11:%.*]]
+// CHECK1:       cond.false10:
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    br label [[COND_END11]]
+// CHECK1:       cond.end11:
+// CHECK1-NEXT:    [[COND12:%.*]] = phi i32 [ 1, [[COND_TRUE9]] ], [ [[TMP11]], [[COND_FALSE10]] ]
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP12]], [[COND12]]
+// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP26:![0-9]+]]
+// CHECK1:       for.end14:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -1197,19 +1048,20 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK1-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
 // CHECK1-NEXT:    store ptr [[ARRAY]], ptr [[__RANGE2]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META27:![0-9]+]]
 // CHECK1-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP0]], i64 0, i64 0
 // CHECK1-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 12
 // CHECK1-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META27]]
 // CHECK1-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP1]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[ARRAYDECAY1]], ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META27]]
 // CHECK1-NEXT:    [[ARRAYDECAY2:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP2]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[ARRAYDECAY2]], ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__END2]], align 8
@@ -1232,56 +1084,51 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK1:       for.cond7:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
-// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
-// CHECK1-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK1:       cond.true:
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
-// CHECK1-NEXT:    br label [[COND_END:%.*]]
-// CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
-// CHECK1-NEXT:    br label [[COND_END]]
-// CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
-// CHECK1-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body14:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
+// CHECK1-NEXT:    br i1 [[CMP8]], label [[FOR_BODY9:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body9:
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
+// CHECK1-NEXT:    store i64 [[ADD10]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP12]], 1
+// CHECK1-NEXT:    [[ADD_PTR11:%.*]] = getelementptr inbounds double, ptr [[TMP11]], i64 [[MUL]]
+// CHECK1-NEXT:    store ptr [[ADD_PTR11]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP14:%.*]] = load double, ptr [[TMP13]], align 8
+// CHECK1-NEXT:    store double [[TMP14]], ptr [[V]], align 8
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
-// CHECK1-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
-// CHECK1-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
-// CHECK1-NEXT:    store double [[TMP17]], ptr [[V]], align 8
-// CHECK1-NEXT:    [[TMP18:%.*]] = load double, ptr [[V]], align 8
-// CHECK1-NEXT:    call void (...) @body(double noundef [[TMP18]])
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP16]], 1
+// CHECK1-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP15]], [[ADD12]]
+// CHECK1-NEXT:    br i1 [[CMP13]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK1:       if.then:
+// CHECK1-NEXT:    [[TMP17:%.*]] = load double, ptr [[V]], align 8
+// CHECK1-NEXT:    call void (...) @body(double noundef [[TMP17]])
+// CHECK1-NEXT:    br label [[IF_END]]
+// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP19]], 1
-// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP25:![0-9]+]]
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP18]], 1
+// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP28:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC16:%.*]]
-// CHECK1:       for.inc16:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP20]], 5
-// CHECK1-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP26:![0-9]+]]
-// CHECK1:       for.end18:
+// CHECK1-NEXT:    br label [[FOR_INC14:%.*]]
+// CHECK1:       for.inc14:
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD15:%.*]] = add nsw i64 [[TMP19]], 5
+// CHECK1-NEXT:    store i64 [[ADD15]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP29:![0-9]+]]
+// CHECK1:       for.end16:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -1295,20 +1142,21 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK1-NEXT:    store double 4.200000e+01, ptr [[C]], align 8
 // CHECK1-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
 // CHECK1-NEXT:    store ptr [[ARRAY]], ptr [[__RANGE2]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META27]]
 // CHECK1-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP0]], i64 0, i64 0
 // CHECK1-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 12
 // CHECK1-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META27]]
 // CHECK1-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP1]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[ARRAYDECAY1]], ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META27]]
 // CHECK1-NEXT:    [[ARRAYDECAY2:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP2]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[ARRAYDECAY2]], ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__END2]], align 8
@@ -1331,57 +1179,52 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK1:       for.cond7:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
-// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
-// CHECK1-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK1:       cond.true:
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
-// CHECK1-NEXT:    br label [[COND_END:%.*]]
-// CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
-// CHECK1-NEXT:    br label [[COND_END]]
-// CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
-// CHECK1-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body14:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
+// CHECK1-NEXT:    br i1 [[CMP8]], label [[FOR_BODY9:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body9:
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
+// CHECK1-NEXT:    store i64 [[ADD10]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP12]], 1
+// CHECK1-NEXT:    [[ADD_PTR11:%.*]] = getelementptr inbounds double, ptr [[TMP11]], i64 [[MUL]]
+// CHECK1-NEXT:    store ptr [[ADD_PTR11]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP14:%.*]] = load double, ptr [[TMP13]], align 8
+// CHECK1-NEXT:    store double [[TMP14]], ptr [[V]], align 8
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
-// CHECK1-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
-// CHECK1-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
-// CHECK1-NEXT:    store double [[TMP17]], ptr [[V]], align 8
-// CHECK1-NEXT:    [[TMP18:%.*]] = load double, ptr [[C]], align 8
-// CHECK1-NEXT:    [[TMP19:%.*]] = load double, ptr [[V]], align 8
-// CHECK1-NEXT:    call void (...) @body(double noundef [[TMP18]], double noundef [[TMP19]])
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP16]], 1
+// CHECK1-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP15]], [[ADD12]]
+// CHECK1-NEXT:    br i1 [[CMP13]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK1:       if.then:
+// CHECK1-NEXT:    [[TMP17:%.*]] = load double, ptr [[C]], align 8
+// CHECK1-NEXT:    [[TMP18:%.*]] = load double, ptr [[V]], align 8
+// CHECK1-NEXT:    call void (...) @body(double noundef [[TMP17]], double noundef [[TMP18]])
+// CHECK1-NEXT:    br label [[IF_END]]
+// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP20]], 1
-// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP27:![0-9]+]]
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP19]], 1
+// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP30:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC16:%.*]]
-// CHECK1:       for.inc16:
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP21]], 5
-// CHECK1-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP28:![0-9]+]]
-// CHECK1:       for.end18:
+// CHECK1-NEXT:    br label [[FOR_INC14:%.*]]
+// CHECK1:       for.inc14:
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD15:%.*]] = add nsw i64 [[TMP20]], 5
+// CHECK1-NEXT:    store i64 [[ADD15]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP31:![0-9]+]]
+// CHECK1:       for.end16:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -1414,8 +1257,9 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:  entry:
 // CHECK2-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[I2:%.*]] = alloca ptr, align 8
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK2-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK2-NEXT:    [[I:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
@@ -1427,50 +1271,47 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END12:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END11:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK2:       for.cond4:
-// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 5
-// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
-// CHECK2-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK2:       cond.true:
-// CHECK2-NEXT:    br label [[COND_END:%.*]]
-// CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP4]], 5
-// CHECK2-NEXT:    br label [[COND_END]]
-// CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
-// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body8:
-// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
-// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK2-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[I2]], align 8
-// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[TMP6]], align 4
-// CHECK2-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[I2]], align 8
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP1]], 5
+// CHECK2-NEXT:    br i1 [[CMP5]], label [[FOR_BODY6:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body6:
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
+// CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP4]], 3
+// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK2-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
+// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[TMP5]], align 4
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP8:%.*]] = icmp slt i32 [[TMP6]], 4
+// CHECK2-NEXT:    br i1 [[CMP8]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK2:       if.then:
+// CHECK2-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2]], !align [[META3]]
 // CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
 // CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP8]])
+// CHECK2-NEXT:    br label [[IF_END]]
+// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP9]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP3:![0-9]+]]
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP4:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC10:%.*]]
-// CHECK2:       for.inc10:
+// CHECK2-NEXT:    br label [[FOR_INC9:%.*]]
+// CHECK2:       for.inc9:
 // CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP10]], 5
-// CHECK2-NEXT:    store i32 [[ADD11]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP5:![0-9]+]]
-// CHECK2:       for.end12:
+// CHECK2-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP10]], 5
+// CHECK2-NEXT:    store i32 [[ADD10]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
+// CHECK2:       for.end11:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -1491,8 +1332,9 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -1521,54 +1363,49 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK2-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP8]], [[ADD5]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END15:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 [[TMP10]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK2:       for.cond6:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[ADD7:%.*]] = add i32 [[TMP12]], 1
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD8:%.*]] = add i32 [[TMP13]], 5
-// CHECK2-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
-// CHECK2-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK2:       cond.true:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[ADD10:%.*]] = add i32 [[TMP14]], 1
-// CHECK2-NEXT:    br label [[COND_END:%.*]]
-// CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD11:%.*]] = add i32 [[TMP15]], 5
-// CHECK2-NEXT:    br label [[COND_END]]
-// CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP11]], [[COND]]
-// CHECK2-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body13:
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP17]], [[TMP18]]
-// CHECK2-NEXT:    [[ADD14:%.*]] = add i32 [[TMP16]], [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP19]])
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP10]], 5
+// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body8:
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD9:%.*]] = add i32 [[TMP11]], [[TMP12]]
+// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP14]], [[TMP15]]
+// CHECK2-NEXT:    [[ADD10:%.*]] = add i32 [[TMP13]], [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD10]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD11:%.*]] = add i32 [[TMP17]], 1
+// CHECK2-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP16]], [[ADD11]]
+// CHECK2-NEXT:    br i1 [[CMP12]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK2:       if.then:
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP18]])
+// CHECK2-NEXT:    br label [[IF_END]]
+// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP20]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP6:![0-9]+]]
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP19]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP7:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC15:%.*]]
-// CHECK2:       for.inc15:
-// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD16:%.*]] = add i32 [[TMP21]], 5
-// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP7:![0-9]+]]
-// CHECK2:       for.end17:
+// CHECK2-NEXT:    br label [[FOR_INC13:%.*]]
+// CHECK2:       for.inc13:
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD14:%.*]] = add i32 [[TMP20]], 5
+// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
+// CHECK2:       for.end15:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -1582,20 +1419,21 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK2-NEXT:    store double 4.200000e+01, ptr [[C]], align 8
 // CHECK2-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
 // CHECK2-NEXT:    store ptr [[ARRAY]], ptr [[__RANGE2]], align 8
-// CHECK2-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META9:![0-9]+]]
 // CHECK2-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP0]], i64 0, i64 0
 // CHECK2-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 12
 // CHECK2-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
-// CHECK2-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META9]]
 // CHECK2-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP1]], i64 0, i64 0
 // CHECK2-NEXT:    store ptr [[ARRAYDECAY1]], ptr [[__BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META9]]
 // CHECK2-NEXT:    [[ARRAYDECAY2:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP2]], i64 0, i64 0
 // CHECK2-NEXT:    store ptr [[ARRAYDECAY2]], ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK2-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__END2]], align 8
@@ -1618,57 +1456,52 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK2:       for.cond7:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
-// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
-// CHECK2-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK2:       cond.true:
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
-// CHECK2-NEXT:    br label [[COND_END:%.*]]
-// CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
-// CHECK2-NEXT:    br label [[COND_END]]
-// CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
-// CHECK2-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body14:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
+// CHECK2-NEXT:    br i1 [[CMP8]], label [[FOR_BODY9:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body9:
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
+// CHECK2-NEXT:    store i64 [[ADD10]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP12]], 1
+// CHECK2-NEXT:    [[ADD_PTR11:%.*]] = getelementptr inbounds double, ptr [[TMP11]], i64 [[MUL]]
+// CHECK2-NEXT:    store ptr [[ADD_PTR11]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP14:%.*]] = load double, ptr [[TMP13]], align 8
+// CHECK2-NEXT:    store double [[TMP14]], ptr [[V]], align 8
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
-// CHECK2-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
-// CHECK2-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
-// CHECK2-NEXT:    store double [[TMP17]], ptr [[V]], align 8
-// CHECK2-NEXT:    [[TMP18:%.*]] = load double, ptr [[C]], align 8
-// CHECK2-NEXT:    [[TMP19:%.*]] = load double, ptr [[V]], align 8
-// CHECK2-NEXT:    call void (...) @body(double noundef [[TMP18]], double noundef [[TMP19]])
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP16]], 1
+// CHECK2-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP15]], [[ADD12]]
+// CHECK2-NEXT:    br i1 [[CMP13]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK2:       if.then:
+// CHECK2-NEXT:    [[TMP17:%.*]] = load double, ptr [[C]], align 8
+// CHECK2-NEXT:    [[TMP18:%.*]] = load double, ptr [[V]], align 8
+// CHECK2-NEXT:    call void (...) @body(double noundef [[TMP17]], double noundef [[TMP18]])
+// CHECK2-NEXT:    br label [[IF_END]]
+// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP20]], 1
-// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP8:![0-9]+]]
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP19]], 1
+// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP10:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC16:%.*]]
-// CHECK2:       for.inc16:
-// CHECK2-NEXT:    [[TMP21:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP21]], 5
-// CHECK2-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP9:![0-9]+]]
-// CHECK2:       for.end18:
+// CHECK2-NEXT:    br label [[FOR_INC14:%.*]]
+// CHECK2:       for.inc14:
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD15:%.*]] = add nsw i64 [[TMP20]], 5
+// CHECK2-NEXT:    store i64 [[ADD15]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP11:![0-9]+]]
+// CHECK2:       for.end16:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -1679,11 +1512,13 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -1694,94 +1529,86 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END30:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
 // CHECK2:       for.body:
 // CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK2:       for.cond1:
 // CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 4
-// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END27:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END21:%.*]]
 // CHECK2:       for.body3:
-// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 [[TMP2]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK2:       for.cond4:
-// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], 5
-// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
-// CHECK2-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK2:       cond.true:
-// CHECK2-NEXT:    br label [[COND_END:%.*]]
-// CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP5]], 5
-// CHECK2-NEXT:    br label [[COND_END]]
-// CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP3]], [[COND]]
-// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END24:%.*]]
-// CHECK2:       for.body8:
-// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
-// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP2]], 5
+// CHECK2-NEXT:    br i1 [[CMP5]], label [[FOR_BODY6:%.*]], label [[FOR_END18:%.*]]
+// CHECK2:       for.body6:
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[TMP4]]
+// CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
+// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND8:%.*]]
+// CHECK2:       for.cond8:
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP6]], 5
+// CHECK2-NEXT:    br i1 [[CMP9]], label [[FOR_BODY10:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body10:
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND10:%.*]]
-// CHECK2:       for.cond10:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP9]], 5
-// CHECK2-NEXT:    [[CMP12:%.*]] = icmp slt i32 4, [[ADD11]]
-// CHECK2-NEXT:    br i1 [[CMP12]], label [[COND_TRUE13:%.*]], label [[COND_FALSE14:%.*]]
-// CHECK2:       cond.true13:
-// CHECK2-NEXT:    br label [[COND_END16:%.*]]
-// CHECK2:       cond.false14:
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP10]], 5
-// CHECK2-NEXT:    br label [[COND_END16]]
-// CHECK2:       cond.end16:
-// CHECK2-NEXT:    [[COND17:%.*]] = phi i32 [ 4, [[COND_TRUE13]] ], [ [[ADD15]], [[COND_FALSE14]] ]
-// CHECK2-NEXT:    [[CMP18:%.*]] = icmp slt i32 [[TMP8]], [[COND17]]
-// CHECK2-NEXT:    br i1 [[CMP18]], label [[FOR_BODY19:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body19:
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP7]], [[TMP8]]
+// CHECK2-NEXT:    store i32 [[ADD11]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[MUL12:%.*]] = mul nsw i32 [[TMP9]], 3
+// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i32 7, [[MUL12]]
+// CHECK2-NEXT:    store i32 [[ADD13]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP10]], 4
+// CHECK2-NEXT:    br i1 [[CMP14]], label [[LAND_LHS_TRUE:%.*]], label [[IF_END:%.*]]
+// CHECK2:       land.lhs.true:
 // CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[TMP11]], 3
-// CHECK2-NEXT:    [[ADD21:%.*]] = add nsw i32 7, [[MUL20]]
-// CHECK2-NEXT:    store i32 [[ADD21]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[CMP15:%.*]] = icmp slt i32 [[TMP11]], 4
+// CHECK2-NEXT:    br i1 [[CMP15]], label [[IF_THEN:%.*]], label [[IF_END]]
+// CHECK2:       if.then:
 // CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
 // CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
 // CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP12]], i32 noundef [[TMP13]])
+// CHECK2-NEXT:    br label [[IF_END]]
+// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND10]], !llvm.loop [[LOOP10:![0-9]+]]
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP12:![0-9]+]]
 // CHECK2:       for.end:
+// CHECK2-NEXT:    br label [[FOR_INC16:%.*]]
+// CHECK2:       for.inc16:
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC17:%.*]] = add nsw i32 [[TMP15]], 1
+// CHECK2-NEXT:    store i32 [[INC17]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP13:![0-9]+]]
+// CHECK2:       for.end18:
+// CHECK2-NEXT:    br label [[FOR_INC19:%.*]]
+// CHECK2:       for.inc19:
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP16]], 5
+// CHECK2-NEXT:    store i32 [[ADD20]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP14:![0-9]+]]
+// CHECK2:       for.end21:
 // CHECK2-NEXT:    br label [[FOR_INC22:%.*]]
 // CHECK2:       for.inc22:
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC23:%.*]] = add nsw i32 [[TMP15]], 1
-// CHECK2-NEXT:    store i32 [[INC23]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP11:![0-9]+]]
-// CHECK2:       for.end24:
-// CHECK2-NEXT:    br label [[FOR_INC25:%.*]]
-// CHECK2:       for.inc25:
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD26:%.*]] = add nsw i32 [[TMP16]], 5
-// CHECK2-NEXT:    store i32 [[ADD26]], ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP12:![0-9]+]]
-// CHECK2:       for.end27:
-// CHECK2-NEXT:    br label [[FOR_INC28:%.*]]
-// CHECK2:       for.inc28:
 // CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD29:%.*]] = add nsw i32 [[TMP17]], 5
-// CHECK2-NEXT:    store i32 [[ADD29]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP13:![0-9]+]]
-// CHECK2:       for.end30:
+// CHECK2-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP17]], 5
+// CHECK2-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP15:![0-9]+]]
+// CHECK2:       for.end24:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -1791,15 +1618,17 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
 // CHECK2-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK2-NEXT:    store i32 7, ptr [[J]], align 4
@@ -1837,87 +1666,79 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP7]], 4
-// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END32:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END22:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 [[TMP8]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND3:%.*]]
 // CHECK2:       for.cond3:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD4:%.*]] = add nsw i32 [[TMP10]], 5
-// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD4]]
-// CHECK2-NEXT:    br i1 [[CMP5]], label [[COND_TRUE6:%.*]], label [[COND_FALSE7:%.*]]
-// CHECK2:       cond.true6:
-// CHECK2-NEXT:    br label [[COND_END9:%.*]]
-// CHECK2:       cond.false7:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP11]], 5
-// CHECK2-NEXT:    br label [[COND_END9]]
-// CHECK2:       cond.end9:
-// CHECK2-NEXT:    [[COND10:%.*]] = phi i32 [ 4, [[COND_TRUE6]] ], [ [[ADD8]], [[COND_FALSE7]] ]
-// CHECK2-NEXT:    [[CMP11:%.*]] = icmp slt i32 [[TMP9]], [[COND10]]
-// CHECK2-NEXT:    br i1 [[CMP11]], label [[FOR_BODY12:%.*]], label [[FOR_END29:%.*]]
-// CHECK2:       for.body12:
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL13:%.*]] = mul nsw i32 [[TMP12]], 3
-// CHECK2-NEXT:    [[ADD14:%.*]] = add nsw i32 7, [[MUL13]]
-// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP8]], 5
+// CHECK2-NEXT:    br i1 [[CMP4]], label [[FOR_BODY5:%.*]], label [[FOR_END19:%.*]]
+// CHECK2:       for.body5:
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
+// CHECK2-NEXT:    store i32 [[ADD6]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL7:%.*]] = mul nsw i32 [[TMP11]], 3
+// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i32 7, [[MUL7]]
+// CHECK2-NEXT:    store i32 [[ADD8]], ptr [[I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND9:%.*]]
+// CHECK2:       for.cond9:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i32 [[TMP12]], 5
+// CHECK2-NEXT:    br i1 [[CMP10]], label [[FOR_BODY11:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body11:
 // CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    store i32 [[TMP13]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND15:%.*]]
-// CHECK2:       for.cond15:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP15]], 5
-// CHECK2-NEXT:    [[CMP17:%.*]] = icmp slt i32 4, [[ADD16]]
-// CHECK2-NEXT:    br i1 [[CMP17]], label [[COND_TRUE18:%.*]], label [[COND_FALSE19:%.*]]
-// CHECK2:       cond.true18:
-// CHECK2-NEXT:    br label [[COND_END21:%.*]]
-// CHECK2:       cond.false19:
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP16]], 5
-// CHECK2-NEXT:    br label [[COND_END21]]
-// CHECK2:       cond.end21:
-// CHECK2-NEXT:    [[COND22:%.*]] = phi i32 [ 4, [[COND_TRUE18]] ], [ [[ADD20]], [[COND_FALSE19]] ]
-// CHECK2-NEXT:    [[CMP23:%.*]] = icmp slt i32 [[TMP14]], [[COND22]]
-// CHECK2-NEXT:    br i1 [[CMP23]], label [[FOR_BODY24:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body24:
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP13]], [[TMP14]]
+// CHECK2-NEXT:    store i32 [[ADD12]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[MUL13:%.*]] = mul nsw i32 [[TMP15]], 3
+// CHECK2-NEXT:    [[ADD14:%.*]] = add nsw i32 7, [[MUL13]]
+// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP15:%.*]] = icmp slt i32 [[TMP16]], 4
+// CHECK2-NEXT:    br i1 [[CMP15]], label [[LAND_LHS_TRUE:%.*]], label [[IF_END:%.*]]
+// CHECK2:       land.lhs.true:
 // CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[MUL25:%.*]] = mul nsw i32 [[TMP17]], 3
-// CHECK2-NEXT:    [[ADD26:%.*]] = add nsw i32 7, [[MUL25]]
-// CHECK2-NEXT:    store i32 [[ADD26]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[CMP16:%.*]] = icmp slt i32 [[TMP17]], 4
+// CHECK2-NEXT:    br i1 [[CMP16]], label [[IF_THEN:%.*]], label [[IF_END]]
+// CHECK2:       if.then:
 // CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
 // CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[J]], align 4
 // CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP18]], i32 noundef [[TMP19]])
+// CHECK2-NEXT:    br label [[IF_END]]
+// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP20]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND15]], !llvm.loop [[LOOP14:![0-9]+]]
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND9]], !llvm.loop [[LOOP16:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC27:%.*]]
-// CHECK2:       for.inc27:
-// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC28:%.*]] = add nsw i32 [[TMP21]], 1
-// CHECK2-NEXT:    store i32 [[INC28]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND3]], !llvm.loop [[LOOP15:![0-9]+]]
-// CHECK2:       for.end29:
-// CHECK2-NEXT:    br label [[FOR_INC30:%.*]]
-// CHECK2:       for.inc30:
+// CHECK2-NEXT:    br label [[FOR_INC17:%.*]]
+// CHECK2:       for.inc17:
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC18:%.*]] = add nsw i32 [[TMP21]], 1
+// CHECK2-NEXT:    store i32 [[INC18]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND3]], !llvm.loop [[LOOP17:![0-9]+]]
+// CHECK2:       for.end19:
+// CHECK2-NEXT:    br label [[FOR_INC20:%.*]]
+// CHECK2:       for.inc20:
 // CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD31:%.*]] = add nsw i32 [[TMP22]], 5
-// CHECK2-NEXT:    store i32 [[ADD31]], ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP16:![0-9]+]]
-// CHECK2:       for.end32:
+// CHECK2-NEXT:    [[ADD21:%.*]] = add nsw i32 [[TMP22]], 5
+// CHECK2-NEXT:    store i32 [[ADD21]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP18:![0-9]+]]
+// CHECK2:       for.end22:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
 // CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[ADD33:%.*]] = add nsw i32 [[TMP23]], 1
-// CHECK2-NEXT:    store i32 [[ADD33]], ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP23]], 1
+// CHECK2-NEXT:    store i32 [[ADD23]], ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -1934,7 +1755,9 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[_TMP1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
@@ -1942,8 +1765,8 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[K:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK2-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK2-NEXT:    store i32 7, ptr [[J]], align 4
@@ -1990,87 +1813,79 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP9]], 4
-// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY:%.*]], label [[FOR_END37:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY:%.*]], label [[FOR_END27:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 [[TMP10]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND8:%.*]]
 // CHECK2:       for.cond8:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i32 [[TMP12]], 5
-// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i32 4, [[ADD9]]
-// CHECK2-NEXT:    br i1 [[CMP10]], label [[COND_TRUE11:%.*]], label [[COND_FALSE12:%.*]]
-// CHECK2:       cond.true11:
-// CHECK2-NEXT:    br label [[COND_END14:%.*]]
-// CHECK2:       cond.false12:
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP13]], 5
-// CHECK2-NEXT:    br label [[COND_END14]]
-// CHECK2:       cond.end14:
-// CHECK2-NEXT:    [[COND15:%.*]] = phi i32 [ 4, [[COND_TRUE11]] ], [ [[ADD13]], [[COND_FALSE12]] ]
-// CHECK2-NEXT:    [[CMP16:%.*]] = icmp slt i32 [[TMP11]], [[COND15]]
-// CHECK2-NEXT:    br i1 [[CMP16]], label [[FOR_BODY17:%.*]], label [[FOR_END34:%.*]]
-// CHECK2:       for.body17:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL18:%.*]] = mul nsw i32 [[TMP14]], 3
-// CHECK2-NEXT:    [[ADD19:%.*]] = add nsw i32 7, [[MUL18]]
-// CHECK2-NEXT:    store i32 [[ADD19]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP10]], 5
+// CHECK2-NEXT:    br i1 [[CMP9]], label [[FOR_BODY10:%.*]], label [[FOR_END24:%.*]]
+// CHECK2:       for.body10:
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP11]], [[TMP12]]
+// CHECK2-NEXT:    store i32 [[ADD11]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL12:%.*]] = mul nsw i32 [[TMP13]], 3
+// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i32 7, [[MUL12]]
+// CHECK2-NEXT:    store i32 [[ADD13]], ptr [[I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND14:%.*]]
+// CHECK2:       for.cond14:
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[CMP15:%.*]] = icmp slt i32 [[TMP14]], 5
+// CHECK2-NEXT:    br i1 [[CMP15]], label [[FOR_BODY16:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body16:
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    store i32 [[TMP15]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND20:%.*]]
-// CHECK2:       for.cond20:
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD21:%.*]] = add nsw i32 [[TMP17]], 5
-// CHECK2-NEXT:    [[CMP22:%.*]] = icmp slt i32 4, [[ADD21]]
-// CHECK2-NEXT:    br i1 [[CMP22]], label [[COND_TRUE23:%.*]], label [[COND_FALSE24:%.*]]
-// CHECK2:       cond.true23:
-// CHECK2-NEXT:    br label [[COND_END26:%.*]]
-// CHECK2:       cond.false24:
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD25:%.*]] = add nsw i32 [[TMP18]], 5
-// CHECK2-NEXT:    br label [[COND_END26]]
-// CHECK2:       cond.end26:
-// CHECK2-NEXT:    [[COND27:%.*]] = phi i32 [ 4, [[COND_TRUE23]] ], [ [[ADD25]], [[COND_FALSE24]] ]
-// CHECK2-NEXT:    [[CMP28:%.*]] = icmp slt i32 [[TMP16]], [[COND27]]
-// CHECK2-NEXT:    br i1 [[CMP28]], label [[FOR_BODY29:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body29:
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD17:%.*]] = add nsw i32 [[TMP15]], [[TMP16]]
+// CHECK2-NEXT:    store i32 [[ADD17]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[MUL18:%.*]] = mul nsw i32 [[TMP17]], 3
+// CHECK2-NEXT:    [[ADD19:%.*]] = add nsw i32 7, [[MUL18]]
+// CHECK2-NEXT:    store i32 [[ADD19]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP20:%.*]] = icmp slt i32 [[TMP18]], 4
+// CHECK2-NEXT:    br i1 [[CMP20]], label [[LAND_LHS_TRUE:%.*]], label [[IF_END:%.*]]
+// CHECK2:       land.lhs.true:
 // CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[MUL30:%.*]] = mul nsw i32 [[TMP19]], 3
-// CHECK2-NEXT:    [[ADD31:%.*]] = add nsw i32 7, [[MUL30]]
-// CHECK2-NEXT:    store i32 [[ADD31]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[CMP21:%.*]] = icmp slt i32 [[TMP19]], 4
+// CHECK2-NEXT:    br i1 [[CMP21]], label [[IF_THEN:%.*]], label [[IF_END]]
+// CHECK2:       if.then:
 // CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[I]], align 4
 // CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[J]], align 4
 // CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP20]], i32 noundef [[TMP21]])
+// CHECK2-NEXT:    br label [[IF_END]]
+// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP22]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND20]], !llvm.loop [[LOOP17:![0-9]+]]
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND14]], !llvm.loop [[LOOP19:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC32:%.*]]
-// CHECK2:       for.inc32:
-// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC33:%.*]] = add nsw i32 [[TMP23]], 1
-// CHECK2-NEXT:    store i32 [[INC33]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP18:![0-9]+]]
-// CHECK2:       for.end34:
-// CHECK2-NEXT:    br label [[FOR_INC35:%.*]]
-// CHECK2:       for.inc35:
+// CHECK2-NEXT:    br label [[FOR_INC22:%.*]]
+// CHECK2:       for.inc22:
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC23:%.*]] = add nsw i32 [[TMP23]], 1
+// CHECK2-NEXT:    store i32 [[INC23]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP20:![0-9]+]]
+// CHECK2:       for.end24:
+// CHECK2-NEXT:    br label [[FOR_INC25:%.*]]
+// CHECK2:       for.inc25:
 // CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD36:%.*]] = add nsw i32 [[TMP24]], 5
-// CHECK2-NEXT:    store i32 [[ADD36]], ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP19:![0-9]+]]
-// CHECK2:       for.end37:
+// CHECK2-NEXT:    [[ADD26:%.*]] = add nsw i32 [[TMP24]], 5
+// CHECK2-NEXT:    store i32 [[ADD26]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP21:![0-9]+]]
+// CHECK2:       for.end27:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
 // CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[ADD38:%.*]] = add nsw i32 [[TMP25]], 1
-// CHECK2-NEXT:    store i32 [[ADD38]], ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[ADD28:%.*]] = add nsw i32 [[TMP25]], 1
+// CHECK2-NEXT:    store i32 [[ADD28]], ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -2083,208 +1898,101 @@ extern "C" void foo10(data_t data) {
 // CHECK2-LABEL: define dso_local void @foo5(
 // CHECK2-SAME: ) #[[ATTR1]] {
 // CHECK2-NEXT:  entry:
-// CHECK2-NEXT:    [[DOTOMP_IV:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[_TMP1:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[_TMP2:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_5:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I11:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I12:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[J13:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK2-NEXT:    store i32 7, ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP]], align 4
-// CHECK2-NEXT:    store i32 [[TMP1]], ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP]], align 4
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], 5
-// CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
+// CHECK2-NEXT:    store i32 4, ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
+// CHECK2-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], 4
 // CHECK2-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK2:       cond.true:
 // CHECK2-NEXT:    br label [[COND_END:%.*]]
 // CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP]], align 4
-// CHECK2-NEXT:    [[ADD4:%.*]] = add nsw i32 [[TMP3]], 5
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
 // CHECK2-NEXT:    br label [[COND_END]]
 // CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD4]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    store i32 [[COND]], ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[SUB:%.*]] = sub i32 [[TMP4]], [[TMP5]]
-// CHECK2-NEXT:    [[SUB6:%.*]] = sub i32 [[SUB]], 1
-// CHECK2-NEXT:    [[ADD7:%.*]] = add i32 [[SUB6]], 1
-// CHECK2-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD7]], 1
-// CHECK2-NEXT:    [[CONV:%.*]] = zext i32 [[DIV]] to i64
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 1, [[CONV]]
-// CHECK2-NEXT:    [[MUL8:%.*]] = mul nsw i64 [[MUL]], 4
-// CHECK2-NEXT:    [[SUB9:%.*]] = sub nsw i64 [[MUL8]], 1
-// CHECK2-NEXT:    store i64 [[SUB9]], ptr [[DOTCAPTURE_EXPR_5]], align 8
-// CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    store i32 [[TMP6]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 7, ptr [[J]], align 4
-// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i32 [[TMP7]], [[TMP8]]
-// CHECK2-NEXT:    br i1 [[CMP10]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END:%.*]]
-// CHECK2:       omp.precond.then:
-// CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_5]], align 8
-// CHECK2-NEXT:    store i64 [[TMP9]], ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    store i64 1, ptr [[DOTOMP_STRIDE]], align 8
-// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
-// CHECK2-NEXT:    call void @__kmpc_for_static_init_8(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i64 1, i64 1)
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_5]], align 8
-// CHECK2-NEXT:    [[CMP14:%.*]] = icmp sgt i64 [[TMP10]], [[TMP11]]
-// CHECK2-NEXT:    br i1 [[CMP14]], label [[COND_TRUE15:%.*]], label [[COND_FALSE16:%.*]]
-// CHECK2:       cond.true15:
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_5]], align 8
-// CHECK2-NEXT:    br label [[COND_END17:%.*]]
-// CHECK2:       cond.false16:
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    br label [[COND_END17]]
-// CHECK2:       cond.end17:
-// CHECK2-NEXT:    [[COND18:%.*]] = phi i64 [ [[TMP12]], [[COND_TRUE15]] ], [ [[TMP13]], [[COND_FALSE16]] ]
-// CHECK2-NEXT:    store i64 [[COND18]], ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
-// CHECK2-NEXT:    store i64 [[TMP14]], ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[TMP2]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
+// CHECK2-NEXT:    store i32 [[TMP3]], ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK2:       omp.inner.for.cond:
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[CMP19:%.*]] = icmp sle i64 [[TMP15]], [[TMP16]]
-// CHECK2-NEXT:    br i1 [[CMP19]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], [[TMP5]]
+// CHECK2-NEXT:    br i1 [[CMP2]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
 // CHECK2:       omp.inner.for.body:
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[SUB20:%.*]] = sub i32 [[TMP18]], [[TMP19]]
-// CHECK2-NEXT:    [[SUB21:%.*]] = sub i32 [[SUB20]], 1
-// CHECK2-NEXT:    [[ADD22:%.*]] = add i32 [[SUB21]], 1
-// CHECK2-NEXT:    [[DIV23:%.*]] = udiv i32 [[ADD22]], 1
-// CHECK2-NEXT:    [[MUL24:%.*]] = mul i32 1, [[DIV23]]
-// CHECK2-NEXT:    [[MUL25:%.*]] = mul i32 [[MUL24]], 4
-// CHECK2-NEXT:    [[CONV26:%.*]] = zext i32 [[MUL25]] to i64
-// CHECK2-NEXT:    [[DIV27:%.*]] = sdiv i64 [[TMP17]], [[CONV26]]
-// CHECK2-NEXT:    [[MUL28:%.*]] = mul nsw i64 [[DIV27]], 5
-// CHECK2-NEXT:    [[ADD29:%.*]] = add nsw i64 0, [[MUL28]]
-// CHECK2-NEXT:    [[CONV30:%.*]] = trunc i64 [[ADD29]] to i32
-// CHECK2-NEXT:    store i32 [[CONV30]], ptr [[DOTFLOOR_0_IV_I11]], align 4
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[CONV31:%.*]] = sext i32 [[TMP20]] to i64
-// CHECK2-NEXT:    [[TMP21:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP22:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[SUB32:%.*]] = sub i32 [[TMP23]], [[TMP24]]
-// CHECK2-NEXT:    [[SUB33:%.*]] = sub i32 [[SUB32]], 1
-// CHECK2-NEXT:    [[ADD34:%.*]] = add i32 [[SUB33]], 1
-// CHECK2-NEXT:    [[DIV35:%.*]] = udiv i32 [[ADD34]], 1
-// CHECK2-NEXT:    [[MUL36:%.*]] = mul i32 1, [[DIV35]]
-// CHECK2-NEXT:    [[MUL37:%.*]] = mul i32 [[MUL36]], 4
-// CHECK2-NEXT:    [[CONV38:%.*]] = zext i32 [[MUL37]] to i64
-// CHECK2-NEXT:    [[DIV39:%.*]] = sdiv i64 [[TMP22]], [[CONV38]]
-// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[SUB40:%.*]] = sub i32 [[TMP25]], [[TMP26]]
-// CHECK2-NEXT:    [[SUB41:%.*]] = sub i32 [[SUB40]], 1
-// CHECK2-NEXT:    [[ADD42:%.*]] = add i32 [[SUB41]], 1
-// CHECK2-NEXT:    [[DIV43:%.*]] = udiv i32 [[ADD42]], 1
-// CHECK2-NEXT:    [[MUL44:%.*]] = mul i32 1, [[DIV43]]
-// CHECK2-NEXT:    [[MUL45:%.*]] = mul i32 [[MUL44]], 4
-// CHECK2-NEXT:    [[CONV46:%.*]] = zext i32 [[MUL45]] to i64
-// CHECK2-NEXT:    [[MUL47:%.*]] = mul nsw i64 [[DIV39]], [[CONV46]]
-// CHECK2-NEXT:    [[SUB48:%.*]] = sub nsw i64 [[TMP21]], [[MUL47]]
-// CHECK2-NEXT:    [[DIV49:%.*]] = sdiv i64 [[SUB48]], 4
-// CHECK2-NEXT:    [[MUL50:%.*]] = mul nsw i64 [[DIV49]], 1
-// CHECK2-NEXT:    [[ADD51:%.*]] = add nsw i64 [[CONV31]], [[MUL50]]
-// CHECK2-NEXT:    [[CONV52:%.*]] = trunc i64 [[ADD51]] to i32
-// CHECK2-NEXT:    store i32 [[CONV52]], ptr [[DOTTILE_0_IV_I12]], align 4
-// CHECK2-NEXT:    [[TMP27:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP28:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[SUB53:%.*]] = sub i32 [[TMP29]], [[TMP30]]
-// CHECK2-NEXT:    [[SUB54:%.*]] = sub i32 [[SUB53]], 1
-// CHECK2-NEXT:    [[ADD55:%.*]] = add i32 [[SUB54]], 1
-// CHECK2-NEXT:    [[DIV56:%.*]] = udiv i32 [[ADD55]], 1
-// CHECK2-NEXT:    [[MUL57:%.*]] = mul i32 1, [[DIV56]]
-// CHECK2-NEXT:    [[MUL58:%.*]] = mul i32 [[MUL57]], 4
-// CHECK2-NEXT:    [[CONV59:%.*]] = zext i32 [[MUL58]] to i64
-// CHECK2-NEXT:    [[DIV60:%.*]] = sdiv i64 [[TMP28]], [[CONV59]]
-// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK2-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[SUB61:%.*]] = sub i32 [[TMP31]], [[TMP32]]
-// CHECK2-NEXT:    [[SUB62:%.*]] = sub i32 [[SUB61]], 1
-// CHECK2-NEXT:    [[ADD63:%.*]] = add i32 [[SUB62]], 1
-// CHECK2-NEXT:    [[DIV64:%.*]] = udiv i32 [[ADD63]], 1
-// CHECK2-NEXT:    [[MUL65:%.*]] = mul i32 1, [[DIV64]]
-// CHECK2-NEXT:    [[MUL66:%.*]] = mul i32 [[MUL65]], 4
-// CHECK2-NEXT:    [[CONV67:%.*]] = zext i32 [[MUL66]] to i64
-// CHECK2-NEXT:    [[MUL68:%.*]] = mul nsw i64 [[DIV60]], [[CONV67]]
-// CHECK2-NEXT:    [[SUB69:%.*]] = sub nsw i64 [[TMP27]], [[MUL68]]
-// CHECK2-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK2-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[SUB70:%.*]] = sub i32 [[TMP35]], [[TMP36]]
-// CHECK2-NEXT:    [[SUB71:%.*]] = sub i32 [[SUB70]], 1
-// CHECK2-NEXT:    [[ADD72:%.*]] = add i32 [[SUB71]], 1
-// CHECK2-NEXT:    [[DIV73:%.*]] = udiv i32 [[ADD72]], 1
-// CHECK2-NEXT:    [[MUL74:%.*]] = mul i32 1, [[DIV73]]
-// CHECK2-NEXT:    [[MUL75:%.*]] = mul i32 [[MUL74]], 4
-// CHECK2-NEXT:    [[CONV76:%.*]] = zext i32 [[MUL75]] to i64
-// CHECK2-NEXT:    [[DIV77:%.*]] = sdiv i64 [[TMP34]], [[CONV76]]
-// CHECK2-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK2-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[SUB78:%.*]] = sub i32 [[TMP37]], [[TMP38]]
-// CHECK2-NEXT:    [[SUB79:%.*]] = sub i32 [[SUB78]], 1
-// CHECK2-NEXT:    [[ADD80:%.*]] = add i32 [[SUB79]], 1
-// CHECK2-NEXT:    [[DIV81:%.*]] = udiv i32 [[ADD80]], 1
-// CHECK2-NEXT:    [[MUL82:%.*]] = mul i32 1, [[DIV81]]
-// CHECK2-NEXT:    [[MUL83:%.*]] = mul i32 [[MUL82]], 4
-// CHECK2-NEXT:    [[CONV84:%.*]] = zext i32 [[MUL83]] to i64
-// CHECK2-NEXT:    [[MUL85:%.*]] = mul nsw i64 [[DIV77]], [[CONV84]]
-// CHECK2-NEXT:    [[SUB86:%.*]] = sub nsw i64 [[TMP33]], [[MUL85]]
-// CHECK2-NEXT:    [[DIV87:%.*]] = sdiv i64 [[SUB86]], 4
-// CHECK2-NEXT:    [[MUL88:%.*]] = mul nsw i64 [[DIV87]], 4
-// CHECK2-NEXT:    [[SUB89:%.*]] = sub nsw i64 [[SUB69]], [[MUL88]]
-// CHECK2-NEXT:    [[MUL90:%.*]] = mul nsw i64 [[SUB89]], 3
-// CHECK2-NEXT:    [[ADD91:%.*]] = add nsw i64 7, [[MUL90]]
-// CHECK2-NEXT:    [[CONV92:%.*]] = trunc i64 [[ADD91]] to i32
-// CHECK2-NEXT:    store i32 [[CONV92]], ptr [[J13]], align 4
-// CHECK2-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTTILE_0_IV_I12]], align 4
-// CHECK2-NEXT:    [[MUL93:%.*]] = mul nsw i32 [[TMP39]], 3
-// CHECK2-NEXT:    [[ADD94:%.*]] = add nsw i32 7, [[MUL93]]
-// CHECK2-NEXT:    store i32 [[ADD94]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP41:%.*]] = load i32, ptr [[J13]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP40]], i32 noundef [[TMP41]])
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP6]], 5
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[DIV]], 5
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[DIV3:%.*]] = sdiv i32 [[TMP8]], 5
+// CHECK2-NEXT:    [[MUL4:%.*]] = mul nsw i32 [[DIV3]], 5
+// CHECK2-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP7]], [[MUL4]]
+// CHECK2-NEXT:    [[MUL5:%.*]] = mul nsw i32 [[SUB]], 1
+// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL5]]
+// CHECK2-NEXT:    store i32 [[ADD6]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
+// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL8:%.*]] = mul nsw i32 [[TMP11]], 3
+// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL8]]
+// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i32 [[TMP12]], 4
+// CHECK2-NEXT:    br i1 [[CMP10]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK2:       if.then:
+// CHECK2-NEXT:    store i32 7, ptr [[J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND:%.*]]
+// CHECK2:       for.cond:
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    [[CMP11:%.*]] = icmp slt i32 [[TMP13]], 17
+// CHECK2-NEXT:    br i1 [[CMP11]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body:
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP14]], i32 noundef [[TMP15]])
+// CHECK2-NEXT:    br label [[FOR_INC:%.*]]
+// CHECK2:       for.inc:
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP16]], 3
+// CHECK2-NEXT:    store i32 [[ADD12]], ptr [[J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP22:![0-9]+]]
+// CHECK2:       for.end:
+// CHECK2-NEXT:    br label [[IF_END]]
+// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
-// CHECK2-NEXT:    [[TMP42:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[ADD95:%.*]] = add nsw i64 [[TMP42]], 1
-// CHECK2-NEXT:    store i64 [[ADD95]], ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP17]], 1
+// CHECK2-NEXT:    store i32 [[ADD13]], ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
 // CHECK2:       omp.loop.exit:
 // CHECK2-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP0]])
-// CHECK2-NEXT:    br label [[OMP_PRECOND_END]]
-// CHECK2:       omp.precond.end:
 // CHECK2-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[TMP0]])
 // CHECK2-NEXT:    ret void
 //
@@ -2304,12 +2012,13 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK2-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
 // CHECK2-NEXT:    store i32 7, ptr [[I]], align 4
@@ -2344,46 +2053,43 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP7]], 5
 // CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 [[TMP8]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND:%.*]]
 // CHECK2:       for.cond:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD2:%.*]] = add nsw i32 [[TMP10]], 5
-// CHECK2-NEXT:    [[CMP3:%.*]] = icmp slt i32 4, [[ADD2]]
-// CHECK2-NEXT:    br i1 [[CMP3]], label [[COND_TRUE4:%.*]], label [[COND_FALSE5:%.*]]
-// CHECK2:       cond.true4:
-// CHECK2-NEXT:    br label [[COND_END7:%.*]]
-// CHECK2:       cond.false5:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP11]], 5
-// CHECK2-NEXT:    br label [[COND_END7]]
-// CHECK2:       cond.end7:
-// CHECK2-NEXT:    [[COND8:%.*]] = phi i32 [ 4, [[COND_TRUE4]] ], [ [[ADD6]], [[COND_FALSE5]] ]
-// CHECK2-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP9]], [[COND8]]
-// CHECK2-NEXT:    br i1 [[CMP9]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP8]], 5
+// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
 // CHECK2:       for.body:
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
+// CHECK2-NEXT:    store i32 [[ADD3]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL4:%.*]] = mul nsw i32 [[TMP11]], 3
+// CHECK2-NEXT:    [[ADD5:%.*]] = add nsw i32 7, [[MUL4]]
+// CHECK2-NEXT:    store i32 [[ADD5]], ptr [[I]], align 4
 // CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL10:%.*]] = mul nsw i32 [[TMP12]], 3
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 7, [[MUL10]]
-// CHECK2-NEXT:    store i32 [[ADD11]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[CMP6:%.*]] = icmp slt i32 [[TMP12]], 4
+// CHECK2-NEXT:    br i1 [[CMP6]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK2:       if.then:
 // CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[I]], align 4
 // CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP13]])
+// CHECK2-NEXT:    br label [[IF_END]]
+// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP20:![0-9]+]]
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP23:![0-9]+]]
 // CHECK2:       for.end:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP15]], 1
-// CHECK2-NEXT:    store i32 [[ADD12]], ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 [[TMP15]], 1
+// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -2397,8 +2103,9 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:  entry:
 // CHECK2-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
 // CHECK2-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
@@ -2406,78 +2113,65 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END14:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK2:       for.cond1:
-// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], 0
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP2]], 0
 // CHECK2-NEXT:    br i1 [[CMP2]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK2:       cond.true:
 // CHECK2-NEXT:    br label [[COND_END:%.*]]
 // CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[A_ADDR]], align 4
 // CHECK2-NEXT:    br label [[COND_END]]
 // CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP5]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[COND]]
-// CHECK2-NEXT:    [[CMP3:%.*]] = icmp slt i32 4, [[ADD]]
-// CHECK2-NEXT:    br i1 [[CMP3]], label [[COND_TRUE4:%.*]], label [[COND_FALSE5:%.*]]
-// CHECK2:       cond.true4:
-// CHECK2-NEXT:    br label [[COND_END12:%.*]]
-// CHECK2:       cond.false5:
-// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    [[CMP6:%.*]] = icmp sle i32 [[TMP7]], 0
-// CHECK2-NEXT:    br i1 [[CMP6]], label [[COND_TRUE7:%.*]], label [[COND_FALSE8:%.*]]
-// CHECK2:       cond.true7:
-// CHECK2-NEXT:    br label [[COND_END9:%.*]]
-// CHECK2:       cond.false8:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    br label [[COND_END9]]
-// CHECK2:       cond.end9:
-// CHECK2-NEXT:    [[COND10:%.*]] = phi i32 [ 1, [[COND_TRUE7]] ], [ [[TMP8]], [[COND_FALSE8]] ]
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP6]], [[COND10]]
-// CHECK2-NEXT:    br label [[COND_END12]]
-// CHECK2:       cond.end12:
-// CHECK2-NEXT:    [[COND13:%.*]] = phi i32 [ 4, [[COND_TRUE4]] ], [ [[ADD11]], [[COND_END9]] ]
-// CHECK2-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP2]], [[COND13]]
-// CHECK2-NEXT:    br i1 [[CMP14]], label [[FOR_BODY15:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body15:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP9]], 3
-// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP10]])
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP3]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[TMP1]], [[COND]]
+// CHECK2-NEXT:    br i1 [[CMP3]], label [[FOR_BODY4:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body4:
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
+// CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
+// CHECK2-NEXT:    [[ADD5:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD5]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP6:%.*]] = icmp slt i32 [[TMP7]], 4
+// CHECK2-NEXT:    br i1 [[CMP6]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK2:       if.then:
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP8]])
+// CHECK2-NEXT:    br label [[IF_END]]
+// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP23:![0-9]+]]
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP9]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP26:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC17:%.*]]
-// CHECK2:       for.inc17:
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    [[CMP18:%.*]] = icmp sle i32 [[TMP12]], 0
-// CHECK2-NEXT:    br i1 [[CMP18]], label [[COND_TRUE19:%.*]], label [[COND_FALSE20:%.*]]
-// CHECK2:       cond.true19:
-// CHECK2-NEXT:    br label [[COND_END21:%.*]]
-// CHECK2:       cond.false20:
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    br label [[COND_END21]]
-// CHECK2:       cond.end21:
-// CHECK2-NEXT:    [[COND22:%.*]] = phi i32 [ 1, [[COND_TRUE19]] ], [ [[TMP13]], [[COND_FALSE20]] ]
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP14]], [[COND22]]
-// CHECK2-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP24:![0-9]+]]
-// CHECK2:       for.end24:
+// CHECK2-NEXT:    br label [[FOR_INC7:%.*]]
+// CHECK2:       for.inc7:
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    [[CMP8:%.*]] = icmp sle i32 [[TMP10]], 0
+// CHECK2-NEXT:    br i1 [[CMP8]], label [[COND_TRUE9:%.*]], label [[COND_FALSE10:%.*]]
+// CHECK2:       cond.true9:
+// CHECK2-NEXT:    br label [[COND_END11:%.*]]
+// CHECK2:       cond.false10:
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    br label [[COND_END11]]
+// CHECK2:       cond.end11:
+// CHECK2-NEXT:    [[COND12:%.*]] = phi i32 [ 1, [[COND_TRUE9]] ], [ [[TMP11]], [[COND_FALSE10]] ]
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP12]], [[COND12]]
+// CHECK2-NEXT:    store i32 [[ADD13]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP27:![0-9]+]]
+// CHECK2:       for.end14:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -2490,19 +2184,20 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK2-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
 // CHECK2-NEXT:    store ptr [[ARRAY]], ptr [[__RANGE2]], align 8
-// CHECK2-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META9]]
 // CHECK2-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP0]], i64 0, i64 0
 // CHECK2-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 12
 // CHECK2-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
-// CHECK2-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META9]]
 // CHECK2-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP1]], i64 0, i64 0
 // CHECK2-NEXT:    store ptr [[ARRAYDECAY1]], ptr [[__BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META9]]
 // CHECK2-NEXT:    [[ARRAYDECAY2:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP2]], i64 0, i64 0
 // CHECK2-NEXT:    store ptr [[ARRAYDECAY2]], ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK2-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__END2]], align 8
@@ -2525,56 +2220,51 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK2:       for.cond7:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
-// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
-// CHECK2-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK2:       cond.true:
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
-// CHECK2-NEXT:    br label [[COND_END:%.*]]
-// CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
-// CHECK2-NEXT:    br label [[COND_END]]
-// CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
-// CHECK2-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body14:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
+// CHECK2-NEXT:    br i1 [[CMP8]], label [[FOR_BODY9:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body9:
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
+// CHECK2-NEXT:    store i64 [[ADD10]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP12]], 1
+// CHECK2-NEXT:    [[ADD_PTR11:%.*]] = getelementptr inbounds double, ptr [[TMP11]], i64 [[MUL]]
+// CHECK2-NEXT:    store ptr [[ADD_PTR11]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP14:%.*]] = load double, ptr [[TMP13]], align 8
+// CHECK2-NEXT:    store double [[TMP14]], ptr [[V]], align 8
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
-// CHECK2-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
-// CHECK2-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
-// CHECK2-NEXT:    store double [[TMP17]], ptr [[V]], align 8
-// CHECK2-NEXT:    [[TMP18:%.*]] = load double, ptr [[V]], align 8
-// CHECK2-NEXT:    call void (...) @body(double noundef [[TMP18]])
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP16]], 1
+// CHECK2-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP15]], [[ADD12]]
+// CHECK2-NEXT:    br i1 [[CMP13]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK2:       if.then:
+// CHECK2-NEXT:    [[TMP17:%.*]] = load double, ptr [[V]], align 8
+// CHECK2-NEXT:    call void (...) @body(double noundef [[TMP17]])
+// CHECK2-NEXT:    br label [[IF_END]]
+// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP19]], 1
-// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP25:![0-9]+]]
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP18]], 1
+// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP28:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC16:%.*]]
-// CHECK2:       for.inc16:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP20]], 5
-// CHECK2-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP26:![0-9]+]]
-// CHECK2:       for.end18:
+// CHECK2-NEXT:    br label [[FOR_INC14:%.*]]
+// CHECK2:       for.inc14:
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD15:%.*]] = add nsw i64 [[TMP19]], 5
+// CHECK2-NEXT:    store i64 [[ADD15]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP29:![0-9]+]]
+// CHECK2:       for.end16:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -2594,8 +2284,9 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[START_ADDR]], align 4
@@ -2619,53 +2310,48 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK2-NEXT:    [[ADD5:%.*]] = add i32 [[TMP6]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP5]], [[ADD5]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END15:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK2:       for.cond6:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[ADD7:%.*]] = add i32 [[TMP9]], 1
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD8:%.*]] = add i32 [[TMP10]], 5
-// CHECK2-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
-// CHECK2-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK2:       cond.true:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[ADD10:%.*]] = add i32 [[TMP11]], 1
-// CHECK2-NEXT:    br label [[COND_END:%.*]]
-// CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD11:%.*]] = add i32 [[TMP12]], 5
-// CHECK2-NEXT:    br label [[COND_END]]
-// CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP8]], [[COND]]
-// CHECK2-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body13:
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP14]], 3
-// CHECK2-NEXT:    [[ADD14:%.*]] = add i32 [[TMP13]], [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP15]])
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP7]], 5
+// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body8:
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD9:%.*]] = add i32 [[TMP8]], [[TMP9]]
+// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP11]], 3
+// CHECK2-NEXT:    [[ADD10:%.*]] = add i32 [[TMP10]], [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD10]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD11:%.*]] = add i32 [[TMP13]], 1
+// CHECK2-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP12]], [[ADD11]]
+// CHECK2-NEXT:    br i1 [[CMP12]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK2:       if.then:
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP14]])
+// CHECK2-NEXT:    br label [[IF_END]]
+// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP16]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP27:![0-9]+]]
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP15]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP30:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC15:%.*]]
-// CHECK2:       for.inc15:
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD16:%.*]] = add i32 [[TMP17]], 5
-// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP28:![0-9]+]]
-// CHECK2:       for.end17:
+// CHECK2-NEXT:    br label [[FOR_INC13:%.*]]
+// CHECK2:       for.inc13:
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD14:%.*]] = add i32 [[TMP16]], 5
+// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP31:![0-9]+]]
+// CHECK2:       for.end15:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -2676,53 +2362,61 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    ret void
 //
 //.
-// CHECK1: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]]}
-// CHECK1: [[META4]] = !{!"llvm.loop.mustprogress"}
-// CHECK1: [[LOOP5]] = distinct !{[[LOOP5]], [[META4]]}
-// CHECK1: [[LOOP6]] = distinct !{[[LOOP6]], [[META4]]}
-// CHECK1: [[LOOP7]] = distinct !{[[LOOP7]], [[META4]]}
-// CHECK1: [[LOOP8]] = distinct !{[[LOOP8]], [[META4]]}
-// CHECK1: [[LOOP9]] = distinct !{[[LOOP9]], [[META4]]}
-// CHECK1: [[LOOP10]] = distinct !{[[LOOP10]], [[META4]]}
-// CHECK1: [[LOOP11]] = distinct !{[[LOOP11]], [[META4]]}
-// CHECK1: [[LOOP12]] = distinct !{[[LOOP12]], [[META4]]}
-// CHECK1: [[LOOP13]] = distinct !{[[LOOP13]], [[META4]]}
-// CHECK1: [[LOOP14]] = distinct !{[[LOOP14]], [[META4]]}
-// CHECK1: [[LOOP15]] = distinct !{[[LOOP15]], [[META4]]}
-// CHECK1: [[LOOP16]] = distinct !{[[LOOP16]], [[META4]]}
-// CHECK1: [[LOOP17]] = distinct !{[[LOOP17]], [[META4]]}
-// CHECK1: [[LOOP18]] = distinct !{[[LOOP18]], [[META4]]}
-// CHECK1: [[LOOP21]] = distinct !{[[LOOP21]], [[META4]]}
-// CHECK1: [[LOOP22]] = distinct !{[[LOOP22]], [[META4]]}
-// CHECK1: [[LOOP23]] = distinct !{[[LOOP23]], [[META4]]}
-// CHECK1: [[LOOP24]] = distinct !{[[LOOP24]], [[META4]]}
-// CHECK1: [[LOOP25]] = distinct !{[[LOOP25]], [[META4]]}
-// CHECK1: [[LOOP26]] = distinct !{[[LOOP26]], [[META4]]}
-// CHECK1: [[LOOP27]] = distinct !{[[LOOP27]], [[META4]]}
-// CHECK1: [[LOOP28]] = distinct !{[[LOOP28]], [[META4]]}
+// CHECK1: [[META2]] = !{}
+// CHECK1: [[META3]] = !{i64 4}
+// CHECK1: [[LOOP4]] = distinct !{[[LOOP4]], [[META5:![0-9]+]]}
+// CHECK1: [[META5]] = !{!"llvm.loop.mustprogress"}
+// CHECK1: [[LOOP6]] = distinct !{[[LOOP6]], [[META5]]}
+// CHECK1: [[LOOP7]] = distinct !{[[LOOP7]], [[META5]]}
+// CHECK1: [[LOOP8]] = distinct !{[[LOOP8]], [[META5]]}
+// CHECK1: [[LOOP9]] = distinct !{[[LOOP9]], [[META5]]}
+// CHECK1: [[LOOP10]] = distinct !{[[LOOP10]], [[META5]]}
+// CHECK1: [[LOOP11]] = distinct !{[[LOOP11]], [[META5]]}
+// CHECK1: [[LOOP12]] = distinct !{[[LOOP12]], [[META5]]}
+// CHECK1: [[LOOP13]] = distinct !{[[LOOP13]], [[META5]]}
+// CHECK1: [[LOOP14]] = distinct !{[[LOOP14]], [[META5]]}
+// CHECK1: [[LOOP15]] = distinct !{[[LOOP15]], [[META5]]}
+// CHECK1: [[LOOP16]] = distinct !{[[LOOP16]], [[META5]]}
+// CHECK1: [[LOOP17]] = distinct !{[[LOOP17]], [[META5]]}
+// CHECK1: [[LOOP18]] = distinct !{[[LOOP18]], [[META5]]}
+// CHECK1: [[LOOP19]] = distinct !{[[LOOP19]], [[META5]]}
+// CHECK1: [[LOOP20]] = distinct !{[[LOOP20]], [[META5]]}
+// CHECK1: [[LOOP23]] = distinct !{[[LOOP23]], [[META5]]}
+// CHECK1: [[LOOP24]] = distinct !{[[LOOP24]], [[META5]]}
+// CHECK1: [[LOOP25]] = distinct !{[[LOOP25]], [[META5]]}
+// CHECK1: [[LOOP26]] = distinct !{[[LOOP26]], [[META5]]}
+// CHECK1: [[META27]] = !{i64 8}
+// CHECK1: [[LOOP28]] = distinct !{[[LOOP28]], [[META5]]}
+// CHECK1: [[LOOP29]] = distinct !{[[LOOP29]], [[META5]]}
+// CHECK1: [[LOOP30]] = distinct !{[[LOOP30]], [[META5]]}
+// CHECK1: [[LOOP31]] = distinct !{[[LOOP31]], [[META5]]}
 //.
-// CHECK2: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]]}
-// CHECK2: [[META4]] = !{!"llvm.loop.mustprogress"}
-// CHECK2: [[LOOP5]] = distinct !{[[LOOP5]], [[META4]]}
-// CHECK2: [[LOOP6]] = distinct !{[[LOOP6]], [[META4]]}
-// CHECK2: [[LOOP7]] = distinct !{[[LOOP7]], [[META4]]}
-// CHECK2: [[LOOP8]] = distinct !{[[LOOP8]], [[META4]]}
-// CHECK2: [[LOOP9]] = distinct !{[[LOOP9]], [[META4]]}
-// CHECK2: [[LOOP10]] = distinct !{[[LOOP10]], [[META4]]}
-// CHECK2: [[LOOP11]] = distinct !{[[LOOP11]], [[META4]]}
-// CHECK2: [[LOOP12]] = distinct !{[[LOOP12]], [[META4]]}
-// CHECK2: [[LOOP13]] = distinct !{[[LOOP13]], [[META4]]}
-// CHECK2: [[LOOP14]] = distinct !{[[LOOP14]], [[META4]]}
-// CHECK2: [[LOOP15]] = distinct !{[[LOOP15]], [[META4]]}
-// CHECK2: [[LOOP16]] = distinct !{[[LOOP16]], [[META4]]}
-// CHECK2: [[LOOP17]] = distinct !{[[LOOP17]], [[META4]]}
-// CHECK2: [[LOOP18]] = distinct !{[[LOOP18]], [[META4]]}
-// CHECK2: [[LOOP19]] = distinct !{[[LOOP19]], [[META4]]}
-// CHECK2: [[LOOP20]] = distinct !{[[LOOP20]], [[META4]]}
-// CHECK2: [[LOOP23]] = distinct !{[[LOOP23]], [[META4]]}
-// CHECK2: [[LOOP24]] = distinct !{[[LOOP24]], [[META4]]}
-// CHECK2: [[LOOP25]] = distinct !{[[LOOP25]], [[META4]]}
-// CHECK2: [[LOOP26]] = distinct !{[[LOOP26]], [[META4]]}
-// CHECK2: [[LOOP27]] = distinct !{[[LOOP27]], [[META4]]}
-// CHECK2: [[LOOP28]] = distinct !{[[LOOP28]], [[META4]]}
+// CHECK2: [[META2]] = !{}
+// CHECK2: [[META3]] = !{i64 4}
+// CHECK2: [[LOOP4]] = distinct !{[[LOOP4]], [[META5:![0-9]+]]}
+// CHECK2: [[META5]] = !{!"llvm.loop.mustprogress"}
+// CHECK2: [[LOOP6]] = distinct !{[[LOOP6]], [[META5]]}
+// CHECK2: [[LOOP7]] = distinct !{[[LOOP7]], [[META5]]}
+// CHECK2: [[LOOP8]] = distinct !{[[LOOP8]], [[META5]]}
+// CHECK2: [[META9]] = !{i64 8}
+// CHECK2: [[LOOP10]] = distinct !{[[LOOP10]], [[META5]]}
+// CHECK2: [[LOOP11]] = distinct !{[[LOOP11]], [[META5]]}
+// CHECK2: [[LOOP12]] = distinct !{[[LOOP12]], [[META5]]}
+// CHECK2: [[LOOP13]] = distinct !{[[LOOP13]], [[META5]]}
+// CHECK2: [[LOOP14]] = distinct !{[[LOOP14]], [[META5]]}
+// CHECK2: [[LOOP15]] = distinct !{[[LOOP15]], [[META5]]}
+// CHECK2: [[LOOP16]] = distinct !{[[LOOP16]], [[META5]]}
+// CHECK2: [[LOOP17]] = distinct !{[[LOOP17]], [[META5]]}
+// CHECK2: [[LOOP18]] = distinct !{[[LOOP18]], [[META5]]}
+// CHECK2: [[LOOP19]] = distinct !{[[LOOP19]], [[META5]]}
+// CHECK2: [[LOOP20]] = distinct !{[[LOOP20]], [[META5]]}
+// CHECK2: [[LOOP21]] = distinct !{[[LOOP21]], [[META5]]}
+// CHECK2: [[LOOP22]] = distinct !{[[LOOP22]], [[META5]]}
+// CHECK2: [[LOOP23]] = distinct !{[[LOOP23]], [[META5]]}
+// CHECK2: [[LOOP26]] = distinct !{[[LOOP26]], [[META5]]}
+// CHECK2: [[LOOP27]] = distinct !{[[LOOP27]], [[META5]]}
+// CHECK2: [[LOOP28]] = distinct !{[[LOOP28]], [[META5]]}
+// CHECK2: [[LOOP29]] = distinct !{[[LOOP29]], [[META5]]}
+// CHECK2: [[LOOP30]] = distinct !{[[LOOP30]], [[META5]]}
+// CHECK2: [[LOOP31]] = distinct !{[[LOOP31]], [[META5]]}
 //.
diff --git a/clang/test/OpenMP/tile_codegen_for_dependent.cpp b/clang/test/OpenMP/tile_codegen_for_dependent.cpp
index 820d33d15287b..aadc8e58bebfc 100644
--- a/clang/test/OpenMP/tile_codegen_for_dependent.cpp
+++ b/clang/test/OpenMP/tile_codegen_for_dependent.cpp
@@ -1,3 +1,4 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
 // Check code generation
 // RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fopenmp -emit-llvm %s -o - | FileCheck %s --check-prefix=IR
 
@@ -17,172 +18,151 @@
 extern "C" void body(...) {}
 
 
-// IR-LABEL: define {{.*}}@func(
-// IR-NEXT:  [[ENTRY:.*]]:
-// IR-NEXT:    %[[START_ADDR:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[END_ADDR:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[STEP_ADDR:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_IV:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[TMP:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[I:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_1:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTNEW_STEP:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_2:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_5:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_7:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTFLOOR_0_IV_I:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_LB:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_UB:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_STRIDE:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_IS_LAST:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTFLOOR_0_IV_I11:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTTILE_0_IV_I:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[TMP0:.+]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:.+]])
-// IR-NEXT:    store i32 %[[START:.+]], ptr %[[START_ADDR]], align 4
-// IR-NEXT:    store i32 %[[END:.+]], ptr %[[END_ADDR]], align 4
-// IR-NEXT:    store i32 %[[STEP:.+]], ptr %[[STEP_ADDR]], align 4
-// IR-NEXT:    %[[TMP1:.+]] = load i32, ptr %[[START_ADDR]], align 4
-// IR-NEXT:    store i32 %[[TMP1]], ptr %[[I]], align 4
-// IR-NEXT:    %[[TMP2:.+]] = load i32, ptr %[[START_ADDR]], align 4
-// IR-NEXT:    store i32 %[[TMP2]], ptr %[[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    %[[TMP3:.+]] = load i32, ptr %[[END_ADDR]], align 4
-// IR-NEXT:    store i32 %[[TMP3]], ptr %[[DOTCAPTURE_EXPR_1]], align 4
-// IR-NEXT:    %[[TMP4:.+]] = load i32, ptr %[[STEP_ADDR]], align 4
-// IR-NEXT:    store i32 %[[TMP4]], ptr %[[DOTNEW_STEP]], align 4
-// IR-NEXT:    %[[TMP5:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_1]], align 4
-// IR-NEXT:    %[[TMP6:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    %[[SUB:.+]] = sub i32 %[[TMP5]], %[[TMP6]]
-// IR-NEXT:    %[[SUB3:.+]] = sub i32 %[[SUB]], 1
-// IR-NEXT:    %[[TMP7:.+]] = load i32, ptr %[[DOTNEW_STEP]], align 4
-// IR-NEXT:    %[[ADD:.+]] = add i32 %[[SUB3]], %[[TMP7]]
-// IR-NEXT:    %[[TMP8:.+]] = load i32, ptr %[[DOTNEW_STEP]], align 4
-// IR-NEXT:    %[[DIV:.+]] = udiv i32 %[[ADD]], %[[TMP8]]
-// IR-NEXT:    %[[SUB4:.+]] = sub i32 %[[DIV]], 1
-// IR-NEXT:    store i32 %[[SUB4]], ptr %[[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    %[[TMP9:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    %[[ADD6:.+]] = add i32 %[[TMP9]], 1
-// IR-NEXT:    store i32 %[[ADD6]], ptr %[[DOTCAPTURE_EXPR_5]], align 4
-// IR-NEXT:    %[[TMP10:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_5]], align 4
-// IR-NEXT:    %[[SUB8:.+]] = sub i32 %[[TMP10]], -3
-// IR-NEXT:    %[[DIV9:.+]] = udiv i32 %[[SUB8]], 4
-// IR-NEXT:    %[[SUB10:.+]] = sub i32 %[[DIV9]], 1
-// IR-NEXT:    store i32 %[[SUB10]], ptr %[[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    store i32 0, ptr %[[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    %[[TMP11:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_5]], align 4
-// IR-NEXT:    %[[CMP:.+]] = icmp ult i32 0, %[[TMP11]]
-// IR-NEXT:    br i1 %[[CMP]], label %[[OMP_PRECOND_THEN:.+]], label %[[OMP_PRECOND_END:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_PRECOND_THEN]]:
-// IR-NEXT:    store i32 0, ptr %[[DOTOMP_LB]], align 4
-// IR-NEXT:    %[[TMP12:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    store i32 %[[TMP12]], ptr %[[DOTOMP_UB]], align 4
-// IR-NEXT:    store i32 1, ptr %[[DOTOMP_STRIDE]], align 4
-// IR-NEXT:    store i32 0, ptr %[[DOTOMP_IS_LAST]], align 4
-// IR-NEXT:    call void @__kmpc_for_static_init_4u(ptr @[[GLOB1:.+]], i32 %[[TMP0]], i32 34, ptr %[[DOTOMP_IS_LAST]], ptr %[[DOTOMP_LB]], ptr %[[DOTOMP_UB]], ptr %[[DOTOMP_STRIDE]], i32 1, i32 1)
-// IR-NEXT:    %[[TMP13:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
-// IR-NEXT:    %[[TMP14:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    %[[CMP12:.+]] = icmp ugt i32 %[[TMP13]], %[[TMP14]]
-// IR-NEXT:    br i1 %[[CMP12]], label %[[COND_TRUE:.+]], label %[[COND_FALSE:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_TRUE]]:
-// IR-NEXT:    %[[TMP15:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    br label %[[COND_END:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_FALSE]]:
-// IR-NEXT:    %[[TMP16:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
+// IR-LABEL: define dso_local void @func(
+// IR-SAME: i32 noundef [[START:%.*]], i32 noundef [[END:%.*]], i32 noundef [[STEP:%.*]]) #[[ATTR0:[0-9]+]] {
+// IR-NEXT:  [[ENTRY:.*:]]
+// IR-NEXT:    [[START_ADDR:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[TMP:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_5:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I11:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
+// IR-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
+// IR-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
+// IR-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// IR-NEXT:    store i32 [[TMP1]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// IR-NEXT:    store i32 [[TMP2]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// IR-NEXT:    store i32 [[TMP3]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// IR-NEXT:    store i32 [[TMP4]], ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    [[SUB:%.*]] = sub i32 [[TMP5]], [[TMP6]]
+// IR-NEXT:    [[SUB3:%.*]] = sub i32 [[SUB]], 1
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add i32 [[SUB3]], [[TMP7]]
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP8]]
+// IR-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
+// IR-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    [[ADD6:%.*]] = add i32 [[TMP9]], 1
+// IR-NEXT:    store i32 [[ADD6]], ptr [[DOTCAPTURE_EXPR_5]], align 4
+// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// IR-NEXT:    [[SUB8:%.*]] = sub i32 [[TMP10]], -3
+// IR-NEXT:    [[DIV9:%.*]] = udiv i32 [[SUB8]], 4
+// IR-NEXT:    [[SUB10:%.*]] = sub i32 [[DIV9]], 1
+// IR-NEXT:    store i32 [[SUB10]], ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// IR-NEXT:    [[CMP:%.*]] = icmp ult i32 0, [[TMP11]]
+// IR-NEXT:    br i1 [[CMP]], label %[[OMP_PRECOND_THEN:.*]], label %[[OMP_PRECOND_END:.*]]
+// IR:       [[OMP_PRECOND_THEN]]:
+// IR-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
+// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    store i32 [[TMP12]], ptr [[DOTOMP_UB]], align 4
+// IR-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
+// IR-NEXT:    call void @__kmpc_for_static_init_4u(ptr @[[GLOB1:[0-9]+]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
+// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    [[CMP12:%.*]] = icmp ugt i32 [[TMP13]], [[TMP14]]
+// IR-NEXT:    br i1 [[CMP12]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
 // IR-NEXT:    br label %[[COND_END]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_END]]:
-// IR-NEXT:    %[[COND:.+]] = phi i32 [ %[[TMP15]], %[[COND_TRUE]] ], [ %[[TMP16]], %[[COND_FALSE]] ]
-// IR-NEXT:    store i32 %[[COND]], ptr %[[DOTOMP_UB]], align 4
-// IR-NEXT:    %[[TMP17:.+]] = load i32, ptr %[[DOTOMP_LB]], align 4
-// IR-NEXT:    store i32 %[[TMP17]], ptr %[[DOTOMP_IV]], align 4
-// IR-NEXT:    br label %[[OMP_INNER_FOR_COND:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_INNER_FOR_COND]]:
-// IR-NEXT:    %[[TMP18:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
-// IR-NEXT:    %[[TMP19:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
-// IR-NEXT:    %[[ADD13:.+]] = add i32 %[[TMP19]], 1
-// IR-NEXT:    %[[CMP14:.+]] = icmp ult i32 %[[TMP18]], %[[ADD13]]
-// IR-NEXT:    br i1 %[[CMP14]], label %[[OMP_INNER_FOR_BODY:.+]], label %[[OMP_INNER_FOR_END:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_INNER_FOR_BODY]]:
-// IR-NEXT:    %[[TMP20:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
-// IR-NEXT:    %[[MUL:.+]] = mul i32 %[[TMP20]], 4
-// IR-NEXT:    %[[ADD15:.+]] = add i32 0, %[[MUL]]
-// IR-NEXT:    store i32 %[[ADD15]], ptr %[[DOTFLOOR_0_IV_I11]], align 4
-// IR-NEXT:    %[[TMP21:.+]] = load i32, ptr %[[DOTFLOOR_0_IV_I11]], align 4
-// IR-NEXT:    store i32 %[[TMP21]], ptr %[[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_COND]]:
-// IR-NEXT:    %[[TMP22:.+]] = load i32, ptr %[[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    %[[TMP23:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    %[[ADD16:.+]] = add i32 %[[TMP23]], 1
-// IR-NEXT:    %[[TMP24:.+]] = load i32, ptr %[[DOTFLOOR_0_IV_I11]], align 4
-// IR-NEXT:    %[[ADD17:.+]] = add i32 %[[TMP24]], 4
-// IR-NEXT:    %[[CMP18:.+]] = icmp ult i32 %[[ADD16]], %[[ADD17]]
-// IR-NEXT:    br i1 %[[CMP18]], label %[[COND_TRUE19:.+]], label %[[COND_FALSE21:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_TRUE19]]:
-// IR-NEXT:    %[[TMP25:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    %[[ADD20:.+]] = add i32 %[[TMP25]], 1
-// IR-NEXT:    br label %[[COND_END23:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_FALSE21]]:
-// IR-NEXT:    %[[TMP26:.+]] = load i32, ptr %[[DOTFLOOR_0_IV_I11]], align 4
-// IR-NEXT:    %[[ADD22:.+]] = add i32 %[[TMP26]], 4
-// IR-NEXT:    br label %[[COND_END23]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_END23]]:
-// IR-NEXT:    %[[COND24:.+]] = phi i32 [ %[[ADD20]], %[[COND_TRUE19]] ], [ %[[ADD22]], %[[COND_FALSE21]] ]
-// IR-NEXT:    %[[CMP25:.+]] = icmp ult i32 %[[TMP22]], %[[COND24]]
-// IR-NEXT:    br i1 %[[CMP25]], label %[[FOR_BODY:.+]], label %[[FOR_END:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_BODY]]:
-// IR-NEXT:    %[[TMP27:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    %[[TMP28:.+]] = load i32, ptr %[[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    %[[TMP29:.+]] = load i32, ptr %[[DOTNEW_STEP]], align 4
-// IR-NEXT:    %[[MUL26:.+]] = mul i32 %[[TMP28]], %[[TMP29]]
-// IR-NEXT:    %[[ADD27:.+]] = add i32 %[[TMP27]], %[[MUL26]]
-// IR-NEXT:    store i32 %[[ADD27]], ptr %[[I]], align 4
-// IR-NEXT:    %[[TMP30:.+]] = load i32, ptr %[[START_ADDR]], align 4
-// IR-NEXT:    %[[TMP31:.+]] = load i32, ptr %[[END_ADDR]], align 4
-// IR-NEXT:    %[[TMP32:.+]] = load i32, ptr %[[STEP_ADDR]], align 4
-// IR-NEXT:    %[[TMP33:.+]] = load i32, ptr %[[I]], align 4
-// IR-NEXT:    call void (...) @body(i32 noundef %[[TMP30]], i32 noundef %[[TMP31]], i32 noundef %[[TMP32]], i32 noundef %[[TMP33]])
-// IR-NEXT:    br label %[[FOR_INC:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_INC]]:
-// IR-NEXT:    %[[TMP34:.+]] = load i32, ptr %[[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    %[[INC:.+]] = add i32 %[[TMP34]], 1
-// IR-NEXT:    store i32 %[[INC]], ptr %[[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop ![[LOOP3:[0-9]+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_END]]:
-// IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_BODY_CONTINUE]]:
-// IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_INNER_FOR_INC]]:
-// IR-NEXT:    %[[TMP35:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
-// IR-NEXT:    %[[ADD28:.+]] = add i32 %[[TMP35]], 1
-// IR-NEXT:    store i32 %[[ADD28]], ptr %[[DOTOMP_IV]], align 4
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP15]], %[[COND_TRUE]] ], [ [[TMP16]], %[[COND_FALSE]] ]
+// IR-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
+// IR-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
+// IR-NEXT:    store i32 [[TMP17]], ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    br label %[[OMP_INNER_FOR_COND:.*]]
+// IR:       [[OMP_INNER_FOR_COND]]:
+// IR-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// IR-NEXT:    [[ADD13:%.*]] = add i32 [[TMP19]], 1
+// IR-NEXT:    [[CMP14:%.*]] = icmp ult i32 [[TMP18]], [[ADD13]]
+// IR-NEXT:    br i1 [[CMP14]], label %[[OMP_INNER_FOR_BODY:.*]], label %[[OMP_INNER_FOR_END:.*]]
+// IR:       [[OMP_INNER_FOR_BODY]]:
+// IR-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul i32 [[TMP20]], 4
+// IR-NEXT:    [[ADD15:%.*]] = add i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I11]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND:.*]]
+// IR:       [[FOR_COND]]:
+// IR-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[CMP16:%.*]] = icmp ult i32 [[TMP21]], 4
+// IR-NEXT:    br i1 [[CMP16]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY]]:
+// IR-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I11]], align 4
+// IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD17:%.*]] = add i32 [[TMP22]], [[TMP23]]
+// IR-NEXT:    store i32 [[ADD17]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[MUL18:%.*]] = mul i32 [[TMP25]], [[TMP26]]
+// IR-NEXT:    [[ADD19:%.*]] = add i32 [[TMP24]], [[MUL18]]
+// IR-NEXT:    store i32 [[ADD19]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    [[ADD20:%.*]] = add i32 [[TMP28]], 1
+// IR-NEXT:    [[CMP21:%.*]] = icmp ult i32 [[TMP27]], [[ADD20]]
+// IR-NEXT:    br i1 [[CMP21]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
+// IR:       [[IF_THEN]]:
+// IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void (...) @body(i32 noundef [[TMP29]], i32 noundef [[TMP30]], i32 noundef [[TMP31]], i32 noundef [[TMP32]])
+// IR-NEXT:    br label %[[IF_END]]
+// IR:       [[IF_END]]:
+// IR-NEXT:    br label %[[FOR_INC:.*]]
+// IR:       [[FOR_INC]]:
+// IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add i32 [[TMP33]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP2:![0-9]+]]
+// IR:       [[FOR_END]]:
+// IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.*]]
+// IR:       [[OMP_BODY_CONTINUE]]:
+// IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.*]]
+// IR:       [[OMP_INNER_FOR_INC]]:
+// IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[ADD22:%.*]] = add i32 [[TMP34]], 1
+// IR-NEXT:    store i32 [[ADD22]], ptr [[DOTOMP_IV]], align 4
 // IR-NEXT:    br label %[[OMP_INNER_FOR_COND]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_INNER_FOR_END]]:
-// IR-NEXT:    br label %[[OMP_LOOP_EXIT:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_LOOP_EXIT]]:
-// IR-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 %[[TMP0]])
+// IR:       [[OMP_INNER_FOR_END]]:
+// IR-NEXT:    br label %[[OMP_LOOP_EXIT:.*]]
+// IR:       [[OMP_LOOP_EXIT]]:
+// IR-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP0]])
 // IR-NEXT:    br label %[[OMP_PRECOND_END]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_PRECOND_END]]:
-// IR-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3:.+]], i32 %[[TMP0]])
+// IR:       [[OMP_PRECOND_END]]:
+// IR-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3:[0-9]+]], i32 [[TMP0]])
 // IR-NEXT:    ret void
-// IR-NEXT:  }
+//
 extern "C" void func(int start, int end, int step) {
 #pragma omp for
 #pragma omp tile sizes(4)
@@ -191,3 +171,7 @@ extern "C" void func(int start, int end, int step) {
 }
 
 #endif /* HEADER */
+//.
+// IR: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]]}
+// IR: [[META3]] = !{!"llvm.loop.mustprogress"}
+//.
diff --git a/clang/test/OpenMP/tile_codegen_tile_for.cpp b/clang/test/OpenMP/tile_codegen_tile_for.cpp
index 8a844c78c54a6..8153f76b7e1a9 100644
--- a/clang/test/OpenMP/tile_codegen_tile_for.cpp
+++ b/clang/test/OpenMP/tile_codegen_tile_for.cpp
@@ -1,3 +1,4 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
 // Check code generation
 // RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fopenmp -emit-llvm %s -o - | FileCheck %s --check-prefix=IR
 
@@ -16,229 +17,197 @@
 extern "C" void body(...) {}
 
 
-// IR-LABEL: define {{.*}}@func(
-// IR-NEXT:  [[ENTRY:.*]]:
-// IR-NEXT:    %[[START_ADDR:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[END_ADDR:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[STEP_ADDR:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_IV:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[TMP:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[I:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_1:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTNEW_STEP:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_2:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTFLOOR_0_IV_I:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_5:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_7:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_11:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_13:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTFLOOR_0_IV__FLOOR_0_IV_I:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_LB:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_UB:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_STRIDE:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_IS_LAST:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTFLOOR_0_IV__FLOOR_0_IV_I17:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTTILE_0_IV__FLOOR_0_IV_I:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTTILE_0_IV_I:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[TMP0:.+]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:.+]])
-// IR-NEXT:    store i32 %[[START:.+]], ptr %[[START_ADDR]], align 4
-// IR-NEXT:    store i32 %[[END:.+]], ptr %[[END_ADDR]], align 4
-// IR-NEXT:    store i32 %[[STEP:.+]], ptr %[[STEP_ADDR]], align 4
-// IR-NEXT:    %[[TMP1:.+]] = load i32, ptr %[[START_ADDR]], align 4
-// IR-NEXT:    store i32 %[[TMP1]], ptr %[[I]], align 4
-// IR-NEXT:    %[[TMP2:.+]] = load i32, ptr %[[START_ADDR]], align 4
-// IR-NEXT:    store i32 %[[TMP2]], ptr %[[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    %[[TMP3:.+]] = load i32, ptr %[[END_ADDR]], align 4
-// IR-NEXT:    store i32 %[[TMP3]], ptr %[[DOTCAPTURE_EXPR_1]], align 4
-// IR-NEXT:    %[[TMP4:.+]] = load i32, ptr %[[STEP_ADDR]], align 4
-// IR-NEXT:    store i32 %[[TMP4]], ptr %[[DOTNEW_STEP]], align 4
-// IR-NEXT:    %[[TMP5:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_1]], align 4
-// IR-NEXT:    %[[TMP6:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    %[[SUB:.+]] = sub i32 %[[TMP5]], %[[TMP6]]
-// IR-NEXT:    %[[SUB3:.+]] = sub i32 %[[SUB]], 1
-// IR-NEXT:    %[[TMP7:.+]] = load i32, ptr %[[DOTNEW_STEP]], align 4
-// IR-NEXT:    %[[ADD:.+]] = add i32 %[[SUB3]], %[[TMP7]]
-// IR-NEXT:    %[[TMP8:.+]] = load i32, ptr %[[DOTNEW_STEP]], align 4
-// IR-NEXT:    %[[DIV:.+]] = udiv i32 %[[ADD]], %[[TMP8]]
-// IR-NEXT:    %[[SUB4:.+]] = sub i32 %[[DIV]], 1
-// IR-NEXT:    store i32 %[[SUB4]], ptr %[[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    store i32 0, ptr %[[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    %[[TMP9:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    %[[ADD6:.+]] = add i32 %[[TMP9]], 1
-// IR-NEXT:    store i32 %[[ADD6]], ptr %[[DOTCAPTURE_EXPR_5]], align 4
-// IR-NEXT:    %[[TMP10:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_5]], align 4
-// IR-NEXT:    %[[SUB8:.+]] = sub i32 %[[TMP10]], -3
-// IR-NEXT:    %[[DIV9:.+]] = udiv i32 %[[SUB8]], 4
-// IR-NEXT:    %[[SUB10:.+]] = sub i32 %[[DIV9]], 1
-// IR-NEXT:    store i32 %[[SUB10]], ptr %[[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    %[[TMP11:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    %[[ADD12:.+]] = add i32 %[[TMP11]], 1
-// IR-NEXT:    store i32 %[[ADD12]], ptr %[[DOTCAPTURE_EXPR_11]], align 4
-// IR-NEXT:    %[[TMP12:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_11]], align 4
-// IR-NEXT:    %[[SUB14:.+]] = sub i32 %[[TMP12]], -2
-// IR-NEXT:    %[[DIV15:.+]] = udiv i32 %[[SUB14]], 3
-// IR-NEXT:    %[[SUB16:.+]] = sub i32 %[[DIV15]], 1
-// IR-NEXT:    store i32 %[[SUB16]], ptr %[[DOTCAPTURE_EXPR_13]], align 4
-// IR-NEXT:    store i32 0, ptr %[[DOTFLOOR_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    %[[TMP13:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_11]], align 4
-// IR-NEXT:    %[[CMP:.+]] = icmp ult i32 0, %[[TMP13]]
-// IR-NEXT:    br i1 %[[CMP]], label %[[OMP_PRECOND_THEN:.+]], label %[[OMP_PRECOND_END:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_PRECOND_THEN]]:
-// IR-NEXT:    store i32 0, ptr %[[DOTOMP_LB]], align 4
-// IR-NEXT:    %[[TMP14:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_13]], align 4
-// IR-NEXT:    store i32 %[[TMP14]], ptr %[[DOTOMP_UB]], align 4
-// IR-NEXT:    store i32 1, ptr %[[DOTOMP_STRIDE]], align 4
-// IR-NEXT:    store i32 0, ptr %[[DOTOMP_IS_LAST]], align 4
-// IR-NEXT:    call void @__kmpc_for_static_init_4u(ptr @[[GLOB1:.+]], i32 %[[TMP0]], i32 34, ptr %[[DOTOMP_IS_LAST]], ptr %[[DOTOMP_LB]], ptr %[[DOTOMP_UB]], ptr %[[DOTOMP_STRIDE]], i32 1, i32 1)
-// IR-NEXT:    %[[TMP15:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
-// IR-NEXT:    %[[TMP16:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_13]], align 4
-// IR-NEXT:    %[[CMP18:.+]] = icmp ugt i32 %[[TMP15]], %[[TMP16]]
-// IR-NEXT:    br i1 %[[CMP18]], label %[[COND_TRUE:.+]], label %[[COND_FALSE:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_TRUE]]:
-// IR-NEXT:    %[[TMP17:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_13]], align 4
-// IR-NEXT:    br label %[[COND_END:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_FALSE]]:
-// IR-NEXT:    %[[TMP18:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
+
+
+// IR-LABEL: define dso_local void @func(
+// IR-SAME: i32 noundef [[START:%.*]], i32 noundef [[END:%.*]], i32 noundef [[STEP:%.*]]) #[[ATTR0:[0-9]+]] {
+// IR-NEXT:  [[ENTRY:.*:]]
+// IR-NEXT:    [[START_ADDR:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[TMP:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_5:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV__FLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_13:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV__FLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV__FLOOR_0_IV_I17:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
+// IR-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
+// IR-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
+// IR-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// IR-NEXT:    store i32 [[TMP1]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// IR-NEXT:    store i32 [[TMP2]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// IR-NEXT:    store i32 [[TMP3]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// IR-NEXT:    store i32 [[TMP4]], ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    [[SUB:%.*]] = sub i32 [[TMP5]], [[TMP6]]
+// IR-NEXT:    [[SUB3:%.*]] = sub i32 [[SUB]], 1
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add i32 [[SUB3]], [[TMP7]]
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP8]]
+// IR-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
+// IR-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    [[ADD6:%.*]] = add i32 [[TMP9]], 1
+// IR-NEXT:    store i32 [[ADD6]], ptr [[DOTCAPTURE_EXPR_5]], align 4
+// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// IR-NEXT:    [[SUB8:%.*]] = sub i32 [[TMP10]], -3
+// IR-NEXT:    [[DIV9:%.*]] = udiv i32 [[SUB8]], 4
+// IR-NEXT:    [[SUB10:%.*]] = sub i32 [[DIV9]], 1
+// IR-NEXT:    store i32 [[SUB10]], ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    [[ADD12:%.*]] = add i32 [[TMP11]], 1
+// IR-NEXT:    store i32 [[ADD12]], ptr [[DOTCAPTURE_EXPR_11]], align 4
+// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// IR-NEXT:    [[SUB14:%.*]] = sub i32 [[TMP12]], -2
+// IR-NEXT:    [[DIV15:%.*]] = udiv i32 [[SUB14]], 3
+// IR-NEXT:    [[SUB16:%.*]] = sub i32 [[DIV15]], 1
+// IR-NEXT:    store i32 [[SUB16]], ptr [[DOTCAPTURE_EXPR_13]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// IR-NEXT:    [[CMP:%.*]] = icmp ult i32 0, [[TMP13]]
+// IR-NEXT:    br i1 [[CMP]], label %[[OMP_PRECOND_THEN:.*]], label %[[OMP_PRECOND_END:.*]]
+// IR:       [[OMP_PRECOND_THEN]]:
+// IR-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
+// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_13]], align 4
+// IR-NEXT:    store i32 [[TMP14]], ptr [[DOTOMP_UB]], align 4
+// IR-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
+// IR-NEXT:    call void @__kmpc_for_static_init_4u(ptr @[[GLOB1:[0-9]+]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
+// IR-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// IR-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_13]], align 4
+// IR-NEXT:    [[CMP18:%.*]] = icmp ugt i32 [[TMP15]], [[TMP16]]
+// IR-NEXT:    br i1 [[CMP18]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_13]], align 4
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
 // IR-NEXT:    br label %[[COND_END]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_END]]:
-// IR-NEXT:    %[[COND:.+]] = phi i32 [ %[[TMP17]], %[[COND_TRUE]] ], [ %[[TMP18]], %[[COND_FALSE]] ]
-// IR-NEXT:    store i32 %[[COND]], ptr %[[DOTOMP_UB]], align 4
-// IR-NEXT:    %[[TMP19:.+]] = load i32, ptr %[[DOTOMP_LB]], align 4
-// IR-NEXT:    store i32 %[[TMP19]], ptr %[[DOTOMP_IV]], align 4
-// IR-NEXT:    br label %[[OMP_INNER_FOR_COND:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_INNER_FOR_COND]]:
-// IR-NEXT:    %[[TMP20:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
-// IR-NEXT:    %[[TMP21:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
-// IR-NEXT:    %[[ADD19:.+]] = add i32 %[[TMP21]], 1
-// IR-NEXT:    %[[CMP20:.+]] = icmp ult i32 %[[TMP20]], %[[ADD19]]
-// IR-NEXT:    br i1 %[[CMP20]], label %[[OMP_INNER_FOR_BODY:.+]], label %[[OMP_INNER_FOR_END:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_INNER_FOR_BODY]]:
-// IR-NEXT:    %[[TMP22:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
-// IR-NEXT:    %[[MUL:.+]] = mul i32 %[[TMP22]], 3
-// IR-NEXT:    %[[ADD21:.+]] = add i32 0, %[[MUL]]
-// IR-NEXT:    store i32 %[[ADD21]], ptr %[[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
-// IR-NEXT:    %[[TMP23:.+]] = load i32, ptr %[[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
-// IR-NEXT:    store i32 %[[TMP23]], ptr %[[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_COND]]:
-// IR-NEXT:    %[[TMP24:.+]] = load i32, ptr %[[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    %[[TMP25:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    %[[ADD22:.+]] = add i32 %[[TMP25]], 1
-// IR-NEXT:    %[[TMP26:.+]] = load i32, ptr %[[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
-// IR-NEXT:    %[[ADD23:.+]] = add i32 %[[TMP26]], 3
-// IR-NEXT:    %[[CMP24:.+]] = icmp ult i32 %[[ADD22]], %[[ADD23]]
-// IR-NEXT:    br i1 %[[CMP24]], label %[[COND_TRUE25:.+]], label %[[COND_FALSE27:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_TRUE25]]:
-// IR-NEXT:    %[[TMP27:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    %[[ADD26:.+]] = add i32 %[[TMP27]], 1
-// IR-NEXT:    br label %[[COND_END29:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_FALSE27]]:
-// IR-NEXT:    %[[TMP28:.+]] = load i32, ptr %[[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
-// IR-NEXT:    %[[ADD28:.+]] = add i32 %[[TMP28]], 3
-// IR-NEXT:    br label %[[COND_END29]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_END29]]:
-// IR-NEXT:    %[[COND30:.+]] = phi i32 [ %[[ADD26]], %[[COND_TRUE25]] ], [ %[[ADD28]], %[[COND_FALSE27]] ]
-// IR-NEXT:    %[[CMP31:.+]] = icmp ult i32 %[[TMP24]], %[[COND30]]
-// IR-NEXT:    br i1 %[[CMP31]], label %[[FOR_BODY:.+]], label %[[FOR_END50:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_BODY]]:
-// IR-NEXT:    %[[TMP29:.+]] = load i32, ptr %[[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    %[[MUL32:.+]] = mul i32 %[[TMP29]], 4
-// IR-NEXT:    %[[ADD33:.+]] = add i32 0, %[[MUL32]]
-// IR-NEXT:    store i32 %[[ADD33]], ptr %[[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    %[[TMP30:.+]] = load i32, ptr %[[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    store i32 %[[TMP30]], ptr %[[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND34:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_COND34]]:
-// IR-NEXT:    %[[TMP31:.+]] = load i32, ptr %[[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    %[[TMP32:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    %[[ADD35:.+]] = add i32 %[[TMP32]], 1
-// IR-NEXT:    %[[TMP33:.+]] = load i32, ptr %[[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    %[[ADD36:.+]] = add i32 %[[TMP33]], 4
-// IR-NEXT:    %[[CMP37:.+]] = icmp ult i32 %[[ADD35]], %[[ADD36]]
-// IR-NEXT:    br i1 %[[CMP37]], label %[[COND_TRUE38:.+]], label %[[COND_FALSE40:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_TRUE38]]:
-// IR-NEXT:    %[[TMP34:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    %[[ADD39:.+]] = add i32 %[[TMP34]], 1
-// IR-NEXT:    br label %[[COND_END42:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_FALSE40]]:
-// IR-NEXT:    %[[TMP35:.+]] = load i32, ptr %[[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    %[[ADD41:.+]] = add i32 %[[TMP35]], 4
-// IR-NEXT:    br label %[[COND_END42]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_END42]]:
-// IR-NEXT:    %[[COND43:.+]] = phi i32 [ %[[ADD39]], %[[COND_TRUE38]] ], [ %[[ADD41]], %[[COND_FALSE40]] ]
-// IR-NEXT:    %[[CMP44:.+]] = icmp ult i32 %[[TMP31]], %[[COND43]]
-// IR-NEXT:    br i1 %[[CMP44]], label %[[FOR_BODY45:.+]], label %[[FOR_END:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_BODY45]]:
-// IR-NEXT:    %[[TMP36:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    %[[TMP37:.+]] = load i32, ptr %[[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    %[[TMP38:.+]] = load i32, ptr %[[DOTNEW_STEP]], align 4
-// IR-NEXT:    %[[MUL46:.+]] = mul i32 %[[TMP37]], %[[TMP38]]
-// IR-NEXT:    %[[ADD47:.+]] = add i32 %[[TMP36]], %[[MUL46]]
-// IR-NEXT:    store i32 %[[ADD47]], ptr %[[I]], align 4
-// IR-NEXT:    %[[TMP39:.+]] = load i32, ptr %[[START_ADDR]], align 4
-// IR-NEXT:    %[[TMP40:.+]] = load i32, ptr %[[END_ADDR]], align 4
-// IR-NEXT:    %[[TMP41:.+]] = load i32, ptr %[[STEP_ADDR]], align 4
-// IR-NEXT:    %[[TMP42:.+]] = load i32, ptr %[[I]], align 4
-// IR-NEXT:    call void (...) @body(i32 noundef %[[TMP39]], i32 noundef %[[TMP40]], i32 noundef %[[TMP41]], i32 noundef %[[TMP42]])
-// IR-NEXT:    br label %[[FOR_INC:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_INC]]:
-// IR-NEXT:    %[[TMP43:.+]] = load i32, ptr %[[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    %[[INC:.+]] = add i32 %[[TMP43]], 1
-// IR-NEXT:    store i32 %[[INC]], ptr %[[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND34]], !llvm.loop ![[LOOP2:[0-9]+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC48:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_INC48]]:
-// IR-NEXT:    %[[TMP44:.+]] = load i32, ptr %[[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    %[[INC49:.+]] = add i32 %[[TMP44]], 1
-// IR-NEXT:    store i32 %[[INC49]], ptr %[[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop ![[LOOP4:[0-9]+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_END50]]:
-// IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_BODY_CONTINUE]]:
-// IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_INNER_FOR_INC]]:
-// IR-NEXT:    %[[TMP45:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
-// IR-NEXT:    %[[ADD51:.+]] = add i32 %[[TMP45]], 1
-// IR-NEXT:    store i32 %[[ADD51]], ptr %[[DOTOMP_IV]], align 4
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP17]], %[[COND_TRUE]] ], [ [[TMP18]], %[[COND_FALSE]] ]
+// IR-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
+// IR-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
+// IR-NEXT:    store i32 [[TMP19]], ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    br label %[[OMP_INNER_FOR_COND:.*]]
+// IR:       [[OMP_INNER_FOR_COND]]:
+// IR-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// IR-NEXT:    [[ADD19:%.*]] = add i32 [[TMP21]], 1
+// IR-NEXT:    [[CMP20:%.*]] = icmp ult i32 [[TMP20]], [[ADD19]]
+// IR-NEXT:    br i1 [[CMP20]], label %[[OMP_INNER_FOR_BODY:.*]], label %[[OMP_INNER_FOR_END:.*]]
+// IR:       [[OMP_INNER_FOR_BODY]]:
+// IR-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul i32 [[TMP22]], 3
+// IR-NEXT:    [[ADD21:%.*]] = add i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD21]], ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND:.*]]
+// IR:       [[FOR_COND]]:
+// IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[CMP22:%.*]] = icmp ult i32 [[TMP23]], 3
+// IR-NEXT:    br i1 [[CMP22]], label %[[FOR_BODY:.*]], label %[[FOR_END40:.*]]
+// IR:       [[FOR_BODY]]:
+// IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
+// IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD23:%.*]] = add i32 [[TMP24]], [[TMP25]]
+// IR-NEXT:    store i32 [[ADD23]], ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[MUL24:%.*]] = mul i32 [[TMP26]], 4
+// IR-NEXT:    [[ADD25:%.*]] = add i32 0, [[MUL24]]
+// IR-NEXT:    store i32 [[ADD25]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    [[ADD26:%.*]] = add i32 [[TMP28]], 1
+// IR-NEXT:    [[CMP27:%.*]] = icmp ult i32 [[TMP27]], [[ADD26]]
+// IR-NEXT:    br i1 [[CMP27]], label %[[IF_THEN:.*]], label %[[IF_END37:.*]]
+// IR:       [[IF_THEN]]:
+// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND28:.*]]
+// IR:       [[FOR_COND28]]:
+// IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[CMP29:%.*]] = icmp ult i32 [[TMP29]], 4
+// IR-NEXT:    br i1 [[CMP29]], label %[[FOR_BODY30:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY30]]:
+// IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD31:%.*]] = add i32 [[TMP30]], [[TMP31]]
+// IR-NEXT:    store i32 [[ADD31]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[MUL32:%.*]] = mul i32 [[TMP33]], [[TMP34]]
+// IR-NEXT:    [[ADD33:%.*]] = add i32 [[TMP32]], [[MUL32]]
+// IR-NEXT:    store i32 [[ADD33]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    [[ADD34:%.*]] = add i32 [[TMP36]], 1
+// IR-NEXT:    [[CMP35:%.*]] = icmp ult i32 [[TMP35]], [[ADD34]]
+// IR-NEXT:    br i1 [[CMP35]], label %[[IF_THEN36:.*]], label %[[IF_END:.*]]
+// IR:       [[IF_THEN36]]:
+// IR-NEXT:    [[TMP37:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// IR-NEXT:    [[TMP38:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// IR-NEXT:    [[TMP39:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// IR-NEXT:    [[TMP40:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void (...) @body(i32 noundef [[TMP37]], i32 noundef [[TMP38]], i32 noundef [[TMP39]], i32 noundef [[TMP40]])
+// IR-NEXT:    br label %[[IF_END]]
+// IR:       [[IF_END]]:
+// IR-NEXT:    br label %[[FOR_INC:.*]]
+// IR:       [[FOR_INC]]:
+// IR-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add i32 [[TMP41]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND28]], !llvm.loop [[LOOP2:![0-9]+]]
+// IR:       [[FOR_END]]:
+// IR-NEXT:    br label %[[IF_END37]]
+// IR:       [[IF_END37]]:
+// IR-NEXT:    br label %[[FOR_INC38:.*]]
+// IR:       [[FOR_INC38]]:
+// IR-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[INC39:%.*]] = add i32 [[TMP42]], 1
+// IR-NEXT:    store i32 [[INC39]], ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP4:![0-9]+]]
+// IR:       [[FOR_END40]]:
+// IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.*]]
+// IR:       [[OMP_BODY_CONTINUE]]:
+// IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.*]]
+// IR:       [[OMP_INNER_FOR_INC]]:
+// IR-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[ADD41:%.*]] = add i32 [[TMP43]], 1
+// IR-NEXT:    store i32 [[ADD41]], ptr [[DOTOMP_IV]], align 4
 // IR-NEXT:    br label %[[OMP_INNER_FOR_COND]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_INNER_FOR_END]]:
-// IR-NEXT:    br label %[[OMP_LOOP_EXIT:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_LOOP_EXIT]]:
-// IR-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 %[[TMP0]])
+// IR:       [[OMP_INNER_FOR_END]]:
+// IR-NEXT:    br label %[[OMP_LOOP_EXIT:.*]]
+// IR:       [[OMP_LOOP_EXIT]]:
+// IR-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP0]])
 // IR-NEXT:    br label %[[OMP_PRECOND_END]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_PRECOND_END]]:
-// IR-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3:.+]], i32 %[[TMP0]])
+// IR:       [[OMP_PRECOND_END]]:
+// IR-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3:[0-9]+]], i32 [[TMP0]])
 // IR-NEXT:    ret void
-// IR-NEXT:  }
-
-
+//
 extern "C" void func(int start, int end, int step) {
 #pragma omp for
 #pragma omp tile sizes(3)
@@ -249,8 +218,8 @@ extern "C" void func(int start, int end, int step) {
 
 #endif /* HEADER */
 
-// IR: ![[META0:[0-9]+]] = !{i32 7, !"openmp", i32 51}
-// IR: ![[META1:[0-9]+]] =
-// IR: ![[LOOP2]] = distinct !{![[LOOP2]], ![[LOOPPROP3:[0-9]+]]}
-// IR: ![[LOOPPROP3]] = !{!"llvm.loop.mustprogress"}
-// IR: ![[LOOP4]] = distinct !{![[LOOP4]], ![[LOOPPROP3]]}
+//.
+// IR: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]]}
+// IR: [[META3]] = !{!"llvm.loop.mustprogress"}
+// IR: [[LOOP4]] = distinct !{[[LOOP4]], [[META3]]}
+//.
diff --git a/clang/test/OpenMP/tile_messages.cpp b/clang/test/OpenMP/tile_messages.cpp
index e1f5155c924e5..8915d1bb3c546 100644
--- a/clang/test/OpenMP/tile_messages.cpp
+++ b/clang/test/OpenMP/tile_messages.cpp
@@ -101,7 +101,7 @@ void func() {
     for (int j = 0; j < i; ++j)
       ;
 
-  // expected-error at +5 {{expected 3 for loops after '#pragma omp for', but found only 2}}
+  // expected-error@* {{expected 3 for loops after '#pragma omp for', but found only 2}}
   // expected-note at +1 {{as specified in 'collapse' clause}}
   #pragma omp for collapse(3)
   #pragma omp tile sizes(5)

>From 5fdb86f0bcdf5b36369d68db94619ec9d4107cf0 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Mon, 13 Apr 2026 03:19:54 -0400
Subject: [PATCH 06/18] ir-regenerate

---
 clang/test/OpenMP/interchange_codegen.cpp     | 4183 +++++++----------
 .../irbuilder_unroll_partial_factor_for.c     |   57 +-
 ...er_unroll_partial_heuristic_constant_for.c |   79 +-
 ...der_unroll_partial_heuristic_runtime_for.c |   79 +-
 .../irbuilder_unroll_unroll_partial_factor.c  |   57 +-
 ...rbuilder_unroll_unroll_partial_heuristic.c |   67 +-
 clang/test/OpenMP/unroll_codegen_tile_for.cpp |  404 +-
 7 files changed, 2096 insertions(+), 2830 deletions(-)

diff --git a/clang/test/OpenMP/interchange_codegen.cpp b/clang/test/OpenMP/interchange_codegen.cpp
index 8e833c9df324c..b062d42c9f162 100644
--- a/clang/test/OpenMP/interchange_codegen.cpp
+++ b/clang/test/OpenMP/interchange_codegen.cpp
@@ -123,6 +123,7 @@ extern "C" void foo10() {
 
 #endif /* HEADER */
 
+
 // CHECK1-LABEL: define {{[^@]+}}@body
 // CHECK1-SAME: (...) #[[ATTR0:[0-9]+]] {
 // CHECK1-NEXT:  entry:
@@ -156,7 +157,7 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP5]], [[TMP4]]
 // CHECK1-NEXT:    store i32 [[ADD]], ptr [[I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP3:![0-9]+]]
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP2:![0-9]+]]
 // CHECK1:       for.end:
 // CHECK1-NEXT:    ret void
 //
@@ -262,14 +263,14 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP28]], 1
 // CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTPERMUTED_1_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND16]], !llvm.loop [[LOOP5:![0-9]+]]
+// CHECK1-NEXT:    br label [[FOR_COND16]], !llvm.loop [[LOOP4:![0-9]+]]
 // CHECK1:       for.end:
 // CHECK1-NEXT:    br label [[FOR_INC22:%.*]]
 // CHECK1:       for.inc22:
 // CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_J]], align 4
 // CHECK1-NEXT:    [[INC23:%.*]] = add i32 [[TMP29]], 1
 // CHECK1-NEXT:    store i32 [[INC23]], ptr [[DOTPERMUTED_0_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP5:![0-9]+]]
 // CHECK1:       for.end24:
 // CHECK1-NEXT:    ret void
 //
@@ -342,7 +343,7 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP12]], 1
 // CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTPERMUTED_1_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP7:![0-9]+]]
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
 // CHECK1:       for.end:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
@@ -439,7 +440,7 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
 // CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTPERMUTED_1_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP7:![0-9]+]]
 // CHECK1:       for.end:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
@@ -754,28 +755,28 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTPERMUTED_3_IV_I]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP12]], 1
 // CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTPERMUTED_3_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND11]], !llvm.loop [[LOOP9:![0-9]+]]
+// CHECK1-NEXT:    br label [[FOR_COND11]], !llvm.loop [[LOOP8:![0-9]+]]
 // CHECK1:       for.end:
 // CHECK1-NEXT:    br label [[FOR_INC16:%.*]]
 // CHECK1:       for.inc16:
 // CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTPERMUTED_2_IV_L]], align 4
 // CHECK1-NEXT:    [[INC17:%.*]] = add nsw i32 [[TMP13]], 1
 // CHECK1-NEXT:    store i32 [[INC17]], ptr [[DOTPERMUTED_2_IV_L]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP10:![0-9]+]]
+// CHECK1-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP9:![0-9]+]]
 // CHECK1:       for.end18:
 // CHECK1-NEXT:    br label [[FOR_INC19:%.*]]
 // CHECK1:       for.inc19:
 // CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_K]], align 4
 // CHECK1-NEXT:    [[INC20:%.*]] = add nsw i32 [[TMP14]], 1
 // CHECK1-NEXT:    store i32 [[INC20]], ptr [[DOTPERMUTED_1_IV_K]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP11:![0-9]+]]
+// CHECK1-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP10:![0-9]+]]
 // CHECK1:       for.end21:
 // CHECK1-NEXT:    br label [[FOR_INC22:%.*]]
 // CHECK1:       for.inc22:
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_J]], align 4
 // CHECK1-NEXT:    [[INC23:%.*]] = add nsw i32 [[TMP15]], 1
 // CHECK1-NEXT:    store i32 [[INC23]], ptr [[DOTPERMUTED_0_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP12:![0-9]+]]
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP11:![0-9]+]]
 // CHECK1:       for.end24:
 // CHECK1-NEXT:    ret void
 //
@@ -810,22 +811,21 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTNEW_STEP10:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_14:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_17:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_22:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_K:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_K:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[I49:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[J50:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV_K51:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_K52:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[I35:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[J36:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV_K37:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_K38:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
@@ -863,630 +863,452 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
 // CHECK1-NEXT:    [[ADD15:%.*]] = add i32 [[TMP15]], 1
 // CHECK1-NEXT:    store i32 [[ADD15]], ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[_TMP2]], align 4
-// CHECK1-NEXT:    store i32 [[TMP16]], ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK1-NEXT:    [[ADD18:%.*]] = add i32 [[TMP17]], 1
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[_TMP2]], align 4
-// CHECK1-NEXT:    [[ADD19:%.*]] = add i32 [[TMP18]], 32
-// CHECK1-NEXT:    [[CMP:%.*]] = icmp ult i32 [[ADD18]], [[ADD19]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK1:       cond.true:
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK1-NEXT:    [[ADD20:%.*]] = add i32 [[TMP19]], 1
-// CHECK1-NEXT:    br label [[COND_END:%.*]]
-// CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[_TMP2]], align 4
-// CHECK1-NEXT:    [[ADD21:%.*]] = add i32 [[TMP20]], 32
-// CHECK1-NEXT:    br label [[COND_END]]
-// CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD20]], [[COND_TRUE]] ], [ [[ADD21]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[SUB23:%.*]] = sub i32 [[TMP21]], [[TMP22]]
-// CHECK1-NEXT:    [[SUB24:%.*]] = sub i32 [[SUB23]], 1
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[ADD25:%.*]] = add i32 [[SUB24]], [[TMP23]]
-// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[DIV26:%.*]] = udiv i32 [[ADD25]], [[TMP24]]
-// CHECK1-NEXT:    [[CONV:%.*]] = zext i32 [[DIV26]] to i64
-// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[SUB17:%.*]] = sub i32 [[TMP16]], [[TMP17]]
+// CHECK1-NEXT:    [[SUB18:%.*]] = sub i32 [[SUB17]], 1
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[ADD19:%.*]] = add i32 [[SUB18]], [[TMP18]]
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[DIV20:%.*]] = udiv i32 [[ADD19]], [[TMP19]]
+// CHECK1-NEXT:    [[CONV:%.*]] = zext i32 [[DIV20]] to i64
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB21:%.*]] = sub i32 [[TMP20]], [[TMP21]]
+// CHECK1-NEXT:    [[SUB22:%.*]] = sub i32 [[SUB21]], 1
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD23:%.*]] = add i32 [[SUB22]], [[TMP22]]
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV24:%.*]] = udiv i32 [[ADD23]], [[TMP23]]
+// CHECK1-NEXT:    [[CONV25:%.*]] = zext i32 [[DIV24]] to i64
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV25]]
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB26:%.*]] = sub i32 [[TMP24]], -31
+// CHECK1-NEXT:    [[DIV27:%.*]] = udiv i32 [[SUB26]], 32
+// CHECK1-NEXT:    [[CONV28:%.*]] = zext i32 [[DIV27]] to i64
+// CHECK1-NEXT:    [[MUL29:%.*]] = mul nsw i64 [[MUL]], [[CONV28]]
+// CHECK1-NEXT:    [[MUL30:%.*]] = mul nsw i64 [[MUL29]], 32
+// CHECK1-NEXT:    [[SUB31:%.*]] = sub nsw i64 [[MUL30]], 1
+// CHECK1-NEXT:    store i64 [[SUB31]], ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    store i32 [[TMP25]], ptr [[I]], align 4
 // CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB27:%.*]] = sub i32 [[TMP25]], [[TMP26]]
-// CHECK1-NEXT:    [[SUB28:%.*]] = sub i32 [[SUB27]], 1
-// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD29:%.*]] = add i32 [[SUB28]], [[TMP27]]
-// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV30:%.*]] = udiv i32 [[ADD29]], [[TMP28]]
-// CHECK1-NEXT:    [[CONV31:%.*]] = zext i32 [[DIV30]] to i64
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV31]]
-// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB32:%.*]] = sub i32 [[TMP29]], -31
-// CHECK1-NEXT:    [[DIV33:%.*]] = udiv i32 [[SUB32]], 32
-// CHECK1-NEXT:    [[CONV34:%.*]] = zext i32 [[DIV33]] to i64
-// CHECK1-NEXT:    [[MUL35:%.*]] = mul nsw i64 [[MUL]], [[CONV34]]
-// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB36:%.*]] = sub i32 [[TMP30]], [[TMP31]]
-// CHECK1-NEXT:    [[SUB37:%.*]] = sub i32 [[SUB36]], 1
-// CHECK1-NEXT:    [[ADD38:%.*]] = add i32 [[SUB37]], 1
-// CHECK1-NEXT:    [[DIV39:%.*]] = udiv i32 [[ADD38]], 1
-// CHECK1-NEXT:    [[CONV40:%.*]] = zext i32 [[DIV39]] to i64
-// CHECK1-NEXT:    [[MUL41:%.*]] = mul nsw i64 [[MUL35]], [[CONV40]]
-// CHECK1-NEXT:    [[SUB42:%.*]] = sub nsw i64 [[MUL41]], 1
-// CHECK1-NEXT:    store i64 [[SUB42]], ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    store i32 [[TMP32]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    store i32 [[TMP33]], ptr [[J]], align 4
+// CHECK1-NEXT:    store i32 [[TMP26]], ptr [[J]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_K]], align 4
-// CHECK1-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    store i32 [[TMP34]], ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK1-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK1-NEXT:    [[CMP43:%.*]] = icmp slt i32 [[TMP35]], [[TMP36]]
-// CHECK1-NEXT:    br i1 [[CMP43]], label [[LAND_LHS_TRUE:%.*]], label [[OMP_PRECOND_END:%.*]]
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_K]], align 4
+// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP27]], [[TMP28]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[LAND_LHS_TRUE:%.*]], label [[OMP_PRECOND_END:%.*]]
 // CHECK1:       land.lhs.true:
-// CHECK1-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[CMP44:%.*]] = icmp slt i32 [[TMP37]], [[TMP38]]
-// CHECK1-NEXT:    br i1 [[CMP44]], label [[LAND_LHS_TRUE45:%.*]], label [[OMP_PRECOND_END]]
-// CHECK1:       land.lhs.true45:
-// CHECK1-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[CMP46:%.*]] = icmp ult i32 0, [[TMP39]]
-// CHECK1-NEXT:    br i1 [[CMP46]], label [[LAND_LHS_TRUE47:%.*]], label [[OMP_PRECOND_END]]
-// CHECK1:       land.lhs.true47:
-// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[CMP48:%.*]] = icmp ult i32 [[TMP40]], [[TMP41]]
-// CHECK1-NEXT:    br i1 [[CMP48]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
+// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[CMP32:%.*]] = icmp slt i32 [[TMP29]], [[TMP30]]
+// CHECK1-NEXT:    br i1 [[CMP32]], label [[LAND_LHS_TRUE33:%.*]], label [[OMP_PRECOND_END]]
+// CHECK1:       land.lhs.true33:
+// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[CMP34:%.*]] = icmp ult i32 0, [[TMP31]]
+// CHECK1-NEXT:    br i1 [[CMP34]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
 // CHECK1:       omp.precond.then:
 // CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
-// CHECK1-NEXT:    [[TMP42:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK1-NEXT:    store i64 [[TMP42]], ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK1-NEXT:    store i64 [[TMP32]], ptr [[DOTOMP_UB]], align 8
 // CHECK1-NEXT:    store i64 1, ptr [[DOTOMP_STRIDE]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
 // CHECK1-NEXT:    call void @__kmpc_for_static_init_8(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i64 1, i64 1)
-// CHECK1-NEXT:    [[TMP43:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[TMP44:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK1-NEXT:    [[CMP53:%.*]] = icmp sgt i64 [[TMP43]], [[TMP44]]
-// CHECK1-NEXT:    br i1 [[CMP53]], label [[COND_TRUE54:%.*]], label [[COND_FALSE55:%.*]]
-// CHECK1:       cond.true54:
-// CHECK1-NEXT:    [[TMP45:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK1-NEXT:    br label [[COND_END56:%.*]]
-// CHECK1:       cond.false55:
-// CHECK1-NEXT:    [[TMP46:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    br label [[COND_END56]]
-// CHECK1:       cond.end56:
-// CHECK1-NEXT:    [[COND57:%.*]] = phi i64 [ [[TMP45]], [[COND_TRUE54]] ], [ [[TMP46]], [[COND_FALSE55]] ]
-// CHECK1-NEXT:    store i64 [[COND57]], ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[TMP47:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
-// CHECK1-NEXT:    store i64 [[TMP47]], ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK1-NEXT:    [[CMP39:%.*]] = icmp sgt i64 [[TMP33]], [[TMP34]]
+// CHECK1-NEXT:    br i1 [[CMP39]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i64 [ [[TMP35]], [[COND_TRUE]] ], [ [[TMP36]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    store i64 [[COND]], ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
+// CHECK1-NEXT:    store i64 [[TMP37]], ptr [[DOTOMP_IV]], align 8
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK1:       omp.inner.for.cond:
-// CHECK1-NEXT:    [[TMP48:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[CMP58:%.*]] = icmp sle i64 [[TMP48]], [[TMP49]]
-// CHECK1-NEXT:    br i1 [[CMP58]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK1-NEXT:    [[TMP38:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP39:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[CMP40:%.*]] = icmp sle i64 [[TMP38]], [[TMP39]]
+// CHECK1-NEXT:    br i1 [[CMP40]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
 // CHECK1:       omp.inner.for.body:
-// CHECK1-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[CONV59:%.*]] = sext i32 [[TMP50]] to i64
-// CHECK1-NEXT:    [[TMP51:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB60:%.*]] = sub i32 [[TMP52]], [[TMP53]]
-// CHECK1-NEXT:    [[SUB61:%.*]] = sub i32 [[SUB60]], 1
+// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[CONV41:%.*]] = sext i32 [[TMP40]] to i64
+// CHECK1-NEXT:    [[TMP41:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB42:%.*]] = sub i32 [[TMP42]], [[TMP43]]
+// CHECK1-NEXT:    [[SUB43:%.*]] = sub i32 [[SUB42]], 1
+// CHECK1-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD44:%.*]] = add i32 [[SUB43]], [[TMP44]]
+// CHECK1-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV45:%.*]] = udiv i32 [[ADD44]], [[TMP45]]
+// CHECK1-NEXT:    [[MUL46:%.*]] = mul i32 1, [[DIV45]]
+// CHECK1-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB47:%.*]] = sub i32 [[TMP46]], -31
+// CHECK1-NEXT:    [[DIV48:%.*]] = udiv i32 [[SUB47]], 32
+// CHECK1-NEXT:    [[MUL49:%.*]] = mul i32 [[MUL46]], [[DIV48]]
+// CHECK1-NEXT:    [[MUL50:%.*]] = mul i32 [[MUL49]], 32
+// CHECK1-NEXT:    [[CONV51:%.*]] = zext i32 [[MUL50]] to i64
+// CHECK1-NEXT:    [[DIV52:%.*]] = sdiv i64 [[TMP41]], [[CONV51]]
+// CHECK1-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[CONV53:%.*]] = sext i32 [[TMP47]] to i64
+// CHECK1-NEXT:    [[MUL54:%.*]] = mul nsw i64 [[DIV52]], [[CONV53]]
+// CHECK1-NEXT:    [[ADD55:%.*]] = add nsw i64 [[CONV41]], [[MUL54]]
+// CHECK1-NEXT:    [[CONV56:%.*]] = trunc i64 [[ADD55]] to i32
+// CHECK1-NEXT:    store i32 [[CONV56]], ptr [[I35]], align 4
+// CHECK1-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP48]] to i64
+// CHECK1-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP50:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB58:%.*]] = sub i32 [[TMP51]], [[TMP52]]
+// CHECK1-NEXT:    [[SUB59:%.*]] = sub i32 [[SUB58]], 1
+// CHECK1-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD60:%.*]] = add i32 [[SUB59]], [[TMP53]]
 // CHECK1-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD62:%.*]] = add i32 [[SUB61]], [[TMP54]]
-// CHECK1-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV63:%.*]] = udiv i32 [[ADD62]], [[TMP55]]
-// CHECK1-NEXT:    [[MUL64:%.*]] = mul i32 1, [[DIV63]]
-// CHECK1-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB65:%.*]] = sub i32 [[TMP56]], -31
-// CHECK1-NEXT:    [[DIV66:%.*]] = udiv i32 [[SUB65]], 32
-// CHECK1-NEXT:    [[MUL67:%.*]] = mul i32 [[MUL64]], [[DIV66]]
-// CHECK1-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB68:%.*]] = sub i32 [[TMP57]], [[TMP58]]
-// CHECK1-NEXT:    [[SUB69:%.*]] = sub i32 [[SUB68]], 1
-// CHECK1-NEXT:    [[ADD70:%.*]] = add i32 [[SUB69]], 1
-// CHECK1-NEXT:    [[DIV71:%.*]] = udiv i32 [[ADD70]], 1
-// CHECK1-NEXT:    [[MUL72:%.*]] = mul i32 [[MUL67]], [[DIV71]]
-// CHECK1-NEXT:    [[CONV73:%.*]] = zext i32 [[MUL72]] to i64
-// CHECK1-NEXT:    [[DIV74:%.*]] = sdiv i64 [[TMP51]], [[CONV73]]
-// CHECK1-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[CONV75:%.*]] = sext i32 [[TMP59]] to i64
-// CHECK1-NEXT:    [[MUL76:%.*]] = mul nsw i64 [[DIV74]], [[CONV75]]
-// CHECK1-NEXT:    [[ADD77:%.*]] = add nsw i64 [[CONV59]], [[MUL76]]
-// CHECK1-NEXT:    [[CONV78:%.*]] = trunc i64 [[ADD77]] to i32
-// CHECK1-NEXT:    store i32 [[CONV78]], ptr [[I49]], align 4
-// CHECK1-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[CONV79:%.*]] = sext i32 [[TMP60]] to i64
-// CHECK1-NEXT:    [[TMP61:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP62:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP63:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP64:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB80:%.*]] = sub i32 [[TMP63]], [[TMP64]]
-// CHECK1-NEXT:    [[SUB81:%.*]] = sub i32 [[SUB80]], 1
-// CHECK1-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD82:%.*]] = add i32 [[SUB81]], [[TMP65]]
-// CHECK1-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV83:%.*]] = udiv i32 [[ADD82]], [[TMP66]]
-// CHECK1-NEXT:    [[MUL84:%.*]] = mul i32 1, [[DIV83]]
-// CHECK1-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB85:%.*]] = sub i32 [[TMP67]], -31
-// CHECK1-NEXT:    [[DIV86:%.*]] = udiv i32 [[SUB85]], 32
-// CHECK1-NEXT:    [[MUL87:%.*]] = mul i32 [[MUL84]], [[DIV86]]
-// CHECK1-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB88:%.*]] = sub i32 [[TMP68]], [[TMP69]]
-// CHECK1-NEXT:    [[SUB89:%.*]] = sub i32 [[SUB88]], 1
-// CHECK1-NEXT:    [[ADD90:%.*]] = add i32 [[SUB89]], 1
-// CHECK1-NEXT:    [[DIV91:%.*]] = udiv i32 [[ADD90]], 1
-// CHECK1-NEXT:    [[MUL92:%.*]] = mul i32 [[MUL87]], [[DIV91]]
-// CHECK1-NEXT:    [[CONV93:%.*]] = zext i32 [[MUL92]] to i64
-// CHECK1-NEXT:    [[DIV94:%.*]] = sdiv i64 [[TMP62]], [[CONV93]]
+// CHECK1-NEXT:    [[DIV61:%.*]] = udiv i32 [[ADD60]], [[TMP54]]
+// CHECK1-NEXT:    [[MUL62:%.*]] = mul i32 1, [[DIV61]]
+// CHECK1-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB63:%.*]] = sub i32 [[TMP55]], -31
+// CHECK1-NEXT:    [[DIV64:%.*]] = udiv i32 [[SUB63]], 32
+// CHECK1-NEXT:    [[MUL65:%.*]] = mul i32 [[MUL62]], [[DIV64]]
+// CHECK1-NEXT:    [[MUL66:%.*]] = mul i32 [[MUL65]], 32
+// CHECK1-NEXT:    [[CONV67:%.*]] = zext i32 [[MUL66]] to i64
+// CHECK1-NEXT:    [[DIV68:%.*]] = sdiv i64 [[TMP50]], [[CONV67]]
+// CHECK1-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB69:%.*]] = sub i32 [[TMP56]], [[TMP57]]
+// CHECK1-NEXT:    [[SUB70:%.*]] = sub i32 [[SUB69]], 1
+// CHECK1-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD71:%.*]] = add i32 [[SUB70]], [[TMP58]]
+// CHECK1-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV72:%.*]] = udiv i32 [[ADD71]], [[TMP59]]
+// CHECK1-NEXT:    [[MUL73:%.*]] = mul i32 1, [[DIV72]]
+// CHECK1-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB74:%.*]] = sub i32 [[TMP60]], -31
+// CHECK1-NEXT:    [[DIV75:%.*]] = udiv i32 [[SUB74]], 32
+// CHECK1-NEXT:    [[MUL76:%.*]] = mul i32 [[MUL73]], [[DIV75]]
+// CHECK1-NEXT:    [[MUL77:%.*]] = mul i32 [[MUL76]], 32
+// CHECK1-NEXT:    [[CONV78:%.*]] = zext i32 [[MUL77]] to i64
+// CHECK1-NEXT:    [[MUL79:%.*]] = mul nsw i64 [[DIV68]], [[CONV78]]
+// CHECK1-NEXT:    [[SUB80:%.*]] = sub nsw i64 [[TMP49]], [[MUL79]]
+// CHECK1-NEXT:    [[TMP61:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB81:%.*]] = sub i32 [[TMP61]], -31
+// CHECK1-NEXT:    [[DIV82:%.*]] = udiv i32 [[SUB81]], 32
+// CHECK1-NEXT:    [[MUL83:%.*]] = mul i32 1, [[DIV82]]
+// CHECK1-NEXT:    [[MUL84:%.*]] = mul i32 [[MUL83]], 32
+// CHECK1-NEXT:    [[CONV85:%.*]] = zext i32 [[MUL84]] to i64
+// CHECK1-NEXT:    [[DIV86:%.*]] = sdiv i64 [[SUB80]], [[CONV85]]
+// CHECK1-NEXT:    [[TMP62:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[CONV87:%.*]] = sext i32 [[TMP62]] to i64
+// CHECK1-NEXT:    [[MUL88:%.*]] = mul nsw i64 [[DIV86]], [[CONV87]]
+// CHECK1-NEXT:    [[ADD89:%.*]] = add nsw i64 [[CONV57]], [[MUL88]]
+// CHECK1-NEXT:    [[CONV90:%.*]] = trunc i64 [[ADD89]] to i32
+// CHECK1-NEXT:    store i32 [[CONV90]], ptr [[J36]], align 4
+// CHECK1-NEXT:    [[TMP63:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP64:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB91:%.*]] = sub i32 [[TMP65]], [[TMP66]]
+// CHECK1-NEXT:    [[SUB92:%.*]] = sub i32 [[SUB91]], 1
+// CHECK1-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD93:%.*]] = add i32 [[SUB92]], [[TMP67]]
+// CHECK1-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV94:%.*]] = udiv i32 [[ADD93]], [[TMP68]]
+// CHECK1-NEXT:    [[MUL95:%.*]] = mul i32 1, [[DIV94]]
+// CHECK1-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB96:%.*]] = sub i32 [[TMP69]], -31
+// CHECK1-NEXT:    [[DIV97:%.*]] = udiv i32 [[SUB96]], 32
+// CHECK1-NEXT:    [[MUL98:%.*]] = mul i32 [[MUL95]], [[DIV97]]
+// CHECK1-NEXT:    [[MUL99:%.*]] = mul i32 [[MUL98]], 32
+// CHECK1-NEXT:    [[CONV100:%.*]] = zext i32 [[MUL99]] to i64
+// CHECK1-NEXT:    [[DIV101:%.*]] = sdiv i64 [[TMP64]], [[CONV100]]
 // CHECK1-NEXT:    [[TMP70:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
 // CHECK1-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB95:%.*]] = sub i32 [[TMP70]], [[TMP71]]
-// CHECK1-NEXT:    [[SUB96:%.*]] = sub i32 [[SUB95]], 1
+// CHECK1-NEXT:    [[SUB102:%.*]] = sub i32 [[TMP70]], [[TMP71]]
+// CHECK1-NEXT:    [[SUB103:%.*]] = sub i32 [[SUB102]], 1
 // CHECK1-NEXT:    [[TMP72:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD97:%.*]] = add i32 [[SUB96]], [[TMP72]]
+// CHECK1-NEXT:    [[ADD104:%.*]] = add i32 [[SUB103]], [[TMP72]]
 // CHECK1-NEXT:    [[TMP73:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV98:%.*]] = udiv i32 [[ADD97]], [[TMP73]]
-// CHECK1-NEXT:    [[MUL99:%.*]] = mul i32 1, [[DIV98]]
+// CHECK1-NEXT:    [[DIV105:%.*]] = udiv i32 [[ADD104]], [[TMP73]]
+// CHECK1-NEXT:    [[MUL106:%.*]] = mul i32 1, [[DIV105]]
 // CHECK1-NEXT:    [[TMP74:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB100:%.*]] = sub i32 [[TMP74]], -31
-// CHECK1-NEXT:    [[DIV101:%.*]] = udiv i32 [[SUB100]], 32
-// CHECK1-NEXT:    [[MUL102:%.*]] = mul i32 [[MUL99]], [[DIV101]]
-// CHECK1-NEXT:    [[TMP75:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP76:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB103:%.*]] = sub i32 [[TMP75]], [[TMP76]]
-// CHECK1-NEXT:    [[SUB104:%.*]] = sub i32 [[SUB103]], 1
-// CHECK1-NEXT:    [[ADD105:%.*]] = add i32 [[SUB104]], 1
-// CHECK1-NEXT:    [[DIV106:%.*]] = udiv i32 [[ADD105]], 1
-// CHECK1-NEXT:    [[MUL107:%.*]] = mul i32 [[MUL102]], [[DIV106]]
-// CHECK1-NEXT:    [[CONV108:%.*]] = zext i32 [[MUL107]] to i64
-// CHECK1-NEXT:    [[MUL109:%.*]] = mul nsw i64 [[DIV94]], [[CONV108]]
-// CHECK1-NEXT:    [[SUB110:%.*]] = sub nsw i64 [[TMP61]], [[MUL109]]
-// CHECK1-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB111:%.*]] = sub i32 [[TMP77]], -31
-// CHECK1-NEXT:    [[DIV112:%.*]] = udiv i32 [[SUB111]], 32
-// CHECK1-NEXT:    [[MUL113:%.*]] = mul i32 1, [[DIV112]]
-// CHECK1-NEXT:    [[TMP78:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP79:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB114:%.*]] = sub i32 [[TMP78]], [[TMP79]]
+// CHECK1-NEXT:    [[SUB107:%.*]] = sub i32 [[TMP74]], -31
+// CHECK1-NEXT:    [[DIV108:%.*]] = udiv i32 [[SUB107]], 32
+// CHECK1-NEXT:    [[MUL109:%.*]] = mul i32 [[MUL106]], [[DIV108]]
+// CHECK1-NEXT:    [[MUL110:%.*]] = mul i32 [[MUL109]], 32
+// CHECK1-NEXT:    [[CONV111:%.*]] = zext i32 [[MUL110]] to i64
+// CHECK1-NEXT:    [[MUL112:%.*]] = mul nsw i64 [[DIV101]], [[CONV111]]
+// CHECK1-NEXT:    [[SUB113:%.*]] = sub nsw i64 [[TMP63]], [[MUL112]]
+// CHECK1-NEXT:    [[TMP75:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP76:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP78:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB114:%.*]] = sub i32 [[TMP77]], [[TMP78]]
 // CHECK1-NEXT:    [[SUB115:%.*]] = sub i32 [[SUB114]], 1
-// CHECK1-NEXT:    [[ADD116:%.*]] = add i32 [[SUB115]], 1
-// CHECK1-NEXT:    [[DIV117:%.*]] = udiv i32 [[ADD116]], 1
-// CHECK1-NEXT:    [[MUL118:%.*]] = mul i32 [[MUL113]], [[DIV117]]
-// CHECK1-NEXT:    [[CONV119:%.*]] = zext i32 [[MUL118]] to i64
-// CHECK1-NEXT:    [[DIV120:%.*]] = sdiv i64 [[SUB110]], [[CONV119]]
+// CHECK1-NEXT:    [[TMP79:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD116:%.*]] = add i32 [[SUB115]], [[TMP79]]
 // CHECK1-NEXT:    [[TMP80:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[CONV121:%.*]] = sext i32 [[TMP80]] to i64
-// CHECK1-NEXT:    [[MUL122:%.*]] = mul nsw i64 [[DIV120]], [[CONV121]]
-// CHECK1-NEXT:    [[ADD123:%.*]] = add nsw i64 [[CONV79]], [[MUL122]]
-// CHECK1-NEXT:    [[CONV124:%.*]] = trunc i64 [[ADD123]] to i32
-// CHECK1-NEXT:    store i32 [[CONV124]], ptr [[J50]], align 4
-// CHECK1-NEXT:    [[TMP81:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP82:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB125:%.*]] = sub i32 [[TMP83]], [[TMP84]]
+// CHECK1-NEXT:    [[DIV117:%.*]] = udiv i32 [[ADD116]], [[TMP80]]
+// CHECK1-NEXT:    [[MUL118:%.*]] = mul i32 1, [[DIV117]]
+// CHECK1-NEXT:    [[TMP81:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB119:%.*]] = sub i32 [[TMP81]], -31
+// CHECK1-NEXT:    [[DIV120:%.*]] = udiv i32 [[SUB119]], 32
+// CHECK1-NEXT:    [[MUL121:%.*]] = mul i32 [[MUL118]], [[DIV120]]
+// CHECK1-NEXT:    [[MUL122:%.*]] = mul i32 [[MUL121]], 32
+// CHECK1-NEXT:    [[CONV123:%.*]] = zext i32 [[MUL122]] to i64
+// CHECK1-NEXT:    [[DIV124:%.*]] = sdiv i64 [[TMP76]], [[CONV123]]
+// CHECK1-NEXT:    [[TMP82:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB125:%.*]] = sub i32 [[TMP82]], [[TMP83]]
 // CHECK1-NEXT:    [[SUB126:%.*]] = sub i32 [[SUB125]], 1
+// CHECK1-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD127:%.*]] = add i32 [[SUB126]], [[TMP84]]
 // CHECK1-NEXT:    [[TMP85:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD127:%.*]] = add i32 [[SUB126]], [[TMP85]]
-// CHECK1-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV128:%.*]] = udiv i32 [[ADD127]], [[TMP86]]
+// CHECK1-NEXT:    [[DIV128:%.*]] = udiv i32 [[ADD127]], [[TMP85]]
 // CHECK1-NEXT:    [[MUL129:%.*]] = mul i32 1, [[DIV128]]
-// CHECK1-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB130:%.*]] = sub i32 [[TMP87]], -31
+// CHECK1-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB130:%.*]] = sub i32 [[TMP86]], -31
 // CHECK1-NEXT:    [[DIV131:%.*]] = udiv i32 [[SUB130]], 32
 // CHECK1-NEXT:    [[MUL132:%.*]] = mul i32 [[MUL129]], [[DIV131]]
-// CHECK1-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP89:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB133:%.*]] = sub i32 [[TMP88]], [[TMP89]]
-// CHECK1-NEXT:    [[SUB134:%.*]] = sub i32 [[SUB133]], 1
-// CHECK1-NEXT:    [[ADD135:%.*]] = add i32 [[SUB134]], 1
-// CHECK1-NEXT:    [[DIV136:%.*]] = udiv i32 [[ADD135]], 1
-// CHECK1-NEXT:    [[MUL137:%.*]] = mul i32 [[MUL132]], [[DIV136]]
-// CHECK1-NEXT:    [[CONV138:%.*]] = zext i32 [[MUL137]] to i64
-// CHECK1-NEXT:    [[DIV139:%.*]] = sdiv i64 [[TMP82]], [[CONV138]]
-// CHECK1-NEXT:    [[TMP90:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB140:%.*]] = sub i32 [[TMP90]], [[TMP91]]
-// CHECK1-NEXT:    [[SUB141:%.*]] = sub i32 [[SUB140]], 1
-// CHECK1-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD142:%.*]] = add i32 [[SUB141]], [[TMP92]]
+// CHECK1-NEXT:    [[MUL133:%.*]] = mul i32 [[MUL132]], 32
+// CHECK1-NEXT:    [[CONV134:%.*]] = zext i32 [[MUL133]] to i64
+// CHECK1-NEXT:    [[MUL135:%.*]] = mul nsw i64 [[DIV124]], [[CONV134]]
+// CHECK1-NEXT:    [[SUB136:%.*]] = sub nsw i64 [[TMP75]], [[MUL135]]
+// CHECK1-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB137:%.*]] = sub i32 [[TMP87]], -31
+// CHECK1-NEXT:    [[DIV138:%.*]] = udiv i32 [[SUB137]], 32
+// CHECK1-NEXT:    [[MUL139:%.*]] = mul i32 1, [[DIV138]]
+// CHECK1-NEXT:    [[MUL140:%.*]] = mul i32 [[MUL139]], 32
+// CHECK1-NEXT:    [[CONV141:%.*]] = zext i32 [[MUL140]] to i64
+// CHECK1-NEXT:    [[DIV142:%.*]] = sdiv i64 [[SUB136]], [[CONV141]]
+// CHECK1-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB143:%.*]] = sub i32 [[TMP88]], -31
+// CHECK1-NEXT:    [[DIV144:%.*]] = udiv i32 [[SUB143]], 32
+// CHECK1-NEXT:    [[MUL145:%.*]] = mul i32 1, [[DIV144]]
+// CHECK1-NEXT:    [[MUL146:%.*]] = mul i32 [[MUL145]], 32
+// CHECK1-NEXT:    [[CONV147:%.*]] = zext i32 [[MUL146]] to i64
+// CHECK1-NEXT:    [[MUL148:%.*]] = mul nsw i64 [[DIV142]], [[CONV147]]
+// CHECK1-NEXT:    [[SUB149:%.*]] = sub nsw i64 [[SUB113]], [[MUL148]]
+// CHECK1-NEXT:    [[DIV150:%.*]] = sdiv i64 [[SUB149]], 32
+// CHECK1-NEXT:    [[MUL151:%.*]] = mul nsw i64 [[DIV150]], 32
+// CHECK1-NEXT:    [[ADD152:%.*]] = add nsw i64 0, [[MUL151]]
+// CHECK1-NEXT:    [[CONV153:%.*]] = trunc i64 [[ADD152]] to i32
+// CHECK1-NEXT:    store i32 [[CONV153]], ptr [[DOTFLOOR_0_IV_K37]], align 4
+// CHECK1-NEXT:    [[TMP89:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP90:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB154:%.*]] = sub i32 [[TMP91]], [[TMP92]]
+// CHECK1-NEXT:    [[SUB155:%.*]] = sub i32 [[SUB154]], 1
 // CHECK1-NEXT:    [[TMP93:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV143:%.*]] = udiv i32 [[ADD142]], [[TMP93]]
-// CHECK1-NEXT:    [[MUL144:%.*]] = mul i32 1, [[DIV143]]
-// CHECK1-NEXT:    [[TMP94:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB145:%.*]] = sub i32 [[TMP94]], -31
-// CHECK1-NEXT:    [[DIV146:%.*]] = udiv i32 [[SUB145]], 32
-// CHECK1-NEXT:    [[MUL147:%.*]] = mul i32 [[MUL144]], [[DIV146]]
-// CHECK1-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB148:%.*]] = sub i32 [[TMP95]], [[TMP96]]
-// CHECK1-NEXT:    [[SUB149:%.*]] = sub i32 [[SUB148]], 1
-// CHECK1-NEXT:    [[ADD150:%.*]] = add i32 [[SUB149]], 1
-// CHECK1-NEXT:    [[DIV151:%.*]] = udiv i32 [[ADD150]], 1
-// CHECK1-NEXT:    [[MUL152:%.*]] = mul i32 [[MUL147]], [[DIV151]]
-// CHECK1-NEXT:    [[CONV153:%.*]] = zext i32 [[MUL152]] to i64
-// CHECK1-NEXT:    [[MUL154:%.*]] = mul nsw i64 [[DIV139]], [[CONV153]]
-// CHECK1-NEXT:    [[SUB155:%.*]] = sub nsw i64 [[TMP81]], [[MUL154]]
-// CHECK1-NEXT:    [[TMP97:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP98:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB156:%.*]] = sub i32 [[TMP99]], [[TMP100]]
-// CHECK1-NEXT:    [[SUB157:%.*]] = sub i32 [[SUB156]], 1
-// CHECK1-NEXT:    [[TMP101:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD158:%.*]] = add i32 [[SUB157]], [[TMP101]]
-// CHECK1-NEXT:    [[TMP102:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV159:%.*]] = udiv i32 [[ADD158]], [[TMP102]]
-// CHECK1-NEXT:    [[MUL160:%.*]] = mul i32 1, [[DIV159]]
-// CHECK1-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB161:%.*]] = sub i32 [[TMP103]], -31
-// CHECK1-NEXT:    [[DIV162:%.*]] = udiv i32 [[SUB161]], 32
-// CHECK1-NEXT:    [[MUL163:%.*]] = mul i32 [[MUL160]], [[DIV162]]
-// CHECK1-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP105:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB164:%.*]] = sub i32 [[TMP104]], [[TMP105]]
-// CHECK1-NEXT:    [[SUB165:%.*]] = sub i32 [[SUB164]], 1
-// CHECK1-NEXT:    [[ADD166:%.*]] = add i32 [[SUB165]], 1
-// CHECK1-NEXT:    [[DIV167:%.*]] = udiv i32 [[ADD166]], 1
-// CHECK1-NEXT:    [[MUL168:%.*]] = mul i32 [[MUL163]], [[DIV167]]
-// CHECK1-NEXT:    [[CONV169:%.*]] = zext i32 [[MUL168]] to i64
-// CHECK1-NEXT:    [[DIV170:%.*]] = sdiv i64 [[TMP98]], [[CONV169]]
-// CHECK1-NEXT:    [[TMP106:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB171:%.*]] = sub i32 [[TMP106]], [[TMP107]]
-// CHECK1-NEXT:    [[SUB172:%.*]] = sub i32 [[SUB171]], 1
-// CHECK1-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD173:%.*]] = add i32 [[SUB172]], [[TMP108]]
-// CHECK1-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV174:%.*]] = udiv i32 [[ADD173]], [[TMP109]]
-// CHECK1-NEXT:    [[MUL175:%.*]] = mul i32 1, [[DIV174]]
-// CHECK1-NEXT:    [[TMP110:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB176:%.*]] = sub i32 [[TMP110]], -31
-// CHECK1-NEXT:    [[DIV177:%.*]] = udiv i32 [[SUB176]], 32
-// CHECK1-NEXT:    [[MUL178:%.*]] = mul i32 [[MUL175]], [[DIV177]]
-// CHECK1-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB179:%.*]] = sub i32 [[TMP111]], [[TMP112]]
-// CHECK1-NEXT:    [[SUB180:%.*]] = sub i32 [[SUB179]], 1
-// CHECK1-NEXT:    [[ADD181:%.*]] = add i32 [[SUB180]], 1
-// CHECK1-NEXT:    [[DIV182:%.*]] = udiv i32 [[ADD181]], 1
-// CHECK1-NEXT:    [[MUL183:%.*]] = mul i32 [[MUL178]], [[DIV182]]
-// CHECK1-NEXT:    [[CONV184:%.*]] = zext i32 [[MUL183]] to i64
-// CHECK1-NEXT:    [[MUL185:%.*]] = mul nsw i64 [[DIV170]], [[CONV184]]
-// CHECK1-NEXT:    [[SUB186:%.*]] = sub nsw i64 [[TMP97]], [[MUL185]]
+// CHECK1-NEXT:    [[ADD156:%.*]] = add i32 [[SUB155]], [[TMP93]]
+// CHECK1-NEXT:    [[TMP94:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV157:%.*]] = udiv i32 [[ADD156]], [[TMP94]]
+// CHECK1-NEXT:    [[MUL158:%.*]] = mul i32 1, [[DIV157]]
+// CHECK1-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB159:%.*]] = sub i32 [[TMP95]], -31
+// CHECK1-NEXT:    [[DIV160:%.*]] = udiv i32 [[SUB159]], 32
+// CHECK1-NEXT:    [[MUL161:%.*]] = mul i32 [[MUL158]], [[DIV160]]
+// CHECK1-NEXT:    [[MUL162:%.*]] = mul i32 [[MUL161]], 32
+// CHECK1-NEXT:    [[CONV163:%.*]] = zext i32 [[MUL162]] to i64
+// CHECK1-NEXT:    [[DIV164:%.*]] = sdiv i64 [[TMP90]], [[CONV163]]
+// CHECK1-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP97:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB165:%.*]] = sub i32 [[TMP96]], [[TMP97]]
+// CHECK1-NEXT:    [[SUB166:%.*]] = sub i32 [[SUB165]], 1
+// CHECK1-NEXT:    [[TMP98:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD167:%.*]] = add i32 [[SUB166]], [[TMP98]]
+// CHECK1-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV168:%.*]] = udiv i32 [[ADD167]], [[TMP99]]
+// CHECK1-NEXT:    [[MUL169:%.*]] = mul i32 1, [[DIV168]]
+// CHECK1-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB170:%.*]] = sub i32 [[TMP100]], -31
+// CHECK1-NEXT:    [[DIV171:%.*]] = udiv i32 [[SUB170]], 32
+// CHECK1-NEXT:    [[MUL172:%.*]] = mul i32 [[MUL169]], [[DIV171]]
+// CHECK1-NEXT:    [[MUL173:%.*]] = mul i32 [[MUL172]], 32
+// CHECK1-NEXT:    [[CONV174:%.*]] = zext i32 [[MUL173]] to i64
+// CHECK1-NEXT:    [[MUL175:%.*]] = mul nsw i64 [[DIV164]], [[CONV174]]
+// CHECK1-NEXT:    [[SUB176:%.*]] = sub nsw i64 [[TMP89]], [[MUL175]]
+// CHECK1-NEXT:    [[TMP101:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP102:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB177:%.*]] = sub i32 [[TMP103]], [[TMP104]]
+// CHECK1-NEXT:    [[SUB178:%.*]] = sub i32 [[SUB177]], 1
+// CHECK1-NEXT:    [[TMP105:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD179:%.*]] = add i32 [[SUB178]], [[TMP105]]
+// CHECK1-NEXT:    [[TMP106:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV180:%.*]] = udiv i32 [[ADD179]], [[TMP106]]
+// CHECK1-NEXT:    [[MUL181:%.*]] = mul i32 1, [[DIV180]]
+// CHECK1-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB182:%.*]] = sub i32 [[TMP107]], -31
+// CHECK1-NEXT:    [[DIV183:%.*]] = udiv i32 [[SUB182]], 32
+// CHECK1-NEXT:    [[MUL184:%.*]] = mul i32 [[MUL181]], [[DIV183]]
+// CHECK1-NEXT:    [[MUL185:%.*]] = mul i32 [[MUL184]], 32
+// CHECK1-NEXT:    [[CONV186:%.*]] = zext i32 [[MUL185]] to i64
+// CHECK1-NEXT:    [[DIV187:%.*]] = sdiv i64 [[TMP102]], [[CONV186]]
+// CHECK1-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB188:%.*]] = sub i32 [[TMP108]], [[TMP109]]
+// CHECK1-NEXT:    [[SUB189:%.*]] = sub i32 [[SUB188]], 1
+// CHECK1-NEXT:    [[TMP110:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD190:%.*]] = add i32 [[SUB189]], [[TMP110]]
+// CHECK1-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV191:%.*]] = udiv i32 [[ADD190]], [[TMP111]]
+// CHECK1-NEXT:    [[MUL192:%.*]] = mul i32 1, [[DIV191]]
+// CHECK1-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB193:%.*]] = sub i32 [[TMP112]], -31
+// CHECK1-NEXT:    [[DIV194:%.*]] = udiv i32 [[SUB193]], 32
+// CHECK1-NEXT:    [[MUL195:%.*]] = mul i32 [[MUL192]], [[DIV194]]
+// CHECK1-NEXT:    [[MUL196:%.*]] = mul i32 [[MUL195]], 32
+// CHECK1-NEXT:    [[CONV197:%.*]] = zext i32 [[MUL196]] to i64
+// CHECK1-NEXT:    [[MUL198:%.*]] = mul nsw i64 [[DIV187]], [[CONV197]]
+// CHECK1-NEXT:    [[SUB199:%.*]] = sub nsw i64 [[TMP101]], [[MUL198]]
 // CHECK1-NEXT:    [[TMP113:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB187:%.*]] = sub i32 [[TMP113]], -31
-// CHECK1-NEXT:    [[DIV188:%.*]] = udiv i32 [[SUB187]], 32
-// CHECK1-NEXT:    [[MUL189:%.*]] = mul i32 1, [[DIV188]]
-// CHECK1-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP115:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB190:%.*]] = sub i32 [[TMP114]], [[TMP115]]
-// CHECK1-NEXT:    [[SUB191:%.*]] = sub i32 [[SUB190]], 1
-// CHECK1-NEXT:    [[ADD192:%.*]] = add i32 [[SUB191]], 1
-// CHECK1-NEXT:    [[DIV193:%.*]] = udiv i32 [[ADD192]], 1
-// CHECK1-NEXT:    [[MUL194:%.*]] = mul i32 [[MUL189]], [[DIV193]]
-// CHECK1-NEXT:    [[CONV195:%.*]] = zext i32 [[MUL194]] to i64
-// CHECK1-NEXT:    [[DIV196:%.*]] = sdiv i64 [[SUB186]], [[CONV195]]
-// CHECK1-NEXT:    [[TMP116:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB197:%.*]] = sub i32 [[TMP116]], -31
-// CHECK1-NEXT:    [[DIV198:%.*]] = udiv i32 [[SUB197]], 32
-// CHECK1-NEXT:    [[MUL199:%.*]] = mul i32 1, [[DIV198]]
-// CHECK1-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB200:%.*]] = sub i32 [[TMP117]], [[TMP118]]
-// CHECK1-NEXT:    [[SUB201:%.*]] = sub i32 [[SUB200]], 1
-// CHECK1-NEXT:    [[ADD202:%.*]] = add i32 [[SUB201]], 1
-// CHECK1-NEXT:    [[DIV203:%.*]] = udiv i32 [[ADD202]], 1
-// CHECK1-NEXT:    [[MUL204:%.*]] = mul i32 [[MUL199]], [[DIV203]]
-// CHECK1-NEXT:    [[CONV205:%.*]] = zext i32 [[MUL204]] to i64
-// CHECK1-NEXT:    [[MUL206:%.*]] = mul nsw i64 [[DIV196]], [[CONV205]]
-// CHECK1-NEXT:    [[SUB207:%.*]] = sub nsw i64 [[SUB155]], [[MUL206]]
-// CHECK1-NEXT:    [[TMP119:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP120:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB208:%.*]] = sub i32 [[TMP119]], [[TMP120]]
-// CHECK1-NEXT:    [[SUB209:%.*]] = sub i32 [[SUB208]], 1
-// CHECK1-NEXT:    [[ADD210:%.*]] = add i32 [[SUB209]], 1
-// CHECK1-NEXT:    [[DIV211:%.*]] = udiv i32 [[ADD210]], 1
-// CHECK1-NEXT:    [[MUL212:%.*]] = mul i32 1, [[DIV211]]
-// CHECK1-NEXT:    [[CONV213:%.*]] = zext i32 [[MUL212]] to i64
-// CHECK1-NEXT:    [[DIV214:%.*]] = sdiv i64 [[SUB207]], [[CONV213]]
-// CHECK1-NEXT:    [[MUL215:%.*]] = mul nsw i64 [[DIV214]], 32
-// CHECK1-NEXT:    [[ADD216:%.*]] = add nsw i64 0, [[MUL215]]
-// CHECK1-NEXT:    [[CONV217:%.*]] = trunc i64 [[ADD216]] to i32
-// CHECK1-NEXT:    store i32 [[CONV217]], ptr [[DOTFLOOR_0_IV_K51]], align 4
-// CHECK1-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[CONV218:%.*]] = zext i32 [[TMP121]] to i64
-// CHECK1-NEXT:    [[TMP122:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP123:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP124:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB219:%.*]] = sub i32 [[TMP124]], [[TMP125]]
-// CHECK1-NEXT:    [[SUB220:%.*]] = sub i32 [[SUB219]], 1
-// CHECK1-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD221:%.*]] = add i32 [[SUB220]], [[TMP126]]
-// CHECK1-NEXT:    [[TMP127:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV222:%.*]] = udiv i32 [[ADD221]], [[TMP127]]
-// CHECK1-NEXT:    [[MUL223:%.*]] = mul i32 1, [[DIV222]]
-// CHECK1-NEXT:    [[TMP128:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB224:%.*]] = sub i32 [[TMP128]], -31
-// CHECK1-NEXT:    [[DIV225:%.*]] = udiv i32 [[SUB224]], 32
-// CHECK1-NEXT:    [[MUL226:%.*]] = mul i32 [[MUL223]], [[DIV225]]
-// CHECK1-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB227:%.*]] = sub i32 [[TMP129]], [[TMP130]]
-// CHECK1-NEXT:    [[SUB228:%.*]] = sub i32 [[SUB227]], 1
-// CHECK1-NEXT:    [[ADD229:%.*]] = add i32 [[SUB228]], 1
-// CHECK1-NEXT:    [[DIV230:%.*]] = udiv i32 [[ADD229]], 1
-// CHECK1-NEXT:    [[MUL231:%.*]] = mul i32 [[MUL226]], [[DIV230]]
-// CHECK1-NEXT:    [[CONV232:%.*]] = zext i32 [[MUL231]] to i64
-// CHECK1-NEXT:    [[DIV233:%.*]] = sdiv i64 [[TMP123]], [[CONV232]]
-// CHECK1-NEXT:    [[TMP131:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP132:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB234:%.*]] = sub i32 [[TMP131]], [[TMP132]]
-// CHECK1-NEXT:    [[SUB235:%.*]] = sub i32 [[SUB234]], 1
-// CHECK1-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD236:%.*]] = add i32 [[SUB235]], [[TMP133]]
-// CHECK1-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV237:%.*]] = udiv i32 [[ADD236]], [[TMP134]]
-// CHECK1-NEXT:    [[MUL238:%.*]] = mul i32 1, [[DIV237]]
-// CHECK1-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB239:%.*]] = sub i32 [[TMP135]], -31
-// CHECK1-NEXT:    [[DIV240:%.*]] = udiv i32 [[SUB239]], 32
-// CHECK1-NEXT:    [[MUL241:%.*]] = mul i32 [[MUL238]], [[DIV240]]
-// CHECK1-NEXT:    [[TMP136:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB242:%.*]] = sub i32 [[TMP136]], [[TMP137]]
-// CHECK1-NEXT:    [[SUB243:%.*]] = sub i32 [[SUB242]], 1
-// CHECK1-NEXT:    [[ADD244:%.*]] = add i32 [[SUB243]], 1
-// CHECK1-NEXT:    [[DIV245:%.*]] = udiv i32 [[ADD244]], 1
-// CHECK1-NEXT:    [[MUL246:%.*]] = mul i32 [[MUL241]], [[DIV245]]
-// CHECK1-NEXT:    [[CONV247:%.*]] = zext i32 [[MUL246]] to i64
-// CHECK1-NEXT:    [[MUL248:%.*]] = mul nsw i64 [[DIV233]], [[CONV247]]
-// CHECK1-NEXT:    [[SUB249:%.*]] = sub nsw i64 [[TMP122]], [[MUL248]]
-// CHECK1-NEXT:    [[TMP138:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP139:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB250:%.*]] = sub i32 [[TMP140]], [[TMP141]]
-// CHECK1-NEXT:    [[SUB251:%.*]] = sub i32 [[SUB250]], 1
-// CHECK1-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD252:%.*]] = add i32 [[SUB251]], [[TMP142]]
-// CHECK1-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV253:%.*]] = udiv i32 [[ADD252]], [[TMP143]]
-// CHECK1-NEXT:    [[MUL254:%.*]] = mul i32 1, [[DIV253]]
-// CHECK1-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB255:%.*]] = sub i32 [[TMP144]], -31
-// CHECK1-NEXT:    [[DIV256:%.*]] = udiv i32 [[SUB255]], 32
-// CHECK1-NEXT:    [[MUL257:%.*]] = mul i32 [[MUL254]], [[DIV256]]
-// CHECK1-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB258:%.*]] = sub i32 [[TMP145]], [[TMP146]]
-// CHECK1-NEXT:    [[SUB259:%.*]] = sub i32 [[SUB258]], 1
-// CHECK1-NEXT:    [[ADD260:%.*]] = add i32 [[SUB259]], 1
-// CHECK1-NEXT:    [[DIV261:%.*]] = udiv i32 [[ADD260]], 1
-// CHECK1-NEXT:    [[MUL262:%.*]] = mul i32 [[MUL257]], [[DIV261]]
+// CHECK1-NEXT:    [[SUB200:%.*]] = sub i32 [[TMP113]], -31
+// CHECK1-NEXT:    [[DIV201:%.*]] = udiv i32 [[SUB200]], 32
+// CHECK1-NEXT:    [[MUL202:%.*]] = mul i32 1, [[DIV201]]
+// CHECK1-NEXT:    [[MUL203:%.*]] = mul i32 [[MUL202]], 32
+// CHECK1-NEXT:    [[CONV204:%.*]] = zext i32 [[MUL203]] to i64
+// CHECK1-NEXT:    [[DIV205:%.*]] = sdiv i64 [[SUB199]], [[CONV204]]
+// CHECK1-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB206:%.*]] = sub i32 [[TMP114]], -31
+// CHECK1-NEXT:    [[DIV207:%.*]] = udiv i32 [[SUB206]], 32
+// CHECK1-NEXT:    [[MUL208:%.*]] = mul i32 1, [[DIV207]]
+// CHECK1-NEXT:    [[MUL209:%.*]] = mul i32 [[MUL208]], 32
+// CHECK1-NEXT:    [[CONV210:%.*]] = zext i32 [[MUL209]] to i64
+// CHECK1-NEXT:    [[MUL211:%.*]] = mul nsw i64 [[DIV205]], [[CONV210]]
+// CHECK1-NEXT:    [[SUB212:%.*]] = sub nsw i64 [[SUB176]], [[MUL211]]
+// CHECK1-NEXT:    [[TMP115:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP116:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB213:%.*]] = sub i32 [[TMP117]], [[TMP118]]
+// CHECK1-NEXT:    [[SUB214:%.*]] = sub i32 [[SUB213]], 1
+// CHECK1-NEXT:    [[TMP119:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD215:%.*]] = add i32 [[SUB214]], [[TMP119]]
+// CHECK1-NEXT:    [[TMP120:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV216:%.*]] = udiv i32 [[ADD215]], [[TMP120]]
+// CHECK1-NEXT:    [[MUL217:%.*]] = mul i32 1, [[DIV216]]
+// CHECK1-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB218:%.*]] = sub i32 [[TMP121]], -31
+// CHECK1-NEXT:    [[DIV219:%.*]] = udiv i32 [[SUB218]], 32
+// CHECK1-NEXT:    [[MUL220:%.*]] = mul i32 [[MUL217]], [[DIV219]]
+// CHECK1-NEXT:    [[MUL221:%.*]] = mul i32 [[MUL220]], 32
+// CHECK1-NEXT:    [[CONV222:%.*]] = zext i32 [[MUL221]] to i64
+// CHECK1-NEXT:    [[DIV223:%.*]] = sdiv i64 [[TMP116]], [[CONV222]]
+// CHECK1-NEXT:    [[TMP122:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP123:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB224:%.*]] = sub i32 [[TMP122]], [[TMP123]]
+// CHECK1-NEXT:    [[SUB225:%.*]] = sub i32 [[SUB224]], 1
+// CHECK1-NEXT:    [[TMP124:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD226:%.*]] = add i32 [[SUB225]], [[TMP124]]
+// CHECK1-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV227:%.*]] = udiv i32 [[ADD226]], [[TMP125]]
+// CHECK1-NEXT:    [[MUL228:%.*]] = mul i32 1, [[DIV227]]
+// CHECK1-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB229:%.*]] = sub i32 [[TMP126]], -31
+// CHECK1-NEXT:    [[DIV230:%.*]] = udiv i32 [[SUB229]], 32
+// CHECK1-NEXT:    [[MUL231:%.*]] = mul i32 [[MUL228]], [[DIV230]]
+// CHECK1-NEXT:    [[MUL232:%.*]] = mul i32 [[MUL231]], 32
+// CHECK1-NEXT:    [[CONV233:%.*]] = zext i32 [[MUL232]] to i64
+// CHECK1-NEXT:    [[MUL234:%.*]] = mul nsw i64 [[DIV223]], [[CONV233]]
+// CHECK1-NEXT:    [[SUB235:%.*]] = sub nsw i64 [[TMP115]], [[MUL234]]
+// CHECK1-NEXT:    [[TMP127:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP128:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB236:%.*]] = sub i32 [[TMP129]], [[TMP130]]
+// CHECK1-NEXT:    [[SUB237:%.*]] = sub i32 [[SUB236]], 1
+// CHECK1-NEXT:    [[TMP131:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD238:%.*]] = add i32 [[SUB237]], [[TMP131]]
+// CHECK1-NEXT:    [[TMP132:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV239:%.*]] = udiv i32 [[ADD238]], [[TMP132]]
+// CHECK1-NEXT:    [[MUL240:%.*]] = mul i32 1, [[DIV239]]
+// CHECK1-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB241:%.*]] = sub i32 [[TMP133]], -31
+// CHECK1-NEXT:    [[DIV242:%.*]] = udiv i32 [[SUB241]], 32
+// CHECK1-NEXT:    [[MUL243:%.*]] = mul i32 [[MUL240]], [[DIV242]]
+// CHECK1-NEXT:    [[MUL244:%.*]] = mul i32 [[MUL243]], 32
+// CHECK1-NEXT:    [[CONV245:%.*]] = zext i32 [[MUL244]] to i64
+// CHECK1-NEXT:    [[DIV246:%.*]] = sdiv i64 [[TMP128]], [[CONV245]]
+// CHECK1-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB247:%.*]] = sub i32 [[TMP134]], [[TMP135]]
+// CHECK1-NEXT:    [[SUB248:%.*]] = sub i32 [[SUB247]], 1
+// CHECK1-NEXT:    [[TMP136:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD249:%.*]] = add i32 [[SUB248]], [[TMP136]]
+// CHECK1-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV250:%.*]] = udiv i32 [[ADD249]], [[TMP137]]
+// CHECK1-NEXT:    [[MUL251:%.*]] = mul i32 1, [[DIV250]]
+// CHECK1-NEXT:    [[TMP138:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB252:%.*]] = sub i32 [[TMP138]], -31
+// CHECK1-NEXT:    [[DIV253:%.*]] = udiv i32 [[SUB252]], 32
+// CHECK1-NEXT:    [[MUL254:%.*]] = mul i32 [[MUL251]], [[DIV253]]
+// CHECK1-NEXT:    [[MUL255:%.*]] = mul i32 [[MUL254]], 32
+// CHECK1-NEXT:    [[CONV256:%.*]] = zext i32 [[MUL255]] to i64
+// CHECK1-NEXT:    [[MUL257:%.*]] = mul nsw i64 [[DIV246]], [[CONV256]]
+// CHECK1-NEXT:    [[SUB258:%.*]] = sub nsw i64 [[TMP127]], [[MUL257]]
+// CHECK1-NEXT:    [[TMP139:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB259:%.*]] = sub i32 [[TMP139]], -31
+// CHECK1-NEXT:    [[DIV260:%.*]] = udiv i32 [[SUB259]], 32
+// CHECK1-NEXT:    [[MUL261:%.*]] = mul i32 1, [[DIV260]]
+// CHECK1-NEXT:    [[MUL262:%.*]] = mul i32 [[MUL261]], 32
 // CHECK1-NEXT:    [[CONV263:%.*]] = zext i32 [[MUL262]] to i64
-// CHECK1-NEXT:    [[DIV264:%.*]] = sdiv i64 [[TMP139]], [[CONV263]]
-// CHECK1-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB265:%.*]] = sub i32 [[TMP147]], [[TMP148]]
-// CHECK1-NEXT:    [[SUB266:%.*]] = sub i32 [[SUB265]], 1
-// CHECK1-NEXT:    [[TMP149:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD267:%.*]] = add i32 [[SUB266]], [[TMP149]]
-// CHECK1-NEXT:    [[TMP150:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV268:%.*]] = udiv i32 [[ADD267]], [[TMP150]]
-// CHECK1-NEXT:    [[MUL269:%.*]] = mul i32 1, [[DIV268]]
-// CHECK1-NEXT:    [[TMP151:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB270:%.*]] = sub i32 [[TMP151]], -31
-// CHECK1-NEXT:    [[DIV271:%.*]] = udiv i32 [[SUB270]], 32
-// CHECK1-NEXT:    [[MUL272:%.*]] = mul i32 [[MUL269]], [[DIV271]]
-// CHECK1-NEXT:    [[TMP152:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP153:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB273:%.*]] = sub i32 [[TMP152]], [[TMP153]]
-// CHECK1-NEXT:    [[SUB274:%.*]] = sub i32 [[SUB273]], 1
-// CHECK1-NEXT:    [[ADD275:%.*]] = add i32 [[SUB274]], 1
-// CHECK1-NEXT:    [[DIV276:%.*]] = udiv i32 [[ADD275]], 1
-// CHECK1-NEXT:    [[MUL277:%.*]] = mul i32 [[MUL272]], [[DIV276]]
-// CHECK1-NEXT:    [[CONV278:%.*]] = zext i32 [[MUL277]] to i64
-// CHECK1-NEXT:    [[MUL279:%.*]] = mul nsw i64 [[DIV264]], [[CONV278]]
-// CHECK1-NEXT:    [[SUB280:%.*]] = sub nsw i64 [[TMP138]], [[MUL279]]
-// CHECK1-NEXT:    [[TMP154:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB281:%.*]] = sub i32 [[TMP154]], -31
-// CHECK1-NEXT:    [[DIV282:%.*]] = udiv i32 [[SUB281]], 32
-// CHECK1-NEXT:    [[MUL283:%.*]] = mul i32 1, [[DIV282]]
-// CHECK1-NEXT:    [[TMP155:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP156:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB284:%.*]] = sub i32 [[TMP155]], [[TMP156]]
-// CHECK1-NEXT:    [[SUB285:%.*]] = sub i32 [[SUB284]], 1
-// CHECK1-NEXT:    [[ADD286:%.*]] = add i32 [[SUB285]], 1
-// CHECK1-NEXT:    [[DIV287:%.*]] = udiv i32 [[ADD286]], 1
-// CHECK1-NEXT:    [[MUL288:%.*]] = mul i32 [[MUL283]], [[DIV287]]
-// CHECK1-NEXT:    [[CONV289:%.*]] = zext i32 [[MUL288]] to i64
-// CHECK1-NEXT:    [[DIV290:%.*]] = sdiv i64 [[SUB280]], [[CONV289]]
-// CHECK1-NEXT:    [[TMP157:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB291:%.*]] = sub i32 [[TMP157]], -31
-// CHECK1-NEXT:    [[DIV292:%.*]] = udiv i32 [[SUB291]], 32
-// CHECK1-NEXT:    [[MUL293:%.*]] = mul i32 1, [[DIV292]]
-// CHECK1-NEXT:    [[TMP158:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP159:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB294:%.*]] = sub i32 [[TMP158]], [[TMP159]]
-// CHECK1-NEXT:    [[SUB295:%.*]] = sub i32 [[SUB294]], 1
-// CHECK1-NEXT:    [[ADD296:%.*]] = add i32 [[SUB295]], 1
-// CHECK1-NEXT:    [[DIV297:%.*]] = udiv i32 [[ADD296]], 1
-// CHECK1-NEXT:    [[MUL298:%.*]] = mul i32 [[MUL293]], [[DIV297]]
-// CHECK1-NEXT:    [[CONV299:%.*]] = zext i32 [[MUL298]] to i64
-// CHECK1-NEXT:    [[MUL300:%.*]] = mul nsw i64 [[DIV290]], [[CONV299]]
-// CHECK1-NEXT:    [[SUB301:%.*]] = sub nsw i64 [[SUB249]], [[MUL300]]
-// CHECK1-NEXT:    [[TMP160:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP161:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP162:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP163:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB302:%.*]] = sub i32 [[TMP162]], [[TMP163]]
-// CHECK1-NEXT:    [[SUB303:%.*]] = sub i32 [[SUB302]], 1
-// CHECK1-NEXT:    [[TMP164:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD304:%.*]] = add i32 [[SUB303]], [[TMP164]]
-// CHECK1-NEXT:    [[TMP165:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV305:%.*]] = udiv i32 [[ADD304]], [[TMP165]]
-// CHECK1-NEXT:    [[MUL306:%.*]] = mul i32 1, [[DIV305]]
-// CHECK1-NEXT:    [[TMP166:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB307:%.*]] = sub i32 [[TMP166]], -31
-// CHECK1-NEXT:    [[DIV308:%.*]] = udiv i32 [[SUB307]], 32
-// CHECK1-NEXT:    [[MUL309:%.*]] = mul i32 [[MUL306]], [[DIV308]]
-// CHECK1-NEXT:    [[TMP167:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP168:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB310:%.*]] = sub i32 [[TMP167]], [[TMP168]]
-// CHECK1-NEXT:    [[SUB311:%.*]] = sub i32 [[SUB310]], 1
-// CHECK1-NEXT:    [[ADD312:%.*]] = add i32 [[SUB311]], 1
-// CHECK1-NEXT:    [[DIV313:%.*]] = udiv i32 [[ADD312]], 1
-// CHECK1-NEXT:    [[MUL314:%.*]] = mul i32 [[MUL309]], [[DIV313]]
-// CHECK1-NEXT:    [[CONV315:%.*]] = zext i32 [[MUL314]] to i64
-// CHECK1-NEXT:    [[DIV316:%.*]] = sdiv i64 [[TMP161]], [[CONV315]]
-// CHECK1-NEXT:    [[TMP169:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP170:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB317:%.*]] = sub i32 [[TMP169]], [[TMP170]]
-// CHECK1-NEXT:    [[SUB318:%.*]] = sub i32 [[SUB317]], 1
-// CHECK1-NEXT:    [[TMP171:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD319:%.*]] = add i32 [[SUB318]], [[TMP171]]
-// CHECK1-NEXT:    [[TMP172:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV320:%.*]] = udiv i32 [[ADD319]], [[TMP172]]
-// CHECK1-NEXT:    [[MUL321:%.*]] = mul i32 1, [[DIV320]]
-// CHECK1-NEXT:    [[TMP173:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB322:%.*]] = sub i32 [[TMP173]], -31
-// CHECK1-NEXT:    [[DIV323:%.*]] = udiv i32 [[SUB322]], 32
-// CHECK1-NEXT:    [[MUL324:%.*]] = mul i32 [[MUL321]], [[DIV323]]
-// CHECK1-NEXT:    [[TMP174:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP175:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB325:%.*]] = sub i32 [[TMP174]], [[TMP175]]
-// CHECK1-NEXT:    [[SUB326:%.*]] = sub i32 [[SUB325]], 1
-// CHECK1-NEXT:    [[ADD327:%.*]] = add i32 [[SUB326]], 1
-// CHECK1-NEXT:    [[DIV328:%.*]] = udiv i32 [[ADD327]], 1
-// CHECK1-NEXT:    [[MUL329:%.*]] = mul i32 [[MUL324]], [[DIV328]]
-// CHECK1-NEXT:    [[CONV330:%.*]] = zext i32 [[MUL329]] to i64
-// CHECK1-NEXT:    [[MUL331:%.*]] = mul nsw i64 [[DIV316]], [[CONV330]]
-// CHECK1-NEXT:    [[SUB332:%.*]] = sub nsw i64 [[TMP160]], [[MUL331]]
-// CHECK1-NEXT:    [[TMP176:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP177:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP178:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP179:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB333:%.*]] = sub i32 [[TMP178]], [[TMP179]]
-// CHECK1-NEXT:    [[SUB334:%.*]] = sub i32 [[SUB333]], 1
-// CHECK1-NEXT:    [[TMP180:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD335:%.*]] = add i32 [[SUB334]], [[TMP180]]
-// CHECK1-NEXT:    [[TMP181:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV336:%.*]] = udiv i32 [[ADD335]], [[TMP181]]
-// CHECK1-NEXT:    [[MUL337:%.*]] = mul i32 1, [[DIV336]]
-// CHECK1-NEXT:    [[TMP182:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB338:%.*]] = sub i32 [[TMP182]], -31
-// CHECK1-NEXT:    [[DIV339:%.*]] = udiv i32 [[SUB338]], 32
-// CHECK1-NEXT:    [[MUL340:%.*]] = mul i32 [[MUL337]], [[DIV339]]
-// CHECK1-NEXT:    [[TMP183:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP184:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB341:%.*]] = sub i32 [[TMP183]], [[TMP184]]
-// CHECK1-NEXT:    [[SUB342:%.*]] = sub i32 [[SUB341]], 1
-// CHECK1-NEXT:    [[ADD343:%.*]] = add i32 [[SUB342]], 1
-// CHECK1-NEXT:    [[DIV344:%.*]] = udiv i32 [[ADD343]], 1
-// CHECK1-NEXT:    [[MUL345:%.*]] = mul i32 [[MUL340]], [[DIV344]]
-// CHECK1-NEXT:    [[CONV346:%.*]] = zext i32 [[MUL345]] to i64
-// CHECK1-NEXT:    [[DIV347:%.*]] = sdiv i64 [[TMP177]], [[CONV346]]
-// CHECK1-NEXT:    [[TMP185:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP186:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB348:%.*]] = sub i32 [[TMP185]], [[TMP186]]
-// CHECK1-NEXT:    [[SUB349:%.*]] = sub i32 [[SUB348]], 1
-// CHECK1-NEXT:    [[TMP187:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD350:%.*]] = add i32 [[SUB349]], [[TMP187]]
-// CHECK1-NEXT:    [[TMP188:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV351:%.*]] = udiv i32 [[ADD350]], [[TMP188]]
-// CHECK1-NEXT:    [[MUL352:%.*]] = mul i32 1, [[DIV351]]
-// CHECK1-NEXT:    [[TMP189:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB353:%.*]] = sub i32 [[TMP189]], -31
-// CHECK1-NEXT:    [[DIV354:%.*]] = udiv i32 [[SUB353]], 32
-// CHECK1-NEXT:    [[MUL355:%.*]] = mul i32 [[MUL352]], [[DIV354]]
-// CHECK1-NEXT:    [[TMP190:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP191:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB356:%.*]] = sub i32 [[TMP190]], [[TMP191]]
-// CHECK1-NEXT:    [[SUB357:%.*]] = sub i32 [[SUB356]], 1
-// CHECK1-NEXT:    [[ADD358:%.*]] = add i32 [[SUB357]], 1
-// CHECK1-NEXT:    [[DIV359:%.*]] = udiv i32 [[ADD358]], 1
-// CHECK1-NEXT:    [[MUL360:%.*]] = mul i32 [[MUL355]], [[DIV359]]
-// CHECK1-NEXT:    [[CONV361:%.*]] = zext i32 [[MUL360]] to i64
-// CHECK1-NEXT:    [[MUL362:%.*]] = mul nsw i64 [[DIV347]], [[CONV361]]
-// CHECK1-NEXT:    [[SUB363:%.*]] = sub nsw i64 [[TMP176]], [[MUL362]]
-// CHECK1-NEXT:    [[TMP192:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB364:%.*]] = sub i32 [[TMP192]], -31
-// CHECK1-NEXT:    [[DIV365:%.*]] = udiv i32 [[SUB364]], 32
-// CHECK1-NEXT:    [[MUL366:%.*]] = mul i32 1, [[DIV365]]
-// CHECK1-NEXT:    [[TMP193:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP194:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB367:%.*]] = sub i32 [[TMP193]], [[TMP194]]
-// CHECK1-NEXT:    [[SUB368:%.*]] = sub i32 [[SUB367]], 1
-// CHECK1-NEXT:    [[ADD369:%.*]] = add i32 [[SUB368]], 1
-// CHECK1-NEXT:    [[DIV370:%.*]] = udiv i32 [[ADD369]], 1
-// CHECK1-NEXT:    [[MUL371:%.*]] = mul i32 [[MUL366]], [[DIV370]]
-// CHECK1-NEXT:    [[CONV372:%.*]] = zext i32 [[MUL371]] to i64
-// CHECK1-NEXT:    [[DIV373:%.*]] = sdiv i64 [[SUB363]], [[CONV372]]
-// CHECK1-NEXT:    [[TMP195:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB374:%.*]] = sub i32 [[TMP195]], -31
-// CHECK1-NEXT:    [[DIV375:%.*]] = udiv i32 [[SUB374]], 32
-// CHECK1-NEXT:    [[MUL376:%.*]] = mul i32 1, [[DIV375]]
-// CHECK1-NEXT:    [[TMP196:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP197:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB377:%.*]] = sub i32 [[TMP196]], [[TMP197]]
-// CHECK1-NEXT:    [[SUB378:%.*]] = sub i32 [[SUB377]], 1
-// CHECK1-NEXT:    [[ADD379:%.*]] = add i32 [[SUB378]], 1
-// CHECK1-NEXT:    [[DIV380:%.*]] = udiv i32 [[ADD379]], 1
-// CHECK1-NEXT:    [[MUL381:%.*]] = mul i32 [[MUL376]], [[DIV380]]
-// CHECK1-NEXT:    [[CONV382:%.*]] = zext i32 [[MUL381]] to i64
-// CHECK1-NEXT:    [[MUL383:%.*]] = mul nsw i64 [[DIV373]], [[CONV382]]
-// CHECK1-NEXT:    [[SUB384:%.*]] = sub nsw i64 [[SUB332]], [[MUL383]]
-// CHECK1-NEXT:    [[TMP198:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP199:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB385:%.*]] = sub i32 [[TMP198]], [[TMP199]]
-// CHECK1-NEXT:    [[SUB386:%.*]] = sub i32 [[SUB385]], 1
-// CHECK1-NEXT:    [[ADD387:%.*]] = add i32 [[SUB386]], 1
-// CHECK1-NEXT:    [[DIV388:%.*]] = udiv i32 [[ADD387]], 1
-// CHECK1-NEXT:    [[MUL389:%.*]] = mul i32 1, [[DIV388]]
-// CHECK1-NEXT:    [[CONV390:%.*]] = zext i32 [[MUL389]] to i64
-// CHECK1-NEXT:    [[DIV391:%.*]] = sdiv i64 [[SUB384]], [[CONV390]]
-// CHECK1-NEXT:    [[TMP200:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP201:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB392:%.*]] = sub i32 [[TMP200]], [[TMP201]]
-// CHECK1-NEXT:    [[SUB393:%.*]] = sub i32 [[SUB392]], 1
-// CHECK1-NEXT:    [[ADD394:%.*]] = add i32 [[SUB393]], 1
-// CHECK1-NEXT:    [[DIV395:%.*]] = udiv i32 [[ADD394]], 1
-// CHECK1-NEXT:    [[MUL396:%.*]] = mul i32 1, [[DIV395]]
-// CHECK1-NEXT:    [[CONV397:%.*]] = zext i32 [[MUL396]] to i64
-// CHECK1-NEXT:    [[MUL398:%.*]] = mul nsw i64 [[DIV391]], [[CONV397]]
-// CHECK1-NEXT:    [[SUB399:%.*]] = sub nsw i64 [[SUB301]], [[MUL398]]
-// CHECK1-NEXT:    [[MUL400:%.*]] = mul nsw i64 [[SUB399]], 1
-// CHECK1-NEXT:    [[ADD401:%.*]] = add nsw i64 [[CONV218]], [[MUL400]]
-// CHECK1-NEXT:    [[CONV402:%.*]] = trunc i64 [[ADD401]] to i32
-// CHECK1-NEXT:    store i32 [[CONV402]], ptr [[DOTTILE_0_IV_K52]], align 4
-// CHECK1-NEXT:    [[TMP202:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
-// CHECK1-NEXT:    [[TMP203:%.*]] = load i32, ptr [[DOTTILE_0_IV_K52]], align 4
-// CHECK1-NEXT:    [[TMP204:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
-// CHECK1-NEXT:    [[MUL403:%.*]] = mul i32 [[TMP203]], [[TMP204]]
-// CHECK1-NEXT:    [[ADD404:%.*]] = add i32 [[TMP202]], [[MUL403]]
-// CHECK1-NEXT:    store i32 [[ADD404]], ptr [[K]], align 4
-// CHECK1-NEXT:    [[TMP205:%.*]] = load i32, ptr [[I49]], align 4
-// CHECK1-NEXT:    [[TMP206:%.*]] = load i32, ptr [[J50]], align 4
-// CHECK1-NEXT:    [[TMP207:%.*]] = load i32, ptr [[K]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP205]], i32 noundef [[TMP206]], i32 noundef [[TMP207]])
+// CHECK1-NEXT:    [[DIV264:%.*]] = sdiv i64 [[SUB258]], [[CONV263]]
+// CHECK1-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB265:%.*]] = sub i32 [[TMP140]], -31
+// CHECK1-NEXT:    [[DIV266:%.*]] = udiv i32 [[SUB265]], 32
+// CHECK1-NEXT:    [[MUL267:%.*]] = mul i32 1, [[DIV266]]
+// CHECK1-NEXT:    [[MUL268:%.*]] = mul i32 [[MUL267]], 32
+// CHECK1-NEXT:    [[CONV269:%.*]] = zext i32 [[MUL268]] to i64
+// CHECK1-NEXT:    [[MUL270:%.*]] = mul nsw i64 [[DIV264]], [[CONV269]]
+// CHECK1-NEXT:    [[SUB271:%.*]] = sub nsw i64 [[SUB235]], [[MUL270]]
+// CHECK1-NEXT:    [[DIV272:%.*]] = sdiv i64 [[SUB271]], 32
+// CHECK1-NEXT:    [[MUL273:%.*]] = mul nsw i64 [[DIV272]], 32
+// CHECK1-NEXT:    [[SUB274:%.*]] = sub nsw i64 [[SUB212]], [[MUL273]]
+// CHECK1-NEXT:    [[MUL275:%.*]] = mul nsw i64 [[SUB274]], 1
+// CHECK1-NEXT:    [[ADD276:%.*]] = add nsw i64 0, [[MUL275]]
+// CHECK1-NEXT:    [[CONV277:%.*]] = trunc i64 [[ADD276]] to i32
+// CHECK1-NEXT:    store i32 [[CONV277]], ptr [[DOTTILE_CNT_0_IV_K38]], align 4
+// CHECK1-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
+// CHECK1-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
+// CHECK1-NEXT:    [[ADD278:%.*]] = add i32 [[TMP141]], [[TMP142]]
+// CHECK1-NEXT:    store i32 [[ADD278]], ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK1-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
+// CHECK1-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK1-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
+// CHECK1-NEXT:    [[MUL279:%.*]] = mul i32 [[TMP144]], [[TMP145]]
+// CHECK1-NEXT:    [[ADD280:%.*]] = add i32 [[TMP143]], [[MUL279]]
+// CHECK1-NEXT:    store i32 [[ADD280]], ptr [[K]], align 4
+// CHECK1-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK1-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// CHECK1-NEXT:    [[ADD281:%.*]] = add i32 [[TMP147]], 1
+// CHECK1-NEXT:    [[CMP282:%.*]] = icmp ult i32 [[TMP146]], [[ADD281]]
+// CHECK1-NEXT:    br i1 [[CMP282]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK1:       if.then:
+// CHECK1-NEXT:    [[TMP148:%.*]] = load i32, ptr [[I35]], align 4
+// CHECK1-NEXT:    [[TMP149:%.*]] = load i32, ptr [[J36]], align 4
+// CHECK1-NEXT:    [[TMP150:%.*]] = load i32, ptr [[K]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP148]], i32 noundef [[TMP149]], i32 noundef [[TMP150]])
+// CHECK1-NEXT:    br label [[IF_END]]
+// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP208:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[ADD405:%.*]] = add nsw i64 [[TMP208]], 1
-// CHECK1-NEXT:    store i64 [[ADD405]], ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP151:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[ADD283:%.*]] = add nsw i64 [[TMP151]], 1
+// CHECK1-NEXT:    store i64 [[ADD283]], ptr [[DOTOMP_IV]], align 8
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -1520,22 +1342,21 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTNEW_STEP10:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_14:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_17:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_22:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_K:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_K:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[I49:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[J50:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV_K51:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_K52:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[I35:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[J36:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV_K37:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_K38:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
@@ -1573,630 +1394,452 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
 // CHECK1-NEXT:    [[ADD15:%.*]] = add i32 [[TMP15]], 1
 // CHECK1-NEXT:    store i32 [[ADD15]], ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[_TMP2]], align 4
-// CHECK1-NEXT:    store i32 [[TMP16]], ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK1-NEXT:    [[ADD18:%.*]] = add i32 [[TMP17]], 1
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[_TMP2]], align 4
-// CHECK1-NEXT:    [[ADD19:%.*]] = add i32 [[TMP18]], 64
-// CHECK1-NEXT:    [[CMP:%.*]] = icmp ult i32 [[ADD18]], [[ADD19]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK1:       cond.true:
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK1-NEXT:    [[ADD20:%.*]] = add i32 [[TMP19]], 1
-// CHECK1-NEXT:    br label [[COND_END:%.*]]
-// CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[_TMP2]], align 4
-// CHECK1-NEXT:    [[ADD21:%.*]] = add i32 [[TMP20]], 64
-// CHECK1-NEXT:    br label [[COND_END]]
-// CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD20]], [[COND_TRUE]] ], [ [[ADD21]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[SUB23:%.*]] = sub i32 [[TMP21]], [[TMP22]]
-// CHECK1-NEXT:    [[SUB24:%.*]] = sub i32 [[SUB23]], 1
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[ADD25:%.*]] = add i32 [[SUB24]], [[TMP23]]
-// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[DIV26:%.*]] = udiv i32 [[ADD25]], [[TMP24]]
-// CHECK1-NEXT:    [[CONV:%.*]] = zext i32 [[DIV26]] to i64
-// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[SUB17:%.*]] = sub i32 [[TMP16]], [[TMP17]]
+// CHECK1-NEXT:    [[SUB18:%.*]] = sub i32 [[SUB17]], 1
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[ADD19:%.*]] = add i32 [[SUB18]], [[TMP18]]
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[DIV20:%.*]] = udiv i32 [[ADD19]], [[TMP19]]
+// CHECK1-NEXT:    [[CONV:%.*]] = zext i32 [[DIV20]] to i64
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB21:%.*]] = sub i32 [[TMP20]], [[TMP21]]
+// CHECK1-NEXT:    [[SUB22:%.*]] = sub i32 [[SUB21]], 1
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD23:%.*]] = add i32 [[SUB22]], [[TMP22]]
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV24:%.*]] = udiv i32 [[ADD23]], [[TMP23]]
+// CHECK1-NEXT:    [[CONV25:%.*]] = zext i32 [[DIV24]] to i64
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV25]]
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB26:%.*]] = sub i32 [[TMP24]], -63
+// CHECK1-NEXT:    [[DIV27:%.*]] = udiv i32 [[SUB26]], 64
+// CHECK1-NEXT:    [[CONV28:%.*]] = zext i32 [[DIV27]] to i64
+// CHECK1-NEXT:    [[MUL29:%.*]] = mul nsw i64 [[MUL]], [[CONV28]]
+// CHECK1-NEXT:    [[MUL30:%.*]] = mul nsw i64 [[MUL29]], 64
+// CHECK1-NEXT:    [[SUB31:%.*]] = sub nsw i64 [[MUL30]], 1
+// CHECK1-NEXT:    store i64 [[SUB31]], ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    store i32 [[TMP25]], ptr [[I]], align 4
 // CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB27:%.*]] = sub i32 [[TMP25]], [[TMP26]]
-// CHECK1-NEXT:    [[SUB28:%.*]] = sub i32 [[SUB27]], 1
-// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD29:%.*]] = add i32 [[SUB28]], [[TMP27]]
-// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV30:%.*]] = udiv i32 [[ADD29]], [[TMP28]]
-// CHECK1-NEXT:    [[CONV31:%.*]] = zext i32 [[DIV30]] to i64
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV31]]
-// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB32:%.*]] = sub i32 [[TMP29]], -63
-// CHECK1-NEXT:    [[DIV33:%.*]] = udiv i32 [[SUB32]], 64
-// CHECK1-NEXT:    [[CONV34:%.*]] = zext i32 [[DIV33]] to i64
-// CHECK1-NEXT:    [[MUL35:%.*]] = mul nsw i64 [[MUL]], [[CONV34]]
-// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB36:%.*]] = sub i32 [[TMP30]], [[TMP31]]
-// CHECK1-NEXT:    [[SUB37:%.*]] = sub i32 [[SUB36]], 1
-// CHECK1-NEXT:    [[ADD38:%.*]] = add i32 [[SUB37]], 1
-// CHECK1-NEXT:    [[DIV39:%.*]] = udiv i32 [[ADD38]], 1
-// CHECK1-NEXT:    [[CONV40:%.*]] = zext i32 [[DIV39]] to i64
-// CHECK1-NEXT:    [[MUL41:%.*]] = mul nsw i64 [[MUL35]], [[CONV40]]
-// CHECK1-NEXT:    [[SUB42:%.*]] = sub nsw i64 [[MUL41]], 1
-// CHECK1-NEXT:    store i64 [[SUB42]], ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    store i32 [[TMP32]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    store i32 [[TMP33]], ptr [[J]], align 4
+// CHECK1-NEXT:    store i32 [[TMP26]], ptr [[J]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_K]], align 4
-// CHECK1-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    store i32 [[TMP34]], ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK1-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK1-NEXT:    [[CMP43:%.*]] = icmp slt i32 [[TMP35]], [[TMP36]]
-// CHECK1-NEXT:    br i1 [[CMP43]], label [[LAND_LHS_TRUE:%.*]], label [[OMP_PRECOND_END:%.*]]
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_K]], align 4
+// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP27]], [[TMP28]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[LAND_LHS_TRUE:%.*]], label [[OMP_PRECOND_END:%.*]]
 // CHECK1:       land.lhs.true:
-// CHECK1-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[CMP44:%.*]] = icmp slt i32 [[TMP37]], [[TMP38]]
-// CHECK1-NEXT:    br i1 [[CMP44]], label [[LAND_LHS_TRUE45:%.*]], label [[OMP_PRECOND_END]]
-// CHECK1:       land.lhs.true45:
-// CHECK1-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[CMP46:%.*]] = icmp ult i32 0, [[TMP39]]
-// CHECK1-NEXT:    br i1 [[CMP46]], label [[LAND_LHS_TRUE47:%.*]], label [[OMP_PRECOND_END]]
-// CHECK1:       land.lhs.true47:
-// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[CMP48:%.*]] = icmp ult i32 [[TMP40]], [[TMP41]]
-// CHECK1-NEXT:    br i1 [[CMP48]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
+// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[CMP32:%.*]] = icmp slt i32 [[TMP29]], [[TMP30]]
+// CHECK1-NEXT:    br i1 [[CMP32]], label [[LAND_LHS_TRUE33:%.*]], label [[OMP_PRECOND_END]]
+// CHECK1:       land.lhs.true33:
+// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[CMP34:%.*]] = icmp ult i32 0, [[TMP31]]
+// CHECK1-NEXT:    br i1 [[CMP34]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
 // CHECK1:       omp.precond.then:
 // CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
-// CHECK1-NEXT:    [[TMP42:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK1-NEXT:    store i64 [[TMP42]], ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK1-NEXT:    store i64 [[TMP32]], ptr [[DOTOMP_UB]], align 8
 // CHECK1-NEXT:    store i64 1, ptr [[DOTOMP_STRIDE]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
 // CHECK1-NEXT:    call void @__kmpc_for_static_init_8(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i64 1, i64 1)
-// CHECK1-NEXT:    [[TMP43:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[TMP44:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK1-NEXT:    [[CMP53:%.*]] = icmp sgt i64 [[TMP43]], [[TMP44]]
-// CHECK1-NEXT:    br i1 [[CMP53]], label [[COND_TRUE54:%.*]], label [[COND_FALSE55:%.*]]
-// CHECK1:       cond.true54:
-// CHECK1-NEXT:    [[TMP45:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK1-NEXT:    br label [[COND_END56:%.*]]
-// CHECK1:       cond.false55:
-// CHECK1-NEXT:    [[TMP46:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    br label [[COND_END56]]
-// CHECK1:       cond.end56:
-// CHECK1-NEXT:    [[COND57:%.*]] = phi i64 [ [[TMP45]], [[COND_TRUE54]] ], [ [[TMP46]], [[COND_FALSE55]] ]
-// CHECK1-NEXT:    store i64 [[COND57]], ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[TMP47:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
-// CHECK1-NEXT:    store i64 [[TMP47]], ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK1-NEXT:    [[CMP39:%.*]] = icmp sgt i64 [[TMP33]], [[TMP34]]
+// CHECK1-NEXT:    br i1 [[CMP39]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i64 [ [[TMP35]], [[COND_TRUE]] ], [ [[TMP36]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    store i64 [[COND]], ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
+// CHECK1-NEXT:    store i64 [[TMP37]], ptr [[DOTOMP_IV]], align 8
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK1:       omp.inner.for.cond:
-// CHECK1-NEXT:    [[TMP48:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[CMP58:%.*]] = icmp sle i64 [[TMP48]], [[TMP49]]
-// CHECK1-NEXT:    br i1 [[CMP58]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK1-NEXT:    [[TMP38:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP39:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[CMP40:%.*]] = icmp sle i64 [[TMP38]], [[TMP39]]
+// CHECK1-NEXT:    br i1 [[CMP40]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
 // CHECK1:       omp.inner.for.body:
-// CHECK1-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[CONV59:%.*]] = sext i32 [[TMP50]] to i64
-// CHECK1-NEXT:    [[TMP51:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB60:%.*]] = sub i32 [[TMP52]], [[TMP53]]
-// CHECK1-NEXT:    [[SUB61:%.*]] = sub i32 [[SUB60]], 1
+// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[CONV41:%.*]] = sext i32 [[TMP40]] to i64
+// CHECK1-NEXT:    [[TMP41:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB42:%.*]] = sub i32 [[TMP42]], [[TMP43]]
+// CHECK1-NEXT:    [[SUB43:%.*]] = sub i32 [[SUB42]], 1
+// CHECK1-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD44:%.*]] = add i32 [[SUB43]], [[TMP44]]
+// CHECK1-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV45:%.*]] = udiv i32 [[ADD44]], [[TMP45]]
+// CHECK1-NEXT:    [[MUL46:%.*]] = mul i32 1, [[DIV45]]
+// CHECK1-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB47:%.*]] = sub i32 [[TMP46]], -63
+// CHECK1-NEXT:    [[DIV48:%.*]] = udiv i32 [[SUB47]], 64
+// CHECK1-NEXT:    [[MUL49:%.*]] = mul i32 [[MUL46]], [[DIV48]]
+// CHECK1-NEXT:    [[MUL50:%.*]] = mul i32 [[MUL49]], 64
+// CHECK1-NEXT:    [[CONV51:%.*]] = zext i32 [[MUL50]] to i64
+// CHECK1-NEXT:    [[DIV52:%.*]] = sdiv i64 [[TMP41]], [[CONV51]]
+// CHECK1-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[CONV53:%.*]] = sext i32 [[TMP47]] to i64
+// CHECK1-NEXT:    [[MUL54:%.*]] = mul nsw i64 [[DIV52]], [[CONV53]]
+// CHECK1-NEXT:    [[ADD55:%.*]] = add nsw i64 [[CONV41]], [[MUL54]]
+// CHECK1-NEXT:    [[CONV56:%.*]] = trunc i64 [[ADD55]] to i32
+// CHECK1-NEXT:    store i32 [[CONV56]], ptr [[I35]], align 4
+// CHECK1-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP48]] to i64
+// CHECK1-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP50:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB58:%.*]] = sub i32 [[TMP51]], [[TMP52]]
+// CHECK1-NEXT:    [[SUB59:%.*]] = sub i32 [[SUB58]], 1
+// CHECK1-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD60:%.*]] = add i32 [[SUB59]], [[TMP53]]
 // CHECK1-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD62:%.*]] = add i32 [[SUB61]], [[TMP54]]
-// CHECK1-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV63:%.*]] = udiv i32 [[ADD62]], [[TMP55]]
-// CHECK1-NEXT:    [[MUL64:%.*]] = mul i32 1, [[DIV63]]
-// CHECK1-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB65:%.*]] = sub i32 [[TMP56]], -63
-// CHECK1-NEXT:    [[DIV66:%.*]] = udiv i32 [[SUB65]], 64
-// CHECK1-NEXT:    [[MUL67:%.*]] = mul i32 [[MUL64]], [[DIV66]]
-// CHECK1-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB68:%.*]] = sub i32 [[TMP57]], [[TMP58]]
-// CHECK1-NEXT:    [[SUB69:%.*]] = sub i32 [[SUB68]], 1
-// CHECK1-NEXT:    [[ADD70:%.*]] = add i32 [[SUB69]], 1
-// CHECK1-NEXT:    [[DIV71:%.*]] = udiv i32 [[ADD70]], 1
-// CHECK1-NEXT:    [[MUL72:%.*]] = mul i32 [[MUL67]], [[DIV71]]
-// CHECK1-NEXT:    [[CONV73:%.*]] = zext i32 [[MUL72]] to i64
-// CHECK1-NEXT:    [[DIV74:%.*]] = sdiv i64 [[TMP51]], [[CONV73]]
-// CHECK1-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[CONV75:%.*]] = sext i32 [[TMP59]] to i64
-// CHECK1-NEXT:    [[MUL76:%.*]] = mul nsw i64 [[DIV74]], [[CONV75]]
-// CHECK1-NEXT:    [[ADD77:%.*]] = add nsw i64 [[CONV59]], [[MUL76]]
-// CHECK1-NEXT:    [[CONV78:%.*]] = trunc i64 [[ADD77]] to i32
-// CHECK1-NEXT:    store i32 [[CONV78]], ptr [[I49]], align 4
-// CHECK1-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[CONV79:%.*]] = sext i32 [[TMP60]] to i64
-// CHECK1-NEXT:    [[TMP61:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP62:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP63:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP64:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB80:%.*]] = sub i32 [[TMP63]], [[TMP64]]
-// CHECK1-NEXT:    [[SUB81:%.*]] = sub i32 [[SUB80]], 1
-// CHECK1-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD82:%.*]] = add i32 [[SUB81]], [[TMP65]]
-// CHECK1-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV83:%.*]] = udiv i32 [[ADD82]], [[TMP66]]
-// CHECK1-NEXT:    [[MUL84:%.*]] = mul i32 1, [[DIV83]]
-// CHECK1-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB85:%.*]] = sub i32 [[TMP67]], -63
-// CHECK1-NEXT:    [[DIV86:%.*]] = udiv i32 [[SUB85]], 64
-// CHECK1-NEXT:    [[MUL87:%.*]] = mul i32 [[MUL84]], [[DIV86]]
-// CHECK1-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB88:%.*]] = sub i32 [[TMP68]], [[TMP69]]
-// CHECK1-NEXT:    [[SUB89:%.*]] = sub i32 [[SUB88]], 1
-// CHECK1-NEXT:    [[ADD90:%.*]] = add i32 [[SUB89]], 1
-// CHECK1-NEXT:    [[DIV91:%.*]] = udiv i32 [[ADD90]], 1
-// CHECK1-NEXT:    [[MUL92:%.*]] = mul i32 [[MUL87]], [[DIV91]]
-// CHECK1-NEXT:    [[CONV93:%.*]] = zext i32 [[MUL92]] to i64
-// CHECK1-NEXT:    [[DIV94:%.*]] = sdiv i64 [[TMP62]], [[CONV93]]
+// CHECK1-NEXT:    [[DIV61:%.*]] = udiv i32 [[ADD60]], [[TMP54]]
+// CHECK1-NEXT:    [[MUL62:%.*]] = mul i32 1, [[DIV61]]
+// CHECK1-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB63:%.*]] = sub i32 [[TMP55]], -63
+// CHECK1-NEXT:    [[DIV64:%.*]] = udiv i32 [[SUB63]], 64
+// CHECK1-NEXT:    [[MUL65:%.*]] = mul i32 [[MUL62]], [[DIV64]]
+// CHECK1-NEXT:    [[MUL66:%.*]] = mul i32 [[MUL65]], 64
+// CHECK1-NEXT:    [[CONV67:%.*]] = zext i32 [[MUL66]] to i64
+// CHECK1-NEXT:    [[DIV68:%.*]] = sdiv i64 [[TMP50]], [[CONV67]]
+// CHECK1-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB69:%.*]] = sub i32 [[TMP56]], [[TMP57]]
+// CHECK1-NEXT:    [[SUB70:%.*]] = sub i32 [[SUB69]], 1
+// CHECK1-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD71:%.*]] = add i32 [[SUB70]], [[TMP58]]
+// CHECK1-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV72:%.*]] = udiv i32 [[ADD71]], [[TMP59]]
+// CHECK1-NEXT:    [[MUL73:%.*]] = mul i32 1, [[DIV72]]
+// CHECK1-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB74:%.*]] = sub i32 [[TMP60]], -63
+// CHECK1-NEXT:    [[DIV75:%.*]] = udiv i32 [[SUB74]], 64
+// CHECK1-NEXT:    [[MUL76:%.*]] = mul i32 [[MUL73]], [[DIV75]]
+// CHECK1-NEXT:    [[MUL77:%.*]] = mul i32 [[MUL76]], 64
+// CHECK1-NEXT:    [[CONV78:%.*]] = zext i32 [[MUL77]] to i64
+// CHECK1-NEXT:    [[MUL79:%.*]] = mul nsw i64 [[DIV68]], [[CONV78]]
+// CHECK1-NEXT:    [[SUB80:%.*]] = sub nsw i64 [[TMP49]], [[MUL79]]
+// CHECK1-NEXT:    [[TMP61:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB81:%.*]] = sub i32 [[TMP61]], -63
+// CHECK1-NEXT:    [[DIV82:%.*]] = udiv i32 [[SUB81]], 64
+// CHECK1-NEXT:    [[MUL83:%.*]] = mul i32 1, [[DIV82]]
+// CHECK1-NEXT:    [[MUL84:%.*]] = mul i32 [[MUL83]], 64
+// CHECK1-NEXT:    [[CONV85:%.*]] = zext i32 [[MUL84]] to i64
+// CHECK1-NEXT:    [[DIV86:%.*]] = sdiv i64 [[SUB80]], [[CONV85]]
+// CHECK1-NEXT:    [[TMP62:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[CONV87:%.*]] = sext i32 [[TMP62]] to i64
+// CHECK1-NEXT:    [[MUL88:%.*]] = mul nsw i64 [[DIV86]], [[CONV87]]
+// CHECK1-NEXT:    [[ADD89:%.*]] = add nsw i64 [[CONV57]], [[MUL88]]
+// CHECK1-NEXT:    [[CONV90:%.*]] = trunc i64 [[ADD89]] to i32
+// CHECK1-NEXT:    store i32 [[CONV90]], ptr [[J36]], align 4
+// CHECK1-NEXT:    [[TMP63:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP64:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB91:%.*]] = sub i32 [[TMP65]], [[TMP66]]
+// CHECK1-NEXT:    [[SUB92:%.*]] = sub i32 [[SUB91]], 1
+// CHECK1-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD93:%.*]] = add i32 [[SUB92]], [[TMP67]]
+// CHECK1-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV94:%.*]] = udiv i32 [[ADD93]], [[TMP68]]
+// CHECK1-NEXT:    [[MUL95:%.*]] = mul i32 1, [[DIV94]]
+// CHECK1-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB96:%.*]] = sub i32 [[TMP69]], -63
+// CHECK1-NEXT:    [[DIV97:%.*]] = udiv i32 [[SUB96]], 64
+// CHECK1-NEXT:    [[MUL98:%.*]] = mul i32 [[MUL95]], [[DIV97]]
+// CHECK1-NEXT:    [[MUL99:%.*]] = mul i32 [[MUL98]], 64
+// CHECK1-NEXT:    [[CONV100:%.*]] = zext i32 [[MUL99]] to i64
+// CHECK1-NEXT:    [[DIV101:%.*]] = sdiv i64 [[TMP64]], [[CONV100]]
 // CHECK1-NEXT:    [[TMP70:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
 // CHECK1-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB95:%.*]] = sub i32 [[TMP70]], [[TMP71]]
-// CHECK1-NEXT:    [[SUB96:%.*]] = sub i32 [[SUB95]], 1
+// CHECK1-NEXT:    [[SUB102:%.*]] = sub i32 [[TMP70]], [[TMP71]]
+// CHECK1-NEXT:    [[SUB103:%.*]] = sub i32 [[SUB102]], 1
 // CHECK1-NEXT:    [[TMP72:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD97:%.*]] = add i32 [[SUB96]], [[TMP72]]
+// CHECK1-NEXT:    [[ADD104:%.*]] = add i32 [[SUB103]], [[TMP72]]
 // CHECK1-NEXT:    [[TMP73:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV98:%.*]] = udiv i32 [[ADD97]], [[TMP73]]
-// CHECK1-NEXT:    [[MUL99:%.*]] = mul i32 1, [[DIV98]]
+// CHECK1-NEXT:    [[DIV105:%.*]] = udiv i32 [[ADD104]], [[TMP73]]
+// CHECK1-NEXT:    [[MUL106:%.*]] = mul i32 1, [[DIV105]]
 // CHECK1-NEXT:    [[TMP74:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB100:%.*]] = sub i32 [[TMP74]], -63
-// CHECK1-NEXT:    [[DIV101:%.*]] = udiv i32 [[SUB100]], 64
-// CHECK1-NEXT:    [[MUL102:%.*]] = mul i32 [[MUL99]], [[DIV101]]
-// CHECK1-NEXT:    [[TMP75:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP76:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB103:%.*]] = sub i32 [[TMP75]], [[TMP76]]
-// CHECK1-NEXT:    [[SUB104:%.*]] = sub i32 [[SUB103]], 1
-// CHECK1-NEXT:    [[ADD105:%.*]] = add i32 [[SUB104]], 1
-// CHECK1-NEXT:    [[DIV106:%.*]] = udiv i32 [[ADD105]], 1
-// CHECK1-NEXT:    [[MUL107:%.*]] = mul i32 [[MUL102]], [[DIV106]]
-// CHECK1-NEXT:    [[CONV108:%.*]] = zext i32 [[MUL107]] to i64
-// CHECK1-NEXT:    [[MUL109:%.*]] = mul nsw i64 [[DIV94]], [[CONV108]]
-// CHECK1-NEXT:    [[SUB110:%.*]] = sub nsw i64 [[TMP61]], [[MUL109]]
-// CHECK1-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB111:%.*]] = sub i32 [[TMP77]], -63
-// CHECK1-NEXT:    [[DIV112:%.*]] = udiv i32 [[SUB111]], 64
-// CHECK1-NEXT:    [[MUL113:%.*]] = mul i32 1, [[DIV112]]
-// CHECK1-NEXT:    [[TMP78:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP79:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB114:%.*]] = sub i32 [[TMP78]], [[TMP79]]
+// CHECK1-NEXT:    [[SUB107:%.*]] = sub i32 [[TMP74]], -63
+// CHECK1-NEXT:    [[DIV108:%.*]] = udiv i32 [[SUB107]], 64
+// CHECK1-NEXT:    [[MUL109:%.*]] = mul i32 [[MUL106]], [[DIV108]]
+// CHECK1-NEXT:    [[MUL110:%.*]] = mul i32 [[MUL109]], 64
+// CHECK1-NEXT:    [[CONV111:%.*]] = zext i32 [[MUL110]] to i64
+// CHECK1-NEXT:    [[MUL112:%.*]] = mul nsw i64 [[DIV101]], [[CONV111]]
+// CHECK1-NEXT:    [[SUB113:%.*]] = sub nsw i64 [[TMP63]], [[MUL112]]
+// CHECK1-NEXT:    [[TMP75:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP76:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP78:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB114:%.*]] = sub i32 [[TMP77]], [[TMP78]]
 // CHECK1-NEXT:    [[SUB115:%.*]] = sub i32 [[SUB114]], 1
-// CHECK1-NEXT:    [[ADD116:%.*]] = add i32 [[SUB115]], 1
-// CHECK1-NEXT:    [[DIV117:%.*]] = udiv i32 [[ADD116]], 1
-// CHECK1-NEXT:    [[MUL118:%.*]] = mul i32 [[MUL113]], [[DIV117]]
-// CHECK1-NEXT:    [[CONV119:%.*]] = zext i32 [[MUL118]] to i64
-// CHECK1-NEXT:    [[DIV120:%.*]] = sdiv i64 [[SUB110]], [[CONV119]]
+// CHECK1-NEXT:    [[TMP79:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD116:%.*]] = add i32 [[SUB115]], [[TMP79]]
 // CHECK1-NEXT:    [[TMP80:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[CONV121:%.*]] = sext i32 [[TMP80]] to i64
-// CHECK1-NEXT:    [[MUL122:%.*]] = mul nsw i64 [[DIV120]], [[CONV121]]
-// CHECK1-NEXT:    [[ADD123:%.*]] = add nsw i64 [[CONV79]], [[MUL122]]
-// CHECK1-NEXT:    [[CONV124:%.*]] = trunc i64 [[ADD123]] to i32
-// CHECK1-NEXT:    store i32 [[CONV124]], ptr [[J50]], align 4
-// CHECK1-NEXT:    [[TMP81:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP82:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB125:%.*]] = sub i32 [[TMP83]], [[TMP84]]
+// CHECK1-NEXT:    [[DIV117:%.*]] = udiv i32 [[ADD116]], [[TMP80]]
+// CHECK1-NEXT:    [[MUL118:%.*]] = mul i32 1, [[DIV117]]
+// CHECK1-NEXT:    [[TMP81:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB119:%.*]] = sub i32 [[TMP81]], -63
+// CHECK1-NEXT:    [[DIV120:%.*]] = udiv i32 [[SUB119]], 64
+// CHECK1-NEXT:    [[MUL121:%.*]] = mul i32 [[MUL118]], [[DIV120]]
+// CHECK1-NEXT:    [[MUL122:%.*]] = mul i32 [[MUL121]], 64
+// CHECK1-NEXT:    [[CONV123:%.*]] = zext i32 [[MUL122]] to i64
+// CHECK1-NEXT:    [[DIV124:%.*]] = sdiv i64 [[TMP76]], [[CONV123]]
+// CHECK1-NEXT:    [[TMP82:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB125:%.*]] = sub i32 [[TMP82]], [[TMP83]]
 // CHECK1-NEXT:    [[SUB126:%.*]] = sub i32 [[SUB125]], 1
+// CHECK1-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD127:%.*]] = add i32 [[SUB126]], [[TMP84]]
 // CHECK1-NEXT:    [[TMP85:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD127:%.*]] = add i32 [[SUB126]], [[TMP85]]
-// CHECK1-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV128:%.*]] = udiv i32 [[ADD127]], [[TMP86]]
+// CHECK1-NEXT:    [[DIV128:%.*]] = udiv i32 [[ADD127]], [[TMP85]]
 // CHECK1-NEXT:    [[MUL129:%.*]] = mul i32 1, [[DIV128]]
-// CHECK1-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB130:%.*]] = sub i32 [[TMP87]], -63
+// CHECK1-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB130:%.*]] = sub i32 [[TMP86]], -63
 // CHECK1-NEXT:    [[DIV131:%.*]] = udiv i32 [[SUB130]], 64
 // CHECK1-NEXT:    [[MUL132:%.*]] = mul i32 [[MUL129]], [[DIV131]]
-// CHECK1-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP89:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB133:%.*]] = sub i32 [[TMP88]], [[TMP89]]
-// CHECK1-NEXT:    [[SUB134:%.*]] = sub i32 [[SUB133]], 1
-// CHECK1-NEXT:    [[ADD135:%.*]] = add i32 [[SUB134]], 1
-// CHECK1-NEXT:    [[DIV136:%.*]] = udiv i32 [[ADD135]], 1
-// CHECK1-NEXT:    [[MUL137:%.*]] = mul i32 [[MUL132]], [[DIV136]]
-// CHECK1-NEXT:    [[CONV138:%.*]] = zext i32 [[MUL137]] to i64
-// CHECK1-NEXT:    [[DIV139:%.*]] = sdiv i64 [[TMP82]], [[CONV138]]
-// CHECK1-NEXT:    [[TMP90:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB140:%.*]] = sub i32 [[TMP90]], [[TMP91]]
-// CHECK1-NEXT:    [[SUB141:%.*]] = sub i32 [[SUB140]], 1
-// CHECK1-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD142:%.*]] = add i32 [[SUB141]], [[TMP92]]
+// CHECK1-NEXT:    [[MUL133:%.*]] = mul i32 [[MUL132]], 64
+// CHECK1-NEXT:    [[CONV134:%.*]] = zext i32 [[MUL133]] to i64
+// CHECK1-NEXT:    [[MUL135:%.*]] = mul nsw i64 [[DIV124]], [[CONV134]]
+// CHECK1-NEXT:    [[SUB136:%.*]] = sub nsw i64 [[TMP75]], [[MUL135]]
+// CHECK1-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB137:%.*]] = sub i32 [[TMP87]], -63
+// CHECK1-NEXT:    [[DIV138:%.*]] = udiv i32 [[SUB137]], 64
+// CHECK1-NEXT:    [[MUL139:%.*]] = mul i32 1, [[DIV138]]
+// CHECK1-NEXT:    [[MUL140:%.*]] = mul i32 [[MUL139]], 64
+// CHECK1-NEXT:    [[CONV141:%.*]] = zext i32 [[MUL140]] to i64
+// CHECK1-NEXT:    [[DIV142:%.*]] = sdiv i64 [[SUB136]], [[CONV141]]
+// CHECK1-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB143:%.*]] = sub i32 [[TMP88]], -63
+// CHECK1-NEXT:    [[DIV144:%.*]] = udiv i32 [[SUB143]], 64
+// CHECK1-NEXT:    [[MUL145:%.*]] = mul i32 1, [[DIV144]]
+// CHECK1-NEXT:    [[MUL146:%.*]] = mul i32 [[MUL145]], 64
+// CHECK1-NEXT:    [[CONV147:%.*]] = zext i32 [[MUL146]] to i64
+// CHECK1-NEXT:    [[MUL148:%.*]] = mul nsw i64 [[DIV142]], [[CONV147]]
+// CHECK1-NEXT:    [[SUB149:%.*]] = sub nsw i64 [[SUB113]], [[MUL148]]
+// CHECK1-NEXT:    [[DIV150:%.*]] = sdiv i64 [[SUB149]], 64
+// CHECK1-NEXT:    [[MUL151:%.*]] = mul nsw i64 [[DIV150]], 64
+// CHECK1-NEXT:    [[ADD152:%.*]] = add nsw i64 0, [[MUL151]]
+// CHECK1-NEXT:    [[CONV153:%.*]] = trunc i64 [[ADD152]] to i32
+// CHECK1-NEXT:    store i32 [[CONV153]], ptr [[DOTFLOOR_0_IV_K37]], align 4
+// CHECK1-NEXT:    [[TMP89:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP90:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB154:%.*]] = sub i32 [[TMP91]], [[TMP92]]
+// CHECK1-NEXT:    [[SUB155:%.*]] = sub i32 [[SUB154]], 1
 // CHECK1-NEXT:    [[TMP93:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV143:%.*]] = udiv i32 [[ADD142]], [[TMP93]]
-// CHECK1-NEXT:    [[MUL144:%.*]] = mul i32 1, [[DIV143]]
-// CHECK1-NEXT:    [[TMP94:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB145:%.*]] = sub i32 [[TMP94]], -63
-// CHECK1-NEXT:    [[DIV146:%.*]] = udiv i32 [[SUB145]], 64
-// CHECK1-NEXT:    [[MUL147:%.*]] = mul i32 [[MUL144]], [[DIV146]]
-// CHECK1-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB148:%.*]] = sub i32 [[TMP95]], [[TMP96]]
-// CHECK1-NEXT:    [[SUB149:%.*]] = sub i32 [[SUB148]], 1
-// CHECK1-NEXT:    [[ADD150:%.*]] = add i32 [[SUB149]], 1
-// CHECK1-NEXT:    [[DIV151:%.*]] = udiv i32 [[ADD150]], 1
-// CHECK1-NEXT:    [[MUL152:%.*]] = mul i32 [[MUL147]], [[DIV151]]
-// CHECK1-NEXT:    [[CONV153:%.*]] = zext i32 [[MUL152]] to i64
-// CHECK1-NEXT:    [[MUL154:%.*]] = mul nsw i64 [[DIV139]], [[CONV153]]
-// CHECK1-NEXT:    [[SUB155:%.*]] = sub nsw i64 [[TMP81]], [[MUL154]]
-// CHECK1-NEXT:    [[TMP97:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP98:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB156:%.*]] = sub i32 [[TMP99]], [[TMP100]]
-// CHECK1-NEXT:    [[SUB157:%.*]] = sub i32 [[SUB156]], 1
-// CHECK1-NEXT:    [[TMP101:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD158:%.*]] = add i32 [[SUB157]], [[TMP101]]
-// CHECK1-NEXT:    [[TMP102:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV159:%.*]] = udiv i32 [[ADD158]], [[TMP102]]
-// CHECK1-NEXT:    [[MUL160:%.*]] = mul i32 1, [[DIV159]]
-// CHECK1-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB161:%.*]] = sub i32 [[TMP103]], -63
-// CHECK1-NEXT:    [[DIV162:%.*]] = udiv i32 [[SUB161]], 64
-// CHECK1-NEXT:    [[MUL163:%.*]] = mul i32 [[MUL160]], [[DIV162]]
-// CHECK1-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP105:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB164:%.*]] = sub i32 [[TMP104]], [[TMP105]]
-// CHECK1-NEXT:    [[SUB165:%.*]] = sub i32 [[SUB164]], 1
-// CHECK1-NEXT:    [[ADD166:%.*]] = add i32 [[SUB165]], 1
-// CHECK1-NEXT:    [[DIV167:%.*]] = udiv i32 [[ADD166]], 1
-// CHECK1-NEXT:    [[MUL168:%.*]] = mul i32 [[MUL163]], [[DIV167]]
-// CHECK1-NEXT:    [[CONV169:%.*]] = zext i32 [[MUL168]] to i64
-// CHECK1-NEXT:    [[DIV170:%.*]] = sdiv i64 [[TMP98]], [[CONV169]]
-// CHECK1-NEXT:    [[TMP106:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB171:%.*]] = sub i32 [[TMP106]], [[TMP107]]
-// CHECK1-NEXT:    [[SUB172:%.*]] = sub i32 [[SUB171]], 1
-// CHECK1-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD173:%.*]] = add i32 [[SUB172]], [[TMP108]]
-// CHECK1-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV174:%.*]] = udiv i32 [[ADD173]], [[TMP109]]
-// CHECK1-NEXT:    [[MUL175:%.*]] = mul i32 1, [[DIV174]]
-// CHECK1-NEXT:    [[TMP110:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB176:%.*]] = sub i32 [[TMP110]], -63
-// CHECK1-NEXT:    [[DIV177:%.*]] = udiv i32 [[SUB176]], 64
-// CHECK1-NEXT:    [[MUL178:%.*]] = mul i32 [[MUL175]], [[DIV177]]
-// CHECK1-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB179:%.*]] = sub i32 [[TMP111]], [[TMP112]]
-// CHECK1-NEXT:    [[SUB180:%.*]] = sub i32 [[SUB179]], 1
-// CHECK1-NEXT:    [[ADD181:%.*]] = add i32 [[SUB180]], 1
-// CHECK1-NEXT:    [[DIV182:%.*]] = udiv i32 [[ADD181]], 1
-// CHECK1-NEXT:    [[MUL183:%.*]] = mul i32 [[MUL178]], [[DIV182]]
-// CHECK1-NEXT:    [[CONV184:%.*]] = zext i32 [[MUL183]] to i64
-// CHECK1-NEXT:    [[MUL185:%.*]] = mul nsw i64 [[DIV170]], [[CONV184]]
-// CHECK1-NEXT:    [[SUB186:%.*]] = sub nsw i64 [[TMP97]], [[MUL185]]
+// CHECK1-NEXT:    [[ADD156:%.*]] = add i32 [[SUB155]], [[TMP93]]
+// CHECK1-NEXT:    [[TMP94:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV157:%.*]] = udiv i32 [[ADD156]], [[TMP94]]
+// CHECK1-NEXT:    [[MUL158:%.*]] = mul i32 1, [[DIV157]]
+// CHECK1-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB159:%.*]] = sub i32 [[TMP95]], -63
+// CHECK1-NEXT:    [[DIV160:%.*]] = udiv i32 [[SUB159]], 64
+// CHECK1-NEXT:    [[MUL161:%.*]] = mul i32 [[MUL158]], [[DIV160]]
+// CHECK1-NEXT:    [[MUL162:%.*]] = mul i32 [[MUL161]], 64
+// CHECK1-NEXT:    [[CONV163:%.*]] = zext i32 [[MUL162]] to i64
+// CHECK1-NEXT:    [[DIV164:%.*]] = sdiv i64 [[TMP90]], [[CONV163]]
+// CHECK1-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP97:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB165:%.*]] = sub i32 [[TMP96]], [[TMP97]]
+// CHECK1-NEXT:    [[SUB166:%.*]] = sub i32 [[SUB165]], 1
+// CHECK1-NEXT:    [[TMP98:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD167:%.*]] = add i32 [[SUB166]], [[TMP98]]
+// CHECK1-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV168:%.*]] = udiv i32 [[ADD167]], [[TMP99]]
+// CHECK1-NEXT:    [[MUL169:%.*]] = mul i32 1, [[DIV168]]
+// CHECK1-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB170:%.*]] = sub i32 [[TMP100]], -63
+// CHECK1-NEXT:    [[DIV171:%.*]] = udiv i32 [[SUB170]], 64
+// CHECK1-NEXT:    [[MUL172:%.*]] = mul i32 [[MUL169]], [[DIV171]]
+// CHECK1-NEXT:    [[MUL173:%.*]] = mul i32 [[MUL172]], 64
+// CHECK1-NEXT:    [[CONV174:%.*]] = zext i32 [[MUL173]] to i64
+// CHECK1-NEXT:    [[MUL175:%.*]] = mul nsw i64 [[DIV164]], [[CONV174]]
+// CHECK1-NEXT:    [[SUB176:%.*]] = sub nsw i64 [[TMP89]], [[MUL175]]
+// CHECK1-NEXT:    [[TMP101:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP102:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB177:%.*]] = sub i32 [[TMP103]], [[TMP104]]
+// CHECK1-NEXT:    [[SUB178:%.*]] = sub i32 [[SUB177]], 1
+// CHECK1-NEXT:    [[TMP105:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD179:%.*]] = add i32 [[SUB178]], [[TMP105]]
+// CHECK1-NEXT:    [[TMP106:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV180:%.*]] = udiv i32 [[ADD179]], [[TMP106]]
+// CHECK1-NEXT:    [[MUL181:%.*]] = mul i32 1, [[DIV180]]
+// CHECK1-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB182:%.*]] = sub i32 [[TMP107]], -63
+// CHECK1-NEXT:    [[DIV183:%.*]] = udiv i32 [[SUB182]], 64
+// CHECK1-NEXT:    [[MUL184:%.*]] = mul i32 [[MUL181]], [[DIV183]]
+// CHECK1-NEXT:    [[MUL185:%.*]] = mul i32 [[MUL184]], 64
+// CHECK1-NEXT:    [[CONV186:%.*]] = zext i32 [[MUL185]] to i64
+// CHECK1-NEXT:    [[DIV187:%.*]] = sdiv i64 [[TMP102]], [[CONV186]]
+// CHECK1-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB188:%.*]] = sub i32 [[TMP108]], [[TMP109]]
+// CHECK1-NEXT:    [[SUB189:%.*]] = sub i32 [[SUB188]], 1
+// CHECK1-NEXT:    [[TMP110:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD190:%.*]] = add i32 [[SUB189]], [[TMP110]]
+// CHECK1-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV191:%.*]] = udiv i32 [[ADD190]], [[TMP111]]
+// CHECK1-NEXT:    [[MUL192:%.*]] = mul i32 1, [[DIV191]]
+// CHECK1-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB193:%.*]] = sub i32 [[TMP112]], -63
+// CHECK1-NEXT:    [[DIV194:%.*]] = udiv i32 [[SUB193]], 64
+// CHECK1-NEXT:    [[MUL195:%.*]] = mul i32 [[MUL192]], [[DIV194]]
+// CHECK1-NEXT:    [[MUL196:%.*]] = mul i32 [[MUL195]], 64
+// CHECK1-NEXT:    [[CONV197:%.*]] = zext i32 [[MUL196]] to i64
+// CHECK1-NEXT:    [[MUL198:%.*]] = mul nsw i64 [[DIV187]], [[CONV197]]
+// CHECK1-NEXT:    [[SUB199:%.*]] = sub nsw i64 [[TMP101]], [[MUL198]]
 // CHECK1-NEXT:    [[TMP113:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB187:%.*]] = sub i32 [[TMP113]], -63
-// CHECK1-NEXT:    [[DIV188:%.*]] = udiv i32 [[SUB187]], 64
-// CHECK1-NEXT:    [[MUL189:%.*]] = mul i32 1, [[DIV188]]
-// CHECK1-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP115:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB190:%.*]] = sub i32 [[TMP114]], [[TMP115]]
-// CHECK1-NEXT:    [[SUB191:%.*]] = sub i32 [[SUB190]], 1
-// CHECK1-NEXT:    [[ADD192:%.*]] = add i32 [[SUB191]], 1
-// CHECK1-NEXT:    [[DIV193:%.*]] = udiv i32 [[ADD192]], 1
-// CHECK1-NEXT:    [[MUL194:%.*]] = mul i32 [[MUL189]], [[DIV193]]
-// CHECK1-NEXT:    [[CONV195:%.*]] = zext i32 [[MUL194]] to i64
-// CHECK1-NEXT:    [[DIV196:%.*]] = sdiv i64 [[SUB186]], [[CONV195]]
-// CHECK1-NEXT:    [[TMP116:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB197:%.*]] = sub i32 [[TMP116]], -63
-// CHECK1-NEXT:    [[DIV198:%.*]] = udiv i32 [[SUB197]], 64
-// CHECK1-NEXT:    [[MUL199:%.*]] = mul i32 1, [[DIV198]]
-// CHECK1-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB200:%.*]] = sub i32 [[TMP117]], [[TMP118]]
-// CHECK1-NEXT:    [[SUB201:%.*]] = sub i32 [[SUB200]], 1
-// CHECK1-NEXT:    [[ADD202:%.*]] = add i32 [[SUB201]], 1
-// CHECK1-NEXT:    [[DIV203:%.*]] = udiv i32 [[ADD202]], 1
-// CHECK1-NEXT:    [[MUL204:%.*]] = mul i32 [[MUL199]], [[DIV203]]
-// CHECK1-NEXT:    [[CONV205:%.*]] = zext i32 [[MUL204]] to i64
-// CHECK1-NEXT:    [[MUL206:%.*]] = mul nsw i64 [[DIV196]], [[CONV205]]
-// CHECK1-NEXT:    [[SUB207:%.*]] = sub nsw i64 [[SUB155]], [[MUL206]]
-// CHECK1-NEXT:    [[TMP119:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP120:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB208:%.*]] = sub i32 [[TMP119]], [[TMP120]]
-// CHECK1-NEXT:    [[SUB209:%.*]] = sub i32 [[SUB208]], 1
-// CHECK1-NEXT:    [[ADD210:%.*]] = add i32 [[SUB209]], 1
-// CHECK1-NEXT:    [[DIV211:%.*]] = udiv i32 [[ADD210]], 1
-// CHECK1-NEXT:    [[MUL212:%.*]] = mul i32 1, [[DIV211]]
-// CHECK1-NEXT:    [[CONV213:%.*]] = zext i32 [[MUL212]] to i64
-// CHECK1-NEXT:    [[DIV214:%.*]] = sdiv i64 [[SUB207]], [[CONV213]]
-// CHECK1-NEXT:    [[MUL215:%.*]] = mul nsw i64 [[DIV214]], 64
-// CHECK1-NEXT:    [[ADD216:%.*]] = add nsw i64 0, [[MUL215]]
-// CHECK1-NEXT:    [[CONV217:%.*]] = trunc i64 [[ADD216]] to i32
-// CHECK1-NEXT:    store i32 [[CONV217]], ptr [[DOTFLOOR_0_IV_K51]], align 4
-// CHECK1-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[CONV218:%.*]] = zext i32 [[TMP121]] to i64
-// CHECK1-NEXT:    [[TMP122:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP123:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP124:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB219:%.*]] = sub i32 [[TMP124]], [[TMP125]]
-// CHECK1-NEXT:    [[SUB220:%.*]] = sub i32 [[SUB219]], 1
-// CHECK1-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD221:%.*]] = add i32 [[SUB220]], [[TMP126]]
-// CHECK1-NEXT:    [[TMP127:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV222:%.*]] = udiv i32 [[ADD221]], [[TMP127]]
-// CHECK1-NEXT:    [[MUL223:%.*]] = mul i32 1, [[DIV222]]
-// CHECK1-NEXT:    [[TMP128:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB224:%.*]] = sub i32 [[TMP128]], -63
-// CHECK1-NEXT:    [[DIV225:%.*]] = udiv i32 [[SUB224]], 64
-// CHECK1-NEXT:    [[MUL226:%.*]] = mul i32 [[MUL223]], [[DIV225]]
-// CHECK1-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB227:%.*]] = sub i32 [[TMP129]], [[TMP130]]
-// CHECK1-NEXT:    [[SUB228:%.*]] = sub i32 [[SUB227]], 1
-// CHECK1-NEXT:    [[ADD229:%.*]] = add i32 [[SUB228]], 1
-// CHECK1-NEXT:    [[DIV230:%.*]] = udiv i32 [[ADD229]], 1
-// CHECK1-NEXT:    [[MUL231:%.*]] = mul i32 [[MUL226]], [[DIV230]]
-// CHECK1-NEXT:    [[CONV232:%.*]] = zext i32 [[MUL231]] to i64
-// CHECK1-NEXT:    [[DIV233:%.*]] = sdiv i64 [[TMP123]], [[CONV232]]
-// CHECK1-NEXT:    [[TMP131:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP132:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB234:%.*]] = sub i32 [[TMP131]], [[TMP132]]
-// CHECK1-NEXT:    [[SUB235:%.*]] = sub i32 [[SUB234]], 1
-// CHECK1-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD236:%.*]] = add i32 [[SUB235]], [[TMP133]]
-// CHECK1-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV237:%.*]] = udiv i32 [[ADD236]], [[TMP134]]
-// CHECK1-NEXT:    [[MUL238:%.*]] = mul i32 1, [[DIV237]]
-// CHECK1-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB239:%.*]] = sub i32 [[TMP135]], -63
-// CHECK1-NEXT:    [[DIV240:%.*]] = udiv i32 [[SUB239]], 64
-// CHECK1-NEXT:    [[MUL241:%.*]] = mul i32 [[MUL238]], [[DIV240]]
-// CHECK1-NEXT:    [[TMP136:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB242:%.*]] = sub i32 [[TMP136]], [[TMP137]]
-// CHECK1-NEXT:    [[SUB243:%.*]] = sub i32 [[SUB242]], 1
-// CHECK1-NEXT:    [[ADD244:%.*]] = add i32 [[SUB243]], 1
-// CHECK1-NEXT:    [[DIV245:%.*]] = udiv i32 [[ADD244]], 1
-// CHECK1-NEXT:    [[MUL246:%.*]] = mul i32 [[MUL241]], [[DIV245]]
-// CHECK1-NEXT:    [[CONV247:%.*]] = zext i32 [[MUL246]] to i64
-// CHECK1-NEXT:    [[MUL248:%.*]] = mul nsw i64 [[DIV233]], [[CONV247]]
-// CHECK1-NEXT:    [[SUB249:%.*]] = sub nsw i64 [[TMP122]], [[MUL248]]
-// CHECK1-NEXT:    [[TMP138:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP139:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB250:%.*]] = sub i32 [[TMP140]], [[TMP141]]
-// CHECK1-NEXT:    [[SUB251:%.*]] = sub i32 [[SUB250]], 1
-// CHECK1-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD252:%.*]] = add i32 [[SUB251]], [[TMP142]]
-// CHECK1-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV253:%.*]] = udiv i32 [[ADD252]], [[TMP143]]
-// CHECK1-NEXT:    [[MUL254:%.*]] = mul i32 1, [[DIV253]]
-// CHECK1-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB255:%.*]] = sub i32 [[TMP144]], -63
-// CHECK1-NEXT:    [[DIV256:%.*]] = udiv i32 [[SUB255]], 64
-// CHECK1-NEXT:    [[MUL257:%.*]] = mul i32 [[MUL254]], [[DIV256]]
-// CHECK1-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB258:%.*]] = sub i32 [[TMP145]], [[TMP146]]
-// CHECK1-NEXT:    [[SUB259:%.*]] = sub i32 [[SUB258]], 1
-// CHECK1-NEXT:    [[ADD260:%.*]] = add i32 [[SUB259]], 1
-// CHECK1-NEXT:    [[DIV261:%.*]] = udiv i32 [[ADD260]], 1
-// CHECK1-NEXT:    [[MUL262:%.*]] = mul i32 [[MUL257]], [[DIV261]]
+// CHECK1-NEXT:    [[SUB200:%.*]] = sub i32 [[TMP113]], -63
+// CHECK1-NEXT:    [[DIV201:%.*]] = udiv i32 [[SUB200]], 64
+// CHECK1-NEXT:    [[MUL202:%.*]] = mul i32 1, [[DIV201]]
+// CHECK1-NEXT:    [[MUL203:%.*]] = mul i32 [[MUL202]], 64
+// CHECK1-NEXT:    [[CONV204:%.*]] = zext i32 [[MUL203]] to i64
+// CHECK1-NEXT:    [[DIV205:%.*]] = sdiv i64 [[SUB199]], [[CONV204]]
+// CHECK1-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB206:%.*]] = sub i32 [[TMP114]], -63
+// CHECK1-NEXT:    [[DIV207:%.*]] = udiv i32 [[SUB206]], 64
+// CHECK1-NEXT:    [[MUL208:%.*]] = mul i32 1, [[DIV207]]
+// CHECK1-NEXT:    [[MUL209:%.*]] = mul i32 [[MUL208]], 64
+// CHECK1-NEXT:    [[CONV210:%.*]] = zext i32 [[MUL209]] to i64
+// CHECK1-NEXT:    [[MUL211:%.*]] = mul nsw i64 [[DIV205]], [[CONV210]]
+// CHECK1-NEXT:    [[SUB212:%.*]] = sub nsw i64 [[SUB176]], [[MUL211]]
+// CHECK1-NEXT:    [[TMP115:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP116:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB213:%.*]] = sub i32 [[TMP117]], [[TMP118]]
+// CHECK1-NEXT:    [[SUB214:%.*]] = sub i32 [[SUB213]], 1
+// CHECK1-NEXT:    [[TMP119:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD215:%.*]] = add i32 [[SUB214]], [[TMP119]]
+// CHECK1-NEXT:    [[TMP120:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV216:%.*]] = udiv i32 [[ADD215]], [[TMP120]]
+// CHECK1-NEXT:    [[MUL217:%.*]] = mul i32 1, [[DIV216]]
+// CHECK1-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB218:%.*]] = sub i32 [[TMP121]], -63
+// CHECK1-NEXT:    [[DIV219:%.*]] = udiv i32 [[SUB218]], 64
+// CHECK1-NEXT:    [[MUL220:%.*]] = mul i32 [[MUL217]], [[DIV219]]
+// CHECK1-NEXT:    [[MUL221:%.*]] = mul i32 [[MUL220]], 64
+// CHECK1-NEXT:    [[CONV222:%.*]] = zext i32 [[MUL221]] to i64
+// CHECK1-NEXT:    [[DIV223:%.*]] = sdiv i64 [[TMP116]], [[CONV222]]
+// CHECK1-NEXT:    [[TMP122:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP123:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB224:%.*]] = sub i32 [[TMP122]], [[TMP123]]
+// CHECK1-NEXT:    [[SUB225:%.*]] = sub i32 [[SUB224]], 1
+// CHECK1-NEXT:    [[TMP124:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD226:%.*]] = add i32 [[SUB225]], [[TMP124]]
+// CHECK1-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV227:%.*]] = udiv i32 [[ADD226]], [[TMP125]]
+// CHECK1-NEXT:    [[MUL228:%.*]] = mul i32 1, [[DIV227]]
+// CHECK1-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB229:%.*]] = sub i32 [[TMP126]], -63
+// CHECK1-NEXT:    [[DIV230:%.*]] = udiv i32 [[SUB229]], 64
+// CHECK1-NEXT:    [[MUL231:%.*]] = mul i32 [[MUL228]], [[DIV230]]
+// CHECK1-NEXT:    [[MUL232:%.*]] = mul i32 [[MUL231]], 64
+// CHECK1-NEXT:    [[CONV233:%.*]] = zext i32 [[MUL232]] to i64
+// CHECK1-NEXT:    [[MUL234:%.*]] = mul nsw i64 [[DIV223]], [[CONV233]]
+// CHECK1-NEXT:    [[SUB235:%.*]] = sub nsw i64 [[TMP115]], [[MUL234]]
+// CHECK1-NEXT:    [[TMP127:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP128:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB236:%.*]] = sub i32 [[TMP129]], [[TMP130]]
+// CHECK1-NEXT:    [[SUB237:%.*]] = sub i32 [[SUB236]], 1
+// CHECK1-NEXT:    [[TMP131:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD238:%.*]] = add i32 [[SUB237]], [[TMP131]]
+// CHECK1-NEXT:    [[TMP132:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV239:%.*]] = udiv i32 [[ADD238]], [[TMP132]]
+// CHECK1-NEXT:    [[MUL240:%.*]] = mul i32 1, [[DIV239]]
+// CHECK1-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB241:%.*]] = sub i32 [[TMP133]], -63
+// CHECK1-NEXT:    [[DIV242:%.*]] = udiv i32 [[SUB241]], 64
+// CHECK1-NEXT:    [[MUL243:%.*]] = mul i32 [[MUL240]], [[DIV242]]
+// CHECK1-NEXT:    [[MUL244:%.*]] = mul i32 [[MUL243]], 64
+// CHECK1-NEXT:    [[CONV245:%.*]] = zext i32 [[MUL244]] to i64
+// CHECK1-NEXT:    [[DIV246:%.*]] = sdiv i64 [[TMP128]], [[CONV245]]
+// CHECK1-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB247:%.*]] = sub i32 [[TMP134]], [[TMP135]]
+// CHECK1-NEXT:    [[SUB248:%.*]] = sub i32 [[SUB247]], 1
+// CHECK1-NEXT:    [[TMP136:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD249:%.*]] = add i32 [[SUB248]], [[TMP136]]
+// CHECK1-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV250:%.*]] = udiv i32 [[ADD249]], [[TMP137]]
+// CHECK1-NEXT:    [[MUL251:%.*]] = mul i32 1, [[DIV250]]
+// CHECK1-NEXT:    [[TMP138:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB252:%.*]] = sub i32 [[TMP138]], -63
+// CHECK1-NEXT:    [[DIV253:%.*]] = udiv i32 [[SUB252]], 64
+// CHECK1-NEXT:    [[MUL254:%.*]] = mul i32 [[MUL251]], [[DIV253]]
+// CHECK1-NEXT:    [[MUL255:%.*]] = mul i32 [[MUL254]], 64
+// CHECK1-NEXT:    [[CONV256:%.*]] = zext i32 [[MUL255]] to i64
+// CHECK1-NEXT:    [[MUL257:%.*]] = mul nsw i64 [[DIV246]], [[CONV256]]
+// CHECK1-NEXT:    [[SUB258:%.*]] = sub nsw i64 [[TMP127]], [[MUL257]]
+// CHECK1-NEXT:    [[TMP139:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB259:%.*]] = sub i32 [[TMP139]], -63
+// CHECK1-NEXT:    [[DIV260:%.*]] = udiv i32 [[SUB259]], 64
+// CHECK1-NEXT:    [[MUL261:%.*]] = mul i32 1, [[DIV260]]
+// CHECK1-NEXT:    [[MUL262:%.*]] = mul i32 [[MUL261]], 64
 // CHECK1-NEXT:    [[CONV263:%.*]] = zext i32 [[MUL262]] to i64
-// CHECK1-NEXT:    [[DIV264:%.*]] = sdiv i64 [[TMP139]], [[CONV263]]
-// CHECK1-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB265:%.*]] = sub i32 [[TMP147]], [[TMP148]]
-// CHECK1-NEXT:    [[SUB266:%.*]] = sub i32 [[SUB265]], 1
-// CHECK1-NEXT:    [[TMP149:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD267:%.*]] = add i32 [[SUB266]], [[TMP149]]
-// CHECK1-NEXT:    [[TMP150:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV268:%.*]] = udiv i32 [[ADD267]], [[TMP150]]
-// CHECK1-NEXT:    [[MUL269:%.*]] = mul i32 1, [[DIV268]]
-// CHECK1-NEXT:    [[TMP151:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB270:%.*]] = sub i32 [[TMP151]], -63
-// CHECK1-NEXT:    [[DIV271:%.*]] = udiv i32 [[SUB270]], 64
-// CHECK1-NEXT:    [[MUL272:%.*]] = mul i32 [[MUL269]], [[DIV271]]
-// CHECK1-NEXT:    [[TMP152:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP153:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB273:%.*]] = sub i32 [[TMP152]], [[TMP153]]
-// CHECK1-NEXT:    [[SUB274:%.*]] = sub i32 [[SUB273]], 1
-// CHECK1-NEXT:    [[ADD275:%.*]] = add i32 [[SUB274]], 1
-// CHECK1-NEXT:    [[DIV276:%.*]] = udiv i32 [[ADD275]], 1
-// CHECK1-NEXT:    [[MUL277:%.*]] = mul i32 [[MUL272]], [[DIV276]]
-// CHECK1-NEXT:    [[CONV278:%.*]] = zext i32 [[MUL277]] to i64
-// CHECK1-NEXT:    [[MUL279:%.*]] = mul nsw i64 [[DIV264]], [[CONV278]]
-// CHECK1-NEXT:    [[SUB280:%.*]] = sub nsw i64 [[TMP138]], [[MUL279]]
-// CHECK1-NEXT:    [[TMP154:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB281:%.*]] = sub i32 [[TMP154]], -63
-// CHECK1-NEXT:    [[DIV282:%.*]] = udiv i32 [[SUB281]], 64
-// CHECK1-NEXT:    [[MUL283:%.*]] = mul i32 1, [[DIV282]]
-// CHECK1-NEXT:    [[TMP155:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP156:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB284:%.*]] = sub i32 [[TMP155]], [[TMP156]]
-// CHECK1-NEXT:    [[SUB285:%.*]] = sub i32 [[SUB284]], 1
-// CHECK1-NEXT:    [[ADD286:%.*]] = add i32 [[SUB285]], 1
-// CHECK1-NEXT:    [[DIV287:%.*]] = udiv i32 [[ADD286]], 1
-// CHECK1-NEXT:    [[MUL288:%.*]] = mul i32 [[MUL283]], [[DIV287]]
-// CHECK1-NEXT:    [[CONV289:%.*]] = zext i32 [[MUL288]] to i64
-// CHECK1-NEXT:    [[DIV290:%.*]] = sdiv i64 [[SUB280]], [[CONV289]]
-// CHECK1-NEXT:    [[TMP157:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB291:%.*]] = sub i32 [[TMP157]], -63
-// CHECK1-NEXT:    [[DIV292:%.*]] = udiv i32 [[SUB291]], 64
-// CHECK1-NEXT:    [[MUL293:%.*]] = mul i32 1, [[DIV292]]
-// CHECK1-NEXT:    [[TMP158:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP159:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB294:%.*]] = sub i32 [[TMP158]], [[TMP159]]
-// CHECK1-NEXT:    [[SUB295:%.*]] = sub i32 [[SUB294]], 1
-// CHECK1-NEXT:    [[ADD296:%.*]] = add i32 [[SUB295]], 1
-// CHECK1-NEXT:    [[DIV297:%.*]] = udiv i32 [[ADD296]], 1
-// CHECK1-NEXT:    [[MUL298:%.*]] = mul i32 [[MUL293]], [[DIV297]]
-// CHECK1-NEXT:    [[CONV299:%.*]] = zext i32 [[MUL298]] to i64
-// CHECK1-NEXT:    [[MUL300:%.*]] = mul nsw i64 [[DIV290]], [[CONV299]]
-// CHECK1-NEXT:    [[SUB301:%.*]] = sub nsw i64 [[SUB249]], [[MUL300]]
-// CHECK1-NEXT:    [[TMP160:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP161:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP162:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP163:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB302:%.*]] = sub i32 [[TMP162]], [[TMP163]]
-// CHECK1-NEXT:    [[SUB303:%.*]] = sub i32 [[SUB302]], 1
-// CHECK1-NEXT:    [[TMP164:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD304:%.*]] = add i32 [[SUB303]], [[TMP164]]
-// CHECK1-NEXT:    [[TMP165:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV305:%.*]] = udiv i32 [[ADD304]], [[TMP165]]
-// CHECK1-NEXT:    [[MUL306:%.*]] = mul i32 1, [[DIV305]]
-// CHECK1-NEXT:    [[TMP166:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB307:%.*]] = sub i32 [[TMP166]], -63
-// CHECK1-NEXT:    [[DIV308:%.*]] = udiv i32 [[SUB307]], 64
-// CHECK1-NEXT:    [[MUL309:%.*]] = mul i32 [[MUL306]], [[DIV308]]
-// CHECK1-NEXT:    [[TMP167:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP168:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB310:%.*]] = sub i32 [[TMP167]], [[TMP168]]
-// CHECK1-NEXT:    [[SUB311:%.*]] = sub i32 [[SUB310]], 1
-// CHECK1-NEXT:    [[ADD312:%.*]] = add i32 [[SUB311]], 1
-// CHECK1-NEXT:    [[DIV313:%.*]] = udiv i32 [[ADD312]], 1
-// CHECK1-NEXT:    [[MUL314:%.*]] = mul i32 [[MUL309]], [[DIV313]]
-// CHECK1-NEXT:    [[CONV315:%.*]] = zext i32 [[MUL314]] to i64
-// CHECK1-NEXT:    [[DIV316:%.*]] = sdiv i64 [[TMP161]], [[CONV315]]
-// CHECK1-NEXT:    [[TMP169:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP170:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB317:%.*]] = sub i32 [[TMP169]], [[TMP170]]
-// CHECK1-NEXT:    [[SUB318:%.*]] = sub i32 [[SUB317]], 1
-// CHECK1-NEXT:    [[TMP171:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD319:%.*]] = add i32 [[SUB318]], [[TMP171]]
-// CHECK1-NEXT:    [[TMP172:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV320:%.*]] = udiv i32 [[ADD319]], [[TMP172]]
-// CHECK1-NEXT:    [[MUL321:%.*]] = mul i32 1, [[DIV320]]
-// CHECK1-NEXT:    [[TMP173:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB322:%.*]] = sub i32 [[TMP173]], -63
-// CHECK1-NEXT:    [[DIV323:%.*]] = udiv i32 [[SUB322]], 64
-// CHECK1-NEXT:    [[MUL324:%.*]] = mul i32 [[MUL321]], [[DIV323]]
-// CHECK1-NEXT:    [[TMP174:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP175:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB325:%.*]] = sub i32 [[TMP174]], [[TMP175]]
-// CHECK1-NEXT:    [[SUB326:%.*]] = sub i32 [[SUB325]], 1
-// CHECK1-NEXT:    [[ADD327:%.*]] = add i32 [[SUB326]], 1
-// CHECK1-NEXT:    [[DIV328:%.*]] = udiv i32 [[ADD327]], 1
-// CHECK1-NEXT:    [[MUL329:%.*]] = mul i32 [[MUL324]], [[DIV328]]
-// CHECK1-NEXT:    [[CONV330:%.*]] = zext i32 [[MUL329]] to i64
-// CHECK1-NEXT:    [[MUL331:%.*]] = mul nsw i64 [[DIV316]], [[CONV330]]
-// CHECK1-NEXT:    [[SUB332:%.*]] = sub nsw i64 [[TMP160]], [[MUL331]]
-// CHECK1-NEXT:    [[TMP176:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP177:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP178:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP179:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB333:%.*]] = sub i32 [[TMP178]], [[TMP179]]
-// CHECK1-NEXT:    [[SUB334:%.*]] = sub i32 [[SUB333]], 1
-// CHECK1-NEXT:    [[TMP180:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD335:%.*]] = add i32 [[SUB334]], [[TMP180]]
-// CHECK1-NEXT:    [[TMP181:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV336:%.*]] = udiv i32 [[ADD335]], [[TMP181]]
-// CHECK1-NEXT:    [[MUL337:%.*]] = mul i32 1, [[DIV336]]
-// CHECK1-NEXT:    [[TMP182:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB338:%.*]] = sub i32 [[TMP182]], -63
-// CHECK1-NEXT:    [[DIV339:%.*]] = udiv i32 [[SUB338]], 64
-// CHECK1-NEXT:    [[MUL340:%.*]] = mul i32 [[MUL337]], [[DIV339]]
-// CHECK1-NEXT:    [[TMP183:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP184:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB341:%.*]] = sub i32 [[TMP183]], [[TMP184]]
-// CHECK1-NEXT:    [[SUB342:%.*]] = sub i32 [[SUB341]], 1
-// CHECK1-NEXT:    [[ADD343:%.*]] = add i32 [[SUB342]], 1
-// CHECK1-NEXT:    [[DIV344:%.*]] = udiv i32 [[ADD343]], 1
-// CHECK1-NEXT:    [[MUL345:%.*]] = mul i32 [[MUL340]], [[DIV344]]
-// CHECK1-NEXT:    [[CONV346:%.*]] = zext i32 [[MUL345]] to i64
-// CHECK1-NEXT:    [[DIV347:%.*]] = sdiv i64 [[TMP177]], [[CONV346]]
-// CHECK1-NEXT:    [[TMP185:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP186:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB348:%.*]] = sub i32 [[TMP185]], [[TMP186]]
-// CHECK1-NEXT:    [[SUB349:%.*]] = sub i32 [[SUB348]], 1
-// CHECK1-NEXT:    [[TMP187:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD350:%.*]] = add i32 [[SUB349]], [[TMP187]]
-// CHECK1-NEXT:    [[TMP188:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV351:%.*]] = udiv i32 [[ADD350]], [[TMP188]]
-// CHECK1-NEXT:    [[MUL352:%.*]] = mul i32 1, [[DIV351]]
-// CHECK1-NEXT:    [[TMP189:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB353:%.*]] = sub i32 [[TMP189]], -63
-// CHECK1-NEXT:    [[DIV354:%.*]] = udiv i32 [[SUB353]], 64
-// CHECK1-NEXT:    [[MUL355:%.*]] = mul i32 [[MUL352]], [[DIV354]]
-// CHECK1-NEXT:    [[TMP190:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP191:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB356:%.*]] = sub i32 [[TMP190]], [[TMP191]]
-// CHECK1-NEXT:    [[SUB357:%.*]] = sub i32 [[SUB356]], 1
-// CHECK1-NEXT:    [[ADD358:%.*]] = add i32 [[SUB357]], 1
-// CHECK1-NEXT:    [[DIV359:%.*]] = udiv i32 [[ADD358]], 1
-// CHECK1-NEXT:    [[MUL360:%.*]] = mul i32 [[MUL355]], [[DIV359]]
-// CHECK1-NEXT:    [[CONV361:%.*]] = zext i32 [[MUL360]] to i64
-// CHECK1-NEXT:    [[MUL362:%.*]] = mul nsw i64 [[DIV347]], [[CONV361]]
-// CHECK1-NEXT:    [[SUB363:%.*]] = sub nsw i64 [[TMP176]], [[MUL362]]
-// CHECK1-NEXT:    [[TMP192:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB364:%.*]] = sub i32 [[TMP192]], -63
-// CHECK1-NEXT:    [[DIV365:%.*]] = udiv i32 [[SUB364]], 64
-// CHECK1-NEXT:    [[MUL366:%.*]] = mul i32 1, [[DIV365]]
-// CHECK1-NEXT:    [[TMP193:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP194:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB367:%.*]] = sub i32 [[TMP193]], [[TMP194]]
-// CHECK1-NEXT:    [[SUB368:%.*]] = sub i32 [[SUB367]], 1
-// CHECK1-NEXT:    [[ADD369:%.*]] = add i32 [[SUB368]], 1
-// CHECK1-NEXT:    [[DIV370:%.*]] = udiv i32 [[ADD369]], 1
-// CHECK1-NEXT:    [[MUL371:%.*]] = mul i32 [[MUL366]], [[DIV370]]
-// CHECK1-NEXT:    [[CONV372:%.*]] = zext i32 [[MUL371]] to i64
-// CHECK1-NEXT:    [[DIV373:%.*]] = sdiv i64 [[SUB363]], [[CONV372]]
-// CHECK1-NEXT:    [[TMP195:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB374:%.*]] = sub i32 [[TMP195]], -63
-// CHECK1-NEXT:    [[DIV375:%.*]] = udiv i32 [[SUB374]], 64
-// CHECK1-NEXT:    [[MUL376:%.*]] = mul i32 1, [[DIV375]]
-// CHECK1-NEXT:    [[TMP196:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP197:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB377:%.*]] = sub i32 [[TMP196]], [[TMP197]]
-// CHECK1-NEXT:    [[SUB378:%.*]] = sub i32 [[SUB377]], 1
-// CHECK1-NEXT:    [[ADD379:%.*]] = add i32 [[SUB378]], 1
-// CHECK1-NEXT:    [[DIV380:%.*]] = udiv i32 [[ADD379]], 1
-// CHECK1-NEXT:    [[MUL381:%.*]] = mul i32 [[MUL376]], [[DIV380]]
-// CHECK1-NEXT:    [[CONV382:%.*]] = zext i32 [[MUL381]] to i64
-// CHECK1-NEXT:    [[MUL383:%.*]] = mul nsw i64 [[DIV373]], [[CONV382]]
-// CHECK1-NEXT:    [[SUB384:%.*]] = sub nsw i64 [[SUB332]], [[MUL383]]
-// CHECK1-NEXT:    [[TMP198:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP199:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB385:%.*]] = sub i32 [[TMP198]], [[TMP199]]
-// CHECK1-NEXT:    [[SUB386:%.*]] = sub i32 [[SUB385]], 1
-// CHECK1-NEXT:    [[ADD387:%.*]] = add i32 [[SUB386]], 1
-// CHECK1-NEXT:    [[DIV388:%.*]] = udiv i32 [[ADD387]], 1
-// CHECK1-NEXT:    [[MUL389:%.*]] = mul i32 1, [[DIV388]]
-// CHECK1-NEXT:    [[CONV390:%.*]] = zext i32 [[MUL389]] to i64
-// CHECK1-NEXT:    [[DIV391:%.*]] = sdiv i64 [[SUB384]], [[CONV390]]
-// CHECK1-NEXT:    [[TMP200:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP201:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB392:%.*]] = sub i32 [[TMP200]], [[TMP201]]
-// CHECK1-NEXT:    [[SUB393:%.*]] = sub i32 [[SUB392]], 1
-// CHECK1-NEXT:    [[ADD394:%.*]] = add i32 [[SUB393]], 1
-// CHECK1-NEXT:    [[DIV395:%.*]] = udiv i32 [[ADD394]], 1
-// CHECK1-NEXT:    [[MUL396:%.*]] = mul i32 1, [[DIV395]]
-// CHECK1-NEXT:    [[CONV397:%.*]] = zext i32 [[MUL396]] to i64
-// CHECK1-NEXT:    [[MUL398:%.*]] = mul nsw i64 [[DIV391]], [[CONV397]]
-// CHECK1-NEXT:    [[SUB399:%.*]] = sub nsw i64 [[SUB301]], [[MUL398]]
-// CHECK1-NEXT:    [[MUL400:%.*]] = mul nsw i64 [[SUB399]], 1
-// CHECK1-NEXT:    [[ADD401:%.*]] = add nsw i64 [[CONV218]], [[MUL400]]
-// CHECK1-NEXT:    [[CONV402:%.*]] = trunc i64 [[ADD401]] to i32
-// CHECK1-NEXT:    store i32 [[CONV402]], ptr [[DOTTILE_0_IV_K52]], align 4
-// CHECK1-NEXT:    [[TMP202:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
-// CHECK1-NEXT:    [[TMP203:%.*]] = load i32, ptr [[DOTTILE_0_IV_K52]], align 4
-// CHECK1-NEXT:    [[TMP204:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
-// CHECK1-NEXT:    [[MUL403:%.*]] = mul i32 [[TMP203]], [[TMP204]]
-// CHECK1-NEXT:    [[ADD404:%.*]] = add i32 [[TMP202]], [[MUL403]]
-// CHECK1-NEXT:    store i32 [[ADD404]], ptr [[K]], align 4
-// CHECK1-NEXT:    [[TMP205:%.*]] = load i32, ptr [[I49]], align 4
-// CHECK1-NEXT:    [[TMP206:%.*]] = load i32, ptr [[J50]], align 4
-// CHECK1-NEXT:    [[TMP207:%.*]] = load i32, ptr [[K]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP205]], i32 noundef [[TMP206]], i32 noundef [[TMP207]])
+// CHECK1-NEXT:    [[DIV264:%.*]] = sdiv i64 [[SUB258]], [[CONV263]]
+// CHECK1-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB265:%.*]] = sub i32 [[TMP140]], -63
+// CHECK1-NEXT:    [[DIV266:%.*]] = udiv i32 [[SUB265]], 64
+// CHECK1-NEXT:    [[MUL267:%.*]] = mul i32 1, [[DIV266]]
+// CHECK1-NEXT:    [[MUL268:%.*]] = mul i32 [[MUL267]], 64
+// CHECK1-NEXT:    [[CONV269:%.*]] = zext i32 [[MUL268]] to i64
+// CHECK1-NEXT:    [[MUL270:%.*]] = mul nsw i64 [[DIV264]], [[CONV269]]
+// CHECK1-NEXT:    [[SUB271:%.*]] = sub nsw i64 [[SUB235]], [[MUL270]]
+// CHECK1-NEXT:    [[DIV272:%.*]] = sdiv i64 [[SUB271]], 64
+// CHECK1-NEXT:    [[MUL273:%.*]] = mul nsw i64 [[DIV272]], 64
+// CHECK1-NEXT:    [[SUB274:%.*]] = sub nsw i64 [[SUB212]], [[MUL273]]
+// CHECK1-NEXT:    [[MUL275:%.*]] = mul nsw i64 [[SUB274]], 1
+// CHECK1-NEXT:    [[ADD276:%.*]] = add nsw i64 0, [[MUL275]]
+// CHECK1-NEXT:    [[CONV277:%.*]] = trunc i64 [[ADD276]] to i32
+// CHECK1-NEXT:    store i32 [[CONV277]], ptr [[DOTTILE_CNT_0_IV_K38]], align 4
+// CHECK1-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
+// CHECK1-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
+// CHECK1-NEXT:    [[ADD278:%.*]] = add i32 [[TMP141]], [[TMP142]]
+// CHECK1-NEXT:    store i32 [[ADD278]], ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK1-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
+// CHECK1-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK1-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
+// CHECK1-NEXT:    [[MUL279:%.*]] = mul i32 [[TMP144]], [[TMP145]]
+// CHECK1-NEXT:    [[ADD280:%.*]] = add i32 [[TMP143]], [[MUL279]]
+// CHECK1-NEXT:    store i32 [[ADD280]], ptr [[K]], align 4
+// CHECK1-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK1-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// CHECK1-NEXT:    [[ADD281:%.*]] = add i32 [[TMP147]], 1
+// CHECK1-NEXT:    [[CMP282:%.*]] = icmp ult i32 [[TMP146]], [[ADD281]]
+// CHECK1-NEXT:    br i1 [[CMP282]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK1:       if.then:
+// CHECK1-NEXT:    [[TMP148:%.*]] = load i32, ptr [[I35]], align 4
+// CHECK1-NEXT:    [[TMP149:%.*]] = load i32, ptr [[J36]], align 4
+// CHECK1-NEXT:    [[TMP150:%.*]] = load i32, ptr [[K]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP148]], i32 noundef [[TMP149]], i32 noundef [[TMP150]])
+// CHECK1-NEXT:    br label [[IF_END]]
+// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP208:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[ADD405:%.*]] = add nsw i64 [[TMP208]], 1
-// CHECK1-NEXT:    store i64 [[ADD405]], ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP151:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[ADD283:%.*]] = add nsw i64 [[TMP151]], 1
+// CHECK1-NEXT:    store i64 [[ADD283]], ptr [[DOTOMP_IV]], align 8
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -2225,14 +1868,14 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[V:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    store double 4.200000e+01, ptr [[C]], align 8
 // CHECK1-NEXT:    store ptr [[ARR]], ptr [[__RANGE2]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META12:![0-9]+]], !align [[META13:![0-9]+]]
 // CHECK1-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP0]], i64 0, i64 0
 // CHECK1-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 128
 // CHECK1-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META12]], !align [[META13]]
 // CHECK1-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP1]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[ARRAYDECAY1]], ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META12]], !align [[META13]]
 // CHECK1-NEXT:    [[ARRAYDECAY2:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP2]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[ARRAYDECAY2]], ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__END2]], align 8
@@ -2277,7 +1920,7 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
 // CHECK1-NEXT:    store ptr [[TMP12]], ptr [[V]], align 8
 // CHECK1-NEXT:    [[TMP13:%.*]] = load double, ptr [[C]], align 8
-// CHECK1-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[V]], align 8
+// CHECK1-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[V]], align 8, !nonnull [[META12]], !align [[META13]]
 // CHECK1-NEXT:    [[TMP15:%.*]] = load double, ptr [[TMP14]], align 8
 // CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    call void (...) @body(double noundef [[TMP13]], double noundef [[TMP15]], i32 noundef [[TMP16]])
@@ -2286,14 +1929,14 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[TMP17:%.*]] = load i64, ptr [[DOTPERMUTED_1_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP17]], 1
 // CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTPERMUTED_1_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP13:![0-9]+]]
+// CHECK1-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP14:![0-9]+]]
 // CHECK1:       for.end:
 // CHECK1-NEXT:    br label [[FOR_INC13:%.*]]
 // CHECK1:       for.inc13:
 // CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_I]], align 4
 // CHECK1-NEXT:    [[INC14:%.*]] = add nsw i32 [[TMP18]], 1
 // CHECK1-NEXT:    store i32 [[INC14]], ptr [[DOTPERMUTED_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP14:![0-9]+]]
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP15:![0-9]+]]
 // CHECK1:       for.end15:
 // CHECK1-NEXT:    ret void
 //
@@ -2342,14 +1985,14 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK1-NEXT:    store double 4.200000e+01, ptr [[C]], align 8
 // CHECK1-NEXT:    store ptr [[A]], ptr [[__RANGE3]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE3]], align 8
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE3]], align 8, !nonnull [[META12]], !align [[META13]]
 // CHECK1-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP1]], i64 0, i64 0
 // CHECK1-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 128
 // CHECK1-NEXT:    store ptr [[ADD_PTR]], ptr [[__END3]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE3]], align 8
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE3]], align 8, !nonnull [[META12]], !align [[META13]]
 // CHECK1-NEXT:    [[ARRAYDECAY4:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP2]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[ARRAYDECAY4]], ptr [[__BEGIN3]], align 8
-// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__RANGE3]], align 8
+// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__RANGE3]], align 8, !nonnull [[META12]], !align [[META13]]
 // CHECK1-NEXT:    [[ARRAYDECAY5:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP3]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[ARRAYDECAY5]], ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[__END3]], align 8
@@ -2367,14 +2010,14 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    store i64 [[SUB8]], ptr [[DOTCAPTURE_EXPR_7]], align 8
 // CHECK1-NEXT:    store double 4.200000e+01, ptr [[D]], align 8
 // CHECK1-NEXT:    store ptr [[B]], ptr [[__RANGE4]], align 8
-// CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[__RANGE4]], align 8
+// CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[__RANGE4]], align 8, !nonnull [[META12]], !align [[META13]]
 // CHECK1-NEXT:    [[ARRAYDECAY9:%.*]] = getelementptr inbounds [16 x double], ptr [[TMP7]], i64 0, i64 0
 // CHECK1-NEXT:    [[ADD_PTR10:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY9]], i64 16
 // CHECK1-NEXT:    store ptr [[ADD_PTR10]], ptr [[__END4]], align 8
-// CHECK1-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[__RANGE4]], align 8
+// CHECK1-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[__RANGE4]], align 8, !nonnull [[META12]], !align [[META13]]
 // CHECK1-NEXT:    [[ARRAYDECAY11:%.*]] = getelementptr inbounds [16 x double], ptr [[TMP8]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[ARRAYDECAY11]], ptr [[__BEGIN4]], align 8
-// CHECK1-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[__RANGE4]], align 8
+// CHECK1-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[__RANGE4]], align 8, !nonnull [[META12]], !align [[META13]]
 // CHECK1-NEXT:    [[ARRAYDECAY13:%.*]] = getelementptr inbounds [16 x double], ptr [[TMP9]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[ARRAYDECAY13]], ptr [[DOTCAPTURE_EXPR_12]], align 8
 // CHECK1-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[__END4]], align 8
@@ -2701,7 +2344,7 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[TMP87:%.*]] = load double, ptr [[C]], align 8
 // CHECK1-NEXT:    [[TMP88:%.*]] = load double, ptr [[AA]], align 8
 // CHECK1-NEXT:    [[TMP89:%.*]] = load double, ptr [[D]], align 8
-// CHECK1-NEXT:    [[TMP90:%.*]] = load ptr, ptr [[BB]], align 8
+// CHECK1-NEXT:    [[TMP90:%.*]] = load ptr, ptr [[BB]], align 8, !nonnull [[META12]], !align [[META13]]
 // CHECK1-NEXT:    [[TMP91:%.*]] = load double, ptr [[TMP90]], align 8
 // CHECK1-NEXT:    [[TMP92:%.*]] = load i32, ptr [[J40]], align 4
 // CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP86]], double noundef [[TMP87]], double noundef [[TMP88]], double noundef [[TMP89]], double noundef [[TMP91]], i32 noundef [[TMP92]])
@@ -2756,7 +2399,7 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[I]], align 4
 // CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP5]], [[TMP4]]
 // CHECK2-NEXT:    store i32 [[ADD]], ptr [[I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP3:![0-9]+]]
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP2:![0-9]+]]
 // CHECK2:       for.end:
 // CHECK2-NEXT:    ret void
 //
@@ -2805,14 +2448,14 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
 // CHECK2-NEXT:    store double 4.200000e+01, ptr [[C]], align 8
 // CHECK2-NEXT:    store ptr [[A]], ptr [[__RANGE3]], align 8
-// CHECK2-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE3]], align 8
+// CHECK2-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE3]], align 8, !nonnull [[META4:![0-9]+]], !align [[META5:![0-9]+]]
 // CHECK2-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP1]], i64 0, i64 0
 // CHECK2-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 128
 // CHECK2-NEXT:    store ptr [[ADD_PTR]], ptr [[__END3]], align 8
-// CHECK2-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE3]], align 8
+// CHECK2-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE3]], align 8, !nonnull [[META4]], !align [[META5]]
 // CHECK2-NEXT:    [[ARRAYDECAY4:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP2]], i64 0, i64 0
 // CHECK2-NEXT:    store ptr [[ARRAYDECAY4]], ptr [[__BEGIN3]], align 8
-// CHECK2-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__RANGE3]], align 8
+// CHECK2-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__RANGE3]], align 8, !nonnull [[META4]], !align [[META5]]
 // CHECK2-NEXT:    [[ARRAYDECAY5:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP3]], i64 0, i64 0
 // CHECK2-NEXT:    store ptr [[ARRAYDECAY5]], ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK2-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[__END3]], align 8
@@ -2830,14 +2473,14 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    store i64 [[SUB8]], ptr [[DOTCAPTURE_EXPR_7]], align 8
 // CHECK2-NEXT:    store double 4.200000e+01, ptr [[D]], align 8
 // CHECK2-NEXT:    store ptr [[B]], ptr [[__RANGE4]], align 8
-// CHECK2-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[__RANGE4]], align 8
+// CHECK2-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[__RANGE4]], align 8, !nonnull [[META4]], !align [[META5]]
 // CHECK2-NEXT:    [[ARRAYDECAY9:%.*]] = getelementptr inbounds [16 x double], ptr [[TMP7]], i64 0, i64 0
 // CHECK2-NEXT:    [[ADD_PTR10:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY9]], i64 16
 // CHECK2-NEXT:    store ptr [[ADD_PTR10]], ptr [[__END4]], align 8
-// CHECK2-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[__RANGE4]], align 8
+// CHECK2-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[__RANGE4]], align 8, !nonnull [[META4]], !align [[META5]]
 // CHECK2-NEXT:    [[ARRAYDECAY11:%.*]] = getelementptr inbounds [16 x double], ptr [[TMP8]], i64 0, i64 0
 // CHECK2-NEXT:    store ptr [[ARRAYDECAY11]], ptr [[__BEGIN4]], align 8
-// CHECK2-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[__RANGE4]], align 8
+// CHECK2-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[__RANGE4]], align 8, !nonnull [[META4]], !align [[META5]]
 // CHECK2-NEXT:    [[ARRAYDECAY13:%.*]] = getelementptr inbounds [16 x double], ptr [[TMP9]], i64 0, i64 0
 // CHECK2-NEXT:    store ptr [[ARRAYDECAY13]], ptr [[DOTCAPTURE_EXPR_12]], align 8
 // CHECK2-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[__END4]], align 8
@@ -3164,7 +2807,7 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[TMP87:%.*]] = load double, ptr [[C]], align 8
 // CHECK2-NEXT:    [[TMP88:%.*]] = load double, ptr [[AA]], align 8
 // CHECK2-NEXT:    [[TMP89:%.*]] = load double, ptr [[D]], align 8
-// CHECK2-NEXT:    [[TMP90:%.*]] = load ptr, ptr [[BB]], align 8
+// CHECK2-NEXT:    [[TMP90:%.*]] = load ptr, ptr [[BB]], align 8, !nonnull [[META4]], !align [[META5]]
 // CHECK2-NEXT:    [[TMP91:%.*]] = load double, ptr [[TMP90]], align 8
 // CHECK2-NEXT:    [[TMP92:%.*]] = load i32, ptr [[J40]], align 4
 // CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP86]], double noundef [[TMP87]], double noundef [[TMP88]], double noundef [[TMP89]], double noundef [[TMP91]], i32 noundef [[TMP92]])
@@ -3287,14 +2930,14 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP28]], 1
 // CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTPERMUTED_1_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND16]], !llvm.loop [[LOOP5:![0-9]+]]
+// CHECK2-NEXT:    br label [[FOR_COND16]], !llvm.loop [[LOOP6:![0-9]+]]
 // CHECK2:       for.end:
 // CHECK2-NEXT:    br label [[FOR_INC22:%.*]]
 // CHECK2:       for.inc22:
 // CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_J]], align 4
 // CHECK2-NEXT:    [[INC23:%.*]] = add i32 [[TMP29]], 1
 // CHECK2-NEXT:    store i32 [[INC23]], ptr [[DOTPERMUTED_0_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP7:![0-9]+]]
 // CHECK2:       for.end24:
 // CHECK2-NEXT:    ret void
 //
@@ -3367,7 +3010,7 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP12]], 1
 // CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTPERMUTED_1_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP7:![0-9]+]]
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
 // CHECK2:       for.end:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
@@ -3464,7 +3107,7 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
 // CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTPERMUTED_1_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP9:![0-9]+]]
 // CHECK2:       for.end:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
@@ -3779,28 +3422,28 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTPERMUTED_3_IV_I]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP12]], 1
 // CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTPERMUTED_3_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND11]], !llvm.loop [[LOOP9:![0-9]+]]
+// CHECK2-NEXT:    br label [[FOR_COND11]], !llvm.loop [[LOOP10:![0-9]+]]
 // CHECK2:       for.end:
 // CHECK2-NEXT:    br label [[FOR_INC16:%.*]]
 // CHECK2:       for.inc16:
 // CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTPERMUTED_2_IV_L]], align 4
 // CHECK2-NEXT:    [[INC17:%.*]] = add nsw i32 [[TMP13]], 1
 // CHECK2-NEXT:    store i32 [[INC17]], ptr [[DOTPERMUTED_2_IV_L]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP10:![0-9]+]]
+// CHECK2-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP11:![0-9]+]]
 // CHECK2:       for.end18:
 // CHECK2-NEXT:    br label [[FOR_INC19:%.*]]
 // CHECK2:       for.inc19:
 // CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_K]], align 4
 // CHECK2-NEXT:    [[INC20:%.*]] = add nsw i32 [[TMP14]], 1
 // CHECK2-NEXT:    store i32 [[INC20]], ptr [[DOTPERMUTED_1_IV_K]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP11:![0-9]+]]
+// CHECK2-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP12:![0-9]+]]
 // CHECK2:       for.end21:
 // CHECK2-NEXT:    br label [[FOR_INC22:%.*]]
 // CHECK2:       for.inc22:
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_J]], align 4
 // CHECK2-NEXT:    [[INC23:%.*]] = add nsw i32 [[TMP15]], 1
 // CHECK2-NEXT:    store i32 [[INC23]], ptr [[DOTPERMUTED_0_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP12:![0-9]+]]
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP13:![0-9]+]]
 // CHECK2:       for.end24:
 // CHECK2-NEXT:    ret void
 //
@@ -3822,14 +3465,14 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[V:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    store double 4.200000e+01, ptr [[C]], align 8
 // CHECK2-NEXT:    store ptr [[ARR]], ptr [[__RANGE2]], align 8
-// CHECK2-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META4]], !align [[META5]]
 // CHECK2-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP0]], i64 0, i64 0
 // CHECK2-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 128
 // CHECK2-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
-// CHECK2-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META4]], !align [[META5]]
 // CHECK2-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP1]], i64 0, i64 0
 // CHECK2-NEXT:    store ptr [[ARRAYDECAY1]], ptr [[__BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META4]], !align [[META5]]
 // CHECK2-NEXT:    [[ARRAYDECAY2:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP2]], i64 0, i64 0
 // CHECK2-NEXT:    store ptr [[ARRAYDECAY2]], ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK2-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__END2]], align 8
@@ -3874,7 +3517,7 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
 // CHECK2-NEXT:    store ptr [[TMP12]], ptr [[V]], align 8
 // CHECK2-NEXT:    [[TMP13:%.*]] = load double, ptr [[C]], align 8
-// CHECK2-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[V]], align 8
+// CHECK2-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[V]], align 8, !nonnull [[META4]], !align [[META5]]
 // CHECK2-NEXT:    [[TMP15:%.*]] = load double, ptr [[TMP14]], align 8
 // CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
 // CHECK2-NEXT:    call void (...) @body(double noundef [[TMP13]], double noundef [[TMP15]], i32 noundef [[TMP16]])
@@ -3883,14 +3526,14 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[TMP17:%.*]] = load i64, ptr [[DOTPERMUTED_1_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP17]], 1
 // CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTPERMUTED_1_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP13:![0-9]+]]
+// CHECK2-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP14:![0-9]+]]
 // CHECK2:       for.end:
 // CHECK2-NEXT:    br label [[FOR_INC13:%.*]]
 // CHECK2:       for.inc13:
 // CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_I]], align 4
 // CHECK2-NEXT:    [[INC14:%.*]] = add nsw i32 [[TMP18]], 1
 // CHECK2-NEXT:    store i32 [[INC14]], ptr [[DOTPERMUTED_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP14:![0-9]+]]
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP15:![0-9]+]]
 // CHECK2:       for.end15:
 // CHECK2-NEXT:    ret void
 //
@@ -3925,22 +3568,21 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTNEW_STEP10:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_14:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_17:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_22:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_K:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_K:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[I49:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[J50:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV_K51:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_K52:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[I35:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[J36:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV_K37:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_K38:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
@@ -3978,630 +3620,452 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
 // CHECK2-NEXT:    [[ADD15:%.*]] = add i32 [[TMP15]], 1
 // CHECK2-NEXT:    store i32 [[ADD15]], ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[_TMP2]], align 4
-// CHECK2-NEXT:    store i32 [[TMP16]], ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK2-NEXT:    [[ADD18:%.*]] = add i32 [[TMP17]], 1
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[_TMP2]], align 4
-// CHECK2-NEXT:    [[ADD19:%.*]] = add i32 [[TMP18]], 32
-// CHECK2-NEXT:    [[CMP:%.*]] = icmp ult i32 [[ADD18]], [[ADD19]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK2:       cond.true:
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK2-NEXT:    [[ADD20:%.*]] = add i32 [[TMP19]], 1
-// CHECK2-NEXT:    br label [[COND_END:%.*]]
-// CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[_TMP2]], align 4
-// CHECK2-NEXT:    [[ADD21:%.*]] = add i32 [[TMP20]], 32
-// CHECK2-NEXT:    br label [[COND_END]]
-// CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD20]], [[COND_TRUE]] ], [ [[ADD21]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    store i32 [[COND]], ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[SUB23:%.*]] = sub i32 [[TMP21]], [[TMP22]]
-// CHECK2-NEXT:    [[SUB24:%.*]] = sub i32 [[SUB23]], 1
-// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[ADD25:%.*]] = add i32 [[SUB24]], [[TMP23]]
-// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[DIV26:%.*]] = udiv i32 [[ADD25]], [[TMP24]]
-// CHECK2-NEXT:    [[CONV:%.*]] = zext i32 [[DIV26]] to i64
-// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[SUB17:%.*]] = sub i32 [[TMP16]], [[TMP17]]
+// CHECK2-NEXT:    [[SUB18:%.*]] = sub i32 [[SUB17]], 1
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[ADD19:%.*]] = add i32 [[SUB18]], [[TMP18]]
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[DIV20:%.*]] = udiv i32 [[ADD19]], [[TMP19]]
+// CHECK2-NEXT:    [[CONV:%.*]] = zext i32 [[DIV20]] to i64
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB21:%.*]] = sub i32 [[TMP20]], [[TMP21]]
+// CHECK2-NEXT:    [[SUB22:%.*]] = sub i32 [[SUB21]], 1
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD23:%.*]] = add i32 [[SUB22]], [[TMP22]]
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV24:%.*]] = udiv i32 [[ADD23]], [[TMP23]]
+// CHECK2-NEXT:    [[CONV25:%.*]] = zext i32 [[DIV24]] to i64
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV25]]
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB26:%.*]] = sub i32 [[TMP24]], -31
+// CHECK2-NEXT:    [[DIV27:%.*]] = udiv i32 [[SUB26]], 32
+// CHECK2-NEXT:    [[CONV28:%.*]] = zext i32 [[DIV27]] to i64
+// CHECK2-NEXT:    [[MUL29:%.*]] = mul nsw i64 [[MUL]], [[CONV28]]
+// CHECK2-NEXT:    [[MUL30:%.*]] = mul nsw i64 [[MUL29]], 32
+// CHECK2-NEXT:    [[SUB31:%.*]] = sub nsw i64 [[MUL30]], 1
+// CHECK2-NEXT:    store i64 [[SUB31]], ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    store i32 [[TMP25]], ptr [[I]], align 4
 // CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB27:%.*]] = sub i32 [[TMP25]], [[TMP26]]
-// CHECK2-NEXT:    [[SUB28:%.*]] = sub i32 [[SUB27]], 1
-// CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD29:%.*]] = add i32 [[SUB28]], [[TMP27]]
-// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV30:%.*]] = udiv i32 [[ADD29]], [[TMP28]]
-// CHECK2-NEXT:    [[CONV31:%.*]] = zext i32 [[DIV30]] to i64
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV31]]
-// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB32:%.*]] = sub i32 [[TMP29]], -31
-// CHECK2-NEXT:    [[DIV33:%.*]] = udiv i32 [[SUB32]], 32
-// CHECK2-NEXT:    [[CONV34:%.*]] = zext i32 [[DIV33]] to i64
-// CHECK2-NEXT:    [[MUL35:%.*]] = mul nsw i64 [[MUL]], [[CONV34]]
-// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB36:%.*]] = sub i32 [[TMP30]], [[TMP31]]
-// CHECK2-NEXT:    [[SUB37:%.*]] = sub i32 [[SUB36]], 1
-// CHECK2-NEXT:    [[ADD38:%.*]] = add i32 [[SUB37]], 1
-// CHECK2-NEXT:    [[DIV39:%.*]] = udiv i32 [[ADD38]], 1
-// CHECK2-NEXT:    [[CONV40:%.*]] = zext i32 [[DIV39]] to i64
-// CHECK2-NEXT:    [[MUL41:%.*]] = mul nsw i64 [[MUL35]], [[CONV40]]
-// CHECK2-NEXT:    [[SUB42:%.*]] = sub nsw i64 [[MUL41]], 1
-// CHECK2-NEXT:    store i64 [[SUB42]], ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK2-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    store i32 [[TMP32]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    store i32 [[TMP33]], ptr [[J]], align 4
+// CHECK2-NEXT:    store i32 [[TMP26]], ptr [[J]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_K]], align 4
-// CHECK2-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    store i32 [[TMP34]], ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK2-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK2-NEXT:    [[CMP43:%.*]] = icmp slt i32 [[TMP35]], [[TMP36]]
-// CHECK2-NEXT:    br i1 [[CMP43]], label [[LAND_LHS_TRUE:%.*]], label [[OMP_PRECOND_END:%.*]]
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_K]], align 4
+// CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP27]], [[TMP28]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[LAND_LHS_TRUE:%.*]], label [[OMP_PRECOND_END:%.*]]
 // CHECK2:       land.lhs.true:
-// CHECK2-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[CMP44:%.*]] = icmp slt i32 [[TMP37]], [[TMP38]]
-// CHECK2-NEXT:    br i1 [[CMP44]], label [[LAND_LHS_TRUE45:%.*]], label [[OMP_PRECOND_END]]
-// CHECK2:       land.lhs.true45:
-// CHECK2-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[CMP46:%.*]] = icmp ult i32 0, [[TMP39]]
-// CHECK2-NEXT:    br i1 [[CMP46]], label [[LAND_LHS_TRUE47:%.*]], label [[OMP_PRECOND_END]]
-// CHECK2:       land.lhs.true47:
-// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[CMP48:%.*]] = icmp ult i32 [[TMP40]], [[TMP41]]
-// CHECK2-NEXT:    br i1 [[CMP48]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
+// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[CMP32:%.*]] = icmp slt i32 [[TMP29]], [[TMP30]]
+// CHECK2-NEXT:    br i1 [[CMP32]], label [[LAND_LHS_TRUE33:%.*]], label [[OMP_PRECOND_END]]
+// CHECK2:       land.lhs.true33:
+// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[CMP34:%.*]] = icmp ult i32 0, [[TMP31]]
+// CHECK2-NEXT:    br i1 [[CMP34]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
 // CHECK2:       omp.precond.then:
 // CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
-// CHECK2-NEXT:    [[TMP42:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK2-NEXT:    store i64 [[TMP42]], ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK2-NEXT:    store i64 [[TMP32]], ptr [[DOTOMP_UB]], align 8
 // CHECK2-NEXT:    store i64 1, ptr [[DOTOMP_STRIDE]], align 8
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
 // CHECK2-NEXT:    call void @__kmpc_for_static_init_8(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i64 1, i64 1)
-// CHECK2-NEXT:    [[TMP43:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[TMP44:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK2-NEXT:    [[CMP53:%.*]] = icmp sgt i64 [[TMP43]], [[TMP44]]
-// CHECK2-NEXT:    br i1 [[CMP53]], label [[COND_TRUE54:%.*]], label [[COND_FALSE55:%.*]]
-// CHECK2:       cond.true54:
-// CHECK2-NEXT:    [[TMP45:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK2-NEXT:    br label [[COND_END56:%.*]]
-// CHECK2:       cond.false55:
-// CHECK2-NEXT:    [[TMP46:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    br label [[COND_END56]]
-// CHECK2:       cond.end56:
-// CHECK2-NEXT:    [[COND57:%.*]] = phi i64 [ [[TMP45]], [[COND_TRUE54]] ], [ [[TMP46]], [[COND_FALSE55]] ]
-// CHECK2-NEXT:    store i64 [[COND57]], ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[TMP47:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
-// CHECK2-NEXT:    store i64 [[TMP47]], ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK2-NEXT:    [[CMP39:%.*]] = icmp sgt i64 [[TMP33]], [[TMP34]]
+// CHECK2-NEXT:    br i1 [[CMP39]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i64 [ [[TMP35]], [[COND_TRUE]] ], [ [[TMP36]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    store i64 [[COND]], ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
+// CHECK2-NEXT:    store i64 [[TMP37]], ptr [[DOTOMP_IV]], align 8
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK2:       omp.inner.for.cond:
-// CHECK2-NEXT:    [[TMP48:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[CMP58:%.*]] = icmp sle i64 [[TMP48]], [[TMP49]]
-// CHECK2-NEXT:    br i1 [[CMP58]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK2-NEXT:    [[TMP38:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP39:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[CMP40:%.*]] = icmp sle i64 [[TMP38]], [[TMP39]]
+// CHECK2-NEXT:    br i1 [[CMP40]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
 // CHECK2:       omp.inner.for.body:
-// CHECK2-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[CONV59:%.*]] = sext i32 [[TMP50]] to i64
-// CHECK2-NEXT:    [[TMP51:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB60:%.*]] = sub i32 [[TMP52]], [[TMP53]]
-// CHECK2-NEXT:    [[SUB61:%.*]] = sub i32 [[SUB60]], 1
+// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[CONV41:%.*]] = sext i32 [[TMP40]] to i64
+// CHECK2-NEXT:    [[TMP41:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB42:%.*]] = sub i32 [[TMP42]], [[TMP43]]
+// CHECK2-NEXT:    [[SUB43:%.*]] = sub i32 [[SUB42]], 1
+// CHECK2-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD44:%.*]] = add i32 [[SUB43]], [[TMP44]]
+// CHECK2-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV45:%.*]] = udiv i32 [[ADD44]], [[TMP45]]
+// CHECK2-NEXT:    [[MUL46:%.*]] = mul i32 1, [[DIV45]]
+// CHECK2-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB47:%.*]] = sub i32 [[TMP46]], -31
+// CHECK2-NEXT:    [[DIV48:%.*]] = udiv i32 [[SUB47]], 32
+// CHECK2-NEXT:    [[MUL49:%.*]] = mul i32 [[MUL46]], [[DIV48]]
+// CHECK2-NEXT:    [[MUL50:%.*]] = mul i32 [[MUL49]], 32
+// CHECK2-NEXT:    [[CONV51:%.*]] = zext i32 [[MUL50]] to i64
+// CHECK2-NEXT:    [[DIV52:%.*]] = sdiv i64 [[TMP41]], [[CONV51]]
+// CHECK2-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[CONV53:%.*]] = sext i32 [[TMP47]] to i64
+// CHECK2-NEXT:    [[MUL54:%.*]] = mul nsw i64 [[DIV52]], [[CONV53]]
+// CHECK2-NEXT:    [[ADD55:%.*]] = add nsw i64 [[CONV41]], [[MUL54]]
+// CHECK2-NEXT:    [[CONV56:%.*]] = trunc i64 [[ADD55]] to i32
+// CHECK2-NEXT:    store i32 [[CONV56]], ptr [[I35]], align 4
+// CHECK2-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP48]] to i64
+// CHECK2-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP50:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB58:%.*]] = sub i32 [[TMP51]], [[TMP52]]
+// CHECK2-NEXT:    [[SUB59:%.*]] = sub i32 [[SUB58]], 1
+// CHECK2-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD60:%.*]] = add i32 [[SUB59]], [[TMP53]]
 // CHECK2-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD62:%.*]] = add i32 [[SUB61]], [[TMP54]]
-// CHECK2-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV63:%.*]] = udiv i32 [[ADD62]], [[TMP55]]
-// CHECK2-NEXT:    [[MUL64:%.*]] = mul i32 1, [[DIV63]]
-// CHECK2-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB65:%.*]] = sub i32 [[TMP56]], -31
-// CHECK2-NEXT:    [[DIV66:%.*]] = udiv i32 [[SUB65]], 32
-// CHECK2-NEXT:    [[MUL67:%.*]] = mul i32 [[MUL64]], [[DIV66]]
-// CHECK2-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB68:%.*]] = sub i32 [[TMP57]], [[TMP58]]
-// CHECK2-NEXT:    [[SUB69:%.*]] = sub i32 [[SUB68]], 1
-// CHECK2-NEXT:    [[ADD70:%.*]] = add i32 [[SUB69]], 1
-// CHECK2-NEXT:    [[DIV71:%.*]] = udiv i32 [[ADD70]], 1
-// CHECK2-NEXT:    [[MUL72:%.*]] = mul i32 [[MUL67]], [[DIV71]]
-// CHECK2-NEXT:    [[CONV73:%.*]] = zext i32 [[MUL72]] to i64
-// CHECK2-NEXT:    [[DIV74:%.*]] = sdiv i64 [[TMP51]], [[CONV73]]
-// CHECK2-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[CONV75:%.*]] = sext i32 [[TMP59]] to i64
-// CHECK2-NEXT:    [[MUL76:%.*]] = mul nsw i64 [[DIV74]], [[CONV75]]
-// CHECK2-NEXT:    [[ADD77:%.*]] = add nsw i64 [[CONV59]], [[MUL76]]
-// CHECK2-NEXT:    [[CONV78:%.*]] = trunc i64 [[ADD77]] to i32
-// CHECK2-NEXT:    store i32 [[CONV78]], ptr [[I49]], align 4
-// CHECK2-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[CONV79:%.*]] = sext i32 [[TMP60]] to i64
-// CHECK2-NEXT:    [[TMP61:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP62:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP63:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP64:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB80:%.*]] = sub i32 [[TMP63]], [[TMP64]]
-// CHECK2-NEXT:    [[SUB81:%.*]] = sub i32 [[SUB80]], 1
-// CHECK2-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD82:%.*]] = add i32 [[SUB81]], [[TMP65]]
-// CHECK2-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV83:%.*]] = udiv i32 [[ADD82]], [[TMP66]]
-// CHECK2-NEXT:    [[MUL84:%.*]] = mul i32 1, [[DIV83]]
-// CHECK2-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB85:%.*]] = sub i32 [[TMP67]], -31
-// CHECK2-NEXT:    [[DIV86:%.*]] = udiv i32 [[SUB85]], 32
-// CHECK2-NEXT:    [[MUL87:%.*]] = mul i32 [[MUL84]], [[DIV86]]
-// CHECK2-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB88:%.*]] = sub i32 [[TMP68]], [[TMP69]]
-// CHECK2-NEXT:    [[SUB89:%.*]] = sub i32 [[SUB88]], 1
-// CHECK2-NEXT:    [[ADD90:%.*]] = add i32 [[SUB89]], 1
-// CHECK2-NEXT:    [[DIV91:%.*]] = udiv i32 [[ADD90]], 1
-// CHECK2-NEXT:    [[MUL92:%.*]] = mul i32 [[MUL87]], [[DIV91]]
-// CHECK2-NEXT:    [[CONV93:%.*]] = zext i32 [[MUL92]] to i64
-// CHECK2-NEXT:    [[DIV94:%.*]] = sdiv i64 [[TMP62]], [[CONV93]]
+// CHECK2-NEXT:    [[DIV61:%.*]] = udiv i32 [[ADD60]], [[TMP54]]
+// CHECK2-NEXT:    [[MUL62:%.*]] = mul i32 1, [[DIV61]]
+// CHECK2-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB63:%.*]] = sub i32 [[TMP55]], -31
+// CHECK2-NEXT:    [[DIV64:%.*]] = udiv i32 [[SUB63]], 32
+// CHECK2-NEXT:    [[MUL65:%.*]] = mul i32 [[MUL62]], [[DIV64]]
+// CHECK2-NEXT:    [[MUL66:%.*]] = mul i32 [[MUL65]], 32
+// CHECK2-NEXT:    [[CONV67:%.*]] = zext i32 [[MUL66]] to i64
+// CHECK2-NEXT:    [[DIV68:%.*]] = sdiv i64 [[TMP50]], [[CONV67]]
+// CHECK2-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB69:%.*]] = sub i32 [[TMP56]], [[TMP57]]
+// CHECK2-NEXT:    [[SUB70:%.*]] = sub i32 [[SUB69]], 1
+// CHECK2-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD71:%.*]] = add i32 [[SUB70]], [[TMP58]]
+// CHECK2-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV72:%.*]] = udiv i32 [[ADD71]], [[TMP59]]
+// CHECK2-NEXT:    [[MUL73:%.*]] = mul i32 1, [[DIV72]]
+// CHECK2-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB74:%.*]] = sub i32 [[TMP60]], -31
+// CHECK2-NEXT:    [[DIV75:%.*]] = udiv i32 [[SUB74]], 32
+// CHECK2-NEXT:    [[MUL76:%.*]] = mul i32 [[MUL73]], [[DIV75]]
+// CHECK2-NEXT:    [[MUL77:%.*]] = mul i32 [[MUL76]], 32
+// CHECK2-NEXT:    [[CONV78:%.*]] = zext i32 [[MUL77]] to i64
+// CHECK2-NEXT:    [[MUL79:%.*]] = mul nsw i64 [[DIV68]], [[CONV78]]
+// CHECK2-NEXT:    [[SUB80:%.*]] = sub nsw i64 [[TMP49]], [[MUL79]]
+// CHECK2-NEXT:    [[TMP61:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB81:%.*]] = sub i32 [[TMP61]], -31
+// CHECK2-NEXT:    [[DIV82:%.*]] = udiv i32 [[SUB81]], 32
+// CHECK2-NEXT:    [[MUL83:%.*]] = mul i32 1, [[DIV82]]
+// CHECK2-NEXT:    [[MUL84:%.*]] = mul i32 [[MUL83]], 32
+// CHECK2-NEXT:    [[CONV85:%.*]] = zext i32 [[MUL84]] to i64
+// CHECK2-NEXT:    [[DIV86:%.*]] = sdiv i64 [[SUB80]], [[CONV85]]
+// CHECK2-NEXT:    [[TMP62:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[CONV87:%.*]] = sext i32 [[TMP62]] to i64
+// CHECK2-NEXT:    [[MUL88:%.*]] = mul nsw i64 [[DIV86]], [[CONV87]]
+// CHECK2-NEXT:    [[ADD89:%.*]] = add nsw i64 [[CONV57]], [[MUL88]]
+// CHECK2-NEXT:    [[CONV90:%.*]] = trunc i64 [[ADD89]] to i32
+// CHECK2-NEXT:    store i32 [[CONV90]], ptr [[J36]], align 4
+// CHECK2-NEXT:    [[TMP63:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP64:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB91:%.*]] = sub i32 [[TMP65]], [[TMP66]]
+// CHECK2-NEXT:    [[SUB92:%.*]] = sub i32 [[SUB91]], 1
+// CHECK2-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD93:%.*]] = add i32 [[SUB92]], [[TMP67]]
+// CHECK2-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV94:%.*]] = udiv i32 [[ADD93]], [[TMP68]]
+// CHECK2-NEXT:    [[MUL95:%.*]] = mul i32 1, [[DIV94]]
+// CHECK2-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB96:%.*]] = sub i32 [[TMP69]], -31
+// CHECK2-NEXT:    [[DIV97:%.*]] = udiv i32 [[SUB96]], 32
+// CHECK2-NEXT:    [[MUL98:%.*]] = mul i32 [[MUL95]], [[DIV97]]
+// CHECK2-NEXT:    [[MUL99:%.*]] = mul i32 [[MUL98]], 32
+// CHECK2-NEXT:    [[CONV100:%.*]] = zext i32 [[MUL99]] to i64
+// CHECK2-NEXT:    [[DIV101:%.*]] = sdiv i64 [[TMP64]], [[CONV100]]
 // CHECK2-NEXT:    [[TMP70:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
 // CHECK2-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB95:%.*]] = sub i32 [[TMP70]], [[TMP71]]
-// CHECK2-NEXT:    [[SUB96:%.*]] = sub i32 [[SUB95]], 1
+// CHECK2-NEXT:    [[SUB102:%.*]] = sub i32 [[TMP70]], [[TMP71]]
+// CHECK2-NEXT:    [[SUB103:%.*]] = sub i32 [[SUB102]], 1
 // CHECK2-NEXT:    [[TMP72:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD97:%.*]] = add i32 [[SUB96]], [[TMP72]]
+// CHECK2-NEXT:    [[ADD104:%.*]] = add i32 [[SUB103]], [[TMP72]]
 // CHECK2-NEXT:    [[TMP73:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV98:%.*]] = udiv i32 [[ADD97]], [[TMP73]]
-// CHECK2-NEXT:    [[MUL99:%.*]] = mul i32 1, [[DIV98]]
+// CHECK2-NEXT:    [[DIV105:%.*]] = udiv i32 [[ADD104]], [[TMP73]]
+// CHECK2-NEXT:    [[MUL106:%.*]] = mul i32 1, [[DIV105]]
 // CHECK2-NEXT:    [[TMP74:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB100:%.*]] = sub i32 [[TMP74]], -31
-// CHECK2-NEXT:    [[DIV101:%.*]] = udiv i32 [[SUB100]], 32
-// CHECK2-NEXT:    [[MUL102:%.*]] = mul i32 [[MUL99]], [[DIV101]]
-// CHECK2-NEXT:    [[TMP75:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP76:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB103:%.*]] = sub i32 [[TMP75]], [[TMP76]]
-// CHECK2-NEXT:    [[SUB104:%.*]] = sub i32 [[SUB103]], 1
-// CHECK2-NEXT:    [[ADD105:%.*]] = add i32 [[SUB104]], 1
-// CHECK2-NEXT:    [[DIV106:%.*]] = udiv i32 [[ADD105]], 1
-// CHECK2-NEXT:    [[MUL107:%.*]] = mul i32 [[MUL102]], [[DIV106]]
-// CHECK2-NEXT:    [[CONV108:%.*]] = zext i32 [[MUL107]] to i64
-// CHECK2-NEXT:    [[MUL109:%.*]] = mul nsw i64 [[DIV94]], [[CONV108]]
-// CHECK2-NEXT:    [[SUB110:%.*]] = sub nsw i64 [[TMP61]], [[MUL109]]
-// CHECK2-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB111:%.*]] = sub i32 [[TMP77]], -31
-// CHECK2-NEXT:    [[DIV112:%.*]] = udiv i32 [[SUB111]], 32
-// CHECK2-NEXT:    [[MUL113:%.*]] = mul i32 1, [[DIV112]]
-// CHECK2-NEXT:    [[TMP78:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP79:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB114:%.*]] = sub i32 [[TMP78]], [[TMP79]]
+// CHECK2-NEXT:    [[SUB107:%.*]] = sub i32 [[TMP74]], -31
+// CHECK2-NEXT:    [[DIV108:%.*]] = udiv i32 [[SUB107]], 32
+// CHECK2-NEXT:    [[MUL109:%.*]] = mul i32 [[MUL106]], [[DIV108]]
+// CHECK2-NEXT:    [[MUL110:%.*]] = mul i32 [[MUL109]], 32
+// CHECK2-NEXT:    [[CONV111:%.*]] = zext i32 [[MUL110]] to i64
+// CHECK2-NEXT:    [[MUL112:%.*]] = mul nsw i64 [[DIV101]], [[CONV111]]
+// CHECK2-NEXT:    [[SUB113:%.*]] = sub nsw i64 [[TMP63]], [[MUL112]]
+// CHECK2-NEXT:    [[TMP75:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP76:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP78:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB114:%.*]] = sub i32 [[TMP77]], [[TMP78]]
 // CHECK2-NEXT:    [[SUB115:%.*]] = sub i32 [[SUB114]], 1
-// CHECK2-NEXT:    [[ADD116:%.*]] = add i32 [[SUB115]], 1
-// CHECK2-NEXT:    [[DIV117:%.*]] = udiv i32 [[ADD116]], 1
-// CHECK2-NEXT:    [[MUL118:%.*]] = mul i32 [[MUL113]], [[DIV117]]
-// CHECK2-NEXT:    [[CONV119:%.*]] = zext i32 [[MUL118]] to i64
-// CHECK2-NEXT:    [[DIV120:%.*]] = sdiv i64 [[SUB110]], [[CONV119]]
+// CHECK2-NEXT:    [[TMP79:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD116:%.*]] = add i32 [[SUB115]], [[TMP79]]
 // CHECK2-NEXT:    [[TMP80:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[CONV121:%.*]] = sext i32 [[TMP80]] to i64
-// CHECK2-NEXT:    [[MUL122:%.*]] = mul nsw i64 [[DIV120]], [[CONV121]]
-// CHECK2-NEXT:    [[ADD123:%.*]] = add nsw i64 [[CONV79]], [[MUL122]]
-// CHECK2-NEXT:    [[CONV124:%.*]] = trunc i64 [[ADD123]] to i32
-// CHECK2-NEXT:    store i32 [[CONV124]], ptr [[J50]], align 4
-// CHECK2-NEXT:    [[TMP81:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP82:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB125:%.*]] = sub i32 [[TMP83]], [[TMP84]]
+// CHECK2-NEXT:    [[DIV117:%.*]] = udiv i32 [[ADD116]], [[TMP80]]
+// CHECK2-NEXT:    [[MUL118:%.*]] = mul i32 1, [[DIV117]]
+// CHECK2-NEXT:    [[TMP81:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB119:%.*]] = sub i32 [[TMP81]], -31
+// CHECK2-NEXT:    [[DIV120:%.*]] = udiv i32 [[SUB119]], 32
+// CHECK2-NEXT:    [[MUL121:%.*]] = mul i32 [[MUL118]], [[DIV120]]
+// CHECK2-NEXT:    [[MUL122:%.*]] = mul i32 [[MUL121]], 32
+// CHECK2-NEXT:    [[CONV123:%.*]] = zext i32 [[MUL122]] to i64
+// CHECK2-NEXT:    [[DIV124:%.*]] = sdiv i64 [[TMP76]], [[CONV123]]
+// CHECK2-NEXT:    [[TMP82:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB125:%.*]] = sub i32 [[TMP82]], [[TMP83]]
 // CHECK2-NEXT:    [[SUB126:%.*]] = sub i32 [[SUB125]], 1
+// CHECK2-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD127:%.*]] = add i32 [[SUB126]], [[TMP84]]
 // CHECK2-NEXT:    [[TMP85:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD127:%.*]] = add i32 [[SUB126]], [[TMP85]]
-// CHECK2-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV128:%.*]] = udiv i32 [[ADD127]], [[TMP86]]
+// CHECK2-NEXT:    [[DIV128:%.*]] = udiv i32 [[ADD127]], [[TMP85]]
 // CHECK2-NEXT:    [[MUL129:%.*]] = mul i32 1, [[DIV128]]
-// CHECK2-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB130:%.*]] = sub i32 [[TMP87]], -31
+// CHECK2-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB130:%.*]] = sub i32 [[TMP86]], -31
 // CHECK2-NEXT:    [[DIV131:%.*]] = udiv i32 [[SUB130]], 32
 // CHECK2-NEXT:    [[MUL132:%.*]] = mul i32 [[MUL129]], [[DIV131]]
-// CHECK2-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP89:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB133:%.*]] = sub i32 [[TMP88]], [[TMP89]]
-// CHECK2-NEXT:    [[SUB134:%.*]] = sub i32 [[SUB133]], 1
-// CHECK2-NEXT:    [[ADD135:%.*]] = add i32 [[SUB134]], 1
-// CHECK2-NEXT:    [[DIV136:%.*]] = udiv i32 [[ADD135]], 1
-// CHECK2-NEXT:    [[MUL137:%.*]] = mul i32 [[MUL132]], [[DIV136]]
-// CHECK2-NEXT:    [[CONV138:%.*]] = zext i32 [[MUL137]] to i64
-// CHECK2-NEXT:    [[DIV139:%.*]] = sdiv i64 [[TMP82]], [[CONV138]]
-// CHECK2-NEXT:    [[TMP90:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB140:%.*]] = sub i32 [[TMP90]], [[TMP91]]
-// CHECK2-NEXT:    [[SUB141:%.*]] = sub i32 [[SUB140]], 1
-// CHECK2-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD142:%.*]] = add i32 [[SUB141]], [[TMP92]]
+// CHECK2-NEXT:    [[MUL133:%.*]] = mul i32 [[MUL132]], 32
+// CHECK2-NEXT:    [[CONV134:%.*]] = zext i32 [[MUL133]] to i64
+// CHECK2-NEXT:    [[MUL135:%.*]] = mul nsw i64 [[DIV124]], [[CONV134]]
+// CHECK2-NEXT:    [[SUB136:%.*]] = sub nsw i64 [[TMP75]], [[MUL135]]
+// CHECK2-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB137:%.*]] = sub i32 [[TMP87]], -31
+// CHECK2-NEXT:    [[DIV138:%.*]] = udiv i32 [[SUB137]], 32
+// CHECK2-NEXT:    [[MUL139:%.*]] = mul i32 1, [[DIV138]]
+// CHECK2-NEXT:    [[MUL140:%.*]] = mul i32 [[MUL139]], 32
+// CHECK2-NEXT:    [[CONV141:%.*]] = zext i32 [[MUL140]] to i64
+// CHECK2-NEXT:    [[DIV142:%.*]] = sdiv i64 [[SUB136]], [[CONV141]]
+// CHECK2-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB143:%.*]] = sub i32 [[TMP88]], -31
+// CHECK2-NEXT:    [[DIV144:%.*]] = udiv i32 [[SUB143]], 32
+// CHECK2-NEXT:    [[MUL145:%.*]] = mul i32 1, [[DIV144]]
+// CHECK2-NEXT:    [[MUL146:%.*]] = mul i32 [[MUL145]], 32
+// CHECK2-NEXT:    [[CONV147:%.*]] = zext i32 [[MUL146]] to i64
+// CHECK2-NEXT:    [[MUL148:%.*]] = mul nsw i64 [[DIV142]], [[CONV147]]
+// CHECK2-NEXT:    [[SUB149:%.*]] = sub nsw i64 [[SUB113]], [[MUL148]]
+// CHECK2-NEXT:    [[DIV150:%.*]] = sdiv i64 [[SUB149]], 32
+// CHECK2-NEXT:    [[MUL151:%.*]] = mul nsw i64 [[DIV150]], 32
+// CHECK2-NEXT:    [[ADD152:%.*]] = add nsw i64 0, [[MUL151]]
+// CHECK2-NEXT:    [[CONV153:%.*]] = trunc i64 [[ADD152]] to i32
+// CHECK2-NEXT:    store i32 [[CONV153]], ptr [[DOTFLOOR_0_IV_K37]], align 4
+// CHECK2-NEXT:    [[TMP89:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP90:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB154:%.*]] = sub i32 [[TMP91]], [[TMP92]]
+// CHECK2-NEXT:    [[SUB155:%.*]] = sub i32 [[SUB154]], 1
 // CHECK2-NEXT:    [[TMP93:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV143:%.*]] = udiv i32 [[ADD142]], [[TMP93]]
-// CHECK2-NEXT:    [[MUL144:%.*]] = mul i32 1, [[DIV143]]
-// CHECK2-NEXT:    [[TMP94:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB145:%.*]] = sub i32 [[TMP94]], -31
-// CHECK2-NEXT:    [[DIV146:%.*]] = udiv i32 [[SUB145]], 32
-// CHECK2-NEXT:    [[MUL147:%.*]] = mul i32 [[MUL144]], [[DIV146]]
-// CHECK2-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB148:%.*]] = sub i32 [[TMP95]], [[TMP96]]
-// CHECK2-NEXT:    [[SUB149:%.*]] = sub i32 [[SUB148]], 1
-// CHECK2-NEXT:    [[ADD150:%.*]] = add i32 [[SUB149]], 1
-// CHECK2-NEXT:    [[DIV151:%.*]] = udiv i32 [[ADD150]], 1
-// CHECK2-NEXT:    [[MUL152:%.*]] = mul i32 [[MUL147]], [[DIV151]]
-// CHECK2-NEXT:    [[CONV153:%.*]] = zext i32 [[MUL152]] to i64
-// CHECK2-NEXT:    [[MUL154:%.*]] = mul nsw i64 [[DIV139]], [[CONV153]]
-// CHECK2-NEXT:    [[SUB155:%.*]] = sub nsw i64 [[TMP81]], [[MUL154]]
-// CHECK2-NEXT:    [[TMP97:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP98:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB156:%.*]] = sub i32 [[TMP99]], [[TMP100]]
-// CHECK2-NEXT:    [[SUB157:%.*]] = sub i32 [[SUB156]], 1
-// CHECK2-NEXT:    [[TMP101:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD158:%.*]] = add i32 [[SUB157]], [[TMP101]]
-// CHECK2-NEXT:    [[TMP102:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV159:%.*]] = udiv i32 [[ADD158]], [[TMP102]]
-// CHECK2-NEXT:    [[MUL160:%.*]] = mul i32 1, [[DIV159]]
-// CHECK2-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB161:%.*]] = sub i32 [[TMP103]], -31
-// CHECK2-NEXT:    [[DIV162:%.*]] = udiv i32 [[SUB161]], 32
-// CHECK2-NEXT:    [[MUL163:%.*]] = mul i32 [[MUL160]], [[DIV162]]
-// CHECK2-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP105:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB164:%.*]] = sub i32 [[TMP104]], [[TMP105]]
-// CHECK2-NEXT:    [[SUB165:%.*]] = sub i32 [[SUB164]], 1
-// CHECK2-NEXT:    [[ADD166:%.*]] = add i32 [[SUB165]], 1
-// CHECK2-NEXT:    [[DIV167:%.*]] = udiv i32 [[ADD166]], 1
-// CHECK2-NEXT:    [[MUL168:%.*]] = mul i32 [[MUL163]], [[DIV167]]
-// CHECK2-NEXT:    [[CONV169:%.*]] = zext i32 [[MUL168]] to i64
-// CHECK2-NEXT:    [[DIV170:%.*]] = sdiv i64 [[TMP98]], [[CONV169]]
-// CHECK2-NEXT:    [[TMP106:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB171:%.*]] = sub i32 [[TMP106]], [[TMP107]]
-// CHECK2-NEXT:    [[SUB172:%.*]] = sub i32 [[SUB171]], 1
-// CHECK2-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD173:%.*]] = add i32 [[SUB172]], [[TMP108]]
-// CHECK2-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV174:%.*]] = udiv i32 [[ADD173]], [[TMP109]]
-// CHECK2-NEXT:    [[MUL175:%.*]] = mul i32 1, [[DIV174]]
-// CHECK2-NEXT:    [[TMP110:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB176:%.*]] = sub i32 [[TMP110]], -31
-// CHECK2-NEXT:    [[DIV177:%.*]] = udiv i32 [[SUB176]], 32
-// CHECK2-NEXT:    [[MUL178:%.*]] = mul i32 [[MUL175]], [[DIV177]]
-// CHECK2-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB179:%.*]] = sub i32 [[TMP111]], [[TMP112]]
-// CHECK2-NEXT:    [[SUB180:%.*]] = sub i32 [[SUB179]], 1
-// CHECK2-NEXT:    [[ADD181:%.*]] = add i32 [[SUB180]], 1
-// CHECK2-NEXT:    [[DIV182:%.*]] = udiv i32 [[ADD181]], 1
-// CHECK2-NEXT:    [[MUL183:%.*]] = mul i32 [[MUL178]], [[DIV182]]
-// CHECK2-NEXT:    [[CONV184:%.*]] = zext i32 [[MUL183]] to i64
-// CHECK2-NEXT:    [[MUL185:%.*]] = mul nsw i64 [[DIV170]], [[CONV184]]
-// CHECK2-NEXT:    [[SUB186:%.*]] = sub nsw i64 [[TMP97]], [[MUL185]]
+// CHECK2-NEXT:    [[ADD156:%.*]] = add i32 [[SUB155]], [[TMP93]]
+// CHECK2-NEXT:    [[TMP94:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV157:%.*]] = udiv i32 [[ADD156]], [[TMP94]]
+// CHECK2-NEXT:    [[MUL158:%.*]] = mul i32 1, [[DIV157]]
+// CHECK2-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB159:%.*]] = sub i32 [[TMP95]], -31
+// CHECK2-NEXT:    [[DIV160:%.*]] = udiv i32 [[SUB159]], 32
+// CHECK2-NEXT:    [[MUL161:%.*]] = mul i32 [[MUL158]], [[DIV160]]
+// CHECK2-NEXT:    [[MUL162:%.*]] = mul i32 [[MUL161]], 32
+// CHECK2-NEXT:    [[CONV163:%.*]] = zext i32 [[MUL162]] to i64
+// CHECK2-NEXT:    [[DIV164:%.*]] = sdiv i64 [[TMP90]], [[CONV163]]
+// CHECK2-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP97:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB165:%.*]] = sub i32 [[TMP96]], [[TMP97]]
+// CHECK2-NEXT:    [[SUB166:%.*]] = sub i32 [[SUB165]], 1
+// CHECK2-NEXT:    [[TMP98:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD167:%.*]] = add i32 [[SUB166]], [[TMP98]]
+// CHECK2-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV168:%.*]] = udiv i32 [[ADD167]], [[TMP99]]
+// CHECK2-NEXT:    [[MUL169:%.*]] = mul i32 1, [[DIV168]]
+// CHECK2-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB170:%.*]] = sub i32 [[TMP100]], -31
+// CHECK2-NEXT:    [[DIV171:%.*]] = udiv i32 [[SUB170]], 32
+// CHECK2-NEXT:    [[MUL172:%.*]] = mul i32 [[MUL169]], [[DIV171]]
+// CHECK2-NEXT:    [[MUL173:%.*]] = mul i32 [[MUL172]], 32
+// CHECK2-NEXT:    [[CONV174:%.*]] = zext i32 [[MUL173]] to i64
+// CHECK2-NEXT:    [[MUL175:%.*]] = mul nsw i64 [[DIV164]], [[CONV174]]
+// CHECK2-NEXT:    [[SUB176:%.*]] = sub nsw i64 [[TMP89]], [[MUL175]]
+// CHECK2-NEXT:    [[TMP101:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP102:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB177:%.*]] = sub i32 [[TMP103]], [[TMP104]]
+// CHECK2-NEXT:    [[SUB178:%.*]] = sub i32 [[SUB177]], 1
+// CHECK2-NEXT:    [[TMP105:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD179:%.*]] = add i32 [[SUB178]], [[TMP105]]
+// CHECK2-NEXT:    [[TMP106:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV180:%.*]] = udiv i32 [[ADD179]], [[TMP106]]
+// CHECK2-NEXT:    [[MUL181:%.*]] = mul i32 1, [[DIV180]]
+// CHECK2-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB182:%.*]] = sub i32 [[TMP107]], -31
+// CHECK2-NEXT:    [[DIV183:%.*]] = udiv i32 [[SUB182]], 32
+// CHECK2-NEXT:    [[MUL184:%.*]] = mul i32 [[MUL181]], [[DIV183]]
+// CHECK2-NEXT:    [[MUL185:%.*]] = mul i32 [[MUL184]], 32
+// CHECK2-NEXT:    [[CONV186:%.*]] = zext i32 [[MUL185]] to i64
+// CHECK2-NEXT:    [[DIV187:%.*]] = sdiv i64 [[TMP102]], [[CONV186]]
+// CHECK2-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB188:%.*]] = sub i32 [[TMP108]], [[TMP109]]
+// CHECK2-NEXT:    [[SUB189:%.*]] = sub i32 [[SUB188]], 1
+// CHECK2-NEXT:    [[TMP110:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD190:%.*]] = add i32 [[SUB189]], [[TMP110]]
+// CHECK2-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV191:%.*]] = udiv i32 [[ADD190]], [[TMP111]]
+// CHECK2-NEXT:    [[MUL192:%.*]] = mul i32 1, [[DIV191]]
+// CHECK2-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB193:%.*]] = sub i32 [[TMP112]], -31
+// CHECK2-NEXT:    [[DIV194:%.*]] = udiv i32 [[SUB193]], 32
+// CHECK2-NEXT:    [[MUL195:%.*]] = mul i32 [[MUL192]], [[DIV194]]
+// CHECK2-NEXT:    [[MUL196:%.*]] = mul i32 [[MUL195]], 32
+// CHECK2-NEXT:    [[CONV197:%.*]] = zext i32 [[MUL196]] to i64
+// CHECK2-NEXT:    [[MUL198:%.*]] = mul nsw i64 [[DIV187]], [[CONV197]]
+// CHECK2-NEXT:    [[SUB199:%.*]] = sub nsw i64 [[TMP101]], [[MUL198]]
 // CHECK2-NEXT:    [[TMP113:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB187:%.*]] = sub i32 [[TMP113]], -31
-// CHECK2-NEXT:    [[DIV188:%.*]] = udiv i32 [[SUB187]], 32
-// CHECK2-NEXT:    [[MUL189:%.*]] = mul i32 1, [[DIV188]]
-// CHECK2-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP115:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB190:%.*]] = sub i32 [[TMP114]], [[TMP115]]
-// CHECK2-NEXT:    [[SUB191:%.*]] = sub i32 [[SUB190]], 1
-// CHECK2-NEXT:    [[ADD192:%.*]] = add i32 [[SUB191]], 1
-// CHECK2-NEXT:    [[DIV193:%.*]] = udiv i32 [[ADD192]], 1
-// CHECK2-NEXT:    [[MUL194:%.*]] = mul i32 [[MUL189]], [[DIV193]]
-// CHECK2-NEXT:    [[CONV195:%.*]] = zext i32 [[MUL194]] to i64
-// CHECK2-NEXT:    [[DIV196:%.*]] = sdiv i64 [[SUB186]], [[CONV195]]
-// CHECK2-NEXT:    [[TMP116:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB197:%.*]] = sub i32 [[TMP116]], -31
-// CHECK2-NEXT:    [[DIV198:%.*]] = udiv i32 [[SUB197]], 32
-// CHECK2-NEXT:    [[MUL199:%.*]] = mul i32 1, [[DIV198]]
-// CHECK2-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB200:%.*]] = sub i32 [[TMP117]], [[TMP118]]
-// CHECK2-NEXT:    [[SUB201:%.*]] = sub i32 [[SUB200]], 1
-// CHECK2-NEXT:    [[ADD202:%.*]] = add i32 [[SUB201]], 1
-// CHECK2-NEXT:    [[DIV203:%.*]] = udiv i32 [[ADD202]], 1
-// CHECK2-NEXT:    [[MUL204:%.*]] = mul i32 [[MUL199]], [[DIV203]]
-// CHECK2-NEXT:    [[CONV205:%.*]] = zext i32 [[MUL204]] to i64
-// CHECK2-NEXT:    [[MUL206:%.*]] = mul nsw i64 [[DIV196]], [[CONV205]]
-// CHECK2-NEXT:    [[SUB207:%.*]] = sub nsw i64 [[SUB155]], [[MUL206]]
-// CHECK2-NEXT:    [[TMP119:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP120:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB208:%.*]] = sub i32 [[TMP119]], [[TMP120]]
-// CHECK2-NEXT:    [[SUB209:%.*]] = sub i32 [[SUB208]], 1
-// CHECK2-NEXT:    [[ADD210:%.*]] = add i32 [[SUB209]], 1
-// CHECK2-NEXT:    [[DIV211:%.*]] = udiv i32 [[ADD210]], 1
-// CHECK2-NEXT:    [[MUL212:%.*]] = mul i32 1, [[DIV211]]
-// CHECK2-NEXT:    [[CONV213:%.*]] = zext i32 [[MUL212]] to i64
-// CHECK2-NEXT:    [[DIV214:%.*]] = sdiv i64 [[SUB207]], [[CONV213]]
-// CHECK2-NEXT:    [[MUL215:%.*]] = mul nsw i64 [[DIV214]], 32
-// CHECK2-NEXT:    [[ADD216:%.*]] = add nsw i64 0, [[MUL215]]
-// CHECK2-NEXT:    [[CONV217:%.*]] = trunc i64 [[ADD216]] to i32
-// CHECK2-NEXT:    store i32 [[CONV217]], ptr [[DOTFLOOR_0_IV_K51]], align 4
-// CHECK2-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[CONV218:%.*]] = zext i32 [[TMP121]] to i64
-// CHECK2-NEXT:    [[TMP122:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP123:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP124:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB219:%.*]] = sub i32 [[TMP124]], [[TMP125]]
-// CHECK2-NEXT:    [[SUB220:%.*]] = sub i32 [[SUB219]], 1
-// CHECK2-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD221:%.*]] = add i32 [[SUB220]], [[TMP126]]
-// CHECK2-NEXT:    [[TMP127:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV222:%.*]] = udiv i32 [[ADD221]], [[TMP127]]
-// CHECK2-NEXT:    [[MUL223:%.*]] = mul i32 1, [[DIV222]]
-// CHECK2-NEXT:    [[TMP128:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB224:%.*]] = sub i32 [[TMP128]], -31
-// CHECK2-NEXT:    [[DIV225:%.*]] = udiv i32 [[SUB224]], 32
-// CHECK2-NEXT:    [[MUL226:%.*]] = mul i32 [[MUL223]], [[DIV225]]
-// CHECK2-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB227:%.*]] = sub i32 [[TMP129]], [[TMP130]]
-// CHECK2-NEXT:    [[SUB228:%.*]] = sub i32 [[SUB227]], 1
-// CHECK2-NEXT:    [[ADD229:%.*]] = add i32 [[SUB228]], 1
-// CHECK2-NEXT:    [[DIV230:%.*]] = udiv i32 [[ADD229]], 1
-// CHECK2-NEXT:    [[MUL231:%.*]] = mul i32 [[MUL226]], [[DIV230]]
-// CHECK2-NEXT:    [[CONV232:%.*]] = zext i32 [[MUL231]] to i64
-// CHECK2-NEXT:    [[DIV233:%.*]] = sdiv i64 [[TMP123]], [[CONV232]]
-// CHECK2-NEXT:    [[TMP131:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP132:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB234:%.*]] = sub i32 [[TMP131]], [[TMP132]]
-// CHECK2-NEXT:    [[SUB235:%.*]] = sub i32 [[SUB234]], 1
-// CHECK2-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD236:%.*]] = add i32 [[SUB235]], [[TMP133]]
-// CHECK2-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV237:%.*]] = udiv i32 [[ADD236]], [[TMP134]]
-// CHECK2-NEXT:    [[MUL238:%.*]] = mul i32 1, [[DIV237]]
-// CHECK2-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB239:%.*]] = sub i32 [[TMP135]], -31
-// CHECK2-NEXT:    [[DIV240:%.*]] = udiv i32 [[SUB239]], 32
-// CHECK2-NEXT:    [[MUL241:%.*]] = mul i32 [[MUL238]], [[DIV240]]
-// CHECK2-NEXT:    [[TMP136:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB242:%.*]] = sub i32 [[TMP136]], [[TMP137]]
-// CHECK2-NEXT:    [[SUB243:%.*]] = sub i32 [[SUB242]], 1
-// CHECK2-NEXT:    [[ADD244:%.*]] = add i32 [[SUB243]], 1
-// CHECK2-NEXT:    [[DIV245:%.*]] = udiv i32 [[ADD244]], 1
-// CHECK2-NEXT:    [[MUL246:%.*]] = mul i32 [[MUL241]], [[DIV245]]
-// CHECK2-NEXT:    [[CONV247:%.*]] = zext i32 [[MUL246]] to i64
-// CHECK2-NEXT:    [[MUL248:%.*]] = mul nsw i64 [[DIV233]], [[CONV247]]
-// CHECK2-NEXT:    [[SUB249:%.*]] = sub nsw i64 [[TMP122]], [[MUL248]]
-// CHECK2-NEXT:    [[TMP138:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP139:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB250:%.*]] = sub i32 [[TMP140]], [[TMP141]]
-// CHECK2-NEXT:    [[SUB251:%.*]] = sub i32 [[SUB250]], 1
-// CHECK2-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD252:%.*]] = add i32 [[SUB251]], [[TMP142]]
-// CHECK2-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV253:%.*]] = udiv i32 [[ADD252]], [[TMP143]]
-// CHECK2-NEXT:    [[MUL254:%.*]] = mul i32 1, [[DIV253]]
-// CHECK2-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB255:%.*]] = sub i32 [[TMP144]], -31
-// CHECK2-NEXT:    [[DIV256:%.*]] = udiv i32 [[SUB255]], 32
-// CHECK2-NEXT:    [[MUL257:%.*]] = mul i32 [[MUL254]], [[DIV256]]
-// CHECK2-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB258:%.*]] = sub i32 [[TMP145]], [[TMP146]]
-// CHECK2-NEXT:    [[SUB259:%.*]] = sub i32 [[SUB258]], 1
-// CHECK2-NEXT:    [[ADD260:%.*]] = add i32 [[SUB259]], 1
-// CHECK2-NEXT:    [[DIV261:%.*]] = udiv i32 [[ADD260]], 1
-// CHECK2-NEXT:    [[MUL262:%.*]] = mul i32 [[MUL257]], [[DIV261]]
+// CHECK2-NEXT:    [[SUB200:%.*]] = sub i32 [[TMP113]], -31
+// CHECK2-NEXT:    [[DIV201:%.*]] = udiv i32 [[SUB200]], 32
+// CHECK2-NEXT:    [[MUL202:%.*]] = mul i32 1, [[DIV201]]
+// CHECK2-NEXT:    [[MUL203:%.*]] = mul i32 [[MUL202]], 32
+// CHECK2-NEXT:    [[CONV204:%.*]] = zext i32 [[MUL203]] to i64
+// CHECK2-NEXT:    [[DIV205:%.*]] = sdiv i64 [[SUB199]], [[CONV204]]
+// CHECK2-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB206:%.*]] = sub i32 [[TMP114]], -31
+// CHECK2-NEXT:    [[DIV207:%.*]] = udiv i32 [[SUB206]], 32
+// CHECK2-NEXT:    [[MUL208:%.*]] = mul i32 1, [[DIV207]]
+// CHECK2-NEXT:    [[MUL209:%.*]] = mul i32 [[MUL208]], 32
+// CHECK2-NEXT:    [[CONV210:%.*]] = zext i32 [[MUL209]] to i64
+// CHECK2-NEXT:    [[MUL211:%.*]] = mul nsw i64 [[DIV205]], [[CONV210]]
+// CHECK2-NEXT:    [[SUB212:%.*]] = sub nsw i64 [[SUB176]], [[MUL211]]
+// CHECK2-NEXT:    [[TMP115:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP116:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB213:%.*]] = sub i32 [[TMP117]], [[TMP118]]
+// CHECK2-NEXT:    [[SUB214:%.*]] = sub i32 [[SUB213]], 1
+// CHECK2-NEXT:    [[TMP119:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD215:%.*]] = add i32 [[SUB214]], [[TMP119]]
+// CHECK2-NEXT:    [[TMP120:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV216:%.*]] = udiv i32 [[ADD215]], [[TMP120]]
+// CHECK2-NEXT:    [[MUL217:%.*]] = mul i32 1, [[DIV216]]
+// CHECK2-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB218:%.*]] = sub i32 [[TMP121]], -31
+// CHECK2-NEXT:    [[DIV219:%.*]] = udiv i32 [[SUB218]], 32
+// CHECK2-NEXT:    [[MUL220:%.*]] = mul i32 [[MUL217]], [[DIV219]]
+// CHECK2-NEXT:    [[MUL221:%.*]] = mul i32 [[MUL220]], 32
+// CHECK2-NEXT:    [[CONV222:%.*]] = zext i32 [[MUL221]] to i64
+// CHECK2-NEXT:    [[DIV223:%.*]] = sdiv i64 [[TMP116]], [[CONV222]]
+// CHECK2-NEXT:    [[TMP122:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP123:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB224:%.*]] = sub i32 [[TMP122]], [[TMP123]]
+// CHECK2-NEXT:    [[SUB225:%.*]] = sub i32 [[SUB224]], 1
+// CHECK2-NEXT:    [[TMP124:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD226:%.*]] = add i32 [[SUB225]], [[TMP124]]
+// CHECK2-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV227:%.*]] = udiv i32 [[ADD226]], [[TMP125]]
+// CHECK2-NEXT:    [[MUL228:%.*]] = mul i32 1, [[DIV227]]
+// CHECK2-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB229:%.*]] = sub i32 [[TMP126]], -31
+// CHECK2-NEXT:    [[DIV230:%.*]] = udiv i32 [[SUB229]], 32
+// CHECK2-NEXT:    [[MUL231:%.*]] = mul i32 [[MUL228]], [[DIV230]]
+// CHECK2-NEXT:    [[MUL232:%.*]] = mul i32 [[MUL231]], 32
+// CHECK2-NEXT:    [[CONV233:%.*]] = zext i32 [[MUL232]] to i64
+// CHECK2-NEXT:    [[MUL234:%.*]] = mul nsw i64 [[DIV223]], [[CONV233]]
+// CHECK2-NEXT:    [[SUB235:%.*]] = sub nsw i64 [[TMP115]], [[MUL234]]
+// CHECK2-NEXT:    [[TMP127:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP128:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB236:%.*]] = sub i32 [[TMP129]], [[TMP130]]
+// CHECK2-NEXT:    [[SUB237:%.*]] = sub i32 [[SUB236]], 1
+// CHECK2-NEXT:    [[TMP131:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD238:%.*]] = add i32 [[SUB237]], [[TMP131]]
+// CHECK2-NEXT:    [[TMP132:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV239:%.*]] = udiv i32 [[ADD238]], [[TMP132]]
+// CHECK2-NEXT:    [[MUL240:%.*]] = mul i32 1, [[DIV239]]
+// CHECK2-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB241:%.*]] = sub i32 [[TMP133]], -31
+// CHECK2-NEXT:    [[DIV242:%.*]] = udiv i32 [[SUB241]], 32
+// CHECK2-NEXT:    [[MUL243:%.*]] = mul i32 [[MUL240]], [[DIV242]]
+// CHECK2-NEXT:    [[MUL244:%.*]] = mul i32 [[MUL243]], 32
+// CHECK2-NEXT:    [[CONV245:%.*]] = zext i32 [[MUL244]] to i64
+// CHECK2-NEXT:    [[DIV246:%.*]] = sdiv i64 [[TMP128]], [[CONV245]]
+// CHECK2-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB247:%.*]] = sub i32 [[TMP134]], [[TMP135]]
+// CHECK2-NEXT:    [[SUB248:%.*]] = sub i32 [[SUB247]], 1
+// CHECK2-NEXT:    [[TMP136:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD249:%.*]] = add i32 [[SUB248]], [[TMP136]]
+// CHECK2-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV250:%.*]] = udiv i32 [[ADD249]], [[TMP137]]
+// CHECK2-NEXT:    [[MUL251:%.*]] = mul i32 1, [[DIV250]]
+// CHECK2-NEXT:    [[TMP138:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB252:%.*]] = sub i32 [[TMP138]], -31
+// CHECK2-NEXT:    [[DIV253:%.*]] = udiv i32 [[SUB252]], 32
+// CHECK2-NEXT:    [[MUL254:%.*]] = mul i32 [[MUL251]], [[DIV253]]
+// CHECK2-NEXT:    [[MUL255:%.*]] = mul i32 [[MUL254]], 32
+// CHECK2-NEXT:    [[CONV256:%.*]] = zext i32 [[MUL255]] to i64
+// CHECK2-NEXT:    [[MUL257:%.*]] = mul nsw i64 [[DIV246]], [[CONV256]]
+// CHECK2-NEXT:    [[SUB258:%.*]] = sub nsw i64 [[TMP127]], [[MUL257]]
+// CHECK2-NEXT:    [[TMP139:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB259:%.*]] = sub i32 [[TMP139]], -31
+// CHECK2-NEXT:    [[DIV260:%.*]] = udiv i32 [[SUB259]], 32
+// CHECK2-NEXT:    [[MUL261:%.*]] = mul i32 1, [[DIV260]]
+// CHECK2-NEXT:    [[MUL262:%.*]] = mul i32 [[MUL261]], 32
 // CHECK2-NEXT:    [[CONV263:%.*]] = zext i32 [[MUL262]] to i64
-// CHECK2-NEXT:    [[DIV264:%.*]] = sdiv i64 [[TMP139]], [[CONV263]]
-// CHECK2-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB265:%.*]] = sub i32 [[TMP147]], [[TMP148]]
-// CHECK2-NEXT:    [[SUB266:%.*]] = sub i32 [[SUB265]], 1
-// CHECK2-NEXT:    [[TMP149:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD267:%.*]] = add i32 [[SUB266]], [[TMP149]]
-// CHECK2-NEXT:    [[TMP150:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV268:%.*]] = udiv i32 [[ADD267]], [[TMP150]]
-// CHECK2-NEXT:    [[MUL269:%.*]] = mul i32 1, [[DIV268]]
-// CHECK2-NEXT:    [[TMP151:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB270:%.*]] = sub i32 [[TMP151]], -31
-// CHECK2-NEXT:    [[DIV271:%.*]] = udiv i32 [[SUB270]], 32
-// CHECK2-NEXT:    [[MUL272:%.*]] = mul i32 [[MUL269]], [[DIV271]]
-// CHECK2-NEXT:    [[TMP152:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP153:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB273:%.*]] = sub i32 [[TMP152]], [[TMP153]]
-// CHECK2-NEXT:    [[SUB274:%.*]] = sub i32 [[SUB273]], 1
-// CHECK2-NEXT:    [[ADD275:%.*]] = add i32 [[SUB274]], 1
-// CHECK2-NEXT:    [[DIV276:%.*]] = udiv i32 [[ADD275]], 1
-// CHECK2-NEXT:    [[MUL277:%.*]] = mul i32 [[MUL272]], [[DIV276]]
-// CHECK2-NEXT:    [[CONV278:%.*]] = zext i32 [[MUL277]] to i64
-// CHECK2-NEXT:    [[MUL279:%.*]] = mul nsw i64 [[DIV264]], [[CONV278]]
-// CHECK2-NEXT:    [[SUB280:%.*]] = sub nsw i64 [[TMP138]], [[MUL279]]
-// CHECK2-NEXT:    [[TMP154:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB281:%.*]] = sub i32 [[TMP154]], -31
-// CHECK2-NEXT:    [[DIV282:%.*]] = udiv i32 [[SUB281]], 32
-// CHECK2-NEXT:    [[MUL283:%.*]] = mul i32 1, [[DIV282]]
-// CHECK2-NEXT:    [[TMP155:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP156:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB284:%.*]] = sub i32 [[TMP155]], [[TMP156]]
-// CHECK2-NEXT:    [[SUB285:%.*]] = sub i32 [[SUB284]], 1
-// CHECK2-NEXT:    [[ADD286:%.*]] = add i32 [[SUB285]], 1
-// CHECK2-NEXT:    [[DIV287:%.*]] = udiv i32 [[ADD286]], 1
-// CHECK2-NEXT:    [[MUL288:%.*]] = mul i32 [[MUL283]], [[DIV287]]
-// CHECK2-NEXT:    [[CONV289:%.*]] = zext i32 [[MUL288]] to i64
-// CHECK2-NEXT:    [[DIV290:%.*]] = sdiv i64 [[SUB280]], [[CONV289]]
-// CHECK2-NEXT:    [[TMP157:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB291:%.*]] = sub i32 [[TMP157]], -31
-// CHECK2-NEXT:    [[DIV292:%.*]] = udiv i32 [[SUB291]], 32
-// CHECK2-NEXT:    [[MUL293:%.*]] = mul i32 1, [[DIV292]]
-// CHECK2-NEXT:    [[TMP158:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP159:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB294:%.*]] = sub i32 [[TMP158]], [[TMP159]]
-// CHECK2-NEXT:    [[SUB295:%.*]] = sub i32 [[SUB294]], 1
-// CHECK2-NEXT:    [[ADD296:%.*]] = add i32 [[SUB295]], 1
-// CHECK2-NEXT:    [[DIV297:%.*]] = udiv i32 [[ADD296]], 1
-// CHECK2-NEXT:    [[MUL298:%.*]] = mul i32 [[MUL293]], [[DIV297]]
-// CHECK2-NEXT:    [[CONV299:%.*]] = zext i32 [[MUL298]] to i64
-// CHECK2-NEXT:    [[MUL300:%.*]] = mul nsw i64 [[DIV290]], [[CONV299]]
-// CHECK2-NEXT:    [[SUB301:%.*]] = sub nsw i64 [[SUB249]], [[MUL300]]
-// CHECK2-NEXT:    [[TMP160:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP161:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP162:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP163:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB302:%.*]] = sub i32 [[TMP162]], [[TMP163]]
-// CHECK2-NEXT:    [[SUB303:%.*]] = sub i32 [[SUB302]], 1
-// CHECK2-NEXT:    [[TMP164:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD304:%.*]] = add i32 [[SUB303]], [[TMP164]]
-// CHECK2-NEXT:    [[TMP165:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV305:%.*]] = udiv i32 [[ADD304]], [[TMP165]]
-// CHECK2-NEXT:    [[MUL306:%.*]] = mul i32 1, [[DIV305]]
-// CHECK2-NEXT:    [[TMP166:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB307:%.*]] = sub i32 [[TMP166]], -31
-// CHECK2-NEXT:    [[DIV308:%.*]] = udiv i32 [[SUB307]], 32
-// CHECK2-NEXT:    [[MUL309:%.*]] = mul i32 [[MUL306]], [[DIV308]]
-// CHECK2-NEXT:    [[TMP167:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP168:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB310:%.*]] = sub i32 [[TMP167]], [[TMP168]]
-// CHECK2-NEXT:    [[SUB311:%.*]] = sub i32 [[SUB310]], 1
-// CHECK2-NEXT:    [[ADD312:%.*]] = add i32 [[SUB311]], 1
-// CHECK2-NEXT:    [[DIV313:%.*]] = udiv i32 [[ADD312]], 1
-// CHECK2-NEXT:    [[MUL314:%.*]] = mul i32 [[MUL309]], [[DIV313]]
-// CHECK2-NEXT:    [[CONV315:%.*]] = zext i32 [[MUL314]] to i64
-// CHECK2-NEXT:    [[DIV316:%.*]] = sdiv i64 [[TMP161]], [[CONV315]]
-// CHECK2-NEXT:    [[TMP169:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP170:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB317:%.*]] = sub i32 [[TMP169]], [[TMP170]]
-// CHECK2-NEXT:    [[SUB318:%.*]] = sub i32 [[SUB317]], 1
-// CHECK2-NEXT:    [[TMP171:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD319:%.*]] = add i32 [[SUB318]], [[TMP171]]
-// CHECK2-NEXT:    [[TMP172:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV320:%.*]] = udiv i32 [[ADD319]], [[TMP172]]
-// CHECK2-NEXT:    [[MUL321:%.*]] = mul i32 1, [[DIV320]]
-// CHECK2-NEXT:    [[TMP173:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB322:%.*]] = sub i32 [[TMP173]], -31
-// CHECK2-NEXT:    [[DIV323:%.*]] = udiv i32 [[SUB322]], 32
-// CHECK2-NEXT:    [[MUL324:%.*]] = mul i32 [[MUL321]], [[DIV323]]
-// CHECK2-NEXT:    [[TMP174:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP175:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB325:%.*]] = sub i32 [[TMP174]], [[TMP175]]
-// CHECK2-NEXT:    [[SUB326:%.*]] = sub i32 [[SUB325]], 1
-// CHECK2-NEXT:    [[ADD327:%.*]] = add i32 [[SUB326]], 1
-// CHECK2-NEXT:    [[DIV328:%.*]] = udiv i32 [[ADD327]], 1
-// CHECK2-NEXT:    [[MUL329:%.*]] = mul i32 [[MUL324]], [[DIV328]]
-// CHECK2-NEXT:    [[CONV330:%.*]] = zext i32 [[MUL329]] to i64
-// CHECK2-NEXT:    [[MUL331:%.*]] = mul nsw i64 [[DIV316]], [[CONV330]]
-// CHECK2-NEXT:    [[SUB332:%.*]] = sub nsw i64 [[TMP160]], [[MUL331]]
-// CHECK2-NEXT:    [[TMP176:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP177:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP178:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP179:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB333:%.*]] = sub i32 [[TMP178]], [[TMP179]]
-// CHECK2-NEXT:    [[SUB334:%.*]] = sub i32 [[SUB333]], 1
-// CHECK2-NEXT:    [[TMP180:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD335:%.*]] = add i32 [[SUB334]], [[TMP180]]
-// CHECK2-NEXT:    [[TMP181:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV336:%.*]] = udiv i32 [[ADD335]], [[TMP181]]
-// CHECK2-NEXT:    [[MUL337:%.*]] = mul i32 1, [[DIV336]]
-// CHECK2-NEXT:    [[TMP182:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB338:%.*]] = sub i32 [[TMP182]], -31
-// CHECK2-NEXT:    [[DIV339:%.*]] = udiv i32 [[SUB338]], 32
-// CHECK2-NEXT:    [[MUL340:%.*]] = mul i32 [[MUL337]], [[DIV339]]
-// CHECK2-NEXT:    [[TMP183:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP184:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB341:%.*]] = sub i32 [[TMP183]], [[TMP184]]
-// CHECK2-NEXT:    [[SUB342:%.*]] = sub i32 [[SUB341]], 1
-// CHECK2-NEXT:    [[ADD343:%.*]] = add i32 [[SUB342]], 1
-// CHECK2-NEXT:    [[DIV344:%.*]] = udiv i32 [[ADD343]], 1
-// CHECK2-NEXT:    [[MUL345:%.*]] = mul i32 [[MUL340]], [[DIV344]]
-// CHECK2-NEXT:    [[CONV346:%.*]] = zext i32 [[MUL345]] to i64
-// CHECK2-NEXT:    [[DIV347:%.*]] = sdiv i64 [[TMP177]], [[CONV346]]
-// CHECK2-NEXT:    [[TMP185:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP186:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB348:%.*]] = sub i32 [[TMP185]], [[TMP186]]
-// CHECK2-NEXT:    [[SUB349:%.*]] = sub i32 [[SUB348]], 1
-// CHECK2-NEXT:    [[TMP187:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD350:%.*]] = add i32 [[SUB349]], [[TMP187]]
-// CHECK2-NEXT:    [[TMP188:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV351:%.*]] = udiv i32 [[ADD350]], [[TMP188]]
-// CHECK2-NEXT:    [[MUL352:%.*]] = mul i32 1, [[DIV351]]
-// CHECK2-NEXT:    [[TMP189:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB353:%.*]] = sub i32 [[TMP189]], -31
-// CHECK2-NEXT:    [[DIV354:%.*]] = udiv i32 [[SUB353]], 32
-// CHECK2-NEXT:    [[MUL355:%.*]] = mul i32 [[MUL352]], [[DIV354]]
-// CHECK2-NEXT:    [[TMP190:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP191:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB356:%.*]] = sub i32 [[TMP190]], [[TMP191]]
-// CHECK2-NEXT:    [[SUB357:%.*]] = sub i32 [[SUB356]], 1
-// CHECK2-NEXT:    [[ADD358:%.*]] = add i32 [[SUB357]], 1
-// CHECK2-NEXT:    [[DIV359:%.*]] = udiv i32 [[ADD358]], 1
-// CHECK2-NEXT:    [[MUL360:%.*]] = mul i32 [[MUL355]], [[DIV359]]
-// CHECK2-NEXT:    [[CONV361:%.*]] = zext i32 [[MUL360]] to i64
-// CHECK2-NEXT:    [[MUL362:%.*]] = mul nsw i64 [[DIV347]], [[CONV361]]
-// CHECK2-NEXT:    [[SUB363:%.*]] = sub nsw i64 [[TMP176]], [[MUL362]]
-// CHECK2-NEXT:    [[TMP192:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB364:%.*]] = sub i32 [[TMP192]], -31
-// CHECK2-NEXT:    [[DIV365:%.*]] = udiv i32 [[SUB364]], 32
-// CHECK2-NEXT:    [[MUL366:%.*]] = mul i32 1, [[DIV365]]
-// CHECK2-NEXT:    [[TMP193:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP194:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB367:%.*]] = sub i32 [[TMP193]], [[TMP194]]
-// CHECK2-NEXT:    [[SUB368:%.*]] = sub i32 [[SUB367]], 1
-// CHECK2-NEXT:    [[ADD369:%.*]] = add i32 [[SUB368]], 1
-// CHECK2-NEXT:    [[DIV370:%.*]] = udiv i32 [[ADD369]], 1
-// CHECK2-NEXT:    [[MUL371:%.*]] = mul i32 [[MUL366]], [[DIV370]]
-// CHECK2-NEXT:    [[CONV372:%.*]] = zext i32 [[MUL371]] to i64
-// CHECK2-NEXT:    [[DIV373:%.*]] = sdiv i64 [[SUB363]], [[CONV372]]
-// CHECK2-NEXT:    [[TMP195:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB374:%.*]] = sub i32 [[TMP195]], -31
-// CHECK2-NEXT:    [[DIV375:%.*]] = udiv i32 [[SUB374]], 32
-// CHECK2-NEXT:    [[MUL376:%.*]] = mul i32 1, [[DIV375]]
-// CHECK2-NEXT:    [[TMP196:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP197:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB377:%.*]] = sub i32 [[TMP196]], [[TMP197]]
-// CHECK2-NEXT:    [[SUB378:%.*]] = sub i32 [[SUB377]], 1
-// CHECK2-NEXT:    [[ADD379:%.*]] = add i32 [[SUB378]], 1
-// CHECK2-NEXT:    [[DIV380:%.*]] = udiv i32 [[ADD379]], 1
-// CHECK2-NEXT:    [[MUL381:%.*]] = mul i32 [[MUL376]], [[DIV380]]
-// CHECK2-NEXT:    [[CONV382:%.*]] = zext i32 [[MUL381]] to i64
-// CHECK2-NEXT:    [[MUL383:%.*]] = mul nsw i64 [[DIV373]], [[CONV382]]
-// CHECK2-NEXT:    [[SUB384:%.*]] = sub nsw i64 [[SUB332]], [[MUL383]]
-// CHECK2-NEXT:    [[TMP198:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP199:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB385:%.*]] = sub i32 [[TMP198]], [[TMP199]]
-// CHECK2-NEXT:    [[SUB386:%.*]] = sub i32 [[SUB385]], 1
-// CHECK2-NEXT:    [[ADD387:%.*]] = add i32 [[SUB386]], 1
-// CHECK2-NEXT:    [[DIV388:%.*]] = udiv i32 [[ADD387]], 1
-// CHECK2-NEXT:    [[MUL389:%.*]] = mul i32 1, [[DIV388]]
-// CHECK2-NEXT:    [[CONV390:%.*]] = zext i32 [[MUL389]] to i64
-// CHECK2-NEXT:    [[DIV391:%.*]] = sdiv i64 [[SUB384]], [[CONV390]]
-// CHECK2-NEXT:    [[TMP200:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP201:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB392:%.*]] = sub i32 [[TMP200]], [[TMP201]]
-// CHECK2-NEXT:    [[SUB393:%.*]] = sub i32 [[SUB392]], 1
-// CHECK2-NEXT:    [[ADD394:%.*]] = add i32 [[SUB393]], 1
-// CHECK2-NEXT:    [[DIV395:%.*]] = udiv i32 [[ADD394]], 1
-// CHECK2-NEXT:    [[MUL396:%.*]] = mul i32 1, [[DIV395]]
-// CHECK2-NEXT:    [[CONV397:%.*]] = zext i32 [[MUL396]] to i64
-// CHECK2-NEXT:    [[MUL398:%.*]] = mul nsw i64 [[DIV391]], [[CONV397]]
-// CHECK2-NEXT:    [[SUB399:%.*]] = sub nsw i64 [[SUB301]], [[MUL398]]
-// CHECK2-NEXT:    [[MUL400:%.*]] = mul nsw i64 [[SUB399]], 1
-// CHECK2-NEXT:    [[ADD401:%.*]] = add nsw i64 [[CONV218]], [[MUL400]]
-// CHECK2-NEXT:    [[CONV402:%.*]] = trunc i64 [[ADD401]] to i32
-// CHECK2-NEXT:    store i32 [[CONV402]], ptr [[DOTTILE_0_IV_K52]], align 4
-// CHECK2-NEXT:    [[TMP202:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
-// CHECK2-NEXT:    [[TMP203:%.*]] = load i32, ptr [[DOTTILE_0_IV_K52]], align 4
-// CHECK2-NEXT:    [[TMP204:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
-// CHECK2-NEXT:    [[MUL403:%.*]] = mul i32 [[TMP203]], [[TMP204]]
-// CHECK2-NEXT:    [[ADD404:%.*]] = add i32 [[TMP202]], [[MUL403]]
-// CHECK2-NEXT:    store i32 [[ADD404]], ptr [[K]], align 4
-// CHECK2-NEXT:    [[TMP205:%.*]] = load i32, ptr [[I49]], align 4
-// CHECK2-NEXT:    [[TMP206:%.*]] = load i32, ptr [[J50]], align 4
-// CHECK2-NEXT:    [[TMP207:%.*]] = load i32, ptr [[K]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP205]], i32 noundef [[TMP206]], i32 noundef [[TMP207]])
+// CHECK2-NEXT:    [[DIV264:%.*]] = sdiv i64 [[SUB258]], [[CONV263]]
+// CHECK2-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB265:%.*]] = sub i32 [[TMP140]], -31
+// CHECK2-NEXT:    [[DIV266:%.*]] = udiv i32 [[SUB265]], 32
+// CHECK2-NEXT:    [[MUL267:%.*]] = mul i32 1, [[DIV266]]
+// CHECK2-NEXT:    [[MUL268:%.*]] = mul i32 [[MUL267]], 32
+// CHECK2-NEXT:    [[CONV269:%.*]] = zext i32 [[MUL268]] to i64
+// CHECK2-NEXT:    [[MUL270:%.*]] = mul nsw i64 [[DIV264]], [[CONV269]]
+// CHECK2-NEXT:    [[SUB271:%.*]] = sub nsw i64 [[SUB235]], [[MUL270]]
+// CHECK2-NEXT:    [[DIV272:%.*]] = sdiv i64 [[SUB271]], 32
+// CHECK2-NEXT:    [[MUL273:%.*]] = mul nsw i64 [[DIV272]], 32
+// CHECK2-NEXT:    [[SUB274:%.*]] = sub nsw i64 [[SUB212]], [[MUL273]]
+// CHECK2-NEXT:    [[MUL275:%.*]] = mul nsw i64 [[SUB274]], 1
+// CHECK2-NEXT:    [[ADD276:%.*]] = add nsw i64 0, [[MUL275]]
+// CHECK2-NEXT:    [[CONV277:%.*]] = trunc i64 [[ADD276]] to i32
+// CHECK2-NEXT:    store i32 [[CONV277]], ptr [[DOTTILE_CNT_0_IV_K38]], align 4
+// CHECK2-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
+// CHECK2-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
+// CHECK2-NEXT:    [[ADD278:%.*]] = add i32 [[TMP141]], [[TMP142]]
+// CHECK2-NEXT:    store i32 [[ADD278]], ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK2-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
+// CHECK2-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK2-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
+// CHECK2-NEXT:    [[MUL279:%.*]] = mul i32 [[TMP144]], [[TMP145]]
+// CHECK2-NEXT:    [[ADD280:%.*]] = add i32 [[TMP143]], [[MUL279]]
+// CHECK2-NEXT:    store i32 [[ADD280]], ptr [[K]], align 4
+// CHECK2-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK2-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// CHECK2-NEXT:    [[ADD281:%.*]] = add i32 [[TMP147]], 1
+// CHECK2-NEXT:    [[CMP282:%.*]] = icmp ult i32 [[TMP146]], [[ADD281]]
+// CHECK2-NEXT:    br i1 [[CMP282]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK2:       if.then:
+// CHECK2-NEXT:    [[TMP148:%.*]] = load i32, ptr [[I35]], align 4
+// CHECK2-NEXT:    [[TMP149:%.*]] = load i32, ptr [[J36]], align 4
+// CHECK2-NEXT:    [[TMP150:%.*]] = load i32, ptr [[K]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP148]], i32 noundef [[TMP149]], i32 noundef [[TMP150]])
+// CHECK2-NEXT:    br label [[IF_END]]
+// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
-// CHECK2-NEXT:    [[TMP208:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[ADD405:%.*]] = add nsw i64 [[TMP208]], 1
-// CHECK2-NEXT:    store i64 [[ADD405]], ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP151:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[ADD283:%.*]] = add nsw i64 [[TMP151]], 1
+// CHECK2-NEXT:    store i64 [[ADD283]], ptr [[DOTOMP_IV]], align 8
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -4635,22 +4099,21 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTNEW_STEP10:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_14:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_17:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_22:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_K:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_K:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[I49:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[J50:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV_K51:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_K52:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[I35:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[J36:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV_K37:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_K38:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
@@ -4688,630 +4151,452 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
 // CHECK2-NEXT:    [[ADD15:%.*]] = add i32 [[TMP15]], 1
 // CHECK2-NEXT:    store i32 [[ADD15]], ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[_TMP2]], align 4
-// CHECK2-NEXT:    store i32 [[TMP16]], ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK2-NEXT:    [[ADD18:%.*]] = add i32 [[TMP17]], 1
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[_TMP2]], align 4
-// CHECK2-NEXT:    [[ADD19:%.*]] = add i32 [[TMP18]], 64
-// CHECK2-NEXT:    [[CMP:%.*]] = icmp ult i32 [[ADD18]], [[ADD19]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK2:       cond.true:
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK2-NEXT:    [[ADD20:%.*]] = add i32 [[TMP19]], 1
-// CHECK2-NEXT:    br label [[COND_END:%.*]]
-// CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[_TMP2]], align 4
-// CHECK2-NEXT:    [[ADD21:%.*]] = add i32 [[TMP20]], 64
-// CHECK2-NEXT:    br label [[COND_END]]
-// CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD20]], [[COND_TRUE]] ], [ [[ADD21]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    store i32 [[COND]], ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[SUB23:%.*]] = sub i32 [[TMP21]], [[TMP22]]
-// CHECK2-NEXT:    [[SUB24:%.*]] = sub i32 [[SUB23]], 1
-// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[ADD25:%.*]] = add i32 [[SUB24]], [[TMP23]]
-// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[DIV26:%.*]] = udiv i32 [[ADD25]], [[TMP24]]
-// CHECK2-NEXT:    [[CONV:%.*]] = zext i32 [[DIV26]] to i64
-// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[SUB17:%.*]] = sub i32 [[TMP16]], [[TMP17]]
+// CHECK2-NEXT:    [[SUB18:%.*]] = sub i32 [[SUB17]], 1
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[ADD19:%.*]] = add i32 [[SUB18]], [[TMP18]]
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[DIV20:%.*]] = udiv i32 [[ADD19]], [[TMP19]]
+// CHECK2-NEXT:    [[CONV:%.*]] = zext i32 [[DIV20]] to i64
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB21:%.*]] = sub i32 [[TMP20]], [[TMP21]]
+// CHECK2-NEXT:    [[SUB22:%.*]] = sub i32 [[SUB21]], 1
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD23:%.*]] = add i32 [[SUB22]], [[TMP22]]
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV24:%.*]] = udiv i32 [[ADD23]], [[TMP23]]
+// CHECK2-NEXT:    [[CONV25:%.*]] = zext i32 [[DIV24]] to i64
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV25]]
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB26:%.*]] = sub i32 [[TMP24]], -63
+// CHECK2-NEXT:    [[DIV27:%.*]] = udiv i32 [[SUB26]], 64
+// CHECK2-NEXT:    [[CONV28:%.*]] = zext i32 [[DIV27]] to i64
+// CHECK2-NEXT:    [[MUL29:%.*]] = mul nsw i64 [[MUL]], [[CONV28]]
+// CHECK2-NEXT:    [[MUL30:%.*]] = mul nsw i64 [[MUL29]], 64
+// CHECK2-NEXT:    [[SUB31:%.*]] = sub nsw i64 [[MUL30]], 1
+// CHECK2-NEXT:    store i64 [[SUB31]], ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    store i32 [[TMP25]], ptr [[I]], align 4
 // CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB27:%.*]] = sub i32 [[TMP25]], [[TMP26]]
-// CHECK2-NEXT:    [[SUB28:%.*]] = sub i32 [[SUB27]], 1
-// CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD29:%.*]] = add i32 [[SUB28]], [[TMP27]]
-// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV30:%.*]] = udiv i32 [[ADD29]], [[TMP28]]
-// CHECK2-NEXT:    [[CONV31:%.*]] = zext i32 [[DIV30]] to i64
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV31]]
-// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB32:%.*]] = sub i32 [[TMP29]], -63
-// CHECK2-NEXT:    [[DIV33:%.*]] = udiv i32 [[SUB32]], 64
-// CHECK2-NEXT:    [[CONV34:%.*]] = zext i32 [[DIV33]] to i64
-// CHECK2-NEXT:    [[MUL35:%.*]] = mul nsw i64 [[MUL]], [[CONV34]]
-// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB36:%.*]] = sub i32 [[TMP30]], [[TMP31]]
-// CHECK2-NEXT:    [[SUB37:%.*]] = sub i32 [[SUB36]], 1
-// CHECK2-NEXT:    [[ADD38:%.*]] = add i32 [[SUB37]], 1
-// CHECK2-NEXT:    [[DIV39:%.*]] = udiv i32 [[ADD38]], 1
-// CHECK2-NEXT:    [[CONV40:%.*]] = zext i32 [[DIV39]] to i64
-// CHECK2-NEXT:    [[MUL41:%.*]] = mul nsw i64 [[MUL35]], [[CONV40]]
-// CHECK2-NEXT:    [[SUB42:%.*]] = sub nsw i64 [[MUL41]], 1
-// CHECK2-NEXT:    store i64 [[SUB42]], ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK2-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    store i32 [[TMP32]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    store i32 [[TMP33]], ptr [[J]], align 4
+// CHECK2-NEXT:    store i32 [[TMP26]], ptr [[J]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_K]], align 4
-// CHECK2-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    store i32 [[TMP34]], ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK2-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK2-NEXT:    [[CMP43:%.*]] = icmp slt i32 [[TMP35]], [[TMP36]]
-// CHECK2-NEXT:    br i1 [[CMP43]], label [[LAND_LHS_TRUE:%.*]], label [[OMP_PRECOND_END:%.*]]
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_K]], align 4
+// CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP27]], [[TMP28]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[LAND_LHS_TRUE:%.*]], label [[OMP_PRECOND_END:%.*]]
 // CHECK2:       land.lhs.true:
-// CHECK2-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[CMP44:%.*]] = icmp slt i32 [[TMP37]], [[TMP38]]
-// CHECK2-NEXT:    br i1 [[CMP44]], label [[LAND_LHS_TRUE45:%.*]], label [[OMP_PRECOND_END]]
-// CHECK2:       land.lhs.true45:
-// CHECK2-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[CMP46:%.*]] = icmp ult i32 0, [[TMP39]]
-// CHECK2-NEXT:    br i1 [[CMP46]], label [[LAND_LHS_TRUE47:%.*]], label [[OMP_PRECOND_END]]
-// CHECK2:       land.lhs.true47:
-// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[CMP48:%.*]] = icmp ult i32 [[TMP40]], [[TMP41]]
-// CHECK2-NEXT:    br i1 [[CMP48]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
+// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[CMP32:%.*]] = icmp slt i32 [[TMP29]], [[TMP30]]
+// CHECK2-NEXT:    br i1 [[CMP32]], label [[LAND_LHS_TRUE33:%.*]], label [[OMP_PRECOND_END]]
+// CHECK2:       land.lhs.true33:
+// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[CMP34:%.*]] = icmp ult i32 0, [[TMP31]]
+// CHECK2-NEXT:    br i1 [[CMP34]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
 // CHECK2:       omp.precond.then:
 // CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
-// CHECK2-NEXT:    [[TMP42:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK2-NEXT:    store i64 [[TMP42]], ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK2-NEXT:    store i64 [[TMP32]], ptr [[DOTOMP_UB]], align 8
 // CHECK2-NEXT:    store i64 1, ptr [[DOTOMP_STRIDE]], align 8
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
 // CHECK2-NEXT:    call void @__kmpc_for_static_init_8(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i64 1, i64 1)
-// CHECK2-NEXT:    [[TMP43:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[TMP44:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK2-NEXT:    [[CMP53:%.*]] = icmp sgt i64 [[TMP43]], [[TMP44]]
-// CHECK2-NEXT:    br i1 [[CMP53]], label [[COND_TRUE54:%.*]], label [[COND_FALSE55:%.*]]
-// CHECK2:       cond.true54:
-// CHECK2-NEXT:    [[TMP45:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK2-NEXT:    br label [[COND_END56:%.*]]
-// CHECK2:       cond.false55:
-// CHECK2-NEXT:    [[TMP46:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    br label [[COND_END56]]
-// CHECK2:       cond.end56:
-// CHECK2-NEXT:    [[COND57:%.*]] = phi i64 [ [[TMP45]], [[COND_TRUE54]] ], [ [[TMP46]], [[COND_FALSE55]] ]
-// CHECK2-NEXT:    store i64 [[COND57]], ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[TMP47:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
-// CHECK2-NEXT:    store i64 [[TMP47]], ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK2-NEXT:    [[CMP39:%.*]] = icmp sgt i64 [[TMP33]], [[TMP34]]
+// CHECK2-NEXT:    br i1 [[CMP39]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i64 [ [[TMP35]], [[COND_TRUE]] ], [ [[TMP36]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    store i64 [[COND]], ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
+// CHECK2-NEXT:    store i64 [[TMP37]], ptr [[DOTOMP_IV]], align 8
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK2:       omp.inner.for.cond:
-// CHECK2-NEXT:    [[TMP48:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[CMP58:%.*]] = icmp sle i64 [[TMP48]], [[TMP49]]
-// CHECK2-NEXT:    br i1 [[CMP58]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK2-NEXT:    [[TMP38:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP39:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[CMP40:%.*]] = icmp sle i64 [[TMP38]], [[TMP39]]
+// CHECK2-NEXT:    br i1 [[CMP40]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
 // CHECK2:       omp.inner.for.body:
-// CHECK2-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[CONV59:%.*]] = sext i32 [[TMP50]] to i64
-// CHECK2-NEXT:    [[TMP51:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB60:%.*]] = sub i32 [[TMP52]], [[TMP53]]
-// CHECK2-NEXT:    [[SUB61:%.*]] = sub i32 [[SUB60]], 1
+// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[CONV41:%.*]] = sext i32 [[TMP40]] to i64
+// CHECK2-NEXT:    [[TMP41:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB42:%.*]] = sub i32 [[TMP42]], [[TMP43]]
+// CHECK2-NEXT:    [[SUB43:%.*]] = sub i32 [[SUB42]], 1
+// CHECK2-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD44:%.*]] = add i32 [[SUB43]], [[TMP44]]
+// CHECK2-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV45:%.*]] = udiv i32 [[ADD44]], [[TMP45]]
+// CHECK2-NEXT:    [[MUL46:%.*]] = mul i32 1, [[DIV45]]
+// CHECK2-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB47:%.*]] = sub i32 [[TMP46]], -63
+// CHECK2-NEXT:    [[DIV48:%.*]] = udiv i32 [[SUB47]], 64
+// CHECK2-NEXT:    [[MUL49:%.*]] = mul i32 [[MUL46]], [[DIV48]]
+// CHECK2-NEXT:    [[MUL50:%.*]] = mul i32 [[MUL49]], 64
+// CHECK2-NEXT:    [[CONV51:%.*]] = zext i32 [[MUL50]] to i64
+// CHECK2-NEXT:    [[DIV52:%.*]] = sdiv i64 [[TMP41]], [[CONV51]]
+// CHECK2-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[CONV53:%.*]] = sext i32 [[TMP47]] to i64
+// CHECK2-NEXT:    [[MUL54:%.*]] = mul nsw i64 [[DIV52]], [[CONV53]]
+// CHECK2-NEXT:    [[ADD55:%.*]] = add nsw i64 [[CONV41]], [[MUL54]]
+// CHECK2-NEXT:    [[CONV56:%.*]] = trunc i64 [[ADD55]] to i32
+// CHECK2-NEXT:    store i32 [[CONV56]], ptr [[I35]], align 4
+// CHECK2-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP48]] to i64
+// CHECK2-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP50:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB58:%.*]] = sub i32 [[TMP51]], [[TMP52]]
+// CHECK2-NEXT:    [[SUB59:%.*]] = sub i32 [[SUB58]], 1
+// CHECK2-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD60:%.*]] = add i32 [[SUB59]], [[TMP53]]
 // CHECK2-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD62:%.*]] = add i32 [[SUB61]], [[TMP54]]
-// CHECK2-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV63:%.*]] = udiv i32 [[ADD62]], [[TMP55]]
-// CHECK2-NEXT:    [[MUL64:%.*]] = mul i32 1, [[DIV63]]
-// CHECK2-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB65:%.*]] = sub i32 [[TMP56]], -63
-// CHECK2-NEXT:    [[DIV66:%.*]] = udiv i32 [[SUB65]], 64
-// CHECK2-NEXT:    [[MUL67:%.*]] = mul i32 [[MUL64]], [[DIV66]]
-// CHECK2-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB68:%.*]] = sub i32 [[TMP57]], [[TMP58]]
-// CHECK2-NEXT:    [[SUB69:%.*]] = sub i32 [[SUB68]], 1
-// CHECK2-NEXT:    [[ADD70:%.*]] = add i32 [[SUB69]], 1
-// CHECK2-NEXT:    [[DIV71:%.*]] = udiv i32 [[ADD70]], 1
-// CHECK2-NEXT:    [[MUL72:%.*]] = mul i32 [[MUL67]], [[DIV71]]
-// CHECK2-NEXT:    [[CONV73:%.*]] = zext i32 [[MUL72]] to i64
-// CHECK2-NEXT:    [[DIV74:%.*]] = sdiv i64 [[TMP51]], [[CONV73]]
-// CHECK2-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[CONV75:%.*]] = sext i32 [[TMP59]] to i64
-// CHECK2-NEXT:    [[MUL76:%.*]] = mul nsw i64 [[DIV74]], [[CONV75]]
-// CHECK2-NEXT:    [[ADD77:%.*]] = add nsw i64 [[CONV59]], [[MUL76]]
-// CHECK2-NEXT:    [[CONV78:%.*]] = trunc i64 [[ADD77]] to i32
-// CHECK2-NEXT:    store i32 [[CONV78]], ptr [[I49]], align 4
-// CHECK2-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[CONV79:%.*]] = sext i32 [[TMP60]] to i64
-// CHECK2-NEXT:    [[TMP61:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP62:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP63:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP64:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB80:%.*]] = sub i32 [[TMP63]], [[TMP64]]
-// CHECK2-NEXT:    [[SUB81:%.*]] = sub i32 [[SUB80]], 1
-// CHECK2-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD82:%.*]] = add i32 [[SUB81]], [[TMP65]]
-// CHECK2-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV83:%.*]] = udiv i32 [[ADD82]], [[TMP66]]
-// CHECK2-NEXT:    [[MUL84:%.*]] = mul i32 1, [[DIV83]]
-// CHECK2-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB85:%.*]] = sub i32 [[TMP67]], -63
-// CHECK2-NEXT:    [[DIV86:%.*]] = udiv i32 [[SUB85]], 64
-// CHECK2-NEXT:    [[MUL87:%.*]] = mul i32 [[MUL84]], [[DIV86]]
-// CHECK2-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB88:%.*]] = sub i32 [[TMP68]], [[TMP69]]
-// CHECK2-NEXT:    [[SUB89:%.*]] = sub i32 [[SUB88]], 1
-// CHECK2-NEXT:    [[ADD90:%.*]] = add i32 [[SUB89]], 1
-// CHECK2-NEXT:    [[DIV91:%.*]] = udiv i32 [[ADD90]], 1
-// CHECK2-NEXT:    [[MUL92:%.*]] = mul i32 [[MUL87]], [[DIV91]]
-// CHECK2-NEXT:    [[CONV93:%.*]] = zext i32 [[MUL92]] to i64
-// CHECK2-NEXT:    [[DIV94:%.*]] = sdiv i64 [[TMP62]], [[CONV93]]
+// CHECK2-NEXT:    [[DIV61:%.*]] = udiv i32 [[ADD60]], [[TMP54]]
+// CHECK2-NEXT:    [[MUL62:%.*]] = mul i32 1, [[DIV61]]
+// CHECK2-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB63:%.*]] = sub i32 [[TMP55]], -63
+// CHECK2-NEXT:    [[DIV64:%.*]] = udiv i32 [[SUB63]], 64
+// CHECK2-NEXT:    [[MUL65:%.*]] = mul i32 [[MUL62]], [[DIV64]]
+// CHECK2-NEXT:    [[MUL66:%.*]] = mul i32 [[MUL65]], 64
+// CHECK2-NEXT:    [[CONV67:%.*]] = zext i32 [[MUL66]] to i64
+// CHECK2-NEXT:    [[DIV68:%.*]] = sdiv i64 [[TMP50]], [[CONV67]]
+// CHECK2-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB69:%.*]] = sub i32 [[TMP56]], [[TMP57]]
+// CHECK2-NEXT:    [[SUB70:%.*]] = sub i32 [[SUB69]], 1
+// CHECK2-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD71:%.*]] = add i32 [[SUB70]], [[TMP58]]
+// CHECK2-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV72:%.*]] = udiv i32 [[ADD71]], [[TMP59]]
+// CHECK2-NEXT:    [[MUL73:%.*]] = mul i32 1, [[DIV72]]
+// CHECK2-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB74:%.*]] = sub i32 [[TMP60]], -63
+// CHECK2-NEXT:    [[DIV75:%.*]] = udiv i32 [[SUB74]], 64
+// CHECK2-NEXT:    [[MUL76:%.*]] = mul i32 [[MUL73]], [[DIV75]]
+// CHECK2-NEXT:    [[MUL77:%.*]] = mul i32 [[MUL76]], 64
+// CHECK2-NEXT:    [[CONV78:%.*]] = zext i32 [[MUL77]] to i64
+// CHECK2-NEXT:    [[MUL79:%.*]] = mul nsw i64 [[DIV68]], [[CONV78]]
+// CHECK2-NEXT:    [[SUB80:%.*]] = sub nsw i64 [[TMP49]], [[MUL79]]
+// CHECK2-NEXT:    [[TMP61:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB81:%.*]] = sub i32 [[TMP61]], -63
+// CHECK2-NEXT:    [[DIV82:%.*]] = udiv i32 [[SUB81]], 64
+// CHECK2-NEXT:    [[MUL83:%.*]] = mul i32 1, [[DIV82]]
+// CHECK2-NEXT:    [[MUL84:%.*]] = mul i32 [[MUL83]], 64
+// CHECK2-NEXT:    [[CONV85:%.*]] = zext i32 [[MUL84]] to i64
+// CHECK2-NEXT:    [[DIV86:%.*]] = sdiv i64 [[SUB80]], [[CONV85]]
+// CHECK2-NEXT:    [[TMP62:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[CONV87:%.*]] = sext i32 [[TMP62]] to i64
+// CHECK2-NEXT:    [[MUL88:%.*]] = mul nsw i64 [[DIV86]], [[CONV87]]
+// CHECK2-NEXT:    [[ADD89:%.*]] = add nsw i64 [[CONV57]], [[MUL88]]
+// CHECK2-NEXT:    [[CONV90:%.*]] = trunc i64 [[ADD89]] to i32
+// CHECK2-NEXT:    store i32 [[CONV90]], ptr [[J36]], align 4
+// CHECK2-NEXT:    [[TMP63:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP64:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB91:%.*]] = sub i32 [[TMP65]], [[TMP66]]
+// CHECK2-NEXT:    [[SUB92:%.*]] = sub i32 [[SUB91]], 1
+// CHECK2-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD93:%.*]] = add i32 [[SUB92]], [[TMP67]]
+// CHECK2-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV94:%.*]] = udiv i32 [[ADD93]], [[TMP68]]
+// CHECK2-NEXT:    [[MUL95:%.*]] = mul i32 1, [[DIV94]]
+// CHECK2-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB96:%.*]] = sub i32 [[TMP69]], -63
+// CHECK2-NEXT:    [[DIV97:%.*]] = udiv i32 [[SUB96]], 64
+// CHECK2-NEXT:    [[MUL98:%.*]] = mul i32 [[MUL95]], [[DIV97]]
+// CHECK2-NEXT:    [[MUL99:%.*]] = mul i32 [[MUL98]], 64
+// CHECK2-NEXT:    [[CONV100:%.*]] = zext i32 [[MUL99]] to i64
+// CHECK2-NEXT:    [[DIV101:%.*]] = sdiv i64 [[TMP64]], [[CONV100]]
 // CHECK2-NEXT:    [[TMP70:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
 // CHECK2-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB95:%.*]] = sub i32 [[TMP70]], [[TMP71]]
-// CHECK2-NEXT:    [[SUB96:%.*]] = sub i32 [[SUB95]], 1
+// CHECK2-NEXT:    [[SUB102:%.*]] = sub i32 [[TMP70]], [[TMP71]]
+// CHECK2-NEXT:    [[SUB103:%.*]] = sub i32 [[SUB102]], 1
 // CHECK2-NEXT:    [[TMP72:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD97:%.*]] = add i32 [[SUB96]], [[TMP72]]
+// CHECK2-NEXT:    [[ADD104:%.*]] = add i32 [[SUB103]], [[TMP72]]
 // CHECK2-NEXT:    [[TMP73:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV98:%.*]] = udiv i32 [[ADD97]], [[TMP73]]
-// CHECK2-NEXT:    [[MUL99:%.*]] = mul i32 1, [[DIV98]]
+// CHECK2-NEXT:    [[DIV105:%.*]] = udiv i32 [[ADD104]], [[TMP73]]
+// CHECK2-NEXT:    [[MUL106:%.*]] = mul i32 1, [[DIV105]]
 // CHECK2-NEXT:    [[TMP74:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB100:%.*]] = sub i32 [[TMP74]], -63
-// CHECK2-NEXT:    [[DIV101:%.*]] = udiv i32 [[SUB100]], 64
-// CHECK2-NEXT:    [[MUL102:%.*]] = mul i32 [[MUL99]], [[DIV101]]
-// CHECK2-NEXT:    [[TMP75:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP76:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB103:%.*]] = sub i32 [[TMP75]], [[TMP76]]
-// CHECK2-NEXT:    [[SUB104:%.*]] = sub i32 [[SUB103]], 1
-// CHECK2-NEXT:    [[ADD105:%.*]] = add i32 [[SUB104]], 1
-// CHECK2-NEXT:    [[DIV106:%.*]] = udiv i32 [[ADD105]], 1
-// CHECK2-NEXT:    [[MUL107:%.*]] = mul i32 [[MUL102]], [[DIV106]]
-// CHECK2-NEXT:    [[CONV108:%.*]] = zext i32 [[MUL107]] to i64
-// CHECK2-NEXT:    [[MUL109:%.*]] = mul nsw i64 [[DIV94]], [[CONV108]]
-// CHECK2-NEXT:    [[SUB110:%.*]] = sub nsw i64 [[TMP61]], [[MUL109]]
-// CHECK2-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB111:%.*]] = sub i32 [[TMP77]], -63
-// CHECK2-NEXT:    [[DIV112:%.*]] = udiv i32 [[SUB111]], 64
-// CHECK2-NEXT:    [[MUL113:%.*]] = mul i32 1, [[DIV112]]
-// CHECK2-NEXT:    [[TMP78:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP79:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB114:%.*]] = sub i32 [[TMP78]], [[TMP79]]
+// CHECK2-NEXT:    [[SUB107:%.*]] = sub i32 [[TMP74]], -63
+// CHECK2-NEXT:    [[DIV108:%.*]] = udiv i32 [[SUB107]], 64
+// CHECK2-NEXT:    [[MUL109:%.*]] = mul i32 [[MUL106]], [[DIV108]]
+// CHECK2-NEXT:    [[MUL110:%.*]] = mul i32 [[MUL109]], 64
+// CHECK2-NEXT:    [[CONV111:%.*]] = zext i32 [[MUL110]] to i64
+// CHECK2-NEXT:    [[MUL112:%.*]] = mul nsw i64 [[DIV101]], [[CONV111]]
+// CHECK2-NEXT:    [[SUB113:%.*]] = sub nsw i64 [[TMP63]], [[MUL112]]
+// CHECK2-NEXT:    [[TMP75:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP76:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP78:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB114:%.*]] = sub i32 [[TMP77]], [[TMP78]]
 // CHECK2-NEXT:    [[SUB115:%.*]] = sub i32 [[SUB114]], 1
-// CHECK2-NEXT:    [[ADD116:%.*]] = add i32 [[SUB115]], 1
-// CHECK2-NEXT:    [[DIV117:%.*]] = udiv i32 [[ADD116]], 1
-// CHECK2-NEXT:    [[MUL118:%.*]] = mul i32 [[MUL113]], [[DIV117]]
-// CHECK2-NEXT:    [[CONV119:%.*]] = zext i32 [[MUL118]] to i64
-// CHECK2-NEXT:    [[DIV120:%.*]] = sdiv i64 [[SUB110]], [[CONV119]]
+// CHECK2-NEXT:    [[TMP79:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD116:%.*]] = add i32 [[SUB115]], [[TMP79]]
 // CHECK2-NEXT:    [[TMP80:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[CONV121:%.*]] = sext i32 [[TMP80]] to i64
-// CHECK2-NEXT:    [[MUL122:%.*]] = mul nsw i64 [[DIV120]], [[CONV121]]
-// CHECK2-NEXT:    [[ADD123:%.*]] = add nsw i64 [[CONV79]], [[MUL122]]
-// CHECK2-NEXT:    [[CONV124:%.*]] = trunc i64 [[ADD123]] to i32
-// CHECK2-NEXT:    store i32 [[CONV124]], ptr [[J50]], align 4
-// CHECK2-NEXT:    [[TMP81:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP82:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB125:%.*]] = sub i32 [[TMP83]], [[TMP84]]
+// CHECK2-NEXT:    [[DIV117:%.*]] = udiv i32 [[ADD116]], [[TMP80]]
+// CHECK2-NEXT:    [[MUL118:%.*]] = mul i32 1, [[DIV117]]
+// CHECK2-NEXT:    [[TMP81:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB119:%.*]] = sub i32 [[TMP81]], -63
+// CHECK2-NEXT:    [[DIV120:%.*]] = udiv i32 [[SUB119]], 64
+// CHECK2-NEXT:    [[MUL121:%.*]] = mul i32 [[MUL118]], [[DIV120]]
+// CHECK2-NEXT:    [[MUL122:%.*]] = mul i32 [[MUL121]], 64
+// CHECK2-NEXT:    [[CONV123:%.*]] = zext i32 [[MUL122]] to i64
+// CHECK2-NEXT:    [[DIV124:%.*]] = sdiv i64 [[TMP76]], [[CONV123]]
+// CHECK2-NEXT:    [[TMP82:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB125:%.*]] = sub i32 [[TMP82]], [[TMP83]]
 // CHECK2-NEXT:    [[SUB126:%.*]] = sub i32 [[SUB125]], 1
+// CHECK2-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD127:%.*]] = add i32 [[SUB126]], [[TMP84]]
 // CHECK2-NEXT:    [[TMP85:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD127:%.*]] = add i32 [[SUB126]], [[TMP85]]
-// CHECK2-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV128:%.*]] = udiv i32 [[ADD127]], [[TMP86]]
+// CHECK2-NEXT:    [[DIV128:%.*]] = udiv i32 [[ADD127]], [[TMP85]]
 // CHECK2-NEXT:    [[MUL129:%.*]] = mul i32 1, [[DIV128]]
-// CHECK2-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB130:%.*]] = sub i32 [[TMP87]], -63
+// CHECK2-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB130:%.*]] = sub i32 [[TMP86]], -63
 // CHECK2-NEXT:    [[DIV131:%.*]] = udiv i32 [[SUB130]], 64
 // CHECK2-NEXT:    [[MUL132:%.*]] = mul i32 [[MUL129]], [[DIV131]]
-// CHECK2-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP89:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB133:%.*]] = sub i32 [[TMP88]], [[TMP89]]
-// CHECK2-NEXT:    [[SUB134:%.*]] = sub i32 [[SUB133]], 1
-// CHECK2-NEXT:    [[ADD135:%.*]] = add i32 [[SUB134]], 1
-// CHECK2-NEXT:    [[DIV136:%.*]] = udiv i32 [[ADD135]], 1
-// CHECK2-NEXT:    [[MUL137:%.*]] = mul i32 [[MUL132]], [[DIV136]]
-// CHECK2-NEXT:    [[CONV138:%.*]] = zext i32 [[MUL137]] to i64
-// CHECK2-NEXT:    [[DIV139:%.*]] = sdiv i64 [[TMP82]], [[CONV138]]
-// CHECK2-NEXT:    [[TMP90:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB140:%.*]] = sub i32 [[TMP90]], [[TMP91]]
-// CHECK2-NEXT:    [[SUB141:%.*]] = sub i32 [[SUB140]], 1
-// CHECK2-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD142:%.*]] = add i32 [[SUB141]], [[TMP92]]
+// CHECK2-NEXT:    [[MUL133:%.*]] = mul i32 [[MUL132]], 64
+// CHECK2-NEXT:    [[CONV134:%.*]] = zext i32 [[MUL133]] to i64
+// CHECK2-NEXT:    [[MUL135:%.*]] = mul nsw i64 [[DIV124]], [[CONV134]]
+// CHECK2-NEXT:    [[SUB136:%.*]] = sub nsw i64 [[TMP75]], [[MUL135]]
+// CHECK2-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB137:%.*]] = sub i32 [[TMP87]], -63
+// CHECK2-NEXT:    [[DIV138:%.*]] = udiv i32 [[SUB137]], 64
+// CHECK2-NEXT:    [[MUL139:%.*]] = mul i32 1, [[DIV138]]
+// CHECK2-NEXT:    [[MUL140:%.*]] = mul i32 [[MUL139]], 64
+// CHECK2-NEXT:    [[CONV141:%.*]] = zext i32 [[MUL140]] to i64
+// CHECK2-NEXT:    [[DIV142:%.*]] = sdiv i64 [[SUB136]], [[CONV141]]
+// CHECK2-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB143:%.*]] = sub i32 [[TMP88]], -63
+// CHECK2-NEXT:    [[DIV144:%.*]] = udiv i32 [[SUB143]], 64
+// CHECK2-NEXT:    [[MUL145:%.*]] = mul i32 1, [[DIV144]]
+// CHECK2-NEXT:    [[MUL146:%.*]] = mul i32 [[MUL145]], 64
+// CHECK2-NEXT:    [[CONV147:%.*]] = zext i32 [[MUL146]] to i64
+// CHECK2-NEXT:    [[MUL148:%.*]] = mul nsw i64 [[DIV142]], [[CONV147]]
+// CHECK2-NEXT:    [[SUB149:%.*]] = sub nsw i64 [[SUB113]], [[MUL148]]
+// CHECK2-NEXT:    [[DIV150:%.*]] = sdiv i64 [[SUB149]], 64
+// CHECK2-NEXT:    [[MUL151:%.*]] = mul nsw i64 [[DIV150]], 64
+// CHECK2-NEXT:    [[ADD152:%.*]] = add nsw i64 0, [[MUL151]]
+// CHECK2-NEXT:    [[CONV153:%.*]] = trunc i64 [[ADD152]] to i32
+// CHECK2-NEXT:    store i32 [[CONV153]], ptr [[DOTFLOOR_0_IV_K37]], align 4
+// CHECK2-NEXT:    [[TMP89:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP90:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB154:%.*]] = sub i32 [[TMP91]], [[TMP92]]
+// CHECK2-NEXT:    [[SUB155:%.*]] = sub i32 [[SUB154]], 1
 // CHECK2-NEXT:    [[TMP93:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV143:%.*]] = udiv i32 [[ADD142]], [[TMP93]]
-// CHECK2-NEXT:    [[MUL144:%.*]] = mul i32 1, [[DIV143]]
-// CHECK2-NEXT:    [[TMP94:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB145:%.*]] = sub i32 [[TMP94]], -63
-// CHECK2-NEXT:    [[DIV146:%.*]] = udiv i32 [[SUB145]], 64
-// CHECK2-NEXT:    [[MUL147:%.*]] = mul i32 [[MUL144]], [[DIV146]]
-// CHECK2-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB148:%.*]] = sub i32 [[TMP95]], [[TMP96]]
-// CHECK2-NEXT:    [[SUB149:%.*]] = sub i32 [[SUB148]], 1
-// CHECK2-NEXT:    [[ADD150:%.*]] = add i32 [[SUB149]], 1
-// CHECK2-NEXT:    [[DIV151:%.*]] = udiv i32 [[ADD150]], 1
-// CHECK2-NEXT:    [[MUL152:%.*]] = mul i32 [[MUL147]], [[DIV151]]
-// CHECK2-NEXT:    [[CONV153:%.*]] = zext i32 [[MUL152]] to i64
-// CHECK2-NEXT:    [[MUL154:%.*]] = mul nsw i64 [[DIV139]], [[CONV153]]
-// CHECK2-NEXT:    [[SUB155:%.*]] = sub nsw i64 [[TMP81]], [[MUL154]]
-// CHECK2-NEXT:    [[TMP97:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP98:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB156:%.*]] = sub i32 [[TMP99]], [[TMP100]]
-// CHECK2-NEXT:    [[SUB157:%.*]] = sub i32 [[SUB156]], 1
-// CHECK2-NEXT:    [[TMP101:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD158:%.*]] = add i32 [[SUB157]], [[TMP101]]
-// CHECK2-NEXT:    [[TMP102:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV159:%.*]] = udiv i32 [[ADD158]], [[TMP102]]
-// CHECK2-NEXT:    [[MUL160:%.*]] = mul i32 1, [[DIV159]]
-// CHECK2-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB161:%.*]] = sub i32 [[TMP103]], -63
-// CHECK2-NEXT:    [[DIV162:%.*]] = udiv i32 [[SUB161]], 64
-// CHECK2-NEXT:    [[MUL163:%.*]] = mul i32 [[MUL160]], [[DIV162]]
-// CHECK2-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP105:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB164:%.*]] = sub i32 [[TMP104]], [[TMP105]]
-// CHECK2-NEXT:    [[SUB165:%.*]] = sub i32 [[SUB164]], 1
-// CHECK2-NEXT:    [[ADD166:%.*]] = add i32 [[SUB165]], 1
-// CHECK2-NEXT:    [[DIV167:%.*]] = udiv i32 [[ADD166]], 1
-// CHECK2-NEXT:    [[MUL168:%.*]] = mul i32 [[MUL163]], [[DIV167]]
-// CHECK2-NEXT:    [[CONV169:%.*]] = zext i32 [[MUL168]] to i64
-// CHECK2-NEXT:    [[DIV170:%.*]] = sdiv i64 [[TMP98]], [[CONV169]]
-// CHECK2-NEXT:    [[TMP106:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB171:%.*]] = sub i32 [[TMP106]], [[TMP107]]
-// CHECK2-NEXT:    [[SUB172:%.*]] = sub i32 [[SUB171]], 1
-// CHECK2-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD173:%.*]] = add i32 [[SUB172]], [[TMP108]]
-// CHECK2-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV174:%.*]] = udiv i32 [[ADD173]], [[TMP109]]
-// CHECK2-NEXT:    [[MUL175:%.*]] = mul i32 1, [[DIV174]]
-// CHECK2-NEXT:    [[TMP110:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB176:%.*]] = sub i32 [[TMP110]], -63
-// CHECK2-NEXT:    [[DIV177:%.*]] = udiv i32 [[SUB176]], 64
-// CHECK2-NEXT:    [[MUL178:%.*]] = mul i32 [[MUL175]], [[DIV177]]
-// CHECK2-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB179:%.*]] = sub i32 [[TMP111]], [[TMP112]]
-// CHECK2-NEXT:    [[SUB180:%.*]] = sub i32 [[SUB179]], 1
-// CHECK2-NEXT:    [[ADD181:%.*]] = add i32 [[SUB180]], 1
-// CHECK2-NEXT:    [[DIV182:%.*]] = udiv i32 [[ADD181]], 1
-// CHECK2-NEXT:    [[MUL183:%.*]] = mul i32 [[MUL178]], [[DIV182]]
-// CHECK2-NEXT:    [[CONV184:%.*]] = zext i32 [[MUL183]] to i64
-// CHECK2-NEXT:    [[MUL185:%.*]] = mul nsw i64 [[DIV170]], [[CONV184]]
-// CHECK2-NEXT:    [[SUB186:%.*]] = sub nsw i64 [[TMP97]], [[MUL185]]
+// CHECK2-NEXT:    [[ADD156:%.*]] = add i32 [[SUB155]], [[TMP93]]
+// CHECK2-NEXT:    [[TMP94:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV157:%.*]] = udiv i32 [[ADD156]], [[TMP94]]
+// CHECK2-NEXT:    [[MUL158:%.*]] = mul i32 1, [[DIV157]]
+// CHECK2-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB159:%.*]] = sub i32 [[TMP95]], -63
+// CHECK2-NEXT:    [[DIV160:%.*]] = udiv i32 [[SUB159]], 64
+// CHECK2-NEXT:    [[MUL161:%.*]] = mul i32 [[MUL158]], [[DIV160]]
+// CHECK2-NEXT:    [[MUL162:%.*]] = mul i32 [[MUL161]], 64
+// CHECK2-NEXT:    [[CONV163:%.*]] = zext i32 [[MUL162]] to i64
+// CHECK2-NEXT:    [[DIV164:%.*]] = sdiv i64 [[TMP90]], [[CONV163]]
+// CHECK2-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP97:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB165:%.*]] = sub i32 [[TMP96]], [[TMP97]]
+// CHECK2-NEXT:    [[SUB166:%.*]] = sub i32 [[SUB165]], 1
+// CHECK2-NEXT:    [[TMP98:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD167:%.*]] = add i32 [[SUB166]], [[TMP98]]
+// CHECK2-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV168:%.*]] = udiv i32 [[ADD167]], [[TMP99]]
+// CHECK2-NEXT:    [[MUL169:%.*]] = mul i32 1, [[DIV168]]
+// CHECK2-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB170:%.*]] = sub i32 [[TMP100]], -63
+// CHECK2-NEXT:    [[DIV171:%.*]] = udiv i32 [[SUB170]], 64
+// CHECK2-NEXT:    [[MUL172:%.*]] = mul i32 [[MUL169]], [[DIV171]]
+// CHECK2-NEXT:    [[MUL173:%.*]] = mul i32 [[MUL172]], 64
+// CHECK2-NEXT:    [[CONV174:%.*]] = zext i32 [[MUL173]] to i64
+// CHECK2-NEXT:    [[MUL175:%.*]] = mul nsw i64 [[DIV164]], [[CONV174]]
+// CHECK2-NEXT:    [[SUB176:%.*]] = sub nsw i64 [[TMP89]], [[MUL175]]
+// CHECK2-NEXT:    [[TMP101:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP102:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB177:%.*]] = sub i32 [[TMP103]], [[TMP104]]
+// CHECK2-NEXT:    [[SUB178:%.*]] = sub i32 [[SUB177]], 1
+// CHECK2-NEXT:    [[TMP105:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD179:%.*]] = add i32 [[SUB178]], [[TMP105]]
+// CHECK2-NEXT:    [[TMP106:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV180:%.*]] = udiv i32 [[ADD179]], [[TMP106]]
+// CHECK2-NEXT:    [[MUL181:%.*]] = mul i32 1, [[DIV180]]
+// CHECK2-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB182:%.*]] = sub i32 [[TMP107]], -63
+// CHECK2-NEXT:    [[DIV183:%.*]] = udiv i32 [[SUB182]], 64
+// CHECK2-NEXT:    [[MUL184:%.*]] = mul i32 [[MUL181]], [[DIV183]]
+// CHECK2-NEXT:    [[MUL185:%.*]] = mul i32 [[MUL184]], 64
+// CHECK2-NEXT:    [[CONV186:%.*]] = zext i32 [[MUL185]] to i64
+// CHECK2-NEXT:    [[DIV187:%.*]] = sdiv i64 [[TMP102]], [[CONV186]]
+// CHECK2-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB188:%.*]] = sub i32 [[TMP108]], [[TMP109]]
+// CHECK2-NEXT:    [[SUB189:%.*]] = sub i32 [[SUB188]], 1
+// CHECK2-NEXT:    [[TMP110:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD190:%.*]] = add i32 [[SUB189]], [[TMP110]]
+// CHECK2-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV191:%.*]] = udiv i32 [[ADD190]], [[TMP111]]
+// CHECK2-NEXT:    [[MUL192:%.*]] = mul i32 1, [[DIV191]]
+// CHECK2-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB193:%.*]] = sub i32 [[TMP112]], -63
+// CHECK2-NEXT:    [[DIV194:%.*]] = udiv i32 [[SUB193]], 64
+// CHECK2-NEXT:    [[MUL195:%.*]] = mul i32 [[MUL192]], [[DIV194]]
+// CHECK2-NEXT:    [[MUL196:%.*]] = mul i32 [[MUL195]], 64
+// CHECK2-NEXT:    [[CONV197:%.*]] = zext i32 [[MUL196]] to i64
+// CHECK2-NEXT:    [[MUL198:%.*]] = mul nsw i64 [[DIV187]], [[CONV197]]
+// CHECK2-NEXT:    [[SUB199:%.*]] = sub nsw i64 [[TMP101]], [[MUL198]]
 // CHECK2-NEXT:    [[TMP113:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB187:%.*]] = sub i32 [[TMP113]], -63
-// CHECK2-NEXT:    [[DIV188:%.*]] = udiv i32 [[SUB187]], 64
-// CHECK2-NEXT:    [[MUL189:%.*]] = mul i32 1, [[DIV188]]
-// CHECK2-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP115:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB190:%.*]] = sub i32 [[TMP114]], [[TMP115]]
-// CHECK2-NEXT:    [[SUB191:%.*]] = sub i32 [[SUB190]], 1
-// CHECK2-NEXT:    [[ADD192:%.*]] = add i32 [[SUB191]], 1
-// CHECK2-NEXT:    [[DIV193:%.*]] = udiv i32 [[ADD192]], 1
-// CHECK2-NEXT:    [[MUL194:%.*]] = mul i32 [[MUL189]], [[DIV193]]
-// CHECK2-NEXT:    [[CONV195:%.*]] = zext i32 [[MUL194]] to i64
-// CHECK2-NEXT:    [[DIV196:%.*]] = sdiv i64 [[SUB186]], [[CONV195]]
-// CHECK2-NEXT:    [[TMP116:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB197:%.*]] = sub i32 [[TMP116]], -63
-// CHECK2-NEXT:    [[DIV198:%.*]] = udiv i32 [[SUB197]], 64
-// CHECK2-NEXT:    [[MUL199:%.*]] = mul i32 1, [[DIV198]]
-// CHECK2-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB200:%.*]] = sub i32 [[TMP117]], [[TMP118]]
-// CHECK2-NEXT:    [[SUB201:%.*]] = sub i32 [[SUB200]], 1
-// CHECK2-NEXT:    [[ADD202:%.*]] = add i32 [[SUB201]], 1
-// CHECK2-NEXT:    [[DIV203:%.*]] = udiv i32 [[ADD202]], 1
-// CHECK2-NEXT:    [[MUL204:%.*]] = mul i32 [[MUL199]], [[DIV203]]
-// CHECK2-NEXT:    [[CONV205:%.*]] = zext i32 [[MUL204]] to i64
-// CHECK2-NEXT:    [[MUL206:%.*]] = mul nsw i64 [[DIV196]], [[CONV205]]
-// CHECK2-NEXT:    [[SUB207:%.*]] = sub nsw i64 [[SUB155]], [[MUL206]]
-// CHECK2-NEXT:    [[TMP119:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP120:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB208:%.*]] = sub i32 [[TMP119]], [[TMP120]]
-// CHECK2-NEXT:    [[SUB209:%.*]] = sub i32 [[SUB208]], 1
-// CHECK2-NEXT:    [[ADD210:%.*]] = add i32 [[SUB209]], 1
-// CHECK2-NEXT:    [[DIV211:%.*]] = udiv i32 [[ADD210]], 1
-// CHECK2-NEXT:    [[MUL212:%.*]] = mul i32 1, [[DIV211]]
-// CHECK2-NEXT:    [[CONV213:%.*]] = zext i32 [[MUL212]] to i64
-// CHECK2-NEXT:    [[DIV214:%.*]] = sdiv i64 [[SUB207]], [[CONV213]]
-// CHECK2-NEXT:    [[MUL215:%.*]] = mul nsw i64 [[DIV214]], 64
-// CHECK2-NEXT:    [[ADD216:%.*]] = add nsw i64 0, [[MUL215]]
-// CHECK2-NEXT:    [[CONV217:%.*]] = trunc i64 [[ADD216]] to i32
-// CHECK2-NEXT:    store i32 [[CONV217]], ptr [[DOTFLOOR_0_IV_K51]], align 4
-// CHECK2-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[CONV218:%.*]] = zext i32 [[TMP121]] to i64
-// CHECK2-NEXT:    [[TMP122:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP123:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP124:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB219:%.*]] = sub i32 [[TMP124]], [[TMP125]]
-// CHECK2-NEXT:    [[SUB220:%.*]] = sub i32 [[SUB219]], 1
-// CHECK2-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD221:%.*]] = add i32 [[SUB220]], [[TMP126]]
-// CHECK2-NEXT:    [[TMP127:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV222:%.*]] = udiv i32 [[ADD221]], [[TMP127]]
-// CHECK2-NEXT:    [[MUL223:%.*]] = mul i32 1, [[DIV222]]
-// CHECK2-NEXT:    [[TMP128:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB224:%.*]] = sub i32 [[TMP128]], -63
-// CHECK2-NEXT:    [[DIV225:%.*]] = udiv i32 [[SUB224]], 64
-// CHECK2-NEXT:    [[MUL226:%.*]] = mul i32 [[MUL223]], [[DIV225]]
-// CHECK2-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB227:%.*]] = sub i32 [[TMP129]], [[TMP130]]
-// CHECK2-NEXT:    [[SUB228:%.*]] = sub i32 [[SUB227]], 1
-// CHECK2-NEXT:    [[ADD229:%.*]] = add i32 [[SUB228]], 1
-// CHECK2-NEXT:    [[DIV230:%.*]] = udiv i32 [[ADD229]], 1
-// CHECK2-NEXT:    [[MUL231:%.*]] = mul i32 [[MUL226]], [[DIV230]]
-// CHECK2-NEXT:    [[CONV232:%.*]] = zext i32 [[MUL231]] to i64
-// CHECK2-NEXT:    [[DIV233:%.*]] = sdiv i64 [[TMP123]], [[CONV232]]
-// CHECK2-NEXT:    [[TMP131:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP132:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB234:%.*]] = sub i32 [[TMP131]], [[TMP132]]
-// CHECK2-NEXT:    [[SUB235:%.*]] = sub i32 [[SUB234]], 1
-// CHECK2-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD236:%.*]] = add i32 [[SUB235]], [[TMP133]]
-// CHECK2-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV237:%.*]] = udiv i32 [[ADD236]], [[TMP134]]
-// CHECK2-NEXT:    [[MUL238:%.*]] = mul i32 1, [[DIV237]]
-// CHECK2-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB239:%.*]] = sub i32 [[TMP135]], -63
-// CHECK2-NEXT:    [[DIV240:%.*]] = udiv i32 [[SUB239]], 64
-// CHECK2-NEXT:    [[MUL241:%.*]] = mul i32 [[MUL238]], [[DIV240]]
-// CHECK2-NEXT:    [[TMP136:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB242:%.*]] = sub i32 [[TMP136]], [[TMP137]]
-// CHECK2-NEXT:    [[SUB243:%.*]] = sub i32 [[SUB242]], 1
-// CHECK2-NEXT:    [[ADD244:%.*]] = add i32 [[SUB243]], 1
-// CHECK2-NEXT:    [[DIV245:%.*]] = udiv i32 [[ADD244]], 1
-// CHECK2-NEXT:    [[MUL246:%.*]] = mul i32 [[MUL241]], [[DIV245]]
-// CHECK2-NEXT:    [[CONV247:%.*]] = zext i32 [[MUL246]] to i64
-// CHECK2-NEXT:    [[MUL248:%.*]] = mul nsw i64 [[DIV233]], [[CONV247]]
-// CHECK2-NEXT:    [[SUB249:%.*]] = sub nsw i64 [[TMP122]], [[MUL248]]
-// CHECK2-NEXT:    [[TMP138:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP139:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB250:%.*]] = sub i32 [[TMP140]], [[TMP141]]
-// CHECK2-NEXT:    [[SUB251:%.*]] = sub i32 [[SUB250]], 1
-// CHECK2-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD252:%.*]] = add i32 [[SUB251]], [[TMP142]]
-// CHECK2-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV253:%.*]] = udiv i32 [[ADD252]], [[TMP143]]
-// CHECK2-NEXT:    [[MUL254:%.*]] = mul i32 1, [[DIV253]]
-// CHECK2-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB255:%.*]] = sub i32 [[TMP144]], -63
-// CHECK2-NEXT:    [[DIV256:%.*]] = udiv i32 [[SUB255]], 64
-// CHECK2-NEXT:    [[MUL257:%.*]] = mul i32 [[MUL254]], [[DIV256]]
-// CHECK2-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB258:%.*]] = sub i32 [[TMP145]], [[TMP146]]
-// CHECK2-NEXT:    [[SUB259:%.*]] = sub i32 [[SUB258]], 1
-// CHECK2-NEXT:    [[ADD260:%.*]] = add i32 [[SUB259]], 1
-// CHECK2-NEXT:    [[DIV261:%.*]] = udiv i32 [[ADD260]], 1
-// CHECK2-NEXT:    [[MUL262:%.*]] = mul i32 [[MUL257]], [[DIV261]]
+// CHECK2-NEXT:    [[SUB200:%.*]] = sub i32 [[TMP113]], -63
+// CHECK2-NEXT:    [[DIV201:%.*]] = udiv i32 [[SUB200]], 64
+// CHECK2-NEXT:    [[MUL202:%.*]] = mul i32 1, [[DIV201]]
+// CHECK2-NEXT:    [[MUL203:%.*]] = mul i32 [[MUL202]], 64
+// CHECK2-NEXT:    [[CONV204:%.*]] = zext i32 [[MUL203]] to i64
+// CHECK2-NEXT:    [[DIV205:%.*]] = sdiv i64 [[SUB199]], [[CONV204]]
+// CHECK2-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB206:%.*]] = sub i32 [[TMP114]], -63
+// CHECK2-NEXT:    [[DIV207:%.*]] = udiv i32 [[SUB206]], 64
+// CHECK2-NEXT:    [[MUL208:%.*]] = mul i32 1, [[DIV207]]
+// CHECK2-NEXT:    [[MUL209:%.*]] = mul i32 [[MUL208]], 64
+// CHECK2-NEXT:    [[CONV210:%.*]] = zext i32 [[MUL209]] to i64
+// CHECK2-NEXT:    [[MUL211:%.*]] = mul nsw i64 [[DIV205]], [[CONV210]]
+// CHECK2-NEXT:    [[SUB212:%.*]] = sub nsw i64 [[SUB176]], [[MUL211]]
+// CHECK2-NEXT:    [[TMP115:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP116:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB213:%.*]] = sub i32 [[TMP117]], [[TMP118]]
+// CHECK2-NEXT:    [[SUB214:%.*]] = sub i32 [[SUB213]], 1
+// CHECK2-NEXT:    [[TMP119:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD215:%.*]] = add i32 [[SUB214]], [[TMP119]]
+// CHECK2-NEXT:    [[TMP120:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV216:%.*]] = udiv i32 [[ADD215]], [[TMP120]]
+// CHECK2-NEXT:    [[MUL217:%.*]] = mul i32 1, [[DIV216]]
+// CHECK2-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB218:%.*]] = sub i32 [[TMP121]], -63
+// CHECK2-NEXT:    [[DIV219:%.*]] = udiv i32 [[SUB218]], 64
+// CHECK2-NEXT:    [[MUL220:%.*]] = mul i32 [[MUL217]], [[DIV219]]
+// CHECK2-NEXT:    [[MUL221:%.*]] = mul i32 [[MUL220]], 64
+// CHECK2-NEXT:    [[CONV222:%.*]] = zext i32 [[MUL221]] to i64
+// CHECK2-NEXT:    [[DIV223:%.*]] = sdiv i64 [[TMP116]], [[CONV222]]
+// CHECK2-NEXT:    [[TMP122:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP123:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB224:%.*]] = sub i32 [[TMP122]], [[TMP123]]
+// CHECK2-NEXT:    [[SUB225:%.*]] = sub i32 [[SUB224]], 1
+// CHECK2-NEXT:    [[TMP124:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD226:%.*]] = add i32 [[SUB225]], [[TMP124]]
+// CHECK2-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV227:%.*]] = udiv i32 [[ADD226]], [[TMP125]]
+// CHECK2-NEXT:    [[MUL228:%.*]] = mul i32 1, [[DIV227]]
+// CHECK2-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB229:%.*]] = sub i32 [[TMP126]], -63
+// CHECK2-NEXT:    [[DIV230:%.*]] = udiv i32 [[SUB229]], 64
+// CHECK2-NEXT:    [[MUL231:%.*]] = mul i32 [[MUL228]], [[DIV230]]
+// CHECK2-NEXT:    [[MUL232:%.*]] = mul i32 [[MUL231]], 64
+// CHECK2-NEXT:    [[CONV233:%.*]] = zext i32 [[MUL232]] to i64
+// CHECK2-NEXT:    [[MUL234:%.*]] = mul nsw i64 [[DIV223]], [[CONV233]]
+// CHECK2-NEXT:    [[SUB235:%.*]] = sub nsw i64 [[TMP115]], [[MUL234]]
+// CHECK2-NEXT:    [[TMP127:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP128:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB236:%.*]] = sub i32 [[TMP129]], [[TMP130]]
+// CHECK2-NEXT:    [[SUB237:%.*]] = sub i32 [[SUB236]], 1
+// CHECK2-NEXT:    [[TMP131:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD238:%.*]] = add i32 [[SUB237]], [[TMP131]]
+// CHECK2-NEXT:    [[TMP132:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV239:%.*]] = udiv i32 [[ADD238]], [[TMP132]]
+// CHECK2-NEXT:    [[MUL240:%.*]] = mul i32 1, [[DIV239]]
+// CHECK2-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB241:%.*]] = sub i32 [[TMP133]], -63
+// CHECK2-NEXT:    [[DIV242:%.*]] = udiv i32 [[SUB241]], 64
+// CHECK2-NEXT:    [[MUL243:%.*]] = mul i32 [[MUL240]], [[DIV242]]
+// CHECK2-NEXT:    [[MUL244:%.*]] = mul i32 [[MUL243]], 64
+// CHECK2-NEXT:    [[CONV245:%.*]] = zext i32 [[MUL244]] to i64
+// CHECK2-NEXT:    [[DIV246:%.*]] = sdiv i64 [[TMP128]], [[CONV245]]
+// CHECK2-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB247:%.*]] = sub i32 [[TMP134]], [[TMP135]]
+// CHECK2-NEXT:    [[SUB248:%.*]] = sub i32 [[SUB247]], 1
+// CHECK2-NEXT:    [[TMP136:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD249:%.*]] = add i32 [[SUB248]], [[TMP136]]
+// CHECK2-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV250:%.*]] = udiv i32 [[ADD249]], [[TMP137]]
+// CHECK2-NEXT:    [[MUL251:%.*]] = mul i32 1, [[DIV250]]
+// CHECK2-NEXT:    [[TMP138:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB252:%.*]] = sub i32 [[TMP138]], -63
+// CHECK2-NEXT:    [[DIV253:%.*]] = udiv i32 [[SUB252]], 64
+// CHECK2-NEXT:    [[MUL254:%.*]] = mul i32 [[MUL251]], [[DIV253]]
+// CHECK2-NEXT:    [[MUL255:%.*]] = mul i32 [[MUL254]], 64
+// CHECK2-NEXT:    [[CONV256:%.*]] = zext i32 [[MUL255]] to i64
+// CHECK2-NEXT:    [[MUL257:%.*]] = mul nsw i64 [[DIV246]], [[CONV256]]
+// CHECK2-NEXT:    [[SUB258:%.*]] = sub nsw i64 [[TMP127]], [[MUL257]]
+// CHECK2-NEXT:    [[TMP139:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB259:%.*]] = sub i32 [[TMP139]], -63
+// CHECK2-NEXT:    [[DIV260:%.*]] = udiv i32 [[SUB259]], 64
+// CHECK2-NEXT:    [[MUL261:%.*]] = mul i32 1, [[DIV260]]
+// CHECK2-NEXT:    [[MUL262:%.*]] = mul i32 [[MUL261]], 64
 // CHECK2-NEXT:    [[CONV263:%.*]] = zext i32 [[MUL262]] to i64
-// CHECK2-NEXT:    [[DIV264:%.*]] = sdiv i64 [[TMP139]], [[CONV263]]
-// CHECK2-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB265:%.*]] = sub i32 [[TMP147]], [[TMP148]]
-// CHECK2-NEXT:    [[SUB266:%.*]] = sub i32 [[SUB265]], 1
-// CHECK2-NEXT:    [[TMP149:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD267:%.*]] = add i32 [[SUB266]], [[TMP149]]
-// CHECK2-NEXT:    [[TMP150:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV268:%.*]] = udiv i32 [[ADD267]], [[TMP150]]
-// CHECK2-NEXT:    [[MUL269:%.*]] = mul i32 1, [[DIV268]]
-// CHECK2-NEXT:    [[TMP151:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB270:%.*]] = sub i32 [[TMP151]], -63
-// CHECK2-NEXT:    [[DIV271:%.*]] = udiv i32 [[SUB270]], 64
-// CHECK2-NEXT:    [[MUL272:%.*]] = mul i32 [[MUL269]], [[DIV271]]
-// CHECK2-NEXT:    [[TMP152:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP153:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB273:%.*]] = sub i32 [[TMP152]], [[TMP153]]
-// CHECK2-NEXT:    [[SUB274:%.*]] = sub i32 [[SUB273]], 1
-// CHECK2-NEXT:    [[ADD275:%.*]] = add i32 [[SUB274]], 1
-// CHECK2-NEXT:    [[DIV276:%.*]] = udiv i32 [[ADD275]], 1
-// CHECK2-NEXT:    [[MUL277:%.*]] = mul i32 [[MUL272]], [[DIV276]]
-// CHECK2-NEXT:    [[CONV278:%.*]] = zext i32 [[MUL277]] to i64
-// CHECK2-NEXT:    [[MUL279:%.*]] = mul nsw i64 [[DIV264]], [[CONV278]]
-// CHECK2-NEXT:    [[SUB280:%.*]] = sub nsw i64 [[TMP138]], [[MUL279]]
-// CHECK2-NEXT:    [[TMP154:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB281:%.*]] = sub i32 [[TMP154]], -63
-// CHECK2-NEXT:    [[DIV282:%.*]] = udiv i32 [[SUB281]], 64
-// CHECK2-NEXT:    [[MUL283:%.*]] = mul i32 1, [[DIV282]]
-// CHECK2-NEXT:    [[TMP155:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP156:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB284:%.*]] = sub i32 [[TMP155]], [[TMP156]]
-// CHECK2-NEXT:    [[SUB285:%.*]] = sub i32 [[SUB284]], 1
-// CHECK2-NEXT:    [[ADD286:%.*]] = add i32 [[SUB285]], 1
-// CHECK2-NEXT:    [[DIV287:%.*]] = udiv i32 [[ADD286]], 1
-// CHECK2-NEXT:    [[MUL288:%.*]] = mul i32 [[MUL283]], [[DIV287]]
-// CHECK2-NEXT:    [[CONV289:%.*]] = zext i32 [[MUL288]] to i64
-// CHECK2-NEXT:    [[DIV290:%.*]] = sdiv i64 [[SUB280]], [[CONV289]]
-// CHECK2-NEXT:    [[TMP157:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB291:%.*]] = sub i32 [[TMP157]], -63
-// CHECK2-NEXT:    [[DIV292:%.*]] = udiv i32 [[SUB291]], 64
-// CHECK2-NEXT:    [[MUL293:%.*]] = mul i32 1, [[DIV292]]
-// CHECK2-NEXT:    [[TMP158:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP159:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB294:%.*]] = sub i32 [[TMP158]], [[TMP159]]
-// CHECK2-NEXT:    [[SUB295:%.*]] = sub i32 [[SUB294]], 1
-// CHECK2-NEXT:    [[ADD296:%.*]] = add i32 [[SUB295]], 1
-// CHECK2-NEXT:    [[DIV297:%.*]] = udiv i32 [[ADD296]], 1
-// CHECK2-NEXT:    [[MUL298:%.*]] = mul i32 [[MUL293]], [[DIV297]]
-// CHECK2-NEXT:    [[CONV299:%.*]] = zext i32 [[MUL298]] to i64
-// CHECK2-NEXT:    [[MUL300:%.*]] = mul nsw i64 [[DIV290]], [[CONV299]]
-// CHECK2-NEXT:    [[SUB301:%.*]] = sub nsw i64 [[SUB249]], [[MUL300]]
-// CHECK2-NEXT:    [[TMP160:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP161:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP162:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP163:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB302:%.*]] = sub i32 [[TMP162]], [[TMP163]]
-// CHECK2-NEXT:    [[SUB303:%.*]] = sub i32 [[SUB302]], 1
-// CHECK2-NEXT:    [[TMP164:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD304:%.*]] = add i32 [[SUB303]], [[TMP164]]
-// CHECK2-NEXT:    [[TMP165:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV305:%.*]] = udiv i32 [[ADD304]], [[TMP165]]
-// CHECK2-NEXT:    [[MUL306:%.*]] = mul i32 1, [[DIV305]]
-// CHECK2-NEXT:    [[TMP166:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB307:%.*]] = sub i32 [[TMP166]], -63
-// CHECK2-NEXT:    [[DIV308:%.*]] = udiv i32 [[SUB307]], 64
-// CHECK2-NEXT:    [[MUL309:%.*]] = mul i32 [[MUL306]], [[DIV308]]
-// CHECK2-NEXT:    [[TMP167:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP168:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB310:%.*]] = sub i32 [[TMP167]], [[TMP168]]
-// CHECK2-NEXT:    [[SUB311:%.*]] = sub i32 [[SUB310]], 1
-// CHECK2-NEXT:    [[ADD312:%.*]] = add i32 [[SUB311]], 1
-// CHECK2-NEXT:    [[DIV313:%.*]] = udiv i32 [[ADD312]], 1
-// CHECK2-NEXT:    [[MUL314:%.*]] = mul i32 [[MUL309]], [[DIV313]]
-// CHECK2-NEXT:    [[CONV315:%.*]] = zext i32 [[MUL314]] to i64
-// CHECK2-NEXT:    [[DIV316:%.*]] = sdiv i64 [[TMP161]], [[CONV315]]
-// CHECK2-NEXT:    [[TMP169:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP170:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB317:%.*]] = sub i32 [[TMP169]], [[TMP170]]
-// CHECK2-NEXT:    [[SUB318:%.*]] = sub i32 [[SUB317]], 1
-// CHECK2-NEXT:    [[TMP171:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD319:%.*]] = add i32 [[SUB318]], [[TMP171]]
-// CHECK2-NEXT:    [[TMP172:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV320:%.*]] = udiv i32 [[ADD319]], [[TMP172]]
-// CHECK2-NEXT:    [[MUL321:%.*]] = mul i32 1, [[DIV320]]
-// CHECK2-NEXT:    [[TMP173:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB322:%.*]] = sub i32 [[TMP173]], -63
-// CHECK2-NEXT:    [[DIV323:%.*]] = udiv i32 [[SUB322]], 64
-// CHECK2-NEXT:    [[MUL324:%.*]] = mul i32 [[MUL321]], [[DIV323]]
-// CHECK2-NEXT:    [[TMP174:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP175:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB325:%.*]] = sub i32 [[TMP174]], [[TMP175]]
-// CHECK2-NEXT:    [[SUB326:%.*]] = sub i32 [[SUB325]], 1
-// CHECK2-NEXT:    [[ADD327:%.*]] = add i32 [[SUB326]], 1
-// CHECK2-NEXT:    [[DIV328:%.*]] = udiv i32 [[ADD327]], 1
-// CHECK2-NEXT:    [[MUL329:%.*]] = mul i32 [[MUL324]], [[DIV328]]
-// CHECK2-NEXT:    [[CONV330:%.*]] = zext i32 [[MUL329]] to i64
-// CHECK2-NEXT:    [[MUL331:%.*]] = mul nsw i64 [[DIV316]], [[CONV330]]
-// CHECK2-NEXT:    [[SUB332:%.*]] = sub nsw i64 [[TMP160]], [[MUL331]]
-// CHECK2-NEXT:    [[TMP176:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP177:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP178:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP179:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB333:%.*]] = sub i32 [[TMP178]], [[TMP179]]
-// CHECK2-NEXT:    [[SUB334:%.*]] = sub i32 [[SUB333]], 1
-// CHECK2-NEXT:    [[TMP180:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD335:%.*]] = add i32 [[SUB334]], [[TMP180]]
-// CHECK2-NEXT:    [[TMP181:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV336:%.*]] = udiv i32 [[ADD335]], [[TMP181]]
-// CHECK2-NEXT:    [[MUL337:%.*]] = mul i32 1, [[DIV336]]
-// CHECK2-NEXT:    [[TMP182:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB338:%.*]] = sub i32 [[TMP182]], -63
-// CHECK2-NEXT:    [[DIV339:%.*]] = udiv i32 [[SUB338]], 64
-// CHECK2-NEXT:    [[MUL340:%.*]] = mul i32 [[MUL337]], [[DIV339]]
-// CHECK2-NEXT:    [[TMP183:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP184:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB341:%.*]] = sub i32 [[TMP183]], [[TMP184]]
-// CHECK2-NEXT:    [[SUB342:%.*]] = sub i32 [[SUB341]], 1
-// CHECK2-NEXT:    [[ADD343:%.*]] = add i32 [[SUB342]], 1
-// CHECK2-NEXT:    [[DIV344:%.*]] = udiv i32 [[ADD343]], 1
-// CHECK2-NEXT:    [[MUL345:%.*]] = mul i32 [[MUL340]], [[DIV344]]
-// CHECK2-NEXT:    [[CONV346:%.*]] = zext i32 [[MUL345]] to i64
-// CHECK2-NEXT:    [[DIV347:%.*]] = sdiv i64 [[TMP177]], [[CONV346]]
-// CHECK2-NEXT:    [[TMP185:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP186:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB348:%.*]] = sub i32 [[TMP185]], [[TMP186]]
-// CHECK2-NEXT:    [[SUB349:%.*]] = sub i32 [[SUB348]], 1
-// CHECK2-NEXT:    [[TMP187:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD350:%.*]] = add i32 [[SUB349]], [[TMP187]]
-// CHECK2-NEXT:    [[TMP188:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV351:%.*]] = udiv i32 [[ADD350]], [[TMP188]]
-// CHECK2-NEXT:    [[MUL352:%.*]] = mul i32 1, [[DIV351]]
-// CHECK2-NEXT:    [[TMP189:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB353:%.*]] = sub i32 [[TMP189]], -63
-// CHECK2-NEXT:    [[DIV354:%.*]] = udiv i32 [[SUB353]], 64
-// CHECK2-NEXT:    [[MUL355:%.*]] = mul i32 [[MUL352]], [[DIV354]]
-// CHECK2-NEXT:    [[TMP190:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP191:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB356:%.*]] = sub i32 [[TMP190]], [[TMP191]]
-// CHECK2-NEXT:    [[SUB357:%.*]] = sub i32 [[SUB356]], 1
-// CHECK2-NEXT:    [[ADD358:%.*]] = add i32 [[SUB357]], 1
-// CHECK2-NEXT:    [[DIV359:%.*]] = udiv i32 [[ADD358]], 1
-// CHECK2-NEXT:    [[MUL360:%.*]] = mul i32 [[MUL355]], [[DIV359]]
-// CHECK2-NEXT:    [[CONV361:%.*]] = zext i32 [[MUL360]] to i64
-// CHECK2-NEXT:    [[MUL362:%.*]] = mul nsw i64 [[DIV347]], [[CONV361]]
-// CHECK2-NEXT:    [[SUB363:%.*]] = sub nsw i64 [[TMP176]], [[MUL362]]
-// CHECK2-NEXT:    [[TMP192:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB364:%.*]] = sub i32 [[TMP192]], -63
-// CHECK2-NEXT:    [[DIV365:%.*]] = udiv i32 [[SUB364]], 64
-// CHECK2-NEXT:    [[MUL366:%.*]] = mul i32 1, [[DIV365]]
-// CHECK2-NEXT:    [[TMP193:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP194:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB367:%.*]] = sub i32 [[TMP193]], [[TMP194]]
-// CHECK2-NEXT:    [[SUB368:%.*]] = sub i32 [[SUB367]], 1
-// CHECK2-NEXT:    [[ADD369:%.*]] = add i32 [[SUB368]], 1
-// CHECK2-NEXT:    [[DIV370:%.*]] = udiv i32 [[ADD369]], 1
-// CHECK2-NEXT:    [[MUL371:%.*]] = mul i32 [[MUL366]], [[DIV370]]
-// CHECK2-NEXT:    [[CONV372:%.*]] = zext i32 [[MUL371]] to i64
-// CHECK2-NEXT:    [[DIV373:%.*]] = sdiv i64 [[SUB363]], [[CONV372]]
-// CHECK2-NEXT:    [[TMP195:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB374:%.*]] = sub i32 [[TMP195]], -63
-// CHECK2-NEXT:    [[DIV375:%.*]] = udiv i32 [[SUB374]], 64
-// CHECK2-NEXT:    [[MUL376:%.*]] = mul i32 1, [[DIV375]]
-// CHECK2-NEXT:    [[TMP196:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP197:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB377:%.*]] = sub i32 [[TMP196]], [[TMP197]]
-// CHECK2-NEXT:    [[SUB378:%.*]] = sub i32 [[SUB377]], 1
-// CHECK2-NEXT:    [[ADD379:%.*]] = add i32 [[SUB378]], 1
-// CHECK2-NEXT:    [[DIV380:%.*]] = udiv i32 [[ADD379]], 1
-// CHECK2-NEXT:    [[MUL381:%.*]] = mul i32 [[MUL376]], [[DIV380]]
-// CHECK2-NEXT:    [[CONV382:%.*]] = zext i32 [[MUL381]] to i64
-// CHECK2-NEXT:    [[MUL383:%.*]] = mul nsw i64 [[DIV373]], [[CONV382]]
-// CHECK2-NEXT:    [[SUB384:%.*]] = sub nsw i64 [[SUB332]], [[MUL383]]
-// CHECK2-NEXT:    [[TMP198:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP199:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB385:%.*]] = sub i32 [[TMP198]], [[TMP199]]
-// CHECK2-NEXT:    [[SUB386:%.*]] = sub i32 [[SUB385]], 1
-// CHECK2-NEXT:    [[ADD387:%.*]] = add i32 [[SUB386]], 1
-// CHECK2-NEXT:    [[DIV388:%.*]] = udiv i32 [[ADD387]], 1
-// CHECK2-NEXT:    [[MUL389:%.*]] = mul i32 1, [[DIV388]]
-// CHECK2-NEXT:    [[CONV390:%.*]] = zext i32 [[MUL389]] to i64
-// CHECK2-NEXT:    [[DIV391:%.*]] = sdiv i64 [[SUB384]], [[CONV390]]
-// CHECK2-NEXT:    [[TMP200:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP201:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB392:%.*]] = sub i32 [[TMP200]], [[TMP201]]
-// CHECK2-NEXT:    [[SUB393:%.*]] = sub i32 [[SUB392]], 1
-// CHECK2-NEXT:    [[ADD394:%.*]] = add i32 [[SUB393]], 1
-// CHECK2-NEXT:    [[DIV395:%.*]] = udiv i32 [[ADD394]], 1
-// CHECK2-NEXT:    [[MUL396:%.*]] = mul i32 1, [[DIV395]]
-// CHECK2-NEXT:    [[CONV397:%.*]] = zext i32 [[MUL396]] to i64
-// CHECK2-NEXT:    [[MUL398:%.*]] = mul nsw i64 [[DIV391]], [[CONV397]]
-// CHECK2-NEXT:    [[SUB399:%.*]] = sub nsw i64 [[SUB301]], [[MUL398]]
-// CHECK2-NEXT:    [[MUL400:%.*]] = mul nsw i64 [[SUB399]], 1
-// CHECK2-NEXT:    [[ADD401:%.*]] = add nsw i64 [[CONV218]], [[MUL400]]
-// CHECK2-NEXT:    [[CONV402:%.*]] = trunc i64 [[ADD401]] to i32
-// CHECK2-NEXT:    store i32 [[CONV402]], ptr [[DOTTILE_0_IV_K52]], align 4
-// CHECK2-NEXT:    [[TMP202:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
-// CHECK2-NEXT:    [[TMP203:%.*]] = load i32, ptr [[DOTTILE_0_IV_K52]], align 4
-// CHECK2-NEXT:    [[TMP204:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
-// CHECK2-NEXT:    [[MUL403:%.*]] = mul i32 [[TMP203]], [[TMP204]]
-// CHECK2-NEXT:    [[ADD404:%.*]] = add i32 [[TMP202]], [[MUL403]]
-// CHECK2-NEXT:    store i32 [[ADD404]], ptr [[K]], align 4
-// CHECK2-NEXT:    [[TMP205:%.*]] = load i32, ptr [[I49]], align 4
-// CHECK2-NEXT:    [[TMP206:%.*]] = load i32, ptr [[J50]], align 4
-// CHECK2-NEXT:    [[TMP207:%.*]] = load i32, ptr [[K]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP205]], i32 noundef [[TMP206]], i32 noundef [[TMP207]])
+// CHECK2-NEXT:    [[DIV264:%.*]] = sdiv i64 [[SUB258]], [[CONV263]]
+// CHECK2-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB265:%.*]] = sub i32 [[TMP140]], -63
+// CHECK2-NEXT:    [[DIV266:%.*]] = udiv i32 [[SUB265]], 64
+// CHECK2-NEXT:    [[MUL267:%.*]] = mul i32 1, [[DIV266]]
+// CHECK2-NEXT:    [[MUL268:%.*]] = mul i32 [[MUL267]], 64
+// CHECK2-NEXT:    [[CONV269:%.*]] = zext i32 [[MUL268]] to i64
+// CHECK2-NEXT:    [[MUL270:%.*]] = mul nsw i64 [[DIV264]], [[CONV269]]
+// CHECK2-NEXT:    [[SUB271:%.*]] = sub nsw i64 [[SUB235]], [[MUL270]]
+// CHECK2-NEXT:    [[DIV272:%.*]] = sdiv i64 [[SUB271]], 64
+// CHECK2-NEXT:    [[MUL273:%.*]] = mul nsw i64 [[DIV272]], 64
+// CHECK2-NEXT:    [[SUB274:%.*]] = sub nsw i64 [[SUB212]], [[MUL273]]
+// CHECK2-NEXT:    [[MUL275:%.*]] = mul nsw i64 [[SUB274]], 1
+// CHECK2-NEXT:    [[ADD276:%.*]] = add nsw i64 0, [[MUL275]]
+// CHECK2-NEXT:    [[CONV277:%.*]] = trunc i64 [[ADD276]] to i32
+// CHECK2-NEXT:    store i32 [[CONV277]], ptr [[DOTTILE_CNT_0_IV_K38]], align 4
+// CHECK2-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
+// CHECK2-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
+// CHECK2-NEXT:    [[ADD278:%.*]] = add i32 [[TMP141]], [[TMP142]]
+// CHECK2-NEXT:    store i32 [[ADD278]], ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK2-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
+// CHECK2-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK2-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
+// CHECK2-NEXT:    [[MUL279:%.*]] = mul i32 [[TMP144]], [[TMP145]]
+// CHECK2-NEXT:    [[ADD280:%.*]] = add i32 [[TMP143]], [[MUL279]]
+// CHECK2-NEXT:    store i32 [[ADD280]], ptr [[K]], align 4
+// CHECK2-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK2-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// CHECK2-NEXT:    [[ADD281:%.*]] = add i32 [[TMP147]], 1
+// CHECK2-NEXT:    [[CMP282:%.*]] = icmp ult i32 [[TMP146]], [[ADD281]]
+// CHECK2-NEXT:    br i1 [[CMP282]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK2:       if.then:
+// CHECK2-NEXT:    [[TMP148:%.*]] = load i32, ptr [[I35]], align 4
+// CHECK2-NEXT:    [[TMP149:%.*]] = load i32, ptr [[J36]], align 4
+// CHECK2-NEXT:    [[TMP150:%.*]] = load i32, ptr [[K]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP148]], i32 noundef [[TMP149]], i32 noundef [[TMP150]])
+// CHECK2-NEXT:    br label [[IF_END]]
+// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
-// CHECK2-NEXT:    [[TMP208:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[ADD405:%.*]] = add nsw i64 [[TMP208]], 1
-// CHECK2-NEXT:    store i64 [[ADD405]], ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP151:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[ADD283:%.*]] = add nsw i64 [[TMP151]], 1
+// CHECK2-NEXT:    store i64 [[ADD283]], ptr [[DOTOMP_IV]], align 8
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -5321,4 +4606,4 @@ extern "C" void foo10() {
 // CHECK2:       omp.precond.end:
 // CHECK2-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[TMP0]])
 // CHECK2-NEXT:    ret void
-
+//
diff --git a/clang/test/OpenMP/irbuilder_unroll_partial_factor_for.c b/clang/test/OpenMP/irbuilder_unroll_partial_factor_for.c
index cf4fa21c9a31c..43711af5f2e33 100644
--- a/clang/test/OpenMP/irbuilder_unroll_partial_factor_for.c
+++ b/clang/test/OpenMP/irbuilder_unroll_partial_factor_for.c
@@ -79,8 +79,6 @@ void unroll_partial_heuristic_for(int n, float *a, float *b, float *c, float *d)
 // CHECK-NEXT:    br i1 [[OMP_FLOOR0_CMP]], label [[OMP_FLOOR0_BODY:%.*]], label [[OMP_FLOOR0_EXIT:%.*]]
 // CHECK:       omp_floor0.body:
 // CHECK-NEXT:    [[TMP13:%.*]] = add i32 [[OMP_FLOOR0_IV]], [[TMP9]]
-// CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP13]], [[TMP4]]
-// CHECK-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i32 [[TMP5]], i32 13
 // CHECK-NEXT:    br label [[OMP_TILE0_PREHEADER:%.*]]
 // CHECK:       omp_tile0.preheader:
 // CHECK-NEXT:    br label [[OMP_TILE0_HEADER:%.*]]
@@ -88,36 +86,39 @@ void unroll_partial_heuristic_for(int n, float *a, float *b, float *c, float *d)
 // CHECK-NEXT:    [[OMP_TILE0_IV:%.*]] = phi i32 [ 0, [[OMP_TILE0_PREHEADER]] ], [ [[OMP_TILE0_NEXT:%.*]], [[OMP_TILE0_INC:%.*]] ]
 // CHECK-NEXT:    br label [[OMP_TILE0_COND:%.*]]
 // CHECK:       omp_tile0.cond:
-// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], [[TMP15]]
+// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], 13
 // CHECK-NEXT:    br i1 [[OMP_TILE0_CMP]], label [[OMP_TILE0_BODY:%.*]], label [[OMP_TILE0_EXIT:%.*]]
 // CHECK:       omp_tile0.body:
-// CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i32 13, [[TMP13]]
-// CHECK-NEXT:    [[TMP17:%.*]] = add nuw i32 [[TMP16]], [[OMP_TILE0_IV]]
-// CHECK-NEXT:    br label [[OMP_LOOP_BODY:%.*]]
+// CHECK-NEXT:    [[TMP14:%.*]] = mul nuw i32 13, [[TMP13]]
+// CHECK-NEXT:    [[TMP15:%.*]] = add nuw i32 [[TMP14]], [[OMP_TILE0_IV]]
+// CHECK-NEXT:    [[OMP_TILE0_INBOUNDS:%.*]] = icmp ult i32 [[TMP15]], [[DOTCOUNT]]
+// CHECK-NEXT:    br i1 [[OMP_TILE0_INBOUNDS]], label [[OMP_LOOP_BODY:%.*]], label [[OMP_TILE_BODY_MERGE:%.*]]
 // CHECK:       omp_loop.body:
-// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP17]], ptr [[AGG_CAPTURED1]])
-// CHECK-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[B_ADDR]], align 8
-// CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP19]] to i64
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[IDXPROM]]
-// CHECK-NEXT:    [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-// CHECK-NEXT:    [[TMP21:%.*]] = load ptr, ptr [[C_ADDR]], align 8
-// CHECK-NEXT:    [[TMP22:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP22]] to i64
-// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP21]], i64 [[IDXPROM2]]
-// CHECK-NEXT:    [[TMP23:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
-// CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP20]], [[TMP23]]
-// CHECK-NEXT:    [[TMP24:%.*]] = load ptr, ptr [[D_ADDR]], align 8
-// CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[TMP25]] to i64
-// CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[TMP24]], i64 [[IDXPROM4]]
-// CHECK-NEXT:    [[TMP26:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
-// CHECK-NEXT:    [[MUL6:%.*]] = fmul float [[MUL]], [[TMP26]]
-// CHECK-NEXT:    [[TMP27:%.*]] = load ptr, ptr [[A_ADDR]], align 8
-// CHECK-NEXT:    [[TMP28:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM7:%.*]] = sext i32 [[TMP28]] to i64
-// CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[TMP27]], i64 [[IDXPROM7]]
+// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP15]], ptr [[AGG_CAPTURED1]])
+// CHECK-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[B_ADDR]], align 8
+// CHECK-NEXT:    [[TMP17:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP17]] to i64
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[IDXPROM]]
+// CHECK-NEXT:    [[TMP18:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+// CHECK-NEXT:    [[TMP19:%.*]] = load ptr, ptr [[C_ADDR]], align 8
+// CHECK-NEXT:    [[TMP20:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP20]] to i64
+// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP19]], i64 [[IDXPROM2]]
+// CHECK-NEXT:    [[TMP21:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
+// CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP18]], [[TMP21]]
+// CHECK-NEXT:    [[TMP22:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK-NEXT:    [[TMP23:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[TMP23]] to i64
+// CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[TMP22]], i64 [[IDXPROM4]]
+// CHECK-NEXT:    [[TMP24:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
+// CHECK-NEXT:    [[MUL6:%.*]] = fmul float [[MUL]], [[TMP24]]
+// CHECK-NEXT:    [[TMP25:%.*]] = load ptr, ptr [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP26:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM7:%.*]] = sext i32 [[TMP26]] to i64
+// CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[TMP25]], i64 [[IDXPROM7]]
 // CHECK-NEXT:    store float [[MUL6]], ptr [[ARRAYIDX8]], align 4
+// CHECK-NEXT:    br label [[OMP_TILE_BODY_MERGE]]
+// CHECK:       omp_tile.body.merge:
 // CHECK-NEXT:    br label [[OMP_TILE0_INC]]
 // CHECK:       omp_tile0.inc:
 // CHECK-NEXT:    [[OMP_TILE0_NEXT]] = add nuw i32 [[OMP_TILE0_IV]], 1
diff --git a/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_constant_for.c b/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_constant_for.c
index acf0e5164411c..b8a993c9cd8a7 100644
--- a/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_constant_for.c
+++ b/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_constant_for.c
@@ -85,8 +85,6 @@ void unroll_partial_heuristic_constant_for(float *a, float *b, float *c, float *
 // CHECK-NEXT:    br i1 [[OMP_FLOOR0_CMP]], label [[OMP_FLOOR0_BODY:%.*]], label [[OMP_FLOOR0_EXIT:%.*]]
 // CHECK:       omp_floor0.body:
 // CHECK-NEXT:    [[TMP12:%.*]] = add i32 [[OMP_FLOOR0_IV]], [[TMP8]]
-// CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP12]], [[TMP3]]
-// CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP4]], i32 4
 // CHECK-NEXT:    br label [[OMP_TILE0_PREHEADER:%.*]]
 // CHECK:       omp_tile0.preheader:
 // CHECK-NEXT:    br label [[OMP_TILE0_HEADER:%.*]]
@@ -94,54 +92,57 @@ void unroll_partial_heuristic_constant_for(float *a, float *b, float *c, float *
 // CHECK-NEXT:    [[OMP_TILE0_IV:%.*]] = phi i32 [ 0, [[OMP_TILE0_PREHEADER]] ], [ [[OMP_TILE0_NEXT:%.*]], [[OMP_TILE0_INC:%.*]] ]
 // CHECK-NEXT:    br label [[OMP_TILE0_COND:%.*]]
 // CHECK:       omp_tile0.cond:
-// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], [[TMP14]]
+// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], 4
 // CHECK-NEXT:    br i1 [[OMP_TILE0_CMP]], label [[OMP_TILE0_BODY:%.*]], label [[OMP_TILE0_EXIT:%.*]]
 // CHECK:       omp_tile0.body:
-// CHECK-NEXT:    [[TMP15:%.*]] = mul nuw i32 4, [[TMP12]]
-// CHECK-NEXT:    [[TMP16:%.*]] = add nuw i32 [[TMP15]], [[OMP_TILE0_IV]]
-// CHECK-NEXT:    br label [[OMP_LOOP_BODY:%.*]]
+// CHECK-NEXT:    [[TMP13:%.*]] = mul nuw i32 4, [[TMP12]]
+// CHECK-NEXT:    [[TMP14:%.*]] = add nuw i32 [[TMP13]], [[OMP_TILE0_IV]]
+// CHECK-NEXT:    [[OMP_TILE0_INBOUNDS:%.*]] = icmp ult i32 [[TMP14]], [[DOTCOUNT]]
+// CHECK-NEXT:    br i1 [[OMP_TILE0_INBOUNDS]], label [[OMP_LOOP_BODY:%.*]], label [[OMP_TILE_BODY_MERGE:%.*]]
 // CHECK:       omp_loop.body:
-// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP16]], ptr [[AGG_CAPTURED1]])
-// CHECK-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[B_ADDR]], align 8
-// CHECK-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP18]] to i64
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[IDXPROM]]
-// CHECK-NEXT:    [[TMP19:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-// CHECK-NEXT:    [[CONV:%.*]] = fpext float [[TMP19]] to double
+// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP14]], ptr [[AGG_CAPTURED1]])
+// CHECK-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[B_ADDR]], align 8
+// CHECK-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP16]] to i64
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP15]], i64 [[IDXPROM]]
+// CHECK-NEXT:    [[TMP17:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+// CHECK-NEXT:    [[CONV:%.*]] = fpext float [[TMP17]] to double
 // CHECK-NEXT:    [[CALL:%.*]] = call double @sind(double noundef [[CONV]])
-// CHECK-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[C_ADDR]], align 8
-// CHECK-NEXT:    [[TMP21:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP21]] to i64
-// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP20]], i64 [[IDXPROM2]]
-// CHECK-NEXT:    [[TMP22:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
-// CHECK-NEXT:    [[CONV4:%.*]] = fpext float [[TMP22]] to double
+// CHECK-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[C_ADDR]], align 8
+// CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP19]] to i64
+// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[IDXPROM2]]
+// CHECK-NEXT:    [[TMP20:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
+// CHECK-NEXT:    [[CONV4:%.*]] = fpext float [[TMP20]] to double
 // CHECK-NEXT:    [[MUL:%.*]] = fmul double [[CALL]], [[CONV4]]
-// CHECK-NEXT:    [[TMP23:%.*]] = load ptr, ptr [[D_ADDR]], align 8
-// CHECK-NEXT:    [[TMP24:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM5:%.*]] = sext i32 [[TMP24]] to i64
-// CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, ptr [[TMP23]], i64 [[IDXPROM5]]
-// CHECK-NEXT:    [[TMP25:%.*]] = load float, ptr [[ARRAYIDX6]], align 4
-// CHECK-NEXT:    [[CONV7:%.*]] = fpext float [[TMP25]] to double
+// CHECK-NEXT:    [[TMP21:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK-NEXT:    [[TMP22:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM5:%.*]] = sext i32 [[TMP22]] to i64
+// CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, ptr [[TMP21]], i64 [[IDXPROM5]]
+// CHECK-NEXT:    [[TMP23:%.*]] = load float, ptr [[ARRAYIDX6]], align 4
+// CHECK-NEXT:    [[CONV7:%.*]] = fpext float [[TMP23]] to double
 // CHECK-NEXT:    [[MUL8:%.*]] = fmul double [[MUL]], [[CONV7]]
-// CHECK-NEXT:    [[TMP26:%.*]] = load ptr, ptr [[E_ADDR]], align 8
-// CHECK-NEXT:    [[TMP27:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM9:%.*]] = sext i32 [[TMP27]] to i64
-// CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[TMP26]], i64 [[IDXPROM9]]
-// CHECK-NEXT:    [[TMP28:%.*]] = load float, ptr [[ARRAYIDX10]], align 4
-// CHECK-NEXT:    [[CONV11:%.*]] = fpext float [[TMP28]] to double
+// CHECK-NEXT:    [[TMP24:%.*]] = load ptr, ptr [[E_ADDR]], align 8
+// CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM9:%.*]] = sext i32 [[TMP25]] to i64
+// CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[TMP24]], i64 [[IDXPROM9]]
+// CHECK-NEXT:    [[TMP26:%.*]] = load float, ptr [[ARRAYIDX10]], align 4
+// CHECK-NEXT:    [[CONV11:%.*]] = fpext float [[TMP26]] to double
 // CHECK-NEXT:    [[MUL12:%.*]] = fmul double [[MUL8]], [[CONV11]]
-// CHECK-NEXT:    [[TMP29:%.*]] = load float, ptr [[OFFSET_ADDR]], align 4
-// CHECK-NEXT:    [[CONV13:%.*]] = fpext float [[TMP29]] to double
+// CHECK-NEXT:    [[TMP27:%.*]] = load float, ptr [[OFFSET_ADDR]], align 4
+// CHECK-NEXT:    [[CONV13:%.*]] = fpext float [[TMP27]] to double
 // CHECK-NEXT:    [[ADD:%.*]] = fadd double [[MUL12]], [[CONV13]]
-// CHECK-NEXT:    [[TMP30:%.*]] = load ptr, ptr [[A_ADDR]], align 8
-// CHECK-NEXT:    [[TMP31:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM14:%.*]] = sext i32 [[TMP31]] to i64
-// CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds float, ptr [[TMP30]], i64 [[IDXPROM14]]
-// CHECK-NEXT:    [[TMP32:%.*]] = load float, ptr [[ARRAYIDX15]], align 4
-// CHECK-NEXT:    [[CONV16:%.*]] = fpext float [[TMP32]] to double
+// CHECK-NEXT:    [[TMP28:%.*]] = load ptr, ptr [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP29:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM14:%.*]] = sext i32 [[TMP29]] to i64
+// CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds float, ptr [[TMP28]], i64 [[IDXPROM14]]
+// CHECK-NEXT:    [[TMP30:%.*]] = load float, ptr [[ARRAYIDX15]], align 4
+// CHECK-NEXT:    [[CONV16:%.*]] = fpext float [[TMP30]] to double
 // CHECK-NEXT:    [[ADD17:%.*]] = fadd double [[CONV16]], [[ADD]]
 // CHECK-NEXT:    [[CONV18:%.*]] = fptrunc double [[ADD17]] to float
 // CHECK-NEXT:    store float [[CONV18]], ptr [[ARRAYIDX15]], align 4
+// CHECK-NEXT:    br label [[OMP_TILE_BODY_MERGE]]
+// CHECK:       omp_tile.body.merge:
 // CHECK-NEXT:    br label [[OMP_TILE0_INC]]
 // CHECK:       omp_tile0.inc:
 // CHECK-NEXT:    [[OMP_TILE0_NEXT]] = add nuw i32 [[OMP_TILE0_IV]], 1
diff --git a/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_runtime_for.c b/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_runtime_for.c
index 8fee5773e53b4..6b48e3ca22888 100644
--- a/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_runtime_for.c
+++ b/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_runtime_for.c
@@ -87,8 +87,6 @@ void unroll_partial_heuristic_runtime_for(int n, float *a, float *b, float *c, f
 // CHECK-NEXT:    br i1 [[OMP_FLOOR0_CMP]], label [[OMP_FLOOR0_BODY:%.*]], label [[OMP_FLOOR0_EXIT:%.*]]
 // CHECK:       omp_floor0.body:
 // CHECK-NEXT:    [[TMP13:%.*]] = add i32 [[OMP_FLOOR0_IV]], [[TMP9]]
-// CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP13]], [[TMP4]]
-// CHECK-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i32 [[TMP5]], i32 4
 // CHECK-NEXT:    br label [[OMP_TILE0_PREHEADER:%.*]]
 // CHECK:       omp_tile0.preheader:
 // CHECK-NEXT:    br label [[OMP_TILE0_HEADER:%.*]]
@@ -96,54 +94,57 @@ void unroll_partial_heuristic_runtime_for(int n, float *a, float *b, float *c, f
 // CHECK-NEXT:    [[OMP_TILE0_IV:%.*]] = phi i32 [ 0, [[OMP_TILE0_PREHEADER]] ], [ [[OMP_TILE0_NEXT:%.*]], [[OMP_TILE0_INC:%.*]] ]
 // CHECK-NEXT:    br label [[OMP_TILE0_COND:%.*]]
 // CHECK:       omp_tile0.cond:
-// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], [[TMP15]]
+// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], 4
 // CHECK-NEXT:    br i1 [[OMP_TILE0_CMP]], label [[OMP_TILE0_BODY:%.*]], label [[OMP_TILE0_EXIT:%.*]]
 // CHECK:       omp_tile0.body:
-// CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i32 4, [[TMP13]]
-// CHECK-NEXT:    [[TMP17:%.*]] = add nuw i32 [[TMP16]], [[OMP_TILE0_IV]]
-// CHECK-NEXT:    br label [[OMP_LOOP_BODY:%.*]]
+// CHECK-NEXT:    [[TMP14:%.*]] = mul nuw i32 4, [[TMP13]]
+// CHECK-NEXT:    [[TMP15:%.*]] = add nuw i32 [[TMP14]], [[OMP_TILE0_IV]]
+// CHECK-NEXT:    [[OMP_TILE0_INBOUNDS:%.*]] = icmp ult i32 [[TMP15]], [[DOTCOUNT]]
+// CHECK-NEXT:    br i1 [[OMP_TILE0_INBOUNDS]], label [[OMP_LOOP_BODY:%.*]], label [[OMP_TILE_BODY_MERGE:%.*]]
 // CHECK:       omp_loop.body:
-// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP17]], ptr [[AGG_CAPTURED1]])
-// CHECK-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[B_ADDR]], align 8
-// CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP19]] to i64
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[IDXPROM]]
-// CHECK-NEXT:    [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-// CHECK-NEXT:    [[CONV:%.*]] = fpext float [[TMP20]] to double
+// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP15]], ptr [[AGG_CAPTURED1]])
+// CHECK-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[B_ADDR]], align 8
+// CHECK-NEXT:    [[TMP17:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP17]] to i64
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[IDXPROM]]
+// CHECK-NEXT:    [[TMP18:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+// CHECK-NEXT:    [[CONV:%.*]] = fpext float [[TMP18]] to double
 // CHECK-NEXT:    [[CALL:%.*]] = call double @sind(double noundef [[CONV]])
-// CHECK-NEXT:    [[TMP21:%.*]] = load ptr, ptr [[C_ADDR]], align 8
-// CHECK-NEXT:    [[TMP22:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP22]] to i64
-// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP21]], i64 [[IDXPROM2]]
-// CHECK-NEXT:    [[TMP23:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
-// CHECK-NEXT:    [[CONV4:%.*]] = fpext float [[TMP23]] to double
+// CHECK-NEXT:    [[TMP19:%.*]] = load ptr, ptr [[C_ADDR]], align 8
+// CHECK-NEXT:    [[TMP20:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP20]] to i64
+// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP19]], i64 [[IDXPROM2]]
+// CHECK-NEXT:    [[TMP21:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
+// CHECK-NEXT:    [[CONV4:%.*]] = fpext float [[TMP21]] to double
 // CHECK-NEXT:    [[MUL:%.*]] = fmul double [[CALL]], [[CONV4]]
-// CHECK-NEXT:    [[TMP24:%.*]] = load ptr, ptr [[D_ADDR]], align 8
-// CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM5:%.*]] = sext i32 [[TMP25]] to i64
-// CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, ptr [[TMP24]], i64 [[IDXPROM5]]
-// CHECK-NEXT:    [[TMP26:%.*]] = load float, ptr [[ARRAYIDX6]], align 4
-// CHECK-NEXT:    [[CONV7:%.*]] = fpext float [[TMP26]] to double
+// CHECK-NEXT:    [[TMP22:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK-NEXT:    [[TMP23:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM5:%.*]] = sext i32 [[TMP23]] to i64
+// CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, ptr [[TMP22]], i64 [[IDXPROM5]]
+// CHECK-NEXT:    [[TMP24:%.*]] = load float, ptr [[ARRAYIDX6]], align 4
+// CHECK-NEXT:    [[CONV7:%.*]] = fpext float [[TMP24]] to double
 // CHECK-NEXT:    [[MUL8:%.*]] = fmul double [[MUL]], [[CONV7]]
-// CHECK-NEXT:    [[TMP27:%.*]] = load ptr, ptr [[E_ADDR]], align 8
-// CHECK-NEXT:    [[TMP28:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM9:%.*]] = sext i32 [[TMP28]] to i64
-// CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[TMP27]], i64 [[IDXPROM9]]
-// CHECK-NEXT:    [[TMP29:%.*]] = load float, ptr [[ARRAYIDX10]], align 4
-// CHECK-NEXT:    [[CONV11:%.*]] = fpext float [[TMP29]] to double
+// CHECK-NEXT:    [[TMP25:%.*]] = load ptr, ptr [[E_ADDR]], align 8
+// CHECK-NEXT:    [[TMP26:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM9:%.*]] = sext i32 [[TMP26]] to i64
+// CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[TMP25]], i64 [[IDXPROM9]]
+// CHECK-NEXT:    [[TMP27:%.*]] = load float, ptr [[ARRAYIDX10]], align 4
+// CHECK-NEXT:    [[CONV11:%.*]] = fpext float [[TMP27]] to double
 // CHECK-NEXT:    [[MUL12:%.*]] = fmul double [[MUL8]], [[CONV11]]
-// CHECK-NEXT:    [[TMP30:%.*]] = load float, ptr [[OFFSET_ADDR]], align 4
-// CHECK-NEXT:    [[CONV13:%.*]] = fpext float [[TMP30]] to double
+// CHECK-NEXT:    [[TMP28:%.*]] = load float, ptr [[OFFSET_ADDR]], align 4
+// CHECK-NEXT:    [[CONV13:%.*]] = fpext float [[TMP28]] to double
 // CHECK-NEXT:    [[ADD:%.*]] = fadd double [[MUL12]], [[CONV13]]
-// CHECK-NEXT:    [[TMP31:%.*]] = load ptr, ptr [[A_ADDR]], align 8
-// CHECK-NEXT:    [[TMP32:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM14:%.*]] = sext i32 [[TMP32]] to i64
-// CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds float, ptr [[TMP31]], i64 [[IDXPROM14]]
-// CHECK-NEXT:    [[TMP33:%.*]] = load float, ptr [[ARRAYIDX15]], align 4
-// CHECK-NEXT:    [[CONV16:%.*]] = fpext float [[TMP33]] to double
+// CHECK-NEXT:    [[TMP29:%.*]] = load ptr, ptr [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP30:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM14:%.*]] = sext i32 [[TMP30]] to i64
+// CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds float, ptr [[TMP29]], i64 [[IDXPROM14]]
+// CHECK-NEXT:    [[TMP31:%.*]] = load float, ptr [[ARRAYIDX15]], align 4
+// CHECK-NEXT:    [[CONV16:%.*]] = fpext float [[TMP31]] to double
 // CHECK-NEXT:    [[ADD17:%.*]] = fadd double [[CONV16]], [[ADD]]
 // CHECK-NEXT:    [[CONV18:%.*]] = fptrunc double [[ADD17]] to float
 // CHECK-NEXT:    store float [[CONV18]], ptr [[ARRAYIDX15]], align 4
+// CHECK-NEXT:    br label [[OMP_TILE_BODY_MERGE]]
+// CHECK:       omp_tile.body.merge:
 // CHECK-NEXT:    br label [[OMP_TILE0_INC]]
 // CHECK:       omp_tile0.inc:
 // CHECK-NEXT:    [[OMP_TILE0_NEXT]] = add nuw i32 [[OMP_TILE0_IV]], 1
diff --git a/clang/test/OpenMP/irbuilder_unroll_unroll_partial_factor.c b/clang/test/OpenMP/irbuilder_unroll_unroll_partial_factor.c
index c4ebcddb097bc..65bd495ca0426 100644
--- a/clang/test/OpenMP/irbuilder_unroll_unroll_partial_factor.c
+++ b/clang/test/OpenMP/irbuilder_unroll_unroll_partial_factor.c
@@ -75,8 +75,6 @@ void unroll_partial_factor_for(float *a, float *b, float *c, float *d) {
 // CHECK-NEXT:    br i1 [[OMP_FLOOR0_CMP]], label [[OMP_FLOOR0_BODY:%.*]], label [[OMP_FLOOR0_EXIT:%.*]]
 // CHECK:       omp_floor0.body:
 // CHECK-NEXT:    [[TMP12:%.*]] = add i32 [[OMP_FLOOR0_IV]], [[TMP8]]
-// CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP12]], [[TMP3]]
-// CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP4]], i32 2
 // CHECK-NEXT:    br label [[OMP_TILE0_PREHEADER:%.*]]
 // CHECK:       omp_tile0.preheader:
 // CHECK-NEXT:    br label [[OMP_TILE0_HEADER:%.*]]
@@ -84,36 +82,39 @@ void unroll_partial_factor_for(float *a, float *b, float *c, float *d) {
 // CHECK-NEXT:    [[OMP_TILE0_IV:%.*]] = phi i32 [ 0, [[OMP_TILE0_PREHEADER]] ], [ [[OMP_TILE0_NEXT:%.*]], [[OMP_TILE0_INC:%.*]] ]
 // CHECK-NEXT:    br label [[OMP_TILE0_COND:%.*]]
 // CHECK:       omp_tile0.cond:
-// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], [[TMP14]]
+// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], 2
 // CHECK-NEXT:    br i1 [[OMP_TILE0_CMP]], label [[OMP_TILE0_BODY:%.*]], label [[OMP_TILE0_EXIT:%.*]]
 // CHECK:       omp_tile0.body:
-// CHECK-NEXT:    [[TMP15:%.*]] = mul nuw i32 2, [[TMP12]]
-// CHECK-NEXT:    [[TMP16:%.*]] = add nuw i32 [[TMP15]], [[OMP_TILE0_IV]]
-// CHECK-NEXT:    br label [[OMP_LOOP_BODY:%.*]]
+// CHECK-NEXT:    [[TMP13:%.*]] = mul nuw i32 2, [[TMP12]]
+// CHECK-NEXT:    [[TMP14:%.*]] = add nuw i32 [[TMP13]], [[OMP_TILE0_IV]]
+// CHECK-NEXT:    [[OMP_TILE0_INBOUNDS:%.*]] = icmp ult i32 [[TMP14]], [[DOTCOUNT]]
+// CHECK-NEXT:    br i1 [[OMP_TILE0_INBOUNDS]], label [[OMP_LOOP_BODY:%.*]], label [[OMP_TILE_BODY_MERGE:%.*]]
 // CHECK:       omp_loop.body:
-// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP16]], ptr [[AGG_CAPTURED1]])
-// CHECK-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[B_ADDR]], align 8
-// CHECK-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP18]] to i64
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[IDXPROM]]
-// CHECK-NEXT:    [[TMP19:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-// CHECK-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[C_ADDR]], align 8
-// CHECK-NEXT:    [[TMP21:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP21]] to i64
-// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP20]], i64 [[IDXPROM2]]
-// CHECK-NEXT:    [[TMP22:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
-// CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP19]], [[TMP22]]
-// CHECK-NEXT:    [[TMP23:%.*]] = load ptr, ptr [[D_ADDR]], align 8
-// CHECK-NEXT:    [[TMP24:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[TMP24]] to i64
-// CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[TMP23]], i64 [[IDXPROM4]]
-// CHECK-NEXT:    [[TMP25:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
-// CHECK-NEXT:    [[MUL6:%.*]] = fmul float [[MUL]], [[TMP25]]
-// CHECK-NEXT:    [[TMP26:%.*]] = load ptr, ptr [[A_ADDR]], align 8
-// CHECK-NEXT:    [[TMP27:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM7:%.*]] = sext i32 [[TMP27]] to i64
-// CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[TMP26]], i64 [[IDXPROM7]]
+// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP14]], ptr [[AGG_CAPTURED1]])
+// CHECK-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[B_ADDR]], align 8
+// CHECK-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP16]] to i64
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP15]], i64 [[IDXPROM]]
+// CHECK-NEXT:    [[TMP17:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+// CHECK-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[C_ADDR]], align 8
+// CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP19]] to i64
+// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[IDXPROM2]]
+// CHECK-NEXT:    [[TMP20:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
+// CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP17]], [[TMP20]]
+// CHECK-NEXT:    [[TMP21:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK-NEXT:    [[TMP22:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[TMP22]] to i64
+// CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[TMP21]], i64 [[IDXPROM4]]
+// CHECK-NEXT:    [[TMP23:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
+// CHECK-NEXT:    [[MUL6:%.*]] = fmul float [[MUL]], [[TMP23]]
+// CHECK-NEXT:    [[TMP24:%.*]] = load ptr, ptr [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM7:%.*]] = sext i32 [[TMP25]] to i64
+// CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[TMP24]], i64 [[IDXPROM7]]
 // CHECK-NEXT:    store float [[MUL6]], ptr [[ARRAYIDX8]], align 4
+// CHECK-NEXT:    br label [[OMP_TILE_BODY_MERGE]]
+// CHECK:       omp_tile.body.merge:
 // CHECK-NEXT:    br label [[OMP_TILE0_INC]]
 // CHECK:       omp_tile0.inc:
 // CHECK-NEXT:    [[OMP_TILE0_NEXT]] = add nuw i32 [[OMP_TILE0_IV]], 1
diff --git a/clang/test/OpenMP/irbuilder_unroll_unroll_partial_heuristic.c b/clang/test/OpenMP/irbuilder_unroll_unroll_partial_heuristic.c
index 24d42d265d6a6..c2b7d46588f66 100644
--- a/clang/test/OpenMP/irbuilder_unroll_unroll_partial_heuristic.c
+++ b/clang/test/OpenMP/irbuilder_unroll_unroll_partial_heuristic.c
@@ -59,8 +59,6 @@ void unroll_unroll_partial_heuristic(float *a, float *b, float *c, float *d) {
 // CHECK-NEXT:    [[OMP_FLOOR0_CMP:%.*]] = icmp ult i32 [[OMP_FLOOR0_IV]], [[OMP_FLOOR0_TRIPCOUNT]]
 // CHECK-NEXT:    br i1 [[OMP_FLOOR0_CMP]], label [[OMP_FLOOR0_BODY:%.*]], label [[OMP_FLOOR0_EXIT:%.*]]
 // CHECK:       omp_floor0.body:
-// CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[OMP_FLOOR0_IV]], [[TMP3]]
-// CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP4]], i32 8
 // CHECK-NEXT:    br label [[OMP_TILE0_PREHEADER:%.*]]
 // CHECK:       omp_tile0.preheader:
 // CHECK-NEXT:    br label [[OMP_TILE0_HEADER:%.*]]
@@ -68,47 +66,50 @@ void unroll_unroll_partial_heuristic(float *a, float *b, float *c, float *d) {
 // CHECK-NEXT:    [[OMP_TILE0_IV:%.*]] = phi i32 [ 0, [[OMP_TILE0_PREHEADER]] ], [ [[OMP_TILE0_NEXT:%.*]], [[OMP_TILE0_INC:%.*]] ]
 // CHECK-NEXT:    br label [[OMP_TILE0_COND:%.*]]
 // CHECK:       omp_tile0.cond:
-// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], [[TMP8]]
+// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], 8
 // CHECK-NEXT:    br i1 [[OMP_TILE0_CMP]], label [[OMP_TILE0_BODY:%.*]], label [[OMP_TILE0_EXIT:%.*]]
 // CHECK:       omp_tile0.body:
-// CHECK-NEXT:    [[TMP9:%.*]] = mul nuw i32 8, [[OMP_FLOOR0_IV]]
-// CHECK-NEXT:    [[TMP10:%.*]] = add nuw i32 [[TMP9]], [[OMP_TILE0_IV]]
-// CHECK-NEXT:    br label [[OMP_LOOP_BODY:%.*]]
+// CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i32 8, [[OMP_FLOOR0_IV]]
+// CHECK-NEXT:    [[TMP8:%.*]] = add nuw i32 [[TMP7]], [[OMP_TILE0_IV]]
+// CHECK-NEXT:    [[OMP_TILE0_INBOUNDS:%.*]] = icmp ult i32 [[TMP8]], [[DOTCOUNT]]
+// CHECK-NEXT:    br i1 [[OMP_TILE0_INBOUNDS]], label [[OMP_LOOP_BODY:%.*]], label [[OMP_TILE_BODY_MERGE:%.*]]
 // CHECK:       omp_loop.body:
-// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP10]], ptr [[AGG_CAPTURED1]])
-// CHECK-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[B_ADDR]], align 8
-// CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP12]] to i64
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP11]], i64 [[IDXPROM]]
-// CHECK-NEXT:    [[TMP13:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-// CHECK-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[C_ADDR]], align 8
-// CHECK-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP15]] to i64
-// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP14]], i64 [[IDXPROM2]]
-// CHECK-NEXT:    [[TMP16:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
-// CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP13]], [[TMP16]]
-// CHECK-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[D_ADDR]], align 8
-// CHECK-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[TMP18]] to i64
-// CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[IDXPROM4]]
-// CHECK-NEXT:    [[TMP19:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
-// CHECK-NEXT:    [[MUL6:%.*]] = fmul float [[MUL]], [[TMP19]]
-// CHECK-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[A_ADDR]], align 8
-// CHECK-NEXT:    [[TMP21:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM7:%.*]] = sext i32 [[TMP21]] to i64
-// CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[TMP20]], i64 [[IDXPROM7]]
+// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP8]], ptr [[AGG_CAPTURED1]])
+// CHECK-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[B_ADDR]], align 8
+// CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP10]] to i64
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP9]], i64 [[IDXPROM]]
+// CHECK-NEXT:    [[TMP11:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+// CHECK-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[C_ADDR]], align 8
+// CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP13]] to i64
+// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP12]], i64 [[IDXPROM2]]
+// CHECK-NEXT:    [[TMP14:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
+// CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP11]], [[TMP14]]
+// CHECK-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[TMP16]] to i64
+// CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[TMP15]], i64 [[IDXPROM4]]
+// CHECK-NEXT:    [[TMP17:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
+// CHECK-NEXT:    [[MUL6:%.*]] = fmul float [[MUL]], [[TMP17]]
+// CHECK-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM7:%.*]] = sext i32 [[TMP19]] to i64
+// CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[IDXPROM7]]
 // CHECK-NEXT:    store float [[MUL6]], ptr [[ARRAYIDX8]], align 4
+// CHECK-NEXT:    br label [[OMP_TILE_BODY_MERGE]]
+// CHECK:       omp_tile.body.merge:
 // CHECK-NEXT:    br label [[OMP_TILE0_INC]]
 // CHECK:       omp_tile0.inc:
 // CHECK-NEXT:    [[OMP_TILE0_NEXT]] = add nuw i32 [[OMP_TILE0_IV]], 1
-// CHECK-NEXT:    br label [[OMP_TILE0_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
+// CHECK-NEXT:    br label [[OMP_TILE0_HEADER]], !llvm.loop [[LOOP2:![0-9]+]]
 // CHECK:       omp_tile0.exit:
 // CHECK-NEXT:    br label [[OMP_TILE0_AFTER:%.*]]
 // CHECK:       omp_tile0.after:
 // CHECK-NEXT:    br label [[OMP_FLOOR0_INC]]
 // CHECK:       omp_floor0.inc:
 // CHECK-NEXT:    [[OMP_FLOOR0_NEXT]] = add nuw i32 [[OMP_FLOOR0_IV]], 1
-// CHECK-NEXT:    br label [[OMP_FLOOR0_HEADER]], !llvm.loop [[LOOP6:![0-9]+]]
+// CHECK-NEXT:    br label [[OMP_FLOOR0_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
 // CHECK:       omp_floor0.exit:
 // CHECK-NEXT:    br label [[OMP_FLOOR0_AFTER:%.*]]
 // CHECK:       omp_floor0.after:
@@ -129,7 +130,7 @@ void unroll_unroll_partial_heuristic(float *a, float *b, float *c, float *d) {
 // CHECK-NEXT:    store ptr [[__CONTEXT]], ptr [[__CONTEXT_ADDR]], align 8
 // CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__CONTEXT_ADDR]], align 8
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw [[STRUCT_ANON:%.*]], ptr [[TMP0]], i32 0, i32 0
-// CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP1]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP1]], align 8, !nonnull [[META6:![0-9]+]], !align [[META7:![0-9]+]]
 // CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
 // CHECK-NEXT:    store i32 [[TMP3]], ptr [[DOTSTART]], align 4
 // CHECK-NEXT:    store i32 2, ptr [[DOTSTOP]], align 4
@@ -152,7 +153,7 @@ void unroll_unroll_partial_heuristic(float *a, float *b, float *c, float *d) {
 // CHECK-NEXT:    br label [[COND_END]]
 // CHECK:       cond.end:
 // CHECK-NEXT:    [[COND:%.*]] = phi i32 [ [[DIV]], [[COND_TRUE]] ], [ 0, [[COND_FALSE]] ]
-// CHECK-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[DISTANCE_ADDR]], align 8
+// CHECK-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[DISTANCE_ADDR]], align 8, !nonnull [[META6]], !align [[META7]]
 // CHECK-NEXT:    store i32 [[COND]], ptr [[TMP10]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -172,7 +173,7 @@ void unroll_unroll_partial_heuristic(float *a, float *b, float *c, float *d) {
 // CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[LOGICAL_ADDR]], align 4
 // CHECK-NEXT:    [[MUL:%.*]] = mul i32 1, [[TMP3]]
 // CHECK-NEXT:    [[ADD:%.*]] = add i32 [[TMP2]], [[MUL]]
-// CHECK-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[LOOPVAR_ADDR]], align 8
+// CHECK-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[LOOPVAR_ADDR]], align 8, !nonnull [[META6]], !align [[META7]]
 // CHECK-NEXT:    store i32 [[ADD]], ptr [[TMP4]], align 4
 // CHECK-NEXT:    ret void
 //
diff --git a/clang/test/OpenMP/unroll_codegen_tile_for.cpp b/clang/test/OpenMP/unroll_codegen_tile_for.cpp
index ccfc9f38f5a24..6f13f6b67c8cd 100644
--- a/clang/test/OpenMP/unroll_codegen_tile_for.cpp
+++ b/clang/test/OpenMP/unroll_codegen_tile_for.cpp
@@ -1,3 +1,4 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
 // Check code generation
 // RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fopenmp -emit-llvm %s -o - | FileCheck %s --check-prefix=IR
 
@@ -13,221 +14,194 @@
 extern "C" void body(...) {}
 
 
-// IR-LABEL: @func(
-// IR-NEXT:  [[ENTRY:.*]]:
-// IR-NEXT:    %[[START_ADDR:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[END_ADDR:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[STEP_ADDR:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_IV:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[TMP:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[I:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_1:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_2:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_3:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTUNROLLED_IV_I:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_6:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_8:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_12:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_14:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTFLOOR_0_IV__UNROLLED_IV_I:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_LB:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_UB:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_STRIDE:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_IS_LAST:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTFLOOR_0_IV__UNROLLED_IV_I18:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTTILE_0_IV__UNROLLED_IV_I:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTUNROLL_INNER_IV_I:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[TMP0:.+]] = call i32 @__kmpc_global_thread_num(ptr @2)
-// IR-NEXT:    store i32 %[[START:.+]], ptr %[[START_ADDR]], align 4
-// IR-NEXT:    store i32 %[[END:.+]], ptr %[[END_ADDR]], align 4
-// IR-NEXT:    store i32 %[[STEP:.+]], ptr %[[STEP_ADDR]], align 4
-// IR-NEXT:    %[[TMP1:.+]] = load i32, ptr %[[START_ADDR]], align 4
-// IR-NEXT:    store i32 %[[TMP1]], ptr %[[I]], align 4
-// IR-NEXT:    %[[TMP2:.+]] = load i32, ptr %[[START_ADDR]], align 4
-// IR-NEXT:    store i32 %[[TMP2]], ptr %[[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    %[[TMP3:.+]] = load i32, ptr %[[END_ADDR]], align 4
-// IR-NEXT:    store i32 %[[TMP3]], ptr %[[DOTCAPTURE_EXPR_1]], align 4
-// IR-NEXT:    %[[TMP4:.+]] = load i32, ptr %[[STEP_ADDR]], align 4
-// IR-NEXT:    store i32 %[[TMP4]], ptr %[[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    %[[TMP5:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_1]], align 4
-// IR-NEXT:    %[[TMP6:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    %[[SUB:.+]] = sub i32 %[[TMP5]], %[[TMP6]]
-// IR-NEXT:    %[[SUB4:.+]] = sub i32 %[[SUB]], 1
-// IR-NEXT:    %[[TMP7:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    %[[ADD:.+]] = add i32 %[[SUB4]], %[[TMP7]]
-// IR-NEXT:    %[[TMP8:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    %[[DIV:.+]] = udiv i32 %[[ADD]], %[[TMP8]]
-// IR-NEXT:    %[[SUB5:.+]] = sub i32 %[[DIV]], 1
-// IR-NEXT:    store i32 %[[SUB5]], ptr %[[DOTCAPTURE_EXPR_3]], align 4
-// IR-NEXT:    store i32 0, ptr %[[DOTUNROLLED_IV_I]], align 4
-// IR-NEXT:    %[[TMP9:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_3]], align 4
-// IR-NEXT:    %[[ADD7:.+]] = add i32 %[[TMP9]], 1
-// IR-NEXT:    store i32 %[[ADD7]], ptr %[[DOTCAPTURE_EXPR_6]], align 4
-// IR-NEXT:    %[[TMP10:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_6]], align 4
-// IR-NEXT:    %[[SUB9:.+]] = sub i32 %[[TMP10]], -1
-// IR-NEXT:    %[[DIV10:.+]] = udiv i32 %[[SUB9]], 2
-// IR-NEXT:    %[[SUB11:.+]] = sub i32 %[[DIV10]], 1
-// IR-NEXT:    store i32 %[[SUB11]], ptr %[[DOTCAPTURE_EXPR_8]], align 4
-// IR-NEXT:    %[[TMP11:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_8]], align 4
-// IR-NEXT:    %[[ADD13:.+]] = add i32 %[[TMP11]], 1
-// IR-NEXT:    store i32 %[[ADD13]], ptr %[[DOTCAPTURE_EXPR_12]], align 4
-// IR-NEXT:    %[[TMP12:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_12]], align 4
-// IR-NEXT:    %[[SUB15:.+]] = sub i32 %[[TMP12]], -3
-// IR-NEXT:    %[[DIV16:.+]] = udiv i32 %[[SUB15]], 4
-// IR-NEXT:    %[[SUB17:.+]] = sub i32 %[[DIV16]], 1
-// IR-NEXT:    store i32 %[[SUB17]], ptr %[[DOTCAPTURE_EXPR_14]], align 4
-// IR-NEXT:    store i32 0, ptr %[[DOTFLOOR_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    %[[TMP13:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_12]], align 4
-// IR-NEXT:    %[[CMP:.+]] = icmp ult i32 0, %[[TMP13]]
-// IR-NEXT:    br i1 %[[CMP]], label %[[OMP_PRECOND_THEN:.+]], label %[[OMP_PRECOND_END:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_PRECOND_THEN]]:
-// IR-NEXT:    store i32 0, ptr %[[DOTOMP_LB]], align 4
-// IR-NEXT:    %[[TMP14:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_14]], align 4
-// IR-NEXT:    store i32 %[[TMP14]], ptr %[[DOTOMP_UB]], align 4
-// IR-NEXT:    store i32 1, ptr %[[DOTOMP_STRIDE]], align 4
-// IR-NEXT:    store i32 0, ptr %[[DOTOMP_IS_LAST]], align 4
-// IR-NEXT:    call void @__kmpc_for_static_init_4u(ptr @1, i32 %[[TMP0]], i32 34, ptr %[[DOTOMP_IS_LAST]], ptr %[[DOTOMP_LB]], ptr %[[DOTOMP_UB]], ptr %[[DOTOMP_STRIDE]], i32 1, i32 1)
-// IR-NEXT:    %[[TMP15:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
-// IR-NEXT:    %[[TMP16:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_14]], align 4
-// IR-NEXT:    %[[CMP19:.+]] = icmp ugt i32 %[[TMP15]], %[[TMP16]]
-// IR-NEXT:    br i1 %[[CMP19]], label %[[COND_TRUE:.+]], label %[[COND_FALSE:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_TRUE]]:
-// IR-NEXT:    %[[TMP17:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_14]], align 4
-// IR-NEXT:    br label %[[COND_END:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_FALSE]]:
-// IR-NEXT:    %[[TMP18:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
+// IR-LABEL: define dso_local void @func(
+// IR-SAME: i32 noundef [[START:%.*]], i32 noundef [[END:%.*]], i32 noundef [[STEP:%.*]]) #[[ATTR0:[0-9]+]] {
+// IR-NEXT:  [[ENTRY:.*:]]
+// IR-NEXT:    [[START_ADDR:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[TMP:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTUNROLLED_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_5:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV__UNROLLED_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_13:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV__UNROLLED_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV__UNROLLED_IV_I17:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_CNT_0_IV__UNROLLED_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTUNROLL_INNER_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
+// IR-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
+// IR-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
+// IR-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// IR-NEXT:    store i32 [[TMP1]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// IR-NEXT:    store i32 [[TMP2]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// IR-NEXT:    store i32 [[TMP3]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// IR-NEXT:    store i32 [[TMP4]], ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    [[SUB:%.*]] = sub i32 [[TMP5]], [[TMP6]]
+// IR-NEXT:    [[SUB3:%.*]] = sub i32 [[SUB]], 1
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add i32 [[SUB3]], [[TMP7]]
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP8]]
+// IR-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
+// IR-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTUNROLLED_IV_I]], align 4
+// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    [[ADD6:%.*]] = add i32 [[TMP9]], 1
+// IR-NEXT:    store i32 [[ADD6]], ptr [[DOTCAPTURE_EXPR_5]], align 4
+// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// IR-NEXT:    [[SUB8:%.*]] = sub i32 [[TMP10]], -1
+// IR-NEXT:    [[DIV9:%.*]] = udiv i32 [[SUB8]], 2
+// IR-NEXT:    [[SUB10:%.*]] = sub i32 [[DIV9]], 1
+// IR-NEXT:    store i32 [[SUB10]], ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    [[ADD12:%.*]] = add i32 [[TMP11]], 1
+// IR-NEXT:    store i32 [[ADD12]], ptr [[DOTCAPTURE_EXPR_11]], align 4
+// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// IR-NEXT:    [[SUB14:%.*]] = sub i32 [[TMP12]], -3
+// IR-NEXT:    [[DIV15:%.*]] = udiv i32 [[SUB14]], 4
+// IR-NEXT:    [[SUB16:%.*]] = sub i32 [[DIV15]], 1
+// IR-NEXT:    store i32 [[SUB16]], ptr [[DOTCAPTURE_EXPR_13]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// IR-NEXT:    [[CMP:%.*]] = icmp ult i32 0, [[TMP13]]
+// IR-NEXT:    br i1 [[CMP]], label %[[OMP_PRECOND_THEN:.*]], label %[[OMP_PRECOND_END:.*]]
+// IR:       [[OMP_PRECOND_THEN]]:
+// IR-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
+// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_13]], align 4
+// IR-NEXT:    store i32 [[TMP14]], ptr [[DOTOMP_UB]], align 4
+// IR-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
+// IR-NEXT:    call void @__kmpc_for_static_init_4u(ptr @[[GLOB1:[0-9]+]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
+// IR-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// IR-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_13]], align 4
+// IR-NEXT:    [[CMP18:%.*]] = icmp ugt i32 [[TMP15]], [[TMP16]]
+// IR-NEXT:    br i1 [[CMP18]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_13]], align 4
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
 // IR-NEXT:    br label %[[COND_END]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_END]]:
-// IR-NEXT:    %[[COND:.+]] = phi i32 [ %[[TMP17]], %[[COND_TRUE]] ], [ %[[TMP18]], %[[COND_FALSE]] ]
-// IR-NEXT:    store i32 %[[COND]], ptr %[[DOTOMP_UB]], align 4
-// IR-NEXT:    %[[TMP19:.+]] = load i32, ptr %[[DOTOMP_LB]], align 4
-// IR-NEXT:    store i32 %[[TMP19]], ptr %[[DOTOMP_IV]], align 4
-// IR-NEXT:    br label %[[OMP_INNER_FOR_COND:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_INNER_FOR_COND]]:
-// IR-NEXT:    %[[TMP20:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
-// IR-NEXT:    %[[TMP21:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
-// IR-NEXT:    %[[ADD20:.+]] = add i32 %[[TMP21]], 1
-// IR-NEXT:    %[[CMP21:.+]] = icmp ult i32 %[[TMP20]], %[[ADD20]]
-// IR-NEXT:    br i1 %[[CMP21]], label %[[OMP_INNER_FOR_BODY:.+]], label %[[OMP_INNER_FOR_END:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_INNER_FOR_BODY]]:
-// IR-NEXT:    %[[TMP22:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
-// IR-NEXT:    %[[MUL:.+]] = mul i32 %[[TMP22]], 4
-// IR-NEXT:    %[[ADD22:.+]] = add i32 0, %[[MUL]]
-// IR-NEXT:    store i32 %[[ADD22]], ptr %[[DOTFLOOR_0_IV__UNROLLED_IV_I18]], align 4
-// IR-NEXT:    %[[TMP23:.+]] = load i32, ptr %[[DOTFLOOR_0_IV__UNROLLED_IV_I18]], align 4
-// IR-NEXT:    store i32 %[[TMP23]], ptr %[[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_COND]]:
-// IR-NEXT:    %[[TMP24:.+]] = load i32, ptr %[[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    %[[TMP25:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_8]], align 4
-// IR-NEXT:    %[[ADD23:.+]] = add i32 %[[TMP25]], 1
-// IR-NEXT:    %[[TMP26:.+]] = load i32, ptr %[[DOTFLOOR_0_IV__UNROLLED_IV_I18]], align 4
-// IR-NEXT:    %[[ADD24:.+]] = add i32 %[[TMP26]], 4
-// IR-NEXT:    %[[CMP25:.+]] = icmp ult i32 %[[ADD23]], %[[ADD24]]
-// IR-NEXT:    br i1 %[[CMP25]], label %[[COND_TRUE26:.+]], label %[[COND_FALSE28:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_TRUE26]]:
-// IR-NEXT:    %[[TMP27:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_8]], align 4
-// IR-NEXT:    %[[ADD27:.+]] = add i32 %[[TMP27]], 1
-// IR-NEXT:    br label %[[COND_END30:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_FALSE28]]:
-// IR-NEXT:    %[[TMP28:.+]] = load i32, ptr %[[DOTFLOOR_0_IV__UNROLLED_IV_I18]], align 4
-// IR-NEXT:    %[[ADD29:.+]] = add i32 %[[TMP28]], 4
-// IR-NEXT:    br label %[[COND_END30]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_END30]]:
-// IR-NEXT:    %[[COND31:.+]] = phi i32 [ %[[ADD27]], %[[COND_TRUE26]] ], [ %[[ADD29]], %[[COND_FALSE28]] ]
-// IR-NEXT:    %[[CMP32:.+]] = icmp ult i32 %[[TMP24]], %[[COND31]]
-// IR-NEXT:    br i1 %[[CMP32]], label %[[FOR_BODY:.+]], label %[[FOR_END45:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_BODY]]:
-// IR-NEXT:    %[[TMP29:.+]] = load i32, ptr %[[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    %[[MUL33:.+]] = mul i32 %[[TMP29]], 2
-// IR-NEXT:    %[[ADD34:.+]] = add i32 0, %[[MUL33]]
-// IR-NEXT:    store i32 %[[ADD34]], ptr %[[DOTUNROLLED_IV_I]], align 4
-// IR-NEXT:    %[[TMP30:.+]] = load i32, ptr %[[DOTUNROLLED_IV_I]], align 4
-// IR-NEXT:    store i32 %[[TMP30]], ptr %[[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND35:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_COND35]]:
-// IR-NEXT:    %[[TMP31:.+]] = load i32, ptr %[[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    %[[TMP32:.+]] = load i32, ptr %[[DOTUNROLLED_IV_I]], align 4
-// IR-NEXT:    %[[ADD36:.+]] = add i32 %[[TMP32]], 2
-// IR-NEXT:    %[[CMP37:.+]] = icmp ult i32 %[[TMP31]], %[[ADD36]]
-// IR-NEXT:    br i1 %[[CMP37]], label %[[LAND_RHS:.+]], label %[[LAND_END:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[LAND_RHS]]:
-// IR-NEXT:    %[[TMP33:.+]] = load i32, ptr %[[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    %[[TMP34:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_3]], align 4
-// IR-NEXT:    %[[ADD38:.+]] = add i32 %[[TMP34]], 1
-// IR-NEXT:    %[[CMP39:.+]] = icmp ult i32 %[[TMP33]], %[[ADD38]]
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP17]], %[[COND_TRUE]] ], [ [[TMP18]], %[[COND_FALSE]] ]
+// IR-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
+// IR-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
+// IR-NEXT:    store i32 [[TMP19]], ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    br label %[[OMP_INNER_FOR_COND:.*]]
+// IR:       [[OMP_INNER_FOR_COND]]:
+// IR-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// IR-NEXT:    [[ADD19:%.*]] = add i32 [[TMP21]], 1
+// IR-NEXT:    [[CMP20:%.*]] = icmp ult i32 [[TMP20]], [[ADD19]]
+// IR-NEXT:    br i1 [[CMP20]], label %[[OMP_INNER_FOR_BODY:.*]], label %[[OMP_INNER_FOR_END:.*]]
+// IR:       [[OMP_INNER_FOR_BODY]]:
+// IR-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul i32 [[TMP22]], 4
+// IR-NEXT:    [[ADD21:%.*]] = add i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD21]], ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I17]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND:.*]]
+// IR:       [[FOR_COND]]:
+// IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    [[CMP22:%.*]] = icmp ult i32 [[TMP23]], 4
+// IR-NEXT:    br i1 [[CMP22]], label %[[FOR_BODY:.*]], label %[[FOR_END38:.*]]
+// IR:       [[FOR_BODY]]:
+// IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I17]], align 4
+// IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    [[ADD23:%.*]] = add i32 [[TMP24]], [[TMP25]]
+// IR-NEXT:    store i32 [[ADD23]], ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    [[MUL24:%.*]] = mul i32 [[TMP26]], 2
+// IR-NEXT:    [[ADD25:%.*]] = add i32 0, [[MUL24]]
+// IR-NEXT:    store i32 [[ADD25]], ptr [[DOTUNROLLED_IV_I]], align 4
+// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    [[ADD26:%.*]] = add i32 [[TMP28]], 1
+// IR-NEXT:    [[CMP27:%.*]] = icmp ult i32 [[TMP27]], [[ADD26]]
+// IR-NEXT:    br i1 [[CMP27]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
+// IR:       [[IF_THEN]]:
+// IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTUNROLLED_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP29]], ptr [[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND28:.*]]
+// IR:       [[FOR_COND28]]:
+// IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTUNROLLED_IV_I]], align 4
+// IR-NEXT:    [[ADD29:%.*]] = add i32 [[TMP31]], 2
+// IR-NEXT:    [[CMP30:%.*]] = icmp ult i32 [[TMP30]], [[ADD29]]
+// IR-NEXT:    br i1 [[CMP30]], label %[[LAND_RHS:.*]], label %[[LAND_END:.*]]
+// IR:       [[LAND_RHS]]:
+// IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    [[ADD31:%.*]] = add i32 [[TMP33]], 1
+// IR-NEXT:    [[CMP32:%.*]] = icmp ult i32 [[TMP32]], [[ADD31]]
 // IR-NEXT:    br label %[[LAND_END]]
-// IR-EMPTY:
-// IR-NEXT:  [[LAND_END]]:
-// IR-NEXT:    %[[TMP35:.+]] = phi i1 [ false, %[[FOR_COND35]] ], [ %[[CMP39]], %[[LAND_RHS]] ]
-// IR-NEXT:    br i1 %[[TMP35]], label %[[FOR_BODY40:.+]], label %[[FOR_END:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_BODY40]]:
-// IR-NEXT:    %[[TMP36:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    %[[TMP37:.+]] = load i32, ptr %[[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    %[[TMP38:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    %[[MUL41:.+]] = mul i32 %[[TMP37]], %[[TMP38]]
-// IR-NEXT:    %[[ADD42:.+]] = add i32 %[[TMP36]], %[[MUL41]]
-// IR-NEXT:    store i32 %[[ADD42]], ptr %[[I]], align 4
-// IR-NEXT:    %[[TMP39:.+]] = load i32, ptr %[[START_ADDR]], align 4
-// IR-NEXT:    %[[TMP40:.+]] = load i32, ptr %[[END_ADDR]], align 4
-// IR-NEXT:    %[[TMP41:.+]] = load i32, ptr %[[STEP_ADDR]], align 4
-// IR-NEXT:    %[[TMP42:.+]] = load i32, ptr %[[I]], align 4
-// IR-NEXT:    call void (...) @body(i32 noundef %[[TMP39]], i32 noundef %[[TMP40]], i32 noundef %[[TMP41]], i32 noundef %[[TMP42]])
-// IR-NEXT:    br label %[[FOR_INC:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_INC]]:
-// IR-NEXT:    %[[TMP43:.+]] = load i32, ptr %[[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    %[[INC:.+]] = add i32 %[[TMP43]], 1
-// IR-NEXT:    store i32 %[[INC]], ptr %[[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND35]], !llvm.loop ![[LOOP2:[0-9]+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC43:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_INC43]]:
-// IR-NEXT:    %[[TMP44:.+]] = load i32, ptr %[[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    %[[INC44:.+]] = add i32 %[[TMP44]], 1
-// IR-NEXT:    store i32 %[[INC44]], ptr %[[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop ![[LOOP5:[0-9]+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_END45]]:
-// IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_BODY_CONTINUE]]:
-// IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_INNER_FOR_INC]]:
-// IR-NEXT:    %[[TMP45:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
-// IR-NEXT:    %[[ADD46:.+]] = add i32 %[[TMP45]], 1
-// IR-NEXT:    store i32 %[[ADD46]], ptr %[[DOTOMP_IV]], align 4
+// IR:       [[LAND_END]]:
+// IR-NEXT:    [[TMP34:%.*]] = phi i1 [ false, %[[FOR_COND28]] ], [ [[CMP32]], %[[LAND_RHS]] ]
+// IR-NEXT:    br i1 [[TMP34]], label %[[FOR_BODY33:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY33]]:
+// IR-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[MUL34:%.*]] = mul i32 [[TMP36]], [[TMP37]]
+// IR-NEXT:    [[ADD35:%.*]] = add i32 [[TMP35]], [[MUL34]]
+// IR-NEXT:    store i32 [[ADD35]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP38:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// IR-NEXT:    [[TMP39:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// IR-NEXT:    [[TMP40:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// IR-NEXT:    [[TMP41:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void (...) @body(i32 noundef [[TMP38]], i32 noundef [[TMP39]], i32 noundef [[TMP40]], i32 noundef [[TMP41]])
+// IR-NEXT:    br label %[[FOR_INC:.*]]
+// IR:       [[FOR_INC]]:
+// IR-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add i32 [[TMP42]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND28]], !llvm.loop [[LOOP2:![0-9]+]]
+// IR:       [[FOR_END]]:
+// IR-NEXT:    br label %[[IF_END]]
+// IR:       [[IF_END]]:
+// IR-NEXT:    br label %[[FOR_INC36:.*]]
+// IR:       [[FOR_INC36]]:
+// IR-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    [[INC37:%.*]] = add i32 [[TMP43]], 1
+// IR-NEXT:    store i32 [[INC37]], ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP5:![0-9]+]]
+// IR:       [[FOR_END38]]:
+// IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.*]]
+// IR:       [[OMP_BODY_CONTINUE]]:
+// IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.*]]
+// IR:       [[OMP_INNER_FOR_INC]]:
+// IR-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[ADD39:%.*]] = add i32 [[TMP44]], 1
+// IR-NEXT:    store i32 [[ADD39]], ptr [[DOTOMP_IV]], align 4
 // IR-NEXT:    br label %[[OMP_INNER_FOR_COND]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_INNER_FOR_END]]:
-// IR-NEXT:    br label %[[OMP_LOOP_EXIT:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_LOOP_EXIT]]:
-// IR-NEXT:    call void @__kmpc_for_static_fini(ptr @1, i32 %[[TMP0]])
+// IR:       [[OMP_INNER_FOR_END]]:
+// IR-NEXT:    br label %[[OMP_LOOP_EXIT:.*]]
+// IR:       [[OMP_LOOP_EXIT]]:
+// IR-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP0]])
 // IR-NEXT:    br label %[[OMP_PRECOND_END]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_PRECOND_END]]:
-// IR-NEXT:    call void @__kmpc_barrier(ptr @3, i32 %[[TMP0]])
+// IR:       [[OMP_PRECOND_END]]:
+// IR-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3:[0-9]+]], i32 [[TMP0]])
 // IR-NEXT:    ret void
-// IR-NEXT:  }
+//
 extern "C" void func(int start, int end, int step) {
   #pragma omp for
   #pragma omp tile sizes(4)
@@ -239,7 +213,9 @@ extern "C" void func(int start, int end, int step) {
 #endif /* HEADER */
 
 
-// IR: ![[LOOP2]] = distinct !{![[LOOP2]], ![[LOOPPROP3:[0-9]+]], ![[LOOPPROP4:[0-9]+]]}
-// IR: ![[LOOPPROP3]] = !{!"llvm.loop.mustprogress"}
-// IR: ![[LOOPPROP4]] = !{!"llvm.loop.unroll.count", i32 2}
-// IR: ![[LOOP5]] = distinct !{![[LOOP5]], ![[LOOPPROP3]]}
+//.
+// IR: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]], [[META4:![0-9]+]]}
+// IR: [[META3]] = !{!"llvm.loop.mustprogress"}
+// IR: [[META4]] = !{!"llvm.loop.unroll.count", i32 2}
+// IR: [[LOOP5]] = distinct !{[[LOOP5]], [[META3]]}
+//.

>From 9548b4a2aaebf93be69f0d6a342539acaca84f66 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 15 Apr 2026 07:10:22 -0400
Subject: [PATCH 07/18] rect-codegen

---
 clang/test/OpenMP/tile_rect_codegen.cpp    | 50 +++++++++++++
 clang/test/OpenMP/tile_rect_codegen_ir.cpp | 84 ++++++++++++++++++++++
 2 files changed, 134 insertions(+)
 create mode 100644 clang/test/OpenMP/tile_rect_codegen.cpp
 create mode 100644 clang/test/OpenMP/tile_rect_codegen_ir.cpp

diff --git a/clang/test/OpenMP/tile_rect_codegen.cpp b/clang/test/OpenMP/tile_rect_codegen.cpp
new file mode 100644
index 0000000000000..589914eb9d63e
--- /dev/null
+++ b/clang/test/OpenMP/tile_rect_codegen.cpp
@@ -0,0 +1,50 @@
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fclang-abi-compat=latest -std=c++20 -fopenmp -ast-print %s | FileCheck %s
+// expected-no-diagnostics
+
+extern "C" void body(...) {}
+
+// CHECK-LABEL: void rect_tile_1d(
+void rect_tile_1d() {
+  // Tile size 5, trip count 4: 4 % 5 != 0, so predicate is needed.
+  // The tile loop should have rectangular bound (the tile size) and
+  // the body should be guarded by a validity predicate.
+  //
+  // CHECK: #pragma omp tile sizes(5)
+  // CHECK-NEXT: for (int i = 7; i < 17; i += 3)
+  #pragma omp tile sizes(5)
+  for (int i = 7; i < 17; i += 3)
+    body(i);
+}
+
+// CHECK-LABEL: void rect_tile_2d(
+void rect_tile_2d() {
+  // CHECK: #pragma omp tile sizes(5, 5)
+  // CHECK-NEXT: for (int i = 7; i < 17; i += 3)
+  // CHECK-NEXT:   for (int j = 7; j < 17; j += 3)
+  #pragma omp tile sizes(5, 5)
+  for (int i = 7; i < 17; i += 3)
+    for (int j = 7; j < 17; j += 3)
+      body(i, j);
+}
+
+// CHECK-LABEL: void rect_tile_exact_div(
+void rect_tile_exact_div() {
+  // Tile size 5, trip count 10: 10 % 5 == 0, so predicate is NOT needed.
+  // CHECK: #pragma omp tile sizes(5)
+  // CHECK-NEXT: for (int i = 0; i < 10; i += 1)
+  #pragma omp tile sizes(5)
+  for (int i = 0; i < 10; i += 1)
+    body(i);
+}
+
+// CHECK-LABEL: void rect_tile_nested_body_loop(
+void rect_tile_nested_body_loop(int n) {
+  // After tiling i, the j loop (from the associated body) should be part
+  // of the same canonical nest as the floor and tile loops.
+  // CHECK: #pragma omp tile sizes(4)
+  // CHECK-NEXT: for (int i = 0; i < 6; i += 1)
+  #pragma omp tile sizes(4)
+  for (int i = 0; i < 6; i += 1)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+}
diff --git a/clang/test/OpenMP/tile_rect_codegen_ir.cpp b/clang/test/OpenMP/tile_rect_codegen_ir.cpp
new file mode 100644
index 0000000000000..f35c581c8b8ed
--- /dev/null
+++ b/clang/test/OpenMP/tile_rect_codegen_ir.cpp
@@ -0,0 +1,84 @@
+// Regression tests for rectangular #pragma omp tile lowering:
+// - Inner loop counter .tile.cnt.* with bound == tile size (constant).
+// - Validity guard (icmp + branch) when remainder iterations exist or trip count
+//   is not proven divisible at compile time.
+//
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fopenmp -emit-llvm -o - %s | FileCheck %s --check-prefix=IR
+// expected-no-diagnostics
+
+extern "C" void body(int);
+
+// Trip count 6, tile 4 → remainder; body must be guarded.
+// IR-LABEL: remainder_6_tile_4v(
+void remainder_6_tile_4(void) {
+  // Rectangular tile counter and constant tile-size bound.
+  // IR-DAG: %.tile.cnt{{.*}} = alloca i32
+  // IR: icmp {{.*}} i32 {{.*}}, 4
+  // IR: icmp {{.*}} i32 {{.*}}, 6
+  // IR: br i1 {{.*}}, label %if.then{{.*}}, label %if.end{{.*}}
+  // IR: call {{.*}} @body(
+#pragma omp tile sizes(4)
+  for (int i = 0; i < 6; ++i)
+    body(i);
+}
+
+// Simple stride-1 loop; tile size matches full tiles only (10, tile 5).
+// IR-LABEL: full_tiles_10_5v(
+void full_tiles_10_5(void) {
+  // IR-DAG: %.tile.cnt{{.*}} = alloca i32
+  // IR: icmp {{.*}} i32 {{.*}}, 5
+  // IR: call {{.*}} @body(
+#pragma omp tile sizes(5)
+  for (int i = 0; i < 10; ++i)
+    body(i);
+}
+
+// Variable tile size: clamp (TS<=0 ? 1 : TS) is inlined; bound compares to %cond.
+// IR-LABEL: var_tilei(
+void var_tile(int ts) {
+  // IR-DAG: %.tile.cnt{{.*}} = alloca i32
+  // IR: %cond = phi i32
+  // IR: icmp {{.*}} i32 {{.*}}, %cond
+  // IR: call {{.*}} @body(
+#pragma omp tile sizes(ts)
+  for (int i = 0; i < 12; ++i)
+    body(i);
+}
+
+// Two tiled dimensions → two tile counters.
+// IR-LABEL: two_dimv(
+void two_dim(void) {
+  // IR-DAG: %.tile.cnt.0.iv.i{{.*}} = alloca i32
+  // IR-DAG: %.tile.cnt.1.iv.j{{.*}} = alloca i32
+  // IR: icmp {{.*}} i32 {{.*}}, 2
+  // IR: icmp {{.*}} i32 {{.*}}, 2
+  // IR: call {{.*}} @body(
+#pragma omp tile sizes(2, 2)
+  for (int i = 0; i < 3; ++i)
+    for (int j = 0; j < 3; ++j)
+      body(i + j);
+}
+
+// Nested loop in tile body: outer nest still uses .tile.cnt for tiled dim.
+// IR-LABEL: nested_inner_ji(
+void nested_inner_j(int n) {
+  // IR-DAG: %.tile.cnt{{.*}} = alloca i32
+  // IR: icmp {{.*}} i32 {{.*}}, 4
+#pragma omp tile sizes(4)
+  for (int i = 0; i < 6; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i + j);
+}
+
+// Predicate elision: when trip count is evenly divisible by constant tile size,
+// no if-guard is needed (no icmp against trip count in tile body).
+// IR-LABEL: elided_predicatev(
+void elided_predicate(void) {
+  // Trip count 12, tile size 4 → 12 % 4 == 0, no predicate.
+  // IR-NOT: if.then
+  // IR-NOT: if.end
+  // IR: call {{.*}} @body(
+#pragma omp tile sizes(4)
+  for (int i = 0; i < 12; ++i)
+    body(i);
+}

>From 8c03f84c5cd9553258779c31e31e85de85b4094b Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 15 Apr 2026 07:29:31 -0400
Subject: [PATCH 08/18] openmp-ir

---
 .../Frontend/OpenMPIRBuilderTest.cpp          | 35 ++++++++++++++++---
 1 file changed, 30 insertions(+), 5 deletions(-)

diff --git a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
index b96103c82c185..2502c37ba28b3 100644
--- a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
@@ -1643,12 +1643,27 @@ TEST_F(OpenMPIRBuilderTest, TileSingleLoop) {
   CanonicalLoopInfo *Floor = GenLoops[0];
   CanonicalLoopInfo *Tile = GenLoops[1];
 
+  // The merge block sits between the body code and the tile latch.
+  BasicBlock *MergeBB = Tile->getLatch()->getSinglePredecessor();
+  ASSERT_NE(MergeBB, nullptr);
+
   BasicBlock *RefOrder[] = {
-      Floor->getPreheader(), Floor->getHeader(),   Floor->getCond(),
-      Floor->getBody(),      Tile->getPreheader(), Tile->getHeader(),
-      Tile->getCond(),       Tile->getBody(),      BodyCode,
-      Tile->getLatch(),      Tile->getExit(),      Tile->getAfter(),
-      Floor->getLatch(),     Floor->getExit(),     Floor->getAfter(),
+      Floor->getPreheader(),
+      Floor->getHeader(),
+      Floor->getCond(),
+      Floor->getBody(),
+      Tile->getPreheader(),
+      Tile->getHeader(),
+      Tile->getCond(),
+      Tile->getBody(),
+      BodyCode,
+      MergeBB,
+      Tile->getLatch(),
+      Tile->getExit(),
+      Tile->getAfter(),
+      Floor->getLatch(),
+      Floor->getExit(),
+      Floor->getAfter(),
   };
   EXPECT_TRUE(verifyDFSOrder(F, RefOrder));
   EXPECT_TRUE(verifyListOrder(F, RefOrder));
@@ -1718,6 +1733,10 @@ TEST_F(OpenMPIRBuilderTest, TileNestedLoops) {
   CanonicalLoopInfo *Tile1 = GenLoops[2];
   CanonicalLoopInfo *Tile2 = GenLoops[3];
 
+  // The merge block sits between the body code and the innermost tile latch.
+  BasicBlock *MergeBB = Tile2->getLatch()->getSinglePredecessor();
+  ASSERT_NE(MergeBB, nullptr);
+
   BasicBlock *RefOrder[] = {
       Floor1->getPreheader(),
       Floor1->getHeader(),
@@ -1736,6 +1755,7 @@ TEST_F(OpenMPIRBuilderTest, TileNestedLoops) {
       Tile2->getCond(),
       Tile2->getBody(),
       BodyCode,
+      MergeBB,
       Tile2->getLatch(),
       Tile2->getExit(),
       Tile2->getAfter(),
@@ -1826,6 +1846,10 @@ TEST_F(OpenMPIRBuilderTest, TileNestedLoopsWithBounds) {
   CanonicalLoopInfo *Tile0 = GenLoops[2];
   CanonicalLoopInfo *Tile1 = GenLoops[3];
 
+  // The merge block sits between the body code and the innermost tile latch.
+  BasicBlock *MergeBB = Tile1->getLatch()->getSinglePredecessor();
+  ASSERT_NE(MergeBB, nullptr);
+
   BasicBlock *RefOrder[] = {
       Floor0->getPreheader(),
       Floor0->getHeader(),
@@ -1844,6 +1868,7 @@ TEST_F(OpenMPIRBuilderTest, TileNestedLoopsWithBounds) {
       Tile1->getCond(),
       Tile1->getBody(),
       BodyCode,
+      MergeBB,
       Tile1->getLatch(),
       Tile1->getExit(),
       Tile1->getAfter(),

>From f7b15a56ed47ef6e68bedd74ccc3cb4a3c927464 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 15 Apr 2026 07:30:29 -0400
Subject: [PATCH 09/18] runtime-tests

---
 .../runtime/test/transform/tile/foreach.cpp   |  36 ++++++
 openmp/runtime/test/transform/tile/intfor.c   |  78 ++++++-------
 .../runtime/test/transform/tile/iterfor.cpp   |  27 +++++
 .../tile/parallel-wsloop-collapse-foreach.cpp | 108 ++++++++++++++++++
 4 files changed, 210 insertions(+), 39 deletions(-)

diff --git a/openmp/runtime/test/transform/tile/foreach.cpp b/openmp/runtime/test/transform/tile/foreach.cpp
index 4fb3595760974..17031a4acf187 100644
--- a/openmp/runtime/test/transform/tile/foreach.cpp
+++ b/openmp/runtime/test/transform/tile/foreach.cpp
@@ -182,6 +182,10 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: v=0 w=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 1
 // CHECK-NEXT: [A] iterator move assign
@@ -191,6 +195,10 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: v=1 w=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -206,6 +214,18 @@ int main() {
 // CHECK-NEXT: v=2 w=1
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 3
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 3
+// CHECK-NEXT: [B] iterator advance: 0 += 0
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 0
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 1
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 1
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 2
@@ -214,6 +234,22 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: v=2 w=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 3
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 3
+// CHECK-NEXT: [B] iterator advance: 0 += 2
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 2
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
diff --git a/openmp/runtime/test/transform/tile/intfor.c b/openmp/runtime/test/transform/tile/intfor.c
index 4a930eab6730a..e980165277b79 100644
--- a/openmp/runtime/test/transform/tile/intfor.c
+++ b/openmp/runtime/test/transform/tile/intfor.c
@@ -30,77 +30,85 @@ int main() {
 // CHECK:      do
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=7 j=7
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=7 j=7
+// CHECK-NEXT: i=7 j=10
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=7 j=13
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=7 j=10
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=10 j=7
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=7 j=13
+// CHECK-NEXT: i=10 j=10
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=10 j=13
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=13 j=7
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=10 j=7
+// CHECK-NEXT: i=13 j=10
+// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=13 j=13
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=10 j=10
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=7 j=16
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=10 j=13
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=7 j=19
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=10 j=16
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=13 j=7
+// CHECK-NEXT: i=10 j=19
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=13 j=10
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=13 j=16
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=13 j=13
+// CHECK-NEXT: i=13 j=19
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
@@ -109,61 +117,57 @@ int main() {
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=7 j=16
+// CHECK-NEXT: i=16 j=7
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=7 j=19
+// CHECK-NEXT: i=16 j=10
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=16 j=13
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=10 j=16
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=10 j=19
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=13 j=16
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=13 j=19
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=16 j=16
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=16 j=7
+// CHECK-NEXT: i=16 j=19
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=16 j=10
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=16 j=13
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
@@ -172,14 +176,10 @@ int main() {
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=16 j=16
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=16 j=19
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
@@ -188,4 +188,4 @@ int main() {
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: done
\ No newline at end of file
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/tile/iterfor.cpp b/openmp/runtime/test/transform/tile/iterfor.cpp
index 05b5677bf9c4f..d07b10b3e841a 100644
--- a/openmp/runtime/test/transform/tile/iterfor.cpp
+++ b/openmp/runtime/test/transform/tile/iterfor.cpp
@@ -190,6 +190,9 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=0 j=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 1
 // CHECK-NEXT: [A] iterator move assign
@@ -199,6 +202,9 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=1 j=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -215,6 +221,15 @@ int main() {
 // CHECK-NEXT: i=2 j=1
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 3
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [B] iterator advance: 0 += 0
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 1
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [B] iterator advance: 0 += 2
@@ -223,6 +238,18 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=2 j=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 3
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [B] iterator advance: 0 += 2
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
diff --git a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-foreach.cpp b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-foreach.cpp
index b1f4d98a52ddc..966ff7f84f548 100644
--- a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-foreach.cpp
+++ b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-foreach.cpp
@@ -182,6 +182,10 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=0 v=0 w=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 1
 // CHECK-NEXT: [A] iterator move assign
@@ -191,6 +195,10 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=0 v=1 w=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -206,6 +214,18 @@ int main() {
 // CHECK-NEXT: i=0 v=2 w=1
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 3
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 3
+// CHECK-NEXT: [B] iterator advance: 0 += 0
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 0
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 1
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 1
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 2
@@ -214,6 +234,22 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=0 v=2 w=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 3
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 3
+// CHECK-NEXT: [B] iterator advance: 0 += 2
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 2
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 0
 // CHECK-NEXT: [A] iterator move assign
@@ -251,6 +287,10 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=1 v=0 w=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 1
 // CHECK-NEXT: [A] iterator move assign
@@ -260,6 +300,10 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=1 v=1 w=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -275,6 +319,18 @@ int main() {
 // CHECK-NEXT: i=1 v=2 w=1
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 3
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 3
+// CHECK-NEXT: [B] iterator advance: 0 += 0
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 0
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 1
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 1
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 2
@@ -283,6 +339,22 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=1 v=2 w=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 3
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 3
+// CHECK-NEXT: [B] iterator advance: 0 += 2
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 2
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 0
 // CHECK-NEXT: [A] iterator move assign
@@ -320,6 +392,10 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=2 v=0 w=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 1
 // CHECK-NEXT: [A] iterator move assign
@@ -329,6 +405,10 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=2 v=1 w=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -344,6 +424,18 @@ int main() {
 // CHECK-NEXT: i=2 v=2 w=1
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 3
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 3
+// CHECK-NEXT: [B] iterator advance: 0 += 0
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 0
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 1
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 1
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 2
@@ -352,6 +444,22 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=2 v=2 w=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 3
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 3
+// CHECK-NEXT: [B] iterator advance: 0 += 2
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 2
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [B] iterator dtor

>From 52b17c217c114fc92e0793ebd0da638010993c60 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Fri, 24 Apr 2026 08:11:24 -0400
Subject: [PATCH 10/18] optimize-guard

---
 clang/lib/Sema/SemaOpenMP.cpp | 150 +++++++++++++++++++++++-----------
 1 file changed, 104 insertions(+), 46 deletions(-)

diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index c8d75a0b333b4..df602caafe5cd 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -15078,13 +15078,22 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
   SmallVector<Stmt *, 4> PreInits;
   CaptureVars CopyTransformer(SemaRef);
 
+  // Emit the intra-tile loop in canonical (rectangular + predicate) form only
+  // when a consuming OpenMP construct requires canonical input; otherwise keep
+  // the original min(.floor.iv + T, N) bound.
+  OpenMPDirectiveKind ParentDirective = DSAStack->getParentDirective();
+  const bool IntraTileMustBeCanonical =
+      isOpenMPLoopTransformationDirective(ParentDirective) ||
+      isOpenMPLoopDirective(ParentDirective);
+
   // Create iteration variables for the generated loops.
   SmallVector<VarDecl *, 4> FloorIndVars;
   SmallVector<VarDecl *, 4> TileIndVars;
   SmallVector<VarDecl *, 4> TileCntVars;
   FloorIndVars.resize(NumLoops);
   TileIndVars.resize(NumLoops);
-  TileCntVars.resize(NumLoops);
+  if (IntraTileMustBeCanonical)
+    TileCntVars.resize(NumLoops);
   for (unsigned I = 0; I < NumLoops; ++I) {
     OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
 
@@ -15117,7 +15126,7 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     }
 
     // Loop counter for the rectangular tile loop [0, TileSize).
-    {
+    if (IntraTileMustBeCanonical) {
       std::string TileCntName =
           (Twine(".tile.cnt.") + llvm::utostr(I) + ".iv." + OrigVarName).str();
       VarDecl *TileCntDecl =
@@ -15130,9 +15139,11 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
 
     // Declare the logical tile IV in PreInits so it is in scope for the
     // entire loop nest (it will be assigned in each tile loop body).
-    Decl *TileIVDeclPtr = TileIndVars[I];
-    PreInits.push_back(new (Context) DeclStmt(
-        DeclGroupRef::Create(Context, &TileIVDeclPtr, 1), {}, {}));
+    if (IntraTileMustBeCanonical) {
+      Decl *TileIVDeclPtr = TileIndVars[I];
+      PreInits.push_back(new (Context) DeclStmt(
+          DeclGroupRef::Create(Context, &TileIVDeclPtr, 1), {}, {}));
+    }
   }
 
   // Once the original iteration values are set, append the innermost body.
@@ -15147,23 +15158,25 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
   // Optimization: if every dimension's trip count is a compile-time constant
   // that is evenly divisible by the corresponding tile size (also a constant),
   // then the remainder tile is empty and the predicate is trivially true.
-  {
+  //
+  // The min-bounded path clamps the tile loop's upper bound directly so no
+  // predicate is needed.
+  if (IntraTileMustBeCanonical) {
     bool PredicateNeeded = false;
     for (unsigned I = 0; I < NumLoops; ++I) {
       Expr *TSExpr = SizesClause->getSizesRefs()[I];
       Expr *NExpr = LoopHelpers[I].NumIterations;
-      llvm::APSInt TileVal, TripVal;
       bool TSConst =
           !TSExpr->containsErrors() && TSExpr->isIntegerConstantExpr(Context);
       bool NConst = NExpr->isIntegerConstantExpr(Context);
       if (TSConst && NConst) {
         Expr::EvalResult TSResult;
         TSExpr->EvaluateAsInt(TSResult, Context);
-        TileVal = TSResult.Val.getInt();
+        llvm::APSInt TileVal = TSResult.Val.getInt();
         Expr::EvalResult NResult;
         NExpr->EvaluateAsInt(NResult, Context);
-        TripVal = NResult.Val.getInt();
-        if (TileVal.isStrictlyPositive() && (TripVal.srem(TileVal)).isZero())
+        llvm::APSInt TripVal = NResult.Val.getInt();
+        if (TileVal.isStrictlyPositive() && TripVal.srem(TileVal).isZero())
           continue;
       }
       PredicateNeeded = true;
@@ -15245,6 +15258,8 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
   // Each tile loop uses .tile.cnt as its counter with rectangular bounds
   // [0, TileSize), and computes .tile.iv = .floor.iv + .tile.cnt to set
   // the logical iteration number for LoopHelper.Updates.
+  // The min-bounded path instead uses .tile.iv directly, initialized to
+  // .floor.iv and bounded by min(.floor.iv + DimTileSize, NumIterations).
   for (int I = NumLoops - 1; I >= 0; --I) {
     OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
     Expr *NumIterations = LoopHelper.NumIterations;
@@ -15266,46 +15281,88 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
                               OrigCntVar->getExprLoc());
     };
 
-    // For init-statement: auto .tile.cnt = 0
-    SemaRef.AddInitializerToDecl(
-        TileCntVars[I],
-        SemaRef.ActOnIntegerConstant(LoopHelper.Init->getExprLoc(), 0).get(),
-        /*DirectInit=*/false);
-    Decl *CounterDecl = TileCntVars[I];
-    StmtResult InitStmt = new (Context)
-        DeclStmt(DeclGroupRef::Create(Context, &CounterDecl, 1),
-                 OrigCntVar->getBeginLoc(), OrigCntVar->getEndLoc());
-    if (!InitStmt.isUsable())
-      return StmtError();
-
-    // For cond-expression: .tile.cnt < DimTileSize  (rectangular bound)
     Expr *DimTileSize = MakeDimTileSize(I);
     if (!DimTileSize)
       return StmtError();
-    ExprResult CondExpr =
-        SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
-                           MakeTileCntRef(), DimTileSize);
-    if (!CondExpr.isUsable())
-      return StmtError();
 
-    // For incr-statement: ++.tile.cnt
-    ExprResult IncrStmt = SemaRef.BuildUnaryOp(
-        CurScope, LoopHelper.Inc->getExprLoc(), UO_PreInc, MakeTileCntRef());
-    if (!IncrStmt.isUsable())
-      return StmtError();
+    StmtResult InitStmt;
+    ExprResult CondExpr;
+    ExprResult IncrStmt;
+    Stmt *TileIVAssignStmt = nullptr;
 
-    // Compute the logical iteration number:
-    //   .tile.iv = .floor.iv + .tile.cnt
-    ExprResult FloorPlusCnt = SemaRef.BuildBinOp(
-        CurScope, OrigCntVar->getExprLoc(), BO_Add,
-        makeFloorIVRef(SemaRef, FloorIndVars, I, IVTy, OrigCntVar),
-        MakeTileCntRef());
-    if (!FloorPlusCnt.isUsable())
-      return StmtError();
-    ExprResult TileIVAssign =
-        SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(), BO_Assign,
-                           MakeTileIVRef(), FloorPlusCnt.get());
-    if (!TileIVAssign.isUsable())
+    if (IntraTileMustBeCanonical) {
+      // For init-statement: auto .tile.cnt = 0
+      SemaRef.AddInitializerToDecl(
+          TileCntVars[I],
+          SemaRef.ActOnIntegerConstant(LoopHelper.Init->getExprLoc(), 0).get(),
+          /*DirectInit=*/false);
+      Decl *CounterDecl = TileCntVars[I];
+      InitStmt = new (Context)
+          DeclStmt(DeclGroupRef::Create(Context, &CounterDecl, 1),
+                   OrigCntVar->getBeginLoc(), OrigCntVar->getEndLoc());
+
+      // For cond-expression: .tile.cnt < DimTileSize  (rectangular bound)
+      CondExpr = SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(),
+                                    BO_LT, MakeTileCntRef(), DimTileSize);
+
+      // For incr-statement: ++.tile.cnt
+      IncrStmt = SemaRef.BuildUnaryOp(CurScope, LoopHelper.Inc->getExprLoc(),
+                                      UO_PreInc, MakeTileCntRef());
+
+      // Compute the logical iteration number:
+      //   .tile.iv = .floor.iv + .tile.cnt
+      ExprResult FloorPlusCnt = SemaRef.BuildBinOp(
+          CurScope, OrigCntVar->getExprLoc(), BO_Add,
+          makeFloorIVRef(SemaRef, FloorIndVars, I, IVTy, OrigCntVar),
+          MakeTileCntRef());
+      if (!FloorPlusCnt.isUsable())
+        return StmtError();
+      ExprResult TileIVAssign =
+          SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(), BO_Assign,
+                             MakeTileIVRef(), FloorPlusCnt.get());
+      if (!TileIVAssign.isUsable())
+        return StmtError();
+      TileIVAssignStmt = TileIVAssign.get();
+    } else {
+      // Min-bounded path: .tile.iv is itself the loop counter, initialized to
+      // .floor.iv and bounded by min(.floor.iv + DimTileSize, NumIterations).
+      SemaRef.AddInitializerToDecl(
+          TileIndVars[I],
+          SemaRef
+              .DefaultLvalueConversion(
+                  makeFloorIVRef(SemaRef, FloorIndVars, I, IVTy, OrigCntVar))
+              .get(),
+          /*DirectInit=*/false);
+      Decl *CounterDecl = TileIndVars[I];
+      InitStmt = new (Context)
+          DeclStmt(DeclGroupRef::Create(Context, &CounterDecl, 1),
+                   OrigCntVar->getBeginLoc(), OrigCntVar->getEndLoc());
+
+      ExprResult EndOfTile = SemaRef.BuildBinOp(
+          CurScope, LoopHelper.Cond->getExprLoc(), BO_Add,
+          makeFloorIVRef(SemaRef, FloorIndVars, I, IVTy, OrigCntVar),
+          DimTileSize);
+      if (!EndOfTile.isUsable())
+        return StmtError();
+      ExprResult IsPartialTile =
+          SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
+                             NumIterations, EndOfTile.get());
+      if (!IsPartialTile.isUsable())
+        return StmtError();
+      ExprResult MinTileAndIterSpace = SemaRef.ActOnConditionalOp(
+          LoopHelper.Cond->getBeginLoc(), LoopHelper.Cond->getEndLoc(),
+          IsPartialTile.get(), NumIterations, EndOfTile.get());
+      if (!MinTileAndIterSpace.isUsable())
+        return StmtError();
+      CondExpr =
+          SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
+                             MakeTileIVRef(), MinTileAndIterSpace.get());
+
+      IncrStmt = SemaRef.BuildUnaryOp(CurScope, LoopHelper.Inc->getExprLoc(),
+                                      UO_PreInc, MakeTileIVRef());
+    }
+
+    if (!InitStmt.isUsable() || !CondExpr.isUsable() || !IncrStmt.isUsable())
       return StmtError();
 
     // Generated for loop is:
@@ -15321,7 +15378,8 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     // for applying tiling again). If this is the case, sink the expressions
     // further into the inner loop.
     SmallVector<Stmt *, 4> BodyParts;
-    BodyParts.push_back(TileIVAssign.get());
+    if (TileIVAssignStmt)
+      BodyParts.push_back(TileIVAssignStmt);
     BodyParts.append(LoopHelper.Updates.begin(), LoopHelper.Updates.end());
     if (auto *SourceCXXFor = dyn_cast<CXXForRangeStmt>(LoopStmt))
       BodyParts.push_back(SourceCXXFor->getLoopVarStmt());

>From 5057f6b2f80bffbc9ff5f072fa496512893945f6 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Fri, 24 Apr 2026 10:08:20 -0400
Subject: [PATCH 11/18] test_updated

---
 clang/test/OpenMP/tile_codegen.cpp         | 1224 +++++++++++---------
 clang/test/OpenMP/tile_rect_codegen_ir.cpp |  485 +++++++-
 2 files changed, 1105 insertions(+), 604 deletions(-)

diff --git a/clang/test/OpenMP/tile_codegen.cpp b/clang/test/OpenMP/tile_codegen.cpp
index 1fd02d74a1209..65fd3a327ad64 100644
--- a/clang/test/OpenMP/tile_codegen.cpp
+++ b/clang/test/OpenMP/tile_codegen.cpp
@@ -136,9 +136,8 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[I2:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[I:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
@@ -150,47 +149,50 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END11:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END12:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK1:       for.cond4:
-// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP1]], 5
-// CHECK1-NEXT:    br i1 [[CMP5]], label [[FOR_BODY6:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body6:
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
-// CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP4]], 3
-// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
-// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[TMP5]], align 4
-// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP8:%.*]] = icmp slt i32 [[TMP6]], 4
-// CHECK1-NEXT:    br i1 [[CMP8]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK1:       if.then:
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 5
+// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK1-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP4]], 5
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
+// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body8:
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
+// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
+// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[TMP6]], align 4
 // CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2]], !align [[META3]]
 // CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
 // CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP8]])
-// CHECK1-NEXT:    br label [[IF_END]]
-// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP9]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP4:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC9:%.*]]
-// CHECK1:       for.inc9:
+// CHECK1-NEXT:    br label [[FOR_INC10:%.*]]
+// CHECK1:       for.inc10:
 // CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP10]], 5
-// CHECK1-NEXT:    store i32 [[ADD10]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP10]], 5
+// CHECK1-NEXT:    store i32 [[ADD11]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
-// CHECK1:       for.end11:
+// CHECK1:       for.end12:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -205,9 +207,8 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -236,49 +237,54 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK1-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP8]], [[ADD5]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END15:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK1:       for.cond6:
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP10]], 5
-// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body8:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD9:%.*]] = add i32 [[TMP11]], [[TMP12]]
-// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP14]], [[TMP15]]
-// CHECK1-NEXT:    [[ADD10:%.*]] = add i32 [[TMP13]], [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD10]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[ADD11:%.*]] = add i32 [[TMP17]], 1
-// CHECK1-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP16]], [[ADD11]]
-// CHECK1-NEXT:    br i1 [[CMP12]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK1:       if.then:
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP18]])
-// CHECK1-NEXT:    br label [[IF_END]]
-// CHECK1:       if.end:
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD7:%.*]] = add i32 [[TMP12]], 1
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD8:%.*]] = add i32 [[TMP13]], 5
+// CHECK1-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
+// CHECK1-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD10:%.*]] = add i32 [[TMP14]], 1
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD11:%.*]] = add i32 [[TMP15]], 5
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP11]], [[COND]]
+// CHECK1-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body13:
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP17]], [[TMP18]]
+// CHECK1-NEXT:    [[ADD14:%.*]] = add i32 [[TMP16]], [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP19]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP19]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP20]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP7:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC13:%.*]]
-// CHECK1:       for.inc13:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD14:%.*]] = add i32 [[TMP20]], 5
-// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_INC15:%.*]]
+// CHECK1:       for.inc15:
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD16:%.*]] = add i32 [[TMP21]], 5
+// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
-// CHECK1:       for.end15:
+// CHECK1:       for.end17:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -289,13 +295,11 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -306,86 +310,94 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END30:%.*]]
 // CHECK1:       for.body:
 // CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK1:       for.cond1:
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 4
-// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END21:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END27:%.*]]
 // CHECK1:       for.body3:
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[TMP2]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK1:       for.cond4:
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP2]], 5
-// CHECK1-NEXT:    br i1 [[CMP5]], label [[FOR_BODY6:%.*]], label [[FOR_END18:%.*]]
-// CHECK1:       for.body6:
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[TMP4]]
-// CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
-// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND8:%.*]]
-// CHECK1:       for.cond8:
-// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP6]], 5
-// CHECK1-NEXT:    br i1 [[CMP9]], label [[FOR_BODY10:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body10:
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], 5
+// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK1-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP5]], 5
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP3]], [[COND]]
+// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END24:%.*]]
+// CHECK1:       for.body8:
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
+// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP7]], [[TMP8]]
-// CHECK1-NEXT:    store i32 [[ADD11]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[MUL12:%.*]] = mul nsw i32 [[TMP9]], 3
-// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i32 7, [[MUL12]]
-// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[J]], align 4
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP10]], 4
-// CHECK1-NEXT:    br i1 [[CMP14]], label [[LAND_LHS_TRUE:%.*]], label [[IF_END:%.*]]
-// CHECK1:       land.lhs.true:
+// CHECK1-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND10:%.*]]
+// CHECK1:       for.cond10:
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP9]], 5
+// CHECK1-NEXT:    [[CMP12:%.*]] = icmp slt i32 4, [[ADD11]]
+// CHECK1-NEXT:    br i1 [[CMP12]], label [[COND_TRUE13:%.*]], label [[COND_FALSE14:%.*]]
+// CHECK1:       cond.true13:
+// CHECK1-NEXT:    br label [[COND_END16:%.*]]
+// CHECK1:       cond.false14:
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP10]], 5
+// CHECK1-NEXT:    br label [[COND_END16]]
+// CHECK1:       cond.end16:
+// CHECK1-NEXT:    [[COND17:%.*]] = phi i32 [ 4, [[COND_TRUE13]] ], [ [[ADD15]], [[COND_FALSE14]] ]
+// CHECK1-NEXT:    [[CMP18:%.*]] = icmp slt i32 [[TMP8]], [[COND17]]
+// CHECK1-NEXT:    br i1 [[CMP18]], label [[FOR_BODY19:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body19:
 // CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[CMP15:%.*]] = icmp slt i32 [[TMP11]], 4
-// CHECK1-NEXT:    br i1 [[CMP15]], label [[IF_THEN:%.*]], label [[IF_END]]
-// CHECK1:       if.then:
+// CHECK1-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[TMP11]], 3
+// CHECK1-NEXT:    [[ADD21:%.*]] = add nsw i32 7, [[MUL20]]
+// CHECK1-NEXT:    store i32 [[ADD21]], ptr [[J]], align 4
 // CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
 // CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP12]], i32 noundef [[TMP13]])
-// CHECK1-NEXT:    br label [[IF_END]]
-// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP9:![0-9]+]]
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND10]], !llvm.loop [[LOOP9:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC16:%.*]]
-// CHECK1:       for.inc16:
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC17:%.*]] = add nsw i32 [[TMP15]], 1
-// CHECK1-NEXT:    store i32 [[INC17]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_INC22:%.*]]
+// CHECK1:       for.inc22:
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC23:%.*]] = add nsw i32 [[TMP15]], 1
+// CHECK1-NEXT:    store i32 [[INC23]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP10:![0-9]+]]
-// CHECK1:       for.end18:
-// CHECK1-NEXT:    br label [[FOR_INC19:%.*]]
-// CHECK1:       for.inc19:
+// CHECK1:       for.end24:
+// CHECK1-NEXT:    br label [[FOR_INC25:%.*]]
+// CHECK1:       for.inc25:
 // CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP16]], 5
-// CHECK1-NEXT:    store i32 [[ADD20]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD26:%.*]] = add nsw i32 [[TMP16]], 5
+// CHECK1-NEXT:    store i32 [[ADD26]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP11:![0-9]+]]
-// CHECK1:       for.end21:
-// CHECK1-NEXT:    br label [[FOR_INC22:%.*]]
-// CHECK1:       for.inc22:
+// CHECK1:       for.end27:
+// CHECK1-NEXT:    br label [[FOR_INC28:%.*]]
+// CHECK1:       for.inc28:
 // CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP17]], 5
-// CHECK1-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD29:%.*]] = add nsw i32 [[TMP17]], 5
+// CHECK1-NEXT:    store i32 [[ADD29]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP12:![0-9]+]]
-// CHECK1:       for.end24:
+// CHECK1:       for.end30:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -891,9 +903,8 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[START_ADDR]], align 4
@@ -917,48 +928,53 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK1-NEXT:    [[ADD5:%.*]] = add i32 [[TMP6]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP5]], [[ADD5]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END15:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK1:       for.cond6:
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP7]], 5
-// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body8:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD9:%.*]] = add i32 [[TMP8]], [[TMP9]]
-// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP11]], 3
-// CHECK1-NEXT:    [[ADD10:%.*]] = add i32 [[TMP10]], [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD10]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[ADD11:%.*]] = add i32 [[TMP13]], 1
-// CHECK1-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP12]], [[ADD11]]
-// CHECK1-NEXT:    br i1 [[CMP12]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK1:       if.then:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP14]])
-// CHECK1-NEXT:    br label [[IF_END]]
-// CHECK1:       if.end:
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD7:%.*]] = add i32 [[TMP9]], 1
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD8:%.*]] = add i32 [[TMP10]], 5
+// CHECK1-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
+// CHECK1-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD10:%.*]] = add i32 [[TMP11]], 1
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD11:%.*]] = add i32 [[TMP12]], 5
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP8]], [[COND]]
+// CHECK1-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body13:
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP14]], 3
+// CHECK1-NEXT:    [[ADD14:%.*]] = add i32 [[TMP13]], [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP15]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP15]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP16]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP23:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC13:%.*]]
-// CHECK1:       for.inc13:
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD14:%.*]] = add i32 [[TMP16]], 5
-// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_INC15:%.*]]
+// CHECK1:       for.inc15:
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD16:%.*]] = add i32 [[TMP17]], 5
+// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP24:![0-9]+]]
-// CHECK1:       for.end15:
+// CHECK1:       for.end17:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -967,9 +983,8 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
 // CHECK1-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
@@ -977,65 +992,78 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END14:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK1:       for.cond1:
-// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP2]], 0
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], 0
 // CHECK1-NEXT:    br i1 [[CMP2]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK1:       cond.true:
 // CHECK1-NEXT:    br label [[COND_END:%.*]]
 // CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[A_ADDR]], align 4
 // CHECK1-NEXT:    br label [[COND_END]]
 // CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP3]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[TMP1]], [[COND]]
-// CHECK1-NEXT:    br i1 [[CMP3]], label [[FOR_BODY4:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body4:
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
-// CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
-// CHECK1-NEXT:    [[ADD5:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD5]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP6:%.*]] = icmp slt i32 [[TMP7]], 4
-// CHECK1-NEXT:    br i1 [[CMP6]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK1:       if.then:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP8]])
-// CHECK1-NEXT:    br label [[IF_END]]
-// CHECK1:       if.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP5]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[COND]]
+// CHECK1-NEXT:    [[CMP3:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK1-NEXT:    br i1 [[CMP3]], label [[COND_TRUE4:%.*]], label [[COND_FALSE5:%.*]]
+// CHECK1:       cond.true4:
+// CHECK1-NEXT:    br label [[COND_END12:%.*]]
+// CHECK1:       cond.false5:
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    [[CMP6:%.*]] = icmp sle i32 [[TMP7]], 0
+// CHECK1-NEXT:    br i1 [[CMP6]], label [[COND_TRUE7:%.*]], label [[COND_FALSE8:%.*]]
+// CHECK1:       cond.true7:
+// CHECK1-NEXT:    br label [[COND_END9:%.*]]
+// CHECK1:       cond.false8:
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    br label [[COND_END9]]
+// CHECK1:       cond.end9:
+// CHECK1-NEXT:    [[COND10:%.*]] = phi i32 [ 1, [[COND_TRUE7]] ], [ [[TMP8]], [[COND_FALSE8]] ]
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP6]], [[COND10]]
+// CHECK1-NEXT:    br label [[COND_END12]]
+// CHECK1:       cond.end12:
+// CHECK1-NEXT:    [[COND13:%.*]] = phi i32 [ 4, [[COND_TRUE4]] ], [ [[ADD11]], [[COND_END9]] ]
+// CHECK1-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP2]], [[COND13]]
+// CHECK1-NEXT:    br i1 [[CMP14]], label [[FOR_BODY15:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body15:
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP9]], 3
+// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP10]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP9]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP25:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC7:%.*]]
-// CHECK1:       for.inc7:
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    [[CMP8:%.*]] = icmp sle i32 [[TMP10]], 0
-// CHECK1-NEXT:    br i1 [[CMP8]], label [[COND_TRUE9:%.*]], label [[COND_FALSE10:%.*]]
-// CHECK1:       cond.true9:
-// CHECK1-NEXT:    br label [[COND_END11:%.*]]
-// CHECK1:       cond.false10:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    br label [[COND_END11]]
-// CHECK1:       cond.end11:
-// CHECK1-NEXT:    [[COND12:%.*]] = phi i32 [ 1, [[COND_TRUE9]] ], [ [[TMP11]], [[COND_FALSE10]] ]
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP12]], [[COND12]]
-// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_INC17:%.*]]
+// CHECK1:       for.inc17:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    [[CMP18:%.*]] = icmp sle i32 [[TMP12]], 0
+// CHECK1-NEXT:    br i1 [[CMP18]], label [[COND_TRUE19:%.*]], label [[COND_FALSE20:%.*]]
+// CHECK1:       cond.true19:
+// CHECK1-NEXT:    br label [[COND_END21:%.*]]
+// CHECK1:       cond.false20:
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    br label [[COND_END21]]
+// CHECK1:       cond.end21:
+// CHECK1-NEXT:    [[COND22:%.*]] = phi i32 [ 1, [[COND_TRUE19]] ], [ [[TMP13]], [[COND_FALSE20]] ]
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP14]], [[COND22]]
+// CHECK1-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP26:![0-9]+]]
-// CHECK1:       for.end14:
+// CHECK1:       for.end24:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -1048,9 +1076,8 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK1-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
 // CHECK1-NEXT:    store ptr [[ARRAY]], ptr [[__RANGE2]], align 8
@@ -1084,51 +1111,56 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK1:       for.cond7:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
-// CHECK1-NEXT:    br i1 [[CMP8]], label [[FOR_BODY9:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body9:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
-// CHECK1-NEXT:    store i64 [[ADD10]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP12]], 1
-// CHECK1-NEXT:    [[ADD_PTR11:%.*]] = getelementptr inbounds double, ptr [[TMP11]], i64 [[MUL]]
-// CHECK1-NEXT:    store ptr [[ADD_PTR11]], ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP14:%.*]] = load double, ptr [[TMP13]], align 8
-// CHECK1-NEXT:    store double [[TMP14]], ptr [[V]], align 8
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
+// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
+// CHECK1-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
+// CHECK1-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body14:
+// CHECK1-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP16]], 1
-// CHECK1-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP15]], [[ADD12]]
-// CHECK1-NEXT:    br i1 [[CMP13]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK1:       if.then:
-// CHECK1-NEXT:    [[TMP17:%.*]] = load double, ptr [[V]], align 8
-// CHECK1-NEXT:    call void (...) @body(double noundef [[TMP17]])
-// CHECK1-NEXT:    br label [[IF_END]]
-// CHECK1:       if.end:
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
+// CHECK1-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
+// CHECK1-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
+// CHECK1-NEXT:    store double [[TMP17]], ptr [[V]], align 8
+// CHECK1-NEXT:    [[TMP18:%.*]] = load double, ptr [[V]], align 8
+// CHECK1-NEXT:    call void (...) @body(double noundef [[TMP18]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP18]], 1
-// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP19]], 1
+// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP28:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC14:%.*]]
-// CHECK1:       for.inc14:
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD15:%.*]] = add nsw i64 [[TMP19]], 5
-// CHECK1-NEXT:    store i64 [[ADD15]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    br label [[FOR_INC16:%.*]]
+// CHECK1:       for.inc16:
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP20]], 5
+// CHECK1-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP29:![0-9]+]]
-// CHECK1:       for.end16:
+// CHECK1:       for.end18:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -1142,9 +1174,8 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK1-NEXT:    store double 4.200000e+01, ptr [[C]], align 8
 // CHECK1-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
@@ -1179,52 +1210,57 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK1:       for.cond7:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
-// CHECK1-NEXT:    br i1 [[CMP8]], label [[FOR_BODY9:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body9:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
-// CHECK1-NEXT:    store i64 [[ADD10]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP12]], 1
-// CHECK1-NEXT:    [[ADD_PTR11:%.*]] = getelementptr inbounds double, ptr [[TMP11]], i64 [[MUL]]
-// CHECK1-NEXT:    store ptr [[ADD_PTR11]], ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP14:%.*]] = load double, ptr [[TMP13]], align 8
-// CHECK1-NEXT:    store double [[TMP14]], ptr [[V]], align 8
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
+// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
+// CHECK1-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
+// CHECK1-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body14:
+// CHECK1-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP16]], 1
-// CHECK1-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP15]], [[ADD12]]
-// CHECK1-NEXT:    br i1 [[CMP13]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK1:       if.then:
-// CHECK1-NEXT:    [[TMP17:%.*]] = load double, ptr [[C]], align 8
-// CHECK1-NEXT:    [[TMP18:%.*]] = load double, ptr [[V]], align 8
-// CHECK1-NEXT:    call void (...) @body(double noundef [[TMP17]], double noundef [[TMP18]])
-// CHECK1-NEXT:    br label [[IF_END]]
-// CHECK1:       if.end:
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
+// CHECK1-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
+// CHECK1-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
+// CHECK1-NEXT:    store double [[TMP17]], ptr [[V]], align 8
+// CHECK1-NEXT:    [[TMP18:%.*]] = load double, ptr [[C]], align 8
+// CHECK1-NEXT:    [[TMP19:%.*]] = load double, ptr [[V]], align 8
+// CHECK1-NEXT:    call void (...) @body(double noundef [[TMP18]], double noundef [[TMP19]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP19]], 1
-// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP20]], 1
+// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP30:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC14:%.*]]
-// CHECK1:       for.inc14:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD15:%.*]] = add nsw i64 [[TMP20]], 5
-// CHECK1-NEXT:    store i64 [[ADD15]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    br label [[FOR_INC16:%.*]]
+// CHECK1:       for.inc16:
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP21]], 5
+// CHECK1-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP31:![0-9]+]]
-// CHECK1:       for.end16:
+// CHECK1:       for.end18:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -1257,9 +1293,8 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:  entry:
 // CHECK2-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[I2:%.*]] = alloca ptr, align 8
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK2-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK2-NEXT:    [[I:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
@@ -1271,47 +1306,50 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END11:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END12:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK2:       for.cond4:
-// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP1]], 5
-// CHECK2-NEXT:    br i1 [[CMP5]], label [[FOR_BODY6:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body6:
-// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
-// CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP4]], 3
-// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK2-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
-// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[TMP5]], align 4
-// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP8:%.*]] = icmp slt i32 [[TMP6]], 4
-// CHECK2-NEXT:    br i1 [[CMP8]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK2:       if.then:
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 5
+// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK2-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP4]], 5
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
+// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body8:
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
+// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK2-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
+// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[TMP6]], align 4
 // CHECK2-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2]], !align [[META3]]
 // CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
 // CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP8]])
-// CHECK2-NEXT:    br label [[IF_END]]
-// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP9]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP4:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC9:%.*]]
-// CHECK2:       for.inc9:
+// CHECK2-NEXT:    br label [[FOR_INC10:%.*]]
+// CHECK2:       for.inc10:
 // CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP10]], 5
-// CHECK2-NEXT:    store i32 [[ADD10]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP10]], 5
+// CHECK2-NEXT:    store i32 [[ADD11]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
-// CHECK2:       for.end11:
+// CHECK2:       for.end12:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -1332,9 +1370,8 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -1363,49 +1400,54 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK2-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP8]], [[ADD5]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END15:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[TMP10]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK2:       for.cond6:
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP10]], 5
-// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body8:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD9:%.*]] = add i32 [[TMP11]], [[TMP12]]
-// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP14]], [[TMP15]]
-// CHECK2-NEXT:    [[ADD10:%.*]] = add i32 [[TMP13]], [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD10]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[ADD11:%.*]] = add i32 [[TMP17]], 1
-// CHECK2-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP16]], [[ADD11]]
-// CHECK2-NEXT:    br i1 [[CMP12]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK2:       if.then:
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP18]])
-// CHECK2-NEXT:    br label [[IF_END]]
-// CHECK2:       if.end:
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD7:%.*]] = add i32 [[TMP12]], 1
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD8:%.*]] = add i32 [[TMP13]], 5
+// CHECK2-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
+// CHECK2-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD10:%.*]] = add i32 [[TMP14]], 1
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD11:%.*]] = add i32 [[TMP15]], 5
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP11]], [[COND]]
+// CHECK2-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body13:
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP17]], [[TMP18]]
+// CHECK2-NEXT:    [[ADD14:%.*]] = add i32 [[TMP16]], [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP19]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP19]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP20]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP7:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC13:%.*]]
-// CHECK2:       for.inc13:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD14:%.*]] = add i32 [[TMP20]], 5
-// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_INC15:%.*]]
+// CHECK2:       for.inc15:
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD16:%.*]] = add i32 [[TMP21]], 5
+// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
-// CHECK2:       for.end15:
+// CHECK2:       for.end17:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -1419,9 +1461,8 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK2-NEXT:    store double 4.200000e+01, ptr [[C]], align 8
 // CHECK2-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
@@ -1456,52 +1497,57 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK2:       for.cond7:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
-// CHECK2-NEXT:    br i1 [[CMP8]], label [[FOR_BODY9:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body9:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
-// CHECK2-NEXT:    store i64 [[ADD10]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP12]], 1
-// CHECK2-NEXT:    [[ADD_PTR11:%.*]] = getelementptr inbounds double, ptr [[TMP11]], i64 [[MUL]]
-// CHECK2-NEXT:    store ptr [[ADD_PTR11]], ptr [[__BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP14:%.*]] = load double, ptr [[TMP13]], align 8
-// CHECK2-NEXT:    store double [[TMP14]], ptr [[V]], align 8
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
+// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
+// CHECK2-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
+// CHECK2-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body14:
+// CHECK2-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP16]], 1
-// CHECK2-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP15]], [[ADD12]]
-// CHECK2-NEXT:    br i1 [[CMP13]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK2:       if.then:
-// CHECK2-NEXT:    [[TMP17:%.*]] = load double, ptr [[C]], align 8
-// CHECK2-NEXT:    [[TMP18:%.*]] = load double, ptr [[V]], align 8
-// CHECK2-NEXT:    call void (...) @body(double noundef [[TMP17]], double noundef [[TMP18]])
-// CHECK2-NEXT:    br label [[IF_END]]
-// CHECK2:       if.end:
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
+// CHECK2-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
+// CHECK2-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
+// CHECK2-NEXT:    store double [[TMP17]], ptr [[V]], align 8
+// CHECK2-NEXT:    [[TMP18:%.*]] = load double, ptr [[C]], align 8
+// CHECK2-NEXT:    [[TMP19:%.*]] = load double, ptr [[V]], align 8
+// CHECK2-NEXT:    call void (...) @body(double noundef [[TMP18]], double noundef [[TMP19]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP19]], 1
-// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP20]], 1
+// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP10:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC14:%.*]]
-// CHECK2:       for.inc14:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD15:%.*]] = add nsw i64 [[TMP20]], 5
-// CHECK2-NEXT:    store i64 [[ADD15]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    br label [[FOR_INC16:%.*]]
+// CHECK2:       for.inc16:
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP21]], 5
+// CHECK2-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP11:![0-9]+]]
-// CHECK2:       for.end16:
+// CHECK2:       for.end18:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -1512,13 +1558,11 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -1529,86 +1573,94 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END30:%.*]]
 // CHECK2:       for.body:
 // CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK2:       for.cond1:
 // CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 4
-// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END21:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END27:%.*]]
 // CHECK2:       for.body3:
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[TMP2]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK2:       for.cond4:
-// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP2]], 5
-// CHECK2-NEXT:    br i1 [[CMP5]], label [[FOR_BODY6:%.*]], label [[FOR_END18:%.*]]
-// CHECK2:       for.body6:
-// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[TMP4]]
-// CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
-// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND8:%.*]]
-// CHECK2:       for.cond8:
-// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP6]], 5
-// CHECK2-NEXT:    br i1 [[CMP9]], label [[FOR_BODY10:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body10:
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], 5
+// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK2-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP5]], 5
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP3]], [[COND]]
+// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END24:%.*]]
+// CHECK2:       for.body8:
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
+// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP7]], [[TMP8]]
-// CHECK2-NEXT:    store i32 [[ADD11]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[MUL12:%.*]] = mul nsw i32 [[TMP9]], 3
-// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i32 7, [[MUL12]]
-// CHECK2-NEXT:    store i32 [[ADD13]], ptr [[J]], align 4
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP10]], 4
-// CHECK2-NEXT:    br i1 [[CMP14]], label [[LAND_LHS_TRUE:%.*]], label [[IF_END:%.*]]
-// CHECK2:       land.lhs.true:
+// CHECK2-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND10:%.*]]
+// CHECK2:       for.cond10:
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP9]], 5
+// CHECK2-NEXT:    [[CMP12:%.*]] = icmp slt i32 4, [[ADD11]]
+// CHECK2-NEXT:    br i1 [[CMP12]], label [[COND_TRUE13:%.*]], label [[COND_FALSE14:%.*]]
+// CHECK2:       cond.true13:
+// CHECK2-NEXT:    br label [[COND_END16:%.*]]
+// CHECK2:       cond.false14:
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP10]], 5
+// CHECK2-NEXT:    br label [[COND_END16]]
+// CHECK2:       cond.end16:
+// CHECK2-NEXT:    [[COND17:%.*]] = phi i32 [ 4, [[COND_TRUE13]] ], [ [[ADD15]], [[COND_FALSE14]] ]
+// CHECK2-NEXT:    [[CMP18:%.*]] = icmp slt i32 [[TMP8]], [[COND17]]
+// CHECK2-NEXT:    br i1 [[CMP18]], label [[FOR_BODY19:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body19:
 // CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[CMP15:%.*]] = icmp slt i32 [[TMP11]], 4
-// CHECK2-NEXT:    br i1 [[CMP15]], label [[IF_THEN:%.*]], label [[IF_END]]
-// CHECK2:       if.then:
+// CHECK2-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[TMP11]], 3
+// CHECK2-NEXT:    [[ADD21:%.*]] = add nsw i32 7, [[MUL20]]
+// CHECK2-NEXT:    store i32 [[ADD21]], ptr [[J]], align 4
 // CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
 // CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
 // CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP12]], i32 noundef [[TMP13]])
-// CHECK2-NEXT:    br label [[IF_END]]
-// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP12:![0-9]+]]
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND10]], !llvm.loop [[LOOP12:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC16:%.*]]
-// CHECK2:       for.inc16:
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC17:%.*]] = add nsw i32 [[TMP15]], 1
-// CHECK2-NEXT:    store i32 [[INC17]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_INC22:%.*]]
+// CHECK2:       for.inc22:
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC23:%.*]] = add nsw i32 [[TMP15]], 1
+// CHECK2-NEXT:    store i32 [[INC23]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP13:![0-9]+]]
-// CHECK2:       for.end18:
-// CHECK2-NEXT:    br label [[FOR_INC19:%.*]]
-// CHECK2:       for.inc19:
+// CHECK2:       for.end24:
+// CHECK2-NEXT:    br label [[FOR_INC25:%.*]]
+// CHECK2:       for.inc25:
 // CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP16]], 5
-// CHECK2-NEXT:    store i32 [[ADD20]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD26:%.*]] = add nsw i32 [[TMP16]], 5
+// CHECK2-NEXT:    store i32 [[ADD26]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP14:![0-9]+]]
-// CHECK2:       for.end21:
-// CHECK2-NEXT:    br label [[FOR_INC22:%.*]]
-// CHECK2:       for.inc22:
+// CHECK2:       for.end27:
+// CHECK2-NEXT:    br label [[FOR_INC28:%.*]]
+// CHECK2:       for.inc28:
 // CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP17]], 5
-// CHECK2-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD29:%.*]] = add nsw i32 [[TMP17]], 5
+// CHECK2-NEXT:    store i32 [[ADD29]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP15:![0-9]+]]
-// CHECK2:       for.end24:
+// CHECK2:       for.end30:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -2103,9 +2155,8 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:  entry:
 // CHECK2-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
 // CHECK2-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
@@ -2113,65 +2164,78 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END14:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK2:       for.cond1:
-// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP2]], 0
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], 0
 // CHECK2-NEXT:    br i1 [[CMP2]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK2:       cond.true:
 // CHECK2-NEXT:    br label [[COND_END:%.*]]
 // CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[A_ADDR]], align 4
 // CHECK2-NEXT:    br label [[COND_END]]
 // CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP3]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[TMP1]], [[COND]]
-// CHECK2-NEXT:    br i1 [[CMP3]], label [[FOR_BODY4:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body4:
-// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
-// CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
-// CHECK2-NEXT:    [[ADD5:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD5]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP6:%.*]] = icmp slt i32 [[TMP7]], 4
-// CHECK2-NEXT:    br i1 [[CMP6]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK2:       if.then:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP8]])
-// CHECK2-NEXT:    br label [[IF_END]]
-// CHECK2:       if.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP5]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[COND]]
+// CHECK2-NEXT:    [[CMP3:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK2-NEXT:    br i1 [[CMP3]], label [[COND_TRUE4:%.*]], label [[COND_FALSE5:%.*]]
+// CHECK2:       cond.true4:
+// CHECK2-NEXT:    br label [[COND_END12:%.*]]
+// CHECK2:       cond.false5:
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    [[CMP6:%.*]] = icmp sle i32 [[TMP7]], 0
+// CHECK2-NEXT:    br i1 [[CMP6]], label [[COND_TRUE7:%.*]], label [[COND_FALSE8:%.*]]
+// CHECK2:       cond.true7:
+// CHECK2-NEXT:    br label [[COND_END9:%.*]]
+// CHECK2:       cond.false8:
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    br label [[COND_END9]]
+// CHECK2:       cond.end9:
+// CHECK2-NEXT:    [[COND10:%.*]] = phi i32 [ 1, [[COND_TRUE7]] ], [ [[TMP8]], [[COND_FALSE8]] ]
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP6]], [[COND10]]
+// CHECK2-NEXT:    br label [[COND_END12]]
+// CHECK2:       cond.end12:
+// CHECK2-NEXT:    [[COND13:%.*]] = phi i32 [ 4, [[COND_TRUE4]] ], [ [[ADD11]], [[COND_END9]] ]
+// CHECK2-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP2]], [[COND13]]
+// CHECK2-NEXT:    br i1 [[CMP14]], label [[FOR_BODY15:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body15:
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP9]], 3
+// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP10]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP9]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP26:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC7:%.*]]
-// CHECK2:       for.inc7:
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    [[CMP8:%.*]] = icmp sle i32 [[TMP10]], 0
-// CHECK2-NEXT:    br i1 [[CMP8]], label [[COND_TRUE9:%.*]], label [[COND_FALSE10:%.*]]
-// CHECK2:       cond.true9:
-// CHECK2-NEXT:    br label [[COND_END11:%.*]]
-// CHECK2:       cond.false10:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    br label [[COND_END11]]
-// CHECK2:       cond.end11:
-// CHECK2-NEXT:    [[COND12:%.*]] = phi i32 [ 1, [[COND_TRUE9]] ], [ [[TMP11]], [[COND_FALSE10]] ]
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP12]], [[COND12]]
-// CHECK2-NEXT:    store i32 [[ADD13]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_INC17:%.*]]
+// CHECK2:       for.inc17:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    [[CMP18:%.*]] = icmp sle i32 [[TMP12]], 0
+// CHECK2-NEXT:    br i1 [[CMP18]], label [[COND_TRUE19:%.*]], label [[COND_FALSE20:%.*]]
+// CHECK2:       cond.true19:
+// CHECK2-NEXT:    br label [[COND_END21:%.*]]
+// CHECK2:       cond.false20:
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    br label [[COND_END21]]
+// CHECK2:       cond.end21:
+// CHECK2-NEXT:    [[COND22:%.*]] = phi i32 [ 1, [[COND_TRUE19]] ], [ [[TMP13]], [[COND_FALSE20]] ]
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP14]], [[COND22]]
+// CHECK2-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP27:![0-9]+]]
-// CHECK2:       for.end14:
+// CHECK2:       for.end24:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -2184,9 +2248,8 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK2-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
 // CHECK2-NEXT:    store ptr [[ARRAY]], ptr [[__RANGE2]], align 8
@@ -2220,51 +2283,56 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK2:       for.cond7:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
-// CHECK2-NEXT:    br i1 [[CMP8]], label [[FOR_BODY9:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body9:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
-// CHECK2-NEXT:    store i64 [[ADD10]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP12]], 1
-// CHECK2-NEXT:    [[ADD_PTR11:%.*]] = getelementptr inbounds double, ptr [[TMP11]], i64 [[MUL]]
-// CHECK2-NEXT:    store ptr [[ADD_PTR11]], ptr [[__BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP14:%.*]] = load double, ptr [[TMP13]], align 8
-// CHECK2-NEXT:    store double [[TMP14]], ptr [[V]], align 8
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
+// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
+// CHECK2-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
+// CHECK2-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body14:
+// CHECK2-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP16]], 1
-// CHECK2-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP15]], [[ADD12]]
-// CHECK2-NEXT:    br i1 [[CMP13]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK2:       if.then:
-// CHECK2-NEXT:    [[TMP17:%.*]] = load double, ptr [[V]], align 8
-// CHECK2-NEXT:    call void (...) @body(double noundef [[TMP17]])
-// CHECK2-NEXT:    br label [[IF_END]]
-// CHECK2:       if.end:
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
+// CHECK2-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
+// CHECK2-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
+// CHECK2-NEXT:    store double [[TMP17]], ptr [[V]], align 8
+// CHECK2-NEXT:    [[TMP18:%.*]] = load double, ptr [[V]], align 8
+// CHECK2-NEXT:    call void (...) @body(double noundef [[TMP18]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP18]], 1
-// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP19]], 1
+// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP28:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC14:%.*]]
-// CHECK2:       for.inc14:
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD15:%.*]] = add nsw i64 [[TMP19]], 5
-// CHECK2-NEXT:    store i64 [[ADD15]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    br label [[FOR_INC16:%.*]]
+// CHECK2:       for.inc16:
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP20]], 5
+// CHECK2-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP29:![0-9]+]]
-// CHECK2:       for.end16:
+// CHECK2:       for.end18:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -2284,9 +2352,8 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[START_ADDR]], align 4
@@ -2310,48 +2377,53 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK2-NEXT:    [[ADD5:%.*]] = add i32 [[TMP6]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP5]], [[ADD5]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END15:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK2:       for.cond6:
-// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP7]], 5
-// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body8:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD9:%.*]] = add i32 [[TMP8]], [[TMP9]]
-// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP11]], 3
-// CHECK2-NEXT:    [[ADD10:%.*]] = add i32 [[TMP10]], [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD10]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[ADD11:%.*]] = add i32 [[TMP13]], 1
-// CHECK2-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP12]], [[ADD11]]
-// CHECK2-NEXT:    br i1 [[CMP12]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK2:       if.then:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP14]])
-// CHECK2-NEXT:    br label [[IF_END]]
-// CHECK2:       if.end:
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD7:%.*]] = add i32 [[TMP9]], 1
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD8:%.*]] = add i32 [[TMP10]], 5
+// CHECK2-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
+// CHECK2-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD10:%.*]] = add i32 [[TMP11]], 1
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD11:%.*]] = add i32 [[TMP12]], 5
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP8]], [[COND]]
+// CHECK2-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body13:
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP14]], 3
+// CHECK2-NEXT:    [[ADD14:%.*]] = add i32 [[TMP13]], [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP15]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP15]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP16]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP30:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC13:%.*]]
-// CHECK2:       for.inc13:
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD14:%.*]] = add i32 [[TMP16]], 5
-// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_INC15:%.*]]
+// CHECK2:       for.inc15:
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD16:%.*]] = add i32 [[TMP17]], 5
+// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP31:![0-9]+]]
-// CHECK2:       for.end15:
+// CHECK2:       for.end17:
 // CHECK2-NEXT:    ret void
 //
 //
diff --git a/clang/test/OpenMP/tile_rect_codegen_ir.cpp b/clang/test/OpenMP/tile_rect_codegen_ir.cpp
index f35c581c8b8ed..808f1cf0a3d0c 100644
--- a/clang/test/OpenMP/tile_rect_codegen_ir.cpp
+++ b/clang/test/OpenMP/tile_rect_codegen_ir.cpp
@@ -9,50 +9,336 @@
 extern "C" void body(int);
 
 // Trip count 6, tile 4 → remainder; body must be guarded.
-// IR-LABEL: remainder_6_tile_4v(
+// IR-LABEL: define dso_local void @_Z18remainder_6_tile_4v(
+// IR-SAME: ) #[[ATTR0:[0-9]+]] {
+// IR-NEXT:  [[ENTRY:.*:]]
+// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    store i32 0, ptr [[I]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND:.*]]
+// IR:       [[FOR_COND]]:
+// IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 6
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END9:.*]]
+// IR:       [[FOR_BODY]]:
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND1:.*]]
+// IR:       [[FOR_COND1]]:
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 6, [[ADD]]
+// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 4
+// IR-NEXT:    br label %[[COND_END]]
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ 6, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
+// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
+// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY5]]:
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
+// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void @body(i32 noundef [[TMP6]])
+// IR-NEXT:    br label %[[FOR_INC:.*]]
+// IR:       [[FOR_INC]]:
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP7]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP2:![0-9]+]]
+// IR:       [[FOR_END]]:
+// IR-NEXT:    br label %[[FOR_INC7:.*]]
+// IR:       [[FOR_INC7]]:
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP8]], 4
+// IR-NEXT:    store i32 [[ADD8]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP4:![0-9]+]]
+// IR:       [[FOR_END9]]:
+// IR-NEXT:    ret void
+//
 void remainder_6_tile_4(void) {
   // Rectangular tile counter and constant tile-size bound.
-  // IR-DAG: %.tile.cnt{{.*}} = alloca i32
-  // IR: icmp {{.*}} i32 {{.*}}, 4
-  // IR: icmp {{.*}} i32 {{.*}}, 6
-  // IR: br i1 {{.*}}, label %if.then{{.*}}, label %if.end{{.*}}
-  // IR: call {{.*}} @body(
 #pragma omp tile sizes(4)
   for (int i = 0; i < 6; ++i)
     body(i);
 }
 
 // Simple stride-1 loop; tile size matches full tiles only (10, tile 5).
-// IR-LABEL: full_tiles_10_5v(
+// IR-LABEL: define dso_local void @_Z15full_tiles_10_5v(
+// IR-SAME: ) #[[ATTR0]] {
+// IR-NEXT:  [[ENTRY:.*:]]
+// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    store i32 0, ptr [[I]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND:.*]]
+// IR:       [[FOR_COND]]:
+// IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 10
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END9:.*]]
+// IR:       [[FOR_BODY]]:
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND1:.*]]
+// IR:       [[FOR_COND1]]:
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 5
+// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 10, [[ADD]]
+// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 5
+// IR-NEXT:    br label %[[COND_END]]
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ 10, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
+// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
+// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY5]]:
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
+// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void @body(i32 noundef [[TMP6]])
+// IR-NEXT:    br label %[[FOR_INC:.*]]
+// IR:       [[FOR_INC]]:
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP7]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP5:![0-9]+]]
+// IR:       [[FOR_END]]:
+// IR-NEXT:    br label %[[FOR_INC7:.*]]
+// IR:       [[FOR_INC7]]:
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP8]], 5
+// IR-NEXT:    store i32 [[ADD8]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
+// IR:       [[FOR_END9]]:
+// IR-NEXT:    ret void
+//
 void full_tiles_10_5(void) {
-  // IR-DAG: %.tile.cnt{{.*}} = alloca i32
-  // IR: icmp {{.*}} i32 {{.*}}, 5
-  // IR: call {{.*}} @body(
 #pragma omp tile sizes(5)
   for (int i = 0; i < 10; ++i)
     body(i);
 }
 
 // Variable tile size: clamp (TS<=0 ? 1 : TS) is inlined; bound compares to %cond.
-// IR-LABEL: var_tilei(
+// IR-LABEL: define dso_local void @_Z8var_tilei(
+// IR-SAME: i32 noundef [[TS:%.*]]) #[[ATTR0]] {
+// IR-NEXT:  [[ENTRY:.*:]]
+// IR-NEXT:    [[TS_ADDR:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    store i32 [[TS]], ptr [[TS_ADDR]], align 4
+// IR-NEXT:    store i32 0, ptr [[I]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND:.*]]
+// IR:       [[FOR_COND]]:
+// IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 12
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END24:.*]]
+// IR:       [[FOR_BODY]]:
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND1:.*]]
+// IR:       [[FOR_COND1]]:
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TS_ADDR]], align 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], 0
+// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TS_ADDR]], align 4
+// IR-NEXT:    br label %[[COND_END]]
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ 1, %[[COND_TRUE]] ], [ [[TMP5]], %[[COND_FALSE]] ]
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[COND]]
+// IR-NEXT:    [[CMP3:%.*]] = icmp slt i32 12, [[ADD]]
+// IR-NEXT:    br i1 [[CMP3]], label %[[COND_TRUE4:.*]], label %[[COND_FALSE5:.*]]
+// IR:       [[COND_TRUE4]]:
+// IR-NEXT:    br label %[[COND_END12:.*]]
+// IR:       [[COND_FALSE5]]:
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TS_ADDR]], align 4
+// IR-NEXT:    [[CMP6:%.*]] = icmp sle i32 [[TMP7]], 0
+// IR-NEXT:    br i1 [[CMP6]], label %[[COND_TRUE7:.*]], label %[[COND_FALSE8:.*]]
+// IR:       [[COND_TRUE7]]:
+// IR-NEXT:    br label %[[COND_END9:.*]]
+// IR:       [[COND_FALSE8]]:
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TS_ADDR]], align 4
+// IR-NEXT:    br label %[[COND_END9]]
+// IR:       [[COND_END9]]:
+// IR-NEXT:    [[COND10:%.*]] = phi i32 [ 1, %[[COND_TRUE7]] ], [ [[TMP8]], %[[COND_FALSE8]] ]
+// IR-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP6]], [[COND10]]
+// IR-NEXT:    br label %[[COND_END12]]
+// IR:       [[COND_END12]]:
+// IR-NEXT:    [[COND13:%.*]] = phi i32 [ 12, %[[COND_TRUE4]] ], [ [[ADD11]], %[[COND_END9]] ]
+// IR-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP2]], [[COND13]]
+// IR-NEXT:    br i1 [[CMP14]], label %[[FOR_BODY15:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY15]]:
+// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP9]], 1
+// IR-NEXT:    [[ADD16:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD16]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void @body(i32 noundef [[TMP10]])
+// IR-NEXT:    br label %[[FOR_INC:.*]]
+// IR:       [[FOR_INC]]:
+// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP7:![0-9]+]]
+// IR:       [[FOR_END]]:
+// IR-NEXT:    br label %[[FOR_INC17:.*]]
+// IR:       [[FOR_INC17]]:
+// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TS_ADDR]], align 4
+// IR-NEXT:    [[CMP18:%.*]] = icmp sle i32 [[TMP12]], 0
+// IR-NEXT:    br i1 [[CMP18]], label %[[COND_TRUE19:.*]], label %[[COND_FALSE20:.*]]
+// IR:       [[COND_TRUE19]]:
+// IR-NEXT:    br label %[[COND_END21:.*]]
+// IR:       [[COND_FALSE20]]:
+// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TS_ADDR]], align 4
+// IR-NEXT:    br label %[[COND_END21]]
+// IR:       [[COND_END21]]:
+// IR-NEXT:    [[COND22:%.*]] = phi i32 [ 1, %[[COND_TRUE19]] ], [ [[TMP13]], %[[COND_FALSE20]] ]
+// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP14]], [[COND22]]
+// IR-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
+// IR:       [[FOR_END24]]:
+// IR-NEXT:    ret void
+//
 void var_tile(int ts) {
-  // IR-DAG: %.tile.cnt{{.*}} = alloca i32
-  // IR: %cond = phi i32
-  // IR: icmp {{.*}} i32 {{.*}}, %cond
-  // IR: call {{.*}} @body(
 #pragma omp tile sizes(ts)
   for (int i = 0; i < 12; ++i)
     body(i);
 }
 
 // Two tiled dimensions → two tile counters.
-// IR-LABEL: two_dimv(
+// IR-LABEL: define dso_local void @_Z7two_dimv(
+// IR-SAME: ) #[[ATTR0]] {
+// IR-NEXT:  [[ENTRY:.*:]]
+// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[J:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
+// IR-NEXT:    store i32 0, ptr [[I]], align 4
+// IR-NEXT:    store i32 0, ptr [[J]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND:.*]]
+// IR:       [[FOR_COND]]:
+// IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 3
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END31:.*]]
+// IR:       [[FOR_BODY]]:
+// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_1_IV_J]], align 4
+// IR-NEXT:    br label %[[FOR_COND1:.*]]
+// IR:       [[FOR_COND1]]:
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 3
+// IR-NEXT:    br i1 [[CMP2]], label %[[FOR_BODY3:.*]], label %[[FOR_END28:.*]]
+// IR:       [[FOR_BODY3]]:
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP2]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND4:.*]]
+// IR:       [[FOR_COND4]]:
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], 2
+// IR-NEXT:    [[CMP5:%.*]] = icmp slt i32 3, [[ADD]]
+// IR-NEXT:    br i1 [[CMP5]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP5]], 2
+// IR-NEXT:    br label %[[COND_END]]
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ 3, %[[COND_TRUE]] ], [ [[ADD6]], %[[COND_FALSE]] ]
+// IR-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP3]], [[COND]]
+// IR-NEXT:    br i1 [[CMP7]], label %[[FOR_BODY8:.*]], label %[[FOR_END25:.*]]
+// IR:       [[FOR_BODY8]]:
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 1
+// IR-NEXT:    [[ADD9:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// IR-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_1_IV_J]], align 4
+// IR-NEXT:    br label %[[FOR_COND10:.*]]
+// IR:       [[FOR_COND10]]:
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// IR-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP9]], 2
+// IR-NEXT:    [[CMP12:%.*]] = icmp slt i32 3, [[ADD11]]
+// IR-NEXT:    br i1 [[CMP12]], label %[[COND_TRUE13:.*]], label %[[COND_FALSE14:.*]]
+// IR:       [[COND_TRUE13]]:
+// IR-NEXT:    br label %[[COND_END16:.*]]
+// IR:       [[COND_FALSE14]]:
+// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// IR-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP10]], 2
+// IR-NEXT:    br label %[[COND_END16]]
+// IR:       [[COND_END16]]:
+// IR-NEXT:    [[COND17:%.*]] = phi i32 [ 3, %[[COND_TRUE13]] ], [ [[ADD15]], %[[COND_FALSE14]] ]
+// IR-NEXT:    [[CMP18:%.*]] = icmp slt i32 [[TMP8]], [[COND17]]
+// IR-NEXT:    br i1 [[CMP18]], label %[[FOR_BODY19:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY19]]:
+// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// IR-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[TMP11]], 1
+// IR-NEXT:    [[ADD21:%.*]] = add nsw i32 0, [[MUL20]]
+// IR-NEXT:    store i32 [[ADD21]], ptr [[J]], align 4
+// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
+// IR-NEXT:    [[ADD22:%.*]] = add nsw i32 [[TMP12]], [[TMP13]]
+// IR-NEXT:    call void @body(i32 noundef [[ADD22]])
+// IR-NEXT:    br label %[[FOR_INC:.*]]
+// IR:       [[FOR_INC]]:
+// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
+// IR-NEXT:    br label %[[FOR_COND10]], !llvm.loop [[LOOP9:![0-9]+]]
+// IR:       [[FOR_END]]:
+// IR-NEXT:    br label %[[FOR_INC23:.*]]
+// IR:       [[FOR_INC23]]:
+// IR-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[INC24:%.*]] = add nsw i32 [[TMP15]], 1
+// IR-NEXT:    store i32 [[INC24]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND4]], !llvm.loop [[LOOP10:![0-9]+]]
+// IR:       [[FOR_END25]]:
+// IR-NEXT:    br label %[[FOR_INC26:.*]]
+// IR:       [[FOR_INC26]]:
+// IR-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// IR-NEXT:    [[ADD27:%.*]] = add nsw i32 [[TMP16]], 2
+// IR-NEXT:    store i32 [[ADD27]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP11:![0-9]+]]
+// IR:       [[FOR_END28]]:
+// IR-NEXT:    br label %[[FOR_INC29:.*]]
+// IR:       [[FOR_INC29]]:
+// IR-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD30:%.*]] = add nsw i32 [[TMP17]], 2
+// IR-NEXT:    store i32 [[ADD30]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP12:![0-9]+]]
+// IR:       [[FOR_END31]]:
+// IR-NEXT:    ret void
+//
 void two_dim(void) {
-  // IR-DAG: %.tile.cnt.0.iv.i{{.*}} = alloca i32
-  // IR-DAG: %.tile.cnt.1.iv.j{{.*}} = alloca i32
-  // IR: icmp {{.*}} i32 {{.*}}, 2
-  // IR: icmp {{.*}} i32 {{.*}}, 2
-  // IR: call {{.*}} @body(
 #pragma omp tile sizes(2, 2)
   for (int i = 0; i < 3; ++i)
     for (int j = 0; j < 3; ++j)
@@ -60,10 +346,83 @@ void two_dim(void) {
 }
 
 // Nested loop in tile body: outer nest still uses .tile.cnt for tiled dim.
-// IR-LABEL: nested_inner_ji(
+// IR-LABEL: define dso_local void @_Z14nested_inner_ji(
+// IR-SAME: i32 noundef [[N:%.*]]) #[[ATTR0]] {
+// IR-NEXT:  [[ENTRY:.*:]]
+// IR-NEXT:    [[N_ADDR:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[J:%.*]] = alloca i32, align 4
+// IR-NEXT:    store i32 [[N]], ptr [[N_ADDR]], align 4
+// IR-NEXT:    store i32 0, ptr [[I]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND:.*]]
+// IR:       [[FOR_COND]]:
+// IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 6
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END16:.*]]
+// IR:       [[FOR_BODY]]:
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND1:.*]]
+// IR:       [[FOR_COND1]]:
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 6, [[ADD]]
+// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 4
+// IR-NEXT:    br label %[[COND_END]]
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ 6, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
+// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
+// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END13:.*]]
+// IR:       [[FOR_BODY5]]:
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
+// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
+// IR-NEXT:    store i32 0, ptr [[J]], align 4
+// IR-NEXT:    br label %[[FOR_COND7:.*]]
+// IR:       [[FOR_COND7]]:
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[J]], align 4
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[N_ADDR]], align 4
+// IR-NEXT:    [[CMP8:%.*]] = icmp slt i32 [[TMP6]], [[TMP7]]
+// IR-NEXT:    br i1 [[CMP8]], label %[[FOR_BODY9:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY9]]:
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[J]], align 4
+// IR-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP8]], [[TMP9]]
+// IR-NEXT:    call void @body(i32 noundef [[ADD10]])
+// IR-NEXT:    br label %[[FOR_INC:.*]]
+// IR:       [[FOR_INC]]:
+// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[J]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[J]], align 4
+// IR-NEXT:    br label %[[FOR_COND7]], !llvm.loop [[LOOP13:![0-9]+]]
+// IR:       [[FOR_END]]:
+// IR-NEXT:    br label %[[FOR_INC11:.*]]
+// IR:       [[FOR_INC11]]:
+// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[INC12:%.*]] = add nsw i32 [[TMP11]], 1
+// IR-NEXT:    store i32 [[INC12]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP14:![0-9]+]]
+// IR:       [[FOR_END13]]:
+// IR-NEXT:    br label %[[FOR_INC14:.*]]
+// IR:       [[FOR_INC14]]:
+// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP12]], 4
+// IR-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP15:![0-9]+]]
+// IR:       [[FOR_END16]]:
+// IR-NEXT:    ret void
+//
 void nested_inner_j(int n) {
-  // IR-DAG: %.tile.cnt{{.*}} = alloca i32
-  // IR: icmp {{.*}} i32 {{.*}}, 4
 #pragma omp tile sizes(4)
   for (int i = 0; i < 6; ++i)
     for (int j = 0; j < n; ++j)
@@ -72,13 +431,83 @@ void nested_inner_j(int n) {
 
 // Predicate elision: when trip count is evenly divisible by constant tile size,
 // no if-guard is needed (no icmp against trip count in tile body).
-// IR-LABEL: elided_predicatev(
+// IR-LABEL: define dso_local void @_Z16elided_predicatev(
+// IR-SAME: ) #[[ATTR0]] {
+// IR-NEXT:  [[ENTRY:.*:]]
+// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    store i32 0, ptr [[I]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND:.*]]
+// IR:       [[FOR_COND]]:
+// IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 12
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END9:.*]]
+// IR:       [[FOR_BODY]]:
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND1:.*]]
+// IR:       [[FOR_COND1]]:
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 12, [[ADD]]
+// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 4
+// IR-NEXT:    br label %[[COND_END]]
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ 12, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
+// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
+// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY5]]:
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
+// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void @body(i32 noundef [[TMP6]])
+// IR-NEXT:    br label %[[FOR_INC:.*]]
+// IR:       [[FOR_INC]]:
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP7]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP16:![0-9]+]]
+// IR:       [[FOR_END]]:
+// IR-NEXT:    br label %[[FOR_INC7:.*]]
+// IR:       [[FOR_INC7]]:
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP8]], 4
+// IR-NEXT:    store i32 [[ADD8]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP17:![0-9]+]]
+// IR:       [[FOR_END9]]:
+// IR-NEXT:    ret void
+//
 void elided_predicate(void) {
   // Trip count 12, tile size 4 → 12 % 4 == 0, no predicate.
-  // IR-NOT: if.then
-  // IR-NOT: if.end
-  // IR: call {{.*}} @body(
 #pragma omp tile sizes(4)
   for (int i = 0; i < 12; ++i)
     body(i);
 }
+//.
+// IR: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]]}
+// IR: [[META3]] = !{!"llvm.loop.mustprogress"}
+// IR: [[LOOP4]] = distinct !{[[LOOP4]], [[META3]]}
+// IR: [[LOOP5]] = distinct !{[[LOOP5]], [[META3]]}
+// IR: [[LOOP6]] = distinct !{[[LOOP6]], [[META3]]}
+// IR: [[LOOP7]] = distinct !{[[LOOP7]], [[META3]]}
+// IR: [[LOOP8]] = distinct !{[[LOOP8]], [[META3]]}
+// IR: [[LOOP9]] = distinct !{[[LOOP9]], [[META3]]}
+// IR: [[LOOP10]] = distinct !{[[LOOP10]], [[META3]]}
+// IR: [[LOOP11]] = distinct !{[[LOOP11]], [[META3]]}
+// IR: [[LOOP12]] = distinct !{[[LOOP12]], [[META3]]}
+// IR: [[LOOP13]] = distinct !{[[LOOP13]], [[META3]]}
+// IR: [[LOOP14]] = distinct !{[[LOOP14]], [[META3]]}
+// IR: [[LOOP15]] = distinct !{[[LOOP15]], [[META3]]}
+// IR: [[LOOP16]] = distinct !{[[LOOP16]], [[META3]]}
+// IR: [[LOOP17]] = distinct !{[[LOOP17]], [[META3]]}
+//.

>From 78bed3e78c5f69cc3e4373366971b4ff2cebfde0 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Tue, 2 Jun 2026 16:18:57 -0400
Subject: [PATCH 12/18] reverted-mlir

---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     |  64 ++++------
 .../Frontend/OpenMPIRBuilderTest.cpp          |  35 +----
 .../test/Target/LLVMIR/openmp-cli-tile01.mlir |  18 ++-
 .../test/Target/LLVMIR/openmp-cli-tile02.mlir |  30 ++---
 .../test/Target/LLVMIR/openmp-cli-tile03.mlir | 120 +++++++++---------
 5 files changed, 109 insertions(+), 158 deletions(-)

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index a38ecede63376..fc3812502fb2e 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -6984,7 +6984,7 @@ OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
   // Compute the trip counts of the floor loops.
   Builder.SetCurrentDebugLocation(DL);
   Builder.restoreIP(OutermostLoop->getPreheaderIP());
-  SmallVector<Value *, 4> FloorCount;
+  SmallVector<Value *, 4> FloorCompleteCount, FloorCount, FloorRems;
   for (int i = 0; i < NumLoops; ++i) {
     Value *TileSize = TileSizes[i];
     Value *OrigTripCount = OrigTripCounts[i];
@@ -7008,7 +7008,10 @@ OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
         Builder.CreateAdd(FloorCompleteTripCount, FloorTripOverflow,
                           "omp_floor" + Twine(i) + ".tripcount", true);
 
+    // Remember some values for later use.
+    FloorCompleteCount.push_back(FloorCompleteTripCount);
     FloorCount.push_back(FloorTripCount);
+    FloorRems.push_back(FloorTripRem);
   }
 
   // Generate the new loop nest, from the outermost to the innermost.
@@ -7052,10 +7055,23 @@ OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
 
   EmbeddNewLoops(FloorCount, "floor");
 
-  // Create the tile loops with rectangular (constant) trip counts equal to
-  // the tile sizes. A validity predicate in the body guards against
-  // out-of-bounds iterations in the remainder tile.
-  SmallVector<Value *, 4> TileCounts(TileSizes.begin(), TileSizes.end());
+  // Within the innermost floor loop, emit the code that computes the tile
+  // sizes.
+  Builder.SetInsertPoint(Enter->getTerminator());
+  SmallVector<Value *, 4> TileCounts;
+  for (int i = 0; i < NumLoops; ++i) {
+    CanonicalLoopInfo *FloorLoop = Result[i];
+    Value *TileSize = TileSizes[i];
+
+    Value *FloorIsEpilogue =
+        Builder.CreateICmpEQ(FloorLoop->getIndVar(), FloorCompleteCount[i]);
+    Value *TileTripCount =
+        Builder.CreateSelect(FloorIsEpilogue, FloorRems[i], TileSize);
+
+    TileCounts.push_back(TileTripCount);
+  }
+
+  // Create the tile loops.
   EmbeddNewLoops(TileCounts, "tile");
 
   // Insert the inbetween code into the body.
@@ -7084,7 +7100,6 @@ OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
   // Replace the original induction variable with an induction variable computed
   // from the tile and floor induction variables.
   Builder.restoreIP(Result.back()->getBodyIP());
-  SmallVector<Value *, 4> ShiftValues;
   for (int i = 0; i < NumLoops; ++i) {
     CanonicalLoopInfo *FloorLoop = Result[i];
     CanonicalLoopInfo *TileLoop = Result[NumLoops + i];
@@ -7096,47 +7111,18 @@ OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
     Value *Shift =
         Builder.CreateAdd(Scale, TileLoop->getIndVar(), {}, /*HasNUW=*/true);
     OrigIndVar->replaceAllUsesWith(Shift);
-    ShiftValues.push_back(Shift);
-  }
-
-  // Build a validity predicate: for each tiled dimension, check that the
-  // computed original index is within the original trip count. This guards
-  // against executing out-of-bounds iterations in the remainder tile.
-  Value *Pred = nullptr;
-  for (int i = 0; i < NumLoops; ++i) {
-    Value *DimPred = Builder.CreateICmpULT(ShiftValues[i], OrigTripCounts[i],
-                                           "omp_tile" + Twine(i) + ".inbounds");
-    Pred = Pred ? Builder.CreateAnd(Pred, DimPred) : DimPred;
   }
 
-  // Insert the conditional guard: split the body flow so that out-of-bounds
-  // iterations skip directly to a merge block before the tile latch.
-  BasicBlock *TileBodyBB = Builder.GetInsertBlock();
-  Instruction *BodyTerm = TileBodyBB->getTerminator();
-  BasicBlock *BodySucc = cast<UncondBrInst>(BodyTerm)->getSuccessor(0);
-  BasicBlock *TileLatch = Result.back()->getLatch();
-
-  BasicBlock *MergeBB =
-      BasicBlock::Create(F->getContext(), "omp_tile.body.merge", F, TileLatch);
-  Builder.SetInsertPoint(MergeBB);
-  Builder.CreateBr(TileLatch);
-
-  // Redirect the body chain's exit from the tile latch to the merge block.
-  for (BasicBlock *PredBB : llvm::make_early_inc_range(predecessors(TileLatch)))
-    if (PredBB != MergeBB)
-      PredBB->getTerminator()->replaceUsesOfWith(TileLatch, MergeBB);
-
-  // Replace the tile body's unconditional branch with a conditional one.
-  BodyTerm->eraseFromParent();
-  Builder.SetInsertPoint(TileBodyBB);
-  Builder.CreateCondBr(Pred, BodySucc, MergeBB);
-
   // Remove unused parts of the original loops.
   removeUnusedBlocksFromParent(OldControlBBs);
 
   for (CanonicalLoopInfo *L : Loops)
     L->invalidate();
 
+#ifndef NDEBUG
+  for (CanonicalLoopInfo *GenL : Result)
+    GenL->assertOK();
+#endif
   return Result;
 }
 
diff --git a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
index 2502c37ba28b3..b96103c82c185 100644
--- a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
@@ -1643,27 +1643,12 @@ TEST_F(OpenMPIRBuilderTest, TileSingleLoop) {
   CanonicalLoopInfo *Floor = GenLoops[0];
   CanonicalLoopInfo *Tile = GenLoops[1];
 
-  // The merge block sits between the body code and the tile latch.
-  BasicBlock *MergeBB = Tile->getLatch()->getSinglePredecessor();
-  ASSERT_NE(MergeBB, nullptr);
-
   BasicBlock *RefOrder[] = {
-      Floor->getPreheader(),
-      Floor->getHeader(),
-      Floor->getCond(),
-      Floor->getBody(),
-      Tile->getPreheader(),
-      Tile->getHeader(),
-      Tile->getCond(),
-      Tile->getBody(),
-      BodyCode,
-      MergeBB,
-      Tile->getLatch(),
-      Tile->getExit(),
-      Tile->getAfter(),
-      Floor->getLatch(),
-      Floor->getExit(),
-      Floor->getAfter(),
+      Floor->getPreheader(), Floor->getHeader(),   Floor->getCond(),
+      Floor->getBody(),      Tile->getPreheader(), Tile->getHeader(),
+      Tile->getCond(),       Tile->getBody(),      BodyCode,
+      Tile->getLatch(),      Tile->getExit(),      Tile->getAfter(),
+      Floor->getLatch(),     Floor->getExit(),     Floor->getAfter(),
   };
   EXPECT_TRUE(verifyDFSOrder(F, RefOrder));
   EXPECT_TRUE(verifyListOrder(F, RefOrder));
@@ -1733,10 +1718,6 @@ TEST_F(OpenMPIRBuilderTest, TileNestedLoops) {
   CanonicalLoopInfo *Tile1 = GenLoops[2];
   CanonicalLoopInfo *Tile2 = GenLoops[3];
 
-  // The merge block sits between the body code and the innermost tile latch.
-  BasicBlock *MergeBB = Tile2->getLatch()->getSinglePredecessor();
-  ASSERT_NE(MergeBB, nullptr);
-
   BasicBlock *RefOrder[] = {
       Floor1->getPreheader(),
       Floor1->getHeader(),
@@ -1755,7 +1736,6 @@ TEST_F(OpenMPIRBuilderTest, TileNestedLoops) {
       Tile2->getCond(),
       Tile2->getBody(),
       BodyCode,
-      MergeBB,
       Tile2->getLatch(),
       Tile2->getExit(),
       Tile2->getAfter(),
@@ -1846,10 +1826,6 @@ TEST_F(OpenMPIRBuilderTest, TileNestedLoopsWithBounds) {
   CanonicalLoopInfo *Tile0 = GenLoops[2];
   CanonicalLoopInfo *Tile1 = GenLoops[3];
 
-  // The merge block sits between the body code and the innermost tile latch.
-  BasicBlock *MergeBB = Tile1->getLatch()->getSinglePredecessor();
-  ASSERT_NE(MergeBB, nullptr);
-
   BasicBlock *RefOrder[] = {
       Floor0->getPreheader(),
       Floor0->getHeader(),
@@ -1868,7 +1844,6 @@ TEST_F(OpenMPIRBuilderTest, TileNestedLoopsWithBounds) {
       Tile1->getCond(),
       Tile1->getBody(),
       BodyCode,
-      MergeBB,
       Tile1->getLatch(),
       Tile1->getExit(),
       Tile1->getAfter(),
diff --git a/mlir/test/Target/LLVMIR/openmp-cli-tile01.mlir b/mlir/test/Target/LLVMIR/openmp-cli-tile01.mlir
index 948a9815851a5..0d559b69a3ad1 100644
--- a/mlir/test/Target/LLVMIR/openmp-cli-tile01.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-cli-tile01.mlir
@@ -38,6 +38,8 @@ llvm.func @tile_trivial_loop(%baseptr: !llvm.ptr, %tc: i32, %ts: i32) -> () {
 // CHECK-NEXT:    br i1 %[[OMP_FLOOR0_CMP:.+]], label %[[OMP_FLOOR0_BODY:.+]], label %[[OMP_FLOOR0_EXIT:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_FLOOR0_BODY]]:
+// CHECK-NEXT:    %[[TMP8:.+]] = icmp eq i32 %[[OMP_FLOOR0_IV:.+]], %[[TMP4:.+]]
+// CHECK-NEXT:    %[[TMP9:.+]] = select i1 %[[TMP8:.+]], i32 %[[TMP5:.+]], i32 %[[TMP2:.+]]
 // CHECK-NEXT:    br label %[[OMP_TILE0_PREHEADER:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_PREHEADER]]:
@@ -48,27 +50,23 @@ llvm.func @tile_trivial_loop(%baseptr: !llvm.ptr, %tc: i32, %ts: i32) -> () {
 // CHECK-NEXT:    br label %[[OMP_TILE0_COND:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_COND]]:
-// CHECK-NEXT:    %[[OMP_TILE0_CMP:.+]] = icmp ult i32 %[[OMP_TILE0_IV:.+]], %[[TMP2:.+]]
+// CHECK-NEXT:    %[[OMP_TILE0_CMP:.+]] = icmp ult i32 %[[OMP_TILE0_IV:.+]], %[[TMP9:.+]]
 // CHECK-NEXT:    br i1 %[[OMP_TILE0_CMP:.+]], label %[[OMP_TILE0_BODY:.+]], label %[[OMP_TILE0_EXIT:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_BODY]]:
-// CHECK-NEXT:    %[[TMP8:.+]] = mul nuw i32 %[[TMP2:.+]], %[[OMP_FLOOR0_IV:.+]]
-// CHECK-NEXT:    %[[TMP9:.+]] = add nuw i32 %[[TMP8:.+]], %[[OMP_TILE0_IV:.+]]
-// CHECK-NEXT:    %[[OMP_TILE0_INBOUNDS:.+]] = icmp ult i32 %[[TMP9:.+]], %[[TMP1:.+]]
-// CHECK-NEXT:    br i1 %[[OMP_TILE0_INBOUNDS:.+]], label %[[OMP_OMP_LOOP_BODY:.+]], label %[[OMP_TILE_BODY_MERGE:.+]]
+// CHECK-NEXT:    %[[TMP10:.+]] = mul nuw i32 %[[TMP2:.+]], %[[OMP_FLOOR0_IV:.+]]
+// CHECK-NEXT:    %[[TMP11:.+]] = add nuw i32 %[[TMP10:.+]], %[[OMP_TILE0_IV:.+]]
+// CHECK-NEXT:    br label %[[OMP_OMP_LOOP_BODY:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_OMP_LOOP_BODY]]:
 // CHECK-NEXT:    br label %[[OMP_LOOP_REGION:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_LOOP_REGION]]:
-// CHECK-NEXT:    %[[TMP10:.+]] = getelementptr inbounds float, ptr %[[TMP0:.+]], i32 %[[TMP9:.+]]
-// CHECK-NEXT:    store float 4.200000e+01, ptr %[[TMP10:.+]], align 4
+// CHECK-NEXT:    %[[TMP12:.+]] = getelementptr inbounds float, ptr %[[TMP0:.+]], i32 %[[TMP11:.+]]
+// CHECK-NEXT:    store float 4.200000e+01, ptr %[[TMP12:.+]], align 4
 // CHECK-NEXT:    br label %[[OMP_REGION_CONT:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_REGION_CONT]]:
-// CHECK-NEXT:    br label %[[OMP_TILE_BODY_MERGE:.+]]
-// CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE_BODY_MERGE]]:
 // CHECK-NEXT:    br label %[[OMP_TILE0_INC:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_INC]]:
diff --git a/mlir/test/Target/LLVMIR/openmp-cli-tile02.mlir b/mlir/test/Target/LLVMIR/openmp-cli-tile02.mlir
index 59ffa50037f4b..22c2973164159 100644
--- a/mlir/test/Target/LLVMIR/openmp-cli-tile02.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-cli-tile02.mlir
@@ -79,6 +79,10 @@ llvm.func @tile_2d_loop(%baseptr: !llvm.ptr, %tc1: i32, %tc2: i32, %ts1: i32, %t
 // CHECK-NEXT:    br i1 %[[OMP_FLOOR1_CMP:.+]], label %[[OMP_FLOOR1_BODY:.+]], label %[[OMP_FLOOR1_EXIT:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_FLOOR1_BODY]]:
+// CHECK-NEXT:    %[[TMP14:.+]] = icmp eq i32 %[[OMP_FLOOR0_IV:.+]], %[[TMP6:.+]]
+// CHECK-NEXT:    %[[TMP15:.+]] = select i1 %[[TMP14:.+]], i32 %[[TMP7:.+]], i32 %[[TMP3:.+]]
+// CHECK-NEXT:    %[[TMP16:.+]] = icmp eq i32 %[[OMP_FLOOR1_IV:.+]], %[[TMP10:.+]]
+// CHECK-NEXT:    %[[TMP17:.+]] = select i1 %[[TMP16:.+]], i32 %[[TMP11:.+]], i32 %[[TMP4:.+]]
 // CHECK-NEXT:    br label %[[OMP_TILE0_PREHEADER:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_PREHEADER]]:
@@ -89,7 +93,7 @@ llvm.func @tile_2d_loop(%baseptr: !llvm.ptr, %tc1: i32, %tc2: i32, %ts1: i32, %t
 // CHECK-NEXT:    br label %[[OMP_TILE0_COND:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_COND]]:
-// CHECK-NEXT:    %[[OMP_TILE0_CMP:.+]] = icmp ult i32 %[[OMP_TILE0_IV:.+]], %[[TMP3:.+]]
+// CHECK-NEXT:    %[[OMP_TILE0_CMP:.+]] = icmp ult i32 %[[OMP_TILE0_IV:.+]], %[[TMP15:.+]]
 // CHECK-NEXT:    br i1 %[[OMP_TILE0_CMP:.+]], label %[[OMP_TILE0_BODY:.+]], label %[[OMP_TILE0_EXIT:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_BODY]]:
@@ -103,32 +107,26 @@ llvm.func @tile_2d_loop(%baseptr: !llvm.ptr, %tc1: i32, %tc2: i32, %ts1: i32, %t
 // CHECK-NEXT:    br label %[[OMP_TILE1_COND:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE1_COND]]:
-// CHECK-NEXT:    %[[OMP_TILE1_CMP:.+]] = icmp ult i32 %[[OMP_TILE1_IV:.+]], %[[TMP4:.+]]
+// CHECK-NEXT:    %[[OMP_TILE1_CMP:.+]] = icmp ult i32 %[[OMP_TILE1_IV:.+]], %[[TMP17:.+]]
 // CHECK-NEXT:    br i1 %[[OMP_TILE1_CMP:.+]], label %[[OMP_TILE1_BODY:.+]], label %[[OMP_TILE1_EXIT:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE1_BODY]]:
-// CHECK-NEXT:    %[[TMP14:.+]] = mul nuw i32 %[[TMP3:.+]], %[[OMP_FLOOR0_IV:.+]]
-// CHECK-NEXT:    %[[TMP15:.+]] = add nuw i32 %[[TMP14:.+]], %[[OMP_TILE0_IV:.+]]
-// CHECK-NEXT:    %[[TMP16:.+]] = mul nuw i32 %[[TMP4:.+]], %[[OMP_FLOOR1_IV:.+]]
-// CHECK-NEXT:    %[[TMP17:.+]] = add nuw i32 %[[TMP16:.+]], %[[OMP_TILE1_IV:.+]]
-// CHECK-NEXT:    %[[OMP_TILE0_INBOUNDS:.+]] = icmp ult i32 %[[TMP15:.+]], %[[TMP1:.+]]
-// CHECK-NEXT:    %[[OMP_TILE1_INBOUNDS:.+]] = icmp ult i32 %[[TMP17:.+]], %[[TMP2:.+]]
-// CHECK-NEXT:    %[[OMP_TILE_PRED:.+]] = and i1 %[[OMP_TILE0_INBOUNDS:.+]], %[[OMP_TILE1_INBOUNDS:.+]]
-// CHECK-NEXT:    br i1 %[[OMP_TILE_PRED:.+]], label %[[OMP_OMP_LOOP_BODY:.+]], label %[[OMP_TILE_BODY_MERGE:.+]]
+// CHECK-NEXT:    %[[TMP18:.+]] = mul nuw i32 %[[TMP3:.+]], %[[OMP_FLOOR0_IV:.+]]
+// CHECK-NEXT:    %[[TMP19:.+]] = add nuw i32 %[[TMP18:.+]], %[[OMP_TILE0_IV:.+]]
+// CHECK-NEXT:    %[[TMP20:.+]] = mul nuw i32 %[[TMP4:.+]], %[[OMP_FLOOR1_IV:.+]]
+// CHECK-NEXT:    %[[TMP21:.+]] = add nuw i32 %[[TMP20:.+]], %[[OMP_TILE1_IV:.+]]
+// CHECK-NEXT:    br label %[[OMP_OMP_LOOP_BODY:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_OMP_LOOP_BODY4]]:
 // CHECK-NEXT:    br label %[[OMP_LOOP_REGION12:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_LOOP_REGION12]]:
-// CHECK-NEXT:    %[[TMP18:.+]] = add i32 %[[TMP15:.+]], %[[TMP17:.+]]
-// CHECK-NEXT:    %[[TMP19:.+]] = getelementptr inbounds float, ptr %[[TMP0:.+]], i32 %[[TMP18:.+]]
-// CHECK-NEXT:    store float 4.200000e+01, ptr %[[TMP19:.+]], align 4
+// CHECK-NEXT:    %[[TMP22:.+]] = add i32 %[[TMP19:.+]], %[[TMP21:.+]]
+// CHECK-NEXT:    %[[TMP23:.+]] = getelementptr inbounds float, ptr %[[TMP0:.+]], i32 %[[TMP22:.+]]
+// CHECK-NEXT:    store float 4.200000e+01, ptr %[[TMP23:.+]], align 4
 // CHECK-NEXT:    br label %[[OMP_REGION_CONT11:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_REGION_CONT11]]:
-// CHECK-NEXT:    br label %[[OMP_TILE_BODY_MERGE:.+]]
-// CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE_BODY_MERGE]]:
 // CHECK-NEXT:    br label %[[OMP_TILE1_INC:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE1_INC]]:
diff --git a/mlir/test/Target/LLVMIR/openmp-cli-tile03.mlir b/mlir/test/Target/LLVMIR/openmp-cli-tile03.mlir
index fcc37af2ca8f3..0747ff8909362 100644
--- a/mlir/test/Target/LLVMIR/openmp-cli-tile03.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-cli-tile03.mlir
@@ -56,6 +56,8 @@ llvm.func @tile_composition(%baseptr: !llvm.ptr, %tc: i32, %ts: i32, %grid_ts: i
 // CHECK-NEXT:    br i1 %[[OMP_FLOOR0_CMP10:.+]], label %[[OMP_FLOOR0_BODY5:.+]], label %[[OMP_FLOOR0_EXIT7:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_FLOOR0_BODY5]]:
+// CHECK-NEXT:    %[[TMP14:.+]] = icmp eq i32 %[[OMP_FLOOR0_IV9:.+]], %[[TMP10:.+]]
+// CHECK-NEXT:    %[[TMP15:.+]] = select i1 %[[TMP14:.+]], i32 %[[TMP11:.+]], i32 %[[TMP3:.+]]
 // CHECK-NEXT:    br label %[[OMP_TILE0_PREHEADER12:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_PREHEADER12]]:
@@ -66,88 +68,83 @@ llvm.func @tile_composition(%baseptr: !llvm.ptr, %tc: i32, %ts: i32, %grid_ts: i
 // CHECK-NEXT:    br label %[[OMP_TILE0_COND14:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_COND14]]:
-// CHECK-NEXT:    %[[OMP_TILE0_CMP20:.+]] = icmp ult i32 %[[OMP_TILE0_IV19:.+]], %[[TMP3:.+]]
+// CHECK-NEXT:    %[[OMP_TILE0_CMP20:.+]] = icmp ult i32 %[[OMP_TILE0_IV19:.+]], %[[TMP15:.+]]
 // CHECK-NEXT:    br i1 %[[OMP_TILE0_CMP20:.+]], label %[[OMP_TILE0_BODY15:.+]], label %[[OMP_TILE0_EXIT17:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_BODY15]]:
-// CHECK-NEXT:    %[[TMP14:.+]] = mul nuw i32 %[[TMP3:.+]], %[[OMP_FLOOR0_IV9:.+]]
-// CHECK-NEXT:    %[[TMP15:.+]] = add nuw i32 %[[TMP14:.+]], %[[OMP_TILE0_IV19:.+]]
-// CHECK-NEXT:    %[[OMP_TILE0_INBOUNDS22:.+]] = icmp ult i32 %[[TMP15:.+]], %[[OMP_FLOOR0_TRIPCOUNT:.+]]
-// CHECK-NEXT:    br i1 %[[OMP_TILE0_INBOUNDS22:.+]], label %[[OMP_FLOOR_INNER_ENTRY:.+]], label %[[OMP_TILE_BODY_MERGE23:.+]]
-// CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_FLOOR_INNER_ENTRY]]:
-// CHECK-NEXT:    br label %[[OMP_TILE0_PREHEADER_INNER:.+]]
-// CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_PREHEADER_INNER]]:
-// CHECK-NEXT:    %[[TMP16:.+]] = udiv i32 %[[TMP2:.+]], %[[TMP4:.+]]
-// CHECK-NEXT:    %[[TMP17:.+]] = urem i32 %[[TMP2:.+]], %[[TMP4:.+]]
-// CHECK-NEXT:    %[[TMP18:.+]] = icmp ne i32 %[[TMP17:.+]], 0
-// CHECK-NEXT:    %[[TMP19:.+]] = zext i1 %[[TMP18:.+]] to i32
-// CHECK-NEXT:    %[[OMP_FLOOR0_TRIPCOUNT24:.+]] = add nuw i32 %[[TMP16:.+]], %[[TMP19:.+]]
-// CHECK-NEXT:    br label %[[OMP_FLOOR0_PREHEADER25:.+]]
-// CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_FLOOR0_PREHEADER25]]:
+// CHECK-NEXT:    %[[TMP16:.+]] = mul nuw i32 %[[TMP3:.+]], %[[OMP_FLOOR0_IV9:.+]]
+// CHECK-NEXT:    %[[TMP17:.+]] = add nuw i32 %[[TMP16:.+]], %[[OMP_TILE0_IV19:.+]]
+// CHECK-NEXT:    br label %[[OMP_FLOOR0_BODY:.+]]
+// CHECK-EMPTY:
+// CHECK-NEXT:  [[OMP_FLOOR0_BODY]]:
+// CHECK-NEXT:    %[[TMP18:.+]] = icmp eq i32 %[[TMP17:.+]], %[[TMP6:.+]]
+// CHECK-NEXT:    %[[TMP19:.+]] = select i1 %[[TMP18:.+]], i32 %[[TMP7:.+]], i32 %[[TMP2:.+]]
+// CHECK-NEXT:    br label %[[OMP_TILE0_PREHEADER:.+]]
+// CHECK-EMPTY:
+// CHECK-NEXT:  [[OMP_TILE0_PREHEADER]]:
+// CHECK-NEXT:    %[[TMP20:.+]] = udiv i32 %[[TMP19:.+]], %[[TMP4:.+]]
+// CHECK-NEXT:    %[[TMP21:.+]] = urem i32 %[[TMP19:.+]], %[[TMP4:.+]]
+// CHECK-NEXT:    %[[TMP22:.+]] = icmp ne i32 %[[TMP21:.+]], 0
+// CHECK-NEXT:    %[[TMP23:.+]] = zext i1 %[[TMP22:.+]] to i32
+// CHECK-NEXT:    %[[OMP_FLOOR0_TRIPCOUNT22:.+]] = add nuw i32 %[[TMP20:.+]], %[[TMP23:.+]]
+// CHECK-NEXT:    br label %[[OMP_FLOOR0_PREHEADER23:.+]]
+// CHECK-EMPTY:
+// CHECK-NEXT:  [[OMP_FLOOR0_PREHEADER23]]:
 // CHECK-NEXT:    br label %[[OMP_FLOOR0_HEADER:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_FLOOR0_HEADER]]:
-// CHECK-NEXT:    %[[OMP_FLOOR0_IV:.+]] = phi i32 [ 0, %[[OMP_FLOOR0_PREHEADER25:.+]] ], [ %[[OMP_FLOOR0_NEXT:.+]], %[[OMP_FLOOR0_INC:.+]] ]
+// CHECK-NEXT:    %[[OMP_FLOOR0_IV:.+]] = phi i32 [ 0, %[[OMP_FLOOR0_PREHEADER23:.+]] ], [ %[[OMP_FLOOR0_NEXT:.+]], %[[OMP_FLOOR0_INC:.+]] ]
 // CHECK-NEXT:    br label %[[OMP_FLOOR0_COND:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_FLOOR0_COND]]:
-// CHECK-NEXT:    %[[OMP_FLOOR0_CMP:.+]] = icmp ult i32 %[[OMP_FLOOR0_IV:.+]], %[[OMP_FLOOR0_TRIPCOUNT24:.+]]
-// CHECK-NEXT:    br i1 %[[OMP_FLOOR0_CMP:.+]], label %[[OMP_FLOOR0_BODY26:.+]], label %[[OMP_FLOOR0_EXIT:.+]]
+// CHECK-NEXT:    %[[OMP_FLOOR0_CMP:.+]] = icmp ult i32 %[[OMP_FLOOR0_IV:.+]], %[[OMP_FLOOR0_TRIPCOUNT22:.+]]
+// CHECK-NEXT:    br i1 %[[OMP_FLOOR0_CMP:.+]], label %[[OMP_FLOOR0_BODY24:.+]], label %[[OMP_FLOOR0_EXIT:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_FLOOR0_BODY26]]:
-// CHECK-NEXT:    br label %[[OMP_TILE0_PREHEADER28:.+]]
+// CHECK-NEXT:  [[OMP_FLOOR0_BODY24]]:
+// CHECK-NEXT:    %[[TMP24:.+]] = icmp eq i32 %[[OMP_FLOOR0_IV:.+]], %[[TMP20:.+]]
+// CHECK-NEXT:    %[[TMP25:.+]] = select i1 %[[TMP24:.+]], i32 %[[TMP21:.+]], i32 %[[TMP4:.+]]
+// CHECK-NEXT:    br label %[[OMP_TILE0_PREHEADER26:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_PREHEADER28]]:
-// CHECK-NEXT:    br label %[[OMP_TILE0_HEADER29:.+]]
+// CHECK-NEXT:  [[OMP_TILE0_PREHEADER26]]:
+// CHECK-NEXT:    br label %[[OMP_TILE0_HEADER27:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_HEADER29]]:
-// CHECK-NEXT:    %[[OMP_TILE0_IV35:.+]] = phi i32 [ 0, %[[OMP_TILE0_PREHEADER28:.+]] ], [ %[[OMP_TILE0_NEXT37:.+]], %[[OMP_TILE0_INC32:.+]] ]
-// CHECK-NEXT:    br label %[[OMP_TILE0_COND30:.+]]
+// CHECK-NEXT:  [[OMP_TILE0_HEADER27]]:
+// CHECK-NEXT:    %[[OMP_TILE0_IV33:.+]] = phi i32 [ 0, %[[OMP_TILE0_PREHEADER26:.+]] ], [ %[[OMP_TILE0_NEXT35:.+]], %[[OMP_TILE0_INC30:.+]] ]
+// CHECK-NEXT:    br label %[[OMP_TILE0_COND28:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_COND30]]:
-// CHECK-NEXT:    %[[OMP_TILE0_CMP36:.+]] = icmp ult i32 %[[OMP_TILE0_IV35:.+]], %[[TMP4:.+]]
-// CHECK-NEXT:    br i1 %[[OMP_TILE0_CMP36:.+]], label %[[OMP_TILE0_BODY31:.+]], label %[[OMP_TILE0_EXIT33:.+]]
+// CHECK-NEXT:  [[OMP_TILE0_COND28]]:
+// CHECK-NEXT:    %[[OMP_TILE0_CMP34:.+]] = icmp ult i32 %[[OMP_TILE0_IV33:.+]], %[[TMP25:.+]]
+// CHECK-NEXT:    br i1 %[[OMP_TILE0_CMP34:.+]], label %[[OMP_TILE0_BODY29:.+]], label %[[OMP_TILE0_EXIT31:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_BODY31]]:
-// CHECK-NEXT:    %[[TMP20:.+]] = mul nuw i32 %[[TMP4:.+]], %[[OMP_FLOOR0_IV:.+]]
-// CHECK-NEXT:    %[[TMP21:.+]] = add nuw i32 %[[TMP20:.+]], %[[OMP_TILE0_IV35:.+]]
-// CHECK-NEXT:    %[[OMP_TILE0_INBOUNDS38:.+]] = icmp ult i32 %[[TMP21:.+]], %[[TMP2:.+]]
-// CHECK-NEXT:    br i1 %[[OMP_TILE0_INBOUNDS38:.+]], label %[[OMP_TILE0_BODY_STORE:.+]], label %[[OMP_TILE_BODY_MERGE39:.+]]
+// CHECK-NEXT:  [[OMP_TILE0_BODY29]]:
+// CHECK-NEXT:    %[[TMP26:.+]] = mul nuw i32 %[[TMP4:.+]], %[[OMP_FLOOR0_IV:.+]]
+// CHECK-NEXT:    %[[TMP27:.+]] = add nuw i32 %[[TMP26:.+]], %[[OMP_TILE0_IV33:.+]]
+// CHECK-NEXT:    br label %[[OMP_TILE0_BODY:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_BODY_STORE]]:
-// CHECK-NEXT:    %[[TMP22:.+]] = mul nuw i32 %[[TMP2:.+]], %[[TMP15:.+]]
-// CHECK-NEXT:    %[[TMP23:.+]] = add nuw i32 %[[TMP22:.+]], %[[TMP21:.+]]
-// CHECK-NEXT:    %[[OMP_TILE0_INBOUNDS:.+]] = icmp ult i32 %[[TMP23:.+]], %[[TMP1:.+]]
-// CHECK-NEXT:    br i1 %[[OMP_TILE0_INBOUNDS:.+]], label %[[OMP_OMP_LOOP_BODY:.+]], label %[[OMP_TILE_BODY_MERGE:.+]]
+// CHECK-NEXT:  [[OMP_TILE0_BODY]]:
+// CHECK-NEXT:    %[[TMP28:.+]] = mul nuw i32 %[[TMP2:.+]], %[[TMP17:.+]]
+// CHECK-NEXT:    %[[TMP29:.+]] = add nuw i32 %[[TMP28:.+]], %[[TMP27:.+]]
+// CHECK-NEXT:    br label %[[OMP_OMP_LOOP_BODY:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_OMP_LOOP_BODY]]:
 // CHECK-NEXT:    br label %[[OMP_LOOP_REGION:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_LOOP_REGION]]:
-// CHECK-NEXT:    %[[TMP24:.+]] = getelementptr inbounds float, ptr %[[TMP0:.+]], i32 %[[TMP23:.+]]
-// CHECK-NEXT:    store float 4.200000e+01, ptr %[[TMP24:.+]], align 4
+// CHECK-NEXT:    %[[TMP30:.+]] = getelementptr inbounds float, ptr %[[TMP0:.+]], i32 %[[TMP29:.+]]
+// CHECK-NEXT:    store float 4.200000e+01, ptr %[[TMP30:.+]], align 4
 // CHECK-NEXT:    br label %[[OMP_REGION_CONT:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_REGION_CONT]]:
-// CHECK-NEXT:    br label %[[OMP_TILE_BODY_MERGE:.+]]
+// CHECK-NEXT:    br label %[[OMP_TILE0_INC30:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE_BODY_MERGE]]:
-// CHECK-NEXT:    br label %[[OMP_TILE_BODY_MERGE39:.+]]
+// CHECK-NEXT:  [[OMP_TILE0_INC30]]:
+// CHECK-NEXT:    %[[OMP_TILE0_NEXT35:.+]] = add nuw i32 %[[OMP_TILE0_IV33:.+]], 1
+// CHECK-NEXT:    br label %[[OMP_TILE0_HEADER27:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE_BODY_MERGE39]]:
-// CHECK-NEXT:    br label %[[OMP_TILE0_INC32:.+]]
+// CHECK-NEXT:  [[OMP_TILE0_EXIT31]]:
+// CHECK-NEXT:    br label %[[OMP_TILE0_AFTER32:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_INC32]]:
-// CHECK-NEXT:    %[[OMP_TILE0_NEXT37:.+]] = add nuw i32 %[[OMP_TILE0_IV35:.+]], 1
-// CHECK-NEXT:    br label %[[OMP_TILE0_HEADER29:.+]]
-// CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_EXIT33]]:
-// CHECK-NEXT:    br label %[[OMP_TILE0_AFTER34:.+]]
-// CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_AFTER34]]:
+// CHECK-NEXT:  [[OMP_TILE0_AFTER32]]:
 // CHECK-NEXT:    br label %[[OMP_FLOOR0_INC:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_FLOOR0_INC]]:
@@ -155,15 +152,12 @@ llvm.func @tile_composition(%baseptr: !llvm.ptr, %tc: i32, %ts: i32, %grid_ts: i
 // CHECK-NEXT:    br label %[[OMP_FLOOR0_HEADER:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_FLOOR0_EXIT]]:
-// CHECK-NEXT:    br label %[[OMP_FLOOR0_AFTER27:.+]]
-// CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_FLOOR0_AFTER27]]:
-// CHECK-NEXT:    br label %[[OMP_TILE0_AFTER_INNER:.+]]
+// CHECK-NEXT:    br label %[[OMP_FLOOR0_AFTER25:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_AFTER_INNER]]:
-// CHECK-NEXT:    br label %[[OMP_TILE_BODY_MERGE23:.+]]
+// CHECK-NEXT:  [[OMP_FLOOR0_AFTER25]]:
+// CHECK-NEXT:    br label %[[OMP_TILE0_AFTER:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE_BODY_MERGE23]]:
+// CHECK-NEXT:  [[OMP_TILE0_AFTER]]:
 // CHECK-NEXT:    br label %[[OMP_TILE0_INC16:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_INC16]]:

>From d48aac101e6486a85f494db321da504b8a0d063c Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 3 Jun 2026 05:40:05 -0400
Subject: [PATCH 13/18] guard-attribute-marker

---
 clang/include/clang/Basic/Attr.td  |   7 +
 clang/lib/AST/StmtOpenMP.cpp       |  46 ++++++-
 clang/lib/CodeGen/CGStmt.cpp       |  48 ++++++-
 clang/lib/CodeGen/CGStmtOpenMP.cpp |  45 ++++++
 clang/lib/Sema/SemaOpenMP.cpp      | 213 ++++++++++++-----------------
 5 files changed, 227 insertions(+), 132 deletions(-)

diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 39c672322d515..22f5e0fd33433 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -4877,6 +4877,13 @@ def OMPAssume : InheritableAttr {
   let Args = [StringArgument<"Assumption">];
 }
 
+def OMPInvariantPredicateBound : StmtAttr {
+  let Spellings = [];
+  let SemaHandler = 0;
+  let Subjects = SubjectList<[Stmt], ErrorDiag>;
+  let Documentation = [InternalOnly];
+}
+
 def InternalLinkage : InheritableAttr {
   let Spellings = [Clang<"internal_linkage">];
   let Subjects = SubjectList<[Var, Function, CXXRecord]>;
diff --git a/clang/lib/AST/StmtOpenMP.cpp b/clang/lib/AST/StmtOpenMP.cpp
index 9d6b315effb41..94eeddc01d1a3 100644
--- a/clang/lib/AST/StmtOpenMP.cpp
+++ b/clang/lib/AST/StmtOpenMP.cpp
@@ -10,12 +10,48 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/AST/ASTContext.h"
 #include "clang/AST/StmtOpenMP.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Attr.h"
+#include "clang/AST/Stmt.h"
 
 using namespace clang;
 using namespace llvm::omp;
 
+// True if the attributed statement carries the internal `omp tile` intra-tile
+// body-guard marker.
+static bool hasOMPInvariantPredicateBound(const AttributedStmt *AS) {
+  for (const Attr *A : AS->getAttrs())
+    if (isa<OMPInvariantPredicateBoundAttr>(A))
+      return true;
+  return false;
+}
+
+// Like `Stmt::IgnoreContainers`, but also treats the `omp tile` body-guard
+static Stmt *IgnoreContainersAndOMPTileBodyGuard(Stmt *S) {
+  while (true) {
+    if (auto *AS = dyn_cast_or_null<AttributedStmt>(S)) {
+      if (hasOMPInvariantPredicateBound(AS)) {
+        if (auto *If = dyn_cast<IfStmt>(AS->getSubStmt());
+            If && !If->getElse()) {
+          S = If->getThen();
+          continue;
+        }
+      }
+      S = AS->getSubStmt();
+      continue;
+    }
+    if (auto *CS = dyn_cast_or_null<CompoundStmt>(S)) {
+      if (CS->size() != 1)
+        break;
+      S = CS->body_back();
+      continue;
+    }
+    break;
+  }
+  return S;
+}
+
 size_t OMPChildren::size(unsigned NumClauses, bool HasAssociatedStmt,
                          unsigned NumChildren) {
   return llvm::alignTo(
@@ -78,7 +114,7 @@ Stmt *
 OMPLoopBasedDirective::tryToFindNextInnerLoop(Stmt *CurStmt,
                                               bool TryImperfectlyNestedLoops) {
   Stmt *OrigStmt = CurStmt;
-  CurStmt = CurStmt->IgnoreContainers();
+  CurStmt = IgnoreContainersAndOMPTileBodyGuard(CurStmt);
   // Additional work for imperfectly nested loops, introduced in OpenMP 5.0.
   if (TryImperfectlyNestedLoops) {
     if (auto *CS = dyn_cast<CompoundStmt>(CurStmt)) {
@@ -94,6 +130,10 @@ OMPLoopBasedDirective::tryToFindNextInnerLoop(Stmt *CurStmt,
             continue;
           if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(S))
             S = CanonLoop->getLoopStmt();
+          // Check for the `omp tile` body-guard marker.
+          if (auto *AS = dyn_cast<AttributedStmt>(S);
+              AS && hasOMPInvariantPredicateBound(AS))
+            S = IgnoreContainersAndOMPTileBodyGuard(S);
           if (isa<ForStmt>(S) || isa<CXXForRangeStmt>(S) ||
               (isa<OMPLoopBasedDirective>(S) && !isa<OMPLoopDirective>(S))) {
             // Only single loop construct is allowed.
@@ -127,7 +167,7 @@ bool OMPLoopBasedDirective::doForAllLoops(
     llvm::function_ref<bool(unsigned, Stmt *)> Callback,
     llvm::function_ref<void(OMPLoopTransformationDirective *)>
         OnTransformationCallback) {
-  CurStmt = CurStmt->IgnoreContainers();
+  CurStmt = IgnoreContainersAndOMPTileBodyGuard(CurStmt);
   for (unsigned Cnt = 0; Cnt < NumLoops; ++Cnt) {
     while (true) {
       auto *Dir = dyn_cast<OMPLoopTransformationDirective>(CurStmt);
diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index 27e74d966eca1..da7a1b561b207 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -1271,9 +1271,35 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S,
     ConvergenceTokenStack.pop_back();
 }
 
+// Helper function to find and return the omp-tile body-guard marker on the
+// for-body's last item.
+static std::pair<const AttributedStmt *, const IfStmt *>
+findOMPTileBodyGuard(const Stmt *Body) {
+  const auto *CS = dyn_cast_or_null<CompoundStmt>(Body);
+  if (!CS || CS->body().empty())
+    return {nullptr, nullptr};
+  const auto *AS = dyn_cast<AttributedStmt>(CS->body_back());
+  if (!AS)
+    return {nullptr, nullptr};
+  bool IsMarker = false;
+  for (const Attr *A : AS->getAttrs()) {
+    if (!isa<OMPInvariantPredicateBoundAttr>(A))
+      continue;
+    IsMarker = true;
+    break;
+  }
+  if (!IsMarker)
+    return {nullptr, nullptr};
+  const auto *If = dyn_cast<IfStmt>(AS->getSubStmt());
+  if (!If || If->getElse())
+    return {nullptr, nullptr};
+  return {AS, If};
+}
+
 void CodeGenFunction::EmitForStmt(const ForStmt &S,
                                   ArrayRef<const Attr *> ForAttrs) {
   JumpDest LoopExit = getJumpDestInCurrentScope("for.end");
+  const auto [MarkerAS, MarkerIf] = findOMPTileBodyGuard(S.getBody());
 
   std::optional<LexicalScope> ForScope;
   if (getLangOpts().C99 || getLangOpts().CPlusPlus)
@@ -1341,6 +1367,14 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S,
     // compares unequal to 0.  The condition must be a scalar type.
     llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
 
+    // If the for-body has a marker, evaluate the predicate and AND it into the
+    // loop condition.
+    if (MarkerIf) {
+      llvm::Value *PredVal = EvaluateExprAsBool(MarkerIf->getCond());
+      BoolCondVal = Builder.CreateAnd(BoolCondVal, PredVal,
+                                      "omp_tile.invariant_pred_bound");
+    }
+
     MaybeEmitDeferredVarDeclInit(S.getConditionVariable());
 
     llvm::MDNode *Weights =
@@ -1377,7 +1411,19 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S,
     // Create a separate cleanup scope for the body, in case it is not
     // a compound statement.
     RunCleanupsScope BodyScope(*this);
-    EmitStmt(S.getBody());
+    if (MarkerAS) {
+      // If the for-body has a marker, walk the body and emit the then-block of
+      // the marker's predicate.
+      const auto *CS = cast<CompoundStmt>(S.getBody());
+      for (const Stmt *Sub : CS->body()) {
+        if (Sub == MarkerAS)
+          EmitStmt(MarkerIf->getThen());
+        else
+          EmitStmt(Sub);
+      }
+    } else {
+      EmitStmt(S.getBody());
+    }
   }
 
   // The last block in the loop's body (which unconditionally branches to the
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 97c43b4c0b384..93805f7b54432 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -2231,6 +2231,34 @@ class OMPTransformDirectiveScopeRAII {
 };
 } // namespace
 
+// Helper function to look through the statement and return the special marked
+// `omp tile` guard `if`, or nullptr if this is not that shape.
+static const IfStmt *getOMPTileBodyGuardIf(const Stmt *S) {
+  while (S) {
+    if (const auto *AS = dyn_cast<AttributedStmt>(S)) {
+      bool IsMarker = llvm::any_of(AS->getAttrs(), [](const Attr *A) {
+        return isa<OMPInvariantPredicateBoundAttr>(A);
+      });
+      if (!IsMarker) {
+        S = AS->getSubStmt();
+        continue;
+      }
+      const auto *If = dyn_cast<IfStmt>(AS->getSubStmt());
+      if (!If || If->getElse())
+        return nullptr;
+      return If;
+    }
+    if (const auto *CS = dyn_cast<CompoundStmt>(S)) {
+      if (CS->size() != 1)
+        return nullptr;
+      S = CS->body_back();
+      continue;
+    }
+    return nullptr;
+  }
+  return nullptr;
+}
+
 static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop,
                      int MaxLevel, int Level = 0) {
   assert(Level < MaxLevel && "Too deep lookup during loop body codegen.");
@@ -2246,6 +2274,23 @@ static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop,
       emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level);
     return;
   }
+
+  // If this is the marked `omp tile` guard, emit its predicate as a runtime
+  // body guard and recurse into `Then` so consumed loops are still walked.
+  if (const IfStmt *Guard = getOMPTileBodyGuardIf(S)) {
+    assert(!Guard->getInit() && !Guard->getConditionVariable() &&
+           "omp tile body-guard must be a plain `if (P) Then`");
+    llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_tile.pred.then");
+    llvm::BasicBlock *EndBlock = CGF.createBasicBlock("omp_tile.pred.end");
+    CGF.EmitBranchOnBoolExpr(
+        Guard->getCond(), ThenBlock, EndBlock, /*TrueCount=*/0,
+        Stmt::getLikelihood(Guard->getThen(), Guard->getElse()));
+    CGF.EmitBlock(ThenBlock);
+    emitBody(CGF, Guard->getThen(), NextLoop, MaxLevel, Level);
+    CGF.EmitBranch(EndBlock);
+    CGF.EmitBlock(EndBlock);
+    return;
+  }
   if (SimplifiedS == NextLoop) {
     if (auto *Dir = dyn_cast<OMPLoopTransformationDirective>(SimplifiedS))
       SimplifiedS = Dir->getTransformedStmt();
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index df602caafe5cd..20b8c99a1def2 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -15078,22 +15078,13 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
   SmallVector<Stmt *, 4> PreInits;
   CaptureVars CopyTransformer(SemaRef);
 
-  // Emit the intra-tile loop in canonical (rectangular + predicate) form only
-  // when a consuming OpenMP construct requires canonical input; otherwise keep
-  // the original min(.floor.iv + T, N) bound.
-  OpenMPDirectiveKind ParentDirective = DSAStack->getParentDirective();
-  const bool IntraTileMustBeCanonical =
-      isOpenMPLoopTransformationDirective(ParentDirective) ||
-      isOpenMPLoopDirective(ParentDirective);
-
   // Create iteration variables for the generated loops.
   SmallVector<VarDecl *, 4> FloorIndVars;
   SmallVector<VarDecl *, 4> TileIndVars;
   SmallVector<VarDecl *, 4> TileCntVars;
   FloorIndVars.resize(NumLoops);
   TileIndVars.resize(NumLoops);
-  if (IntraTileMustBeCanonical)
-    TileCntVars.resize(NumLoops);
+  TileCntVars.resize(NumLoops);
   for (unsigned I = 0; I < NumLoops; ++I) {
     OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
 
@@ -15126,7 +15117,7 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     }
 
     // Loop counter for the rectangular tile loop [0, TileSize).
-    if (IntraTileMustBeCanonical) {
+    {
       std::string TileCntName =
           (Twine(".tile.cnt.") + llvm::utostr(I) + ".iv." + OrigVarName).str();
       VarDecl *TileCntDecl =
@@ -15139,7 +15130,7 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
 
     // Declare the logical tile IV in PreInits so it is in scope for the
     // entire loop nest (it will be assigned in each tile loop body).
-    if (IntraTileMustBeCanonical) {
+    {
       Decl *TileIVDeclPtr = TileIndVars[I];
       PreInits.push_back(new (Context) DeclStmt(
           DeclGroupRef::Create(Context, &TileIVDeclPtr, 1), {}, {}));
@@ -15159,57 +15150,62 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
   // that is evenly divisible by the corresponding tile size (also a constant),
   // then the remainder tile is empty and the predicate is trivially true.
   //
-  // The min-bounded path clamps the tile loop's upper bound directly so no
-  // predicate is needed.
-  if (IntraTileMustBeCanonical) {
-    bool PredicateNeeded = false;
-    for (unsigned I = 0; I < NumLoops; ++I) {
-      Expr *TSExpr = SizesClause->getSizesRefs()[I];
-      Expr *NExpr = LoopHelpers[I].NumIterations;
-      bool TSConst =
-          !TSExpr->containsErrors() && TSExpr->isIntegerConstantExpr(Context);
-      bool NConst = NExpr->isIntegerConstantExpr(Context);
-      if (TSConst && NConst) {
-        Expr::EvalResult TSResult;
-        TSExpr->EvaluateAsInt(TSResult, Context);
-        llvm::APSInt TileVal = TSResult.Val.getInt();
-        Expr::EvalResult NResult;
-        NExpr->EvaluateAsInt(NResult, Context);
-        llvm::APSInt TripVal = NResult.Val.getInt();
-        if (TileVal.isStrictlyPositive() && TripVal.srem(TileVal).isZero())
-          continue;
-      }
-      PredicateNeeded = true;
-      break;
+  bool PredicateNeeded = false;
+  for (unsigned I = 0; I < NumLoops; ++I) {
+    Expr *TSExpr = SizesClause->getSizesRefs()[I];
+    Expr *NExpr = LoopHelpers[I].NumIterations;
+    Expr::EvalResult TSResult;
+    Expr::EvalResult NResult;
+    if (TSExpr->EvaluateAsInt(TSResult, Context) &&
+        NExpr->EvaluateAsInt(NResult, Context)) {
+      llvm::APSInt TileVal = TSResult.Val.getInt();
+      llvm::APSInt TripVal = NResult.Val.getInt();
+      if (TileVal.isStrictlyPositive() && TripVal.srem(TileVal).isZero())
+        continue;
     }
+    PredicateNeeded = true;
+    break;
+  }
 
-    if (PredicateNeeded) {
-      Expr *CombinedPred = nullptr;
-      for (unsigned I = 0; I < NumLoops; ++I) {
-        auto *OrigCntVar = cast<DeclRefExpr>(LoopHelpers[I].Counters[0]);
-        QualType IVTy = LoopHelpers[I].NumIterations->getType();
-        Expr *TileIVRef = buildDeclRefExpr(SemaRef, TileIndVars[I], IVTy,
-                                           OrigCntVar->getExprLoc());
-        ExprResult DimPred =
-            SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(), BO_LT,
-                               TileIVRef, LoopHelpers[I].NumIterations);
-        if (!DimPred.isUsable())
+  if (PredicateNeeded) {
+    Expr *CombinedPred = nullptr;
+    for (unsigned I = 0; I < NumLoops; ++I) {
+      auto *OrigCntVar = cast<DeclRefExpr>(LoopHelpers[I].Counters[0]);
+      QualType IVTy = LoopHelpers[I].NumIterations->getType();
+      Expr *FloorRef = buildDeclRefExpr(SemaRef, FloorIndVars[I], IVTy,
+                                        OrigCntVar->getExprLoc());
+      Expr *TileCntRef = buildDeclRefExpr(SemaRef, TileCntVars[I], IVTy,
+                                          OrigCntVar->getExprLoc());
+      ExprResult Sum = SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(),
+                                          BO_Add, FloorRef, TileCntRef);
+      if (!Sum.isUsable())
+        return StmtError();
+      ExprResult DimPred =
+          SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(), BO_LT,
+                             Sum.get(), LoopHelpers[I].NumIterations);
+      if (!DimPred.isUsable())
+        return StmtError();
+      if (CombinedPred) {
+        ExprResult Combined =
+            SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(), BO_LAnd,
+                               CombinedPred, DimPred.get());
+        if (!Combined.isUsable())
           return StmtError();
-        if (CombinedPred) {
-          ExprResult Combined =
-              SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(), BO_LAnd,
-                                 CombinedPred, DimPred.get());
-          if (!Combined.isUsable())
-            return StmtError();
-          CombinedPred = Combined.get();
-        } else {
-          CombinedPred = DimPred.get();
-        }
+        CombinedPred = Combined.get();
+      } else {
+        CombinedPred = DimPred.get();
       }
-      Inner = IfStmt::Create(
-          Context, SourceLocation(), IfStatementKind::Ordinary, nullptr,
-          nullptr, CombinedPred, SourceLocation(), SourceLocation(), Inner);
     }
+    // Use the source body's location for the synthetic if-statement and
+    // AttributedStmt so any later diagnostic points at the original body
+    // location.
+    SourceLocation SyntheticLoc =
+        Body ? Body->getBeginLoc() : Inner->getBeginLoc();
+    Stmt *PredIf = IfStmt::Create(
+        Context, SyntheticLoc, IfStatementKind::Ordinary, nullptr, nullptr,
+        CombinedPred, SyntheticLoc, SyntheticLoc, Inner);
+    auto *Marker = OMPInvariantPredicateBoundAttr::CreateImplicit(Context);
+    Inner = AttributedStmt::Create(Context, SyntheticLoc, {Marker}, PredIf);
   }
 
   auto MakeDimTileSize = [&SemaRef = this->SemaRef, &CopyTransformer, &Context,
@@ -15290,77 +15286,38 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     ExprResult IncrStmt;
     Stmt *TileIVAssignStmt = nullptr;
 
-    if (IntraTileMustBeCanonical) {
-      // For init-statement: auto .tile.cnt = 0
-      SemaRef.AddInitializerToDecl(
-          TileCntVars[I],
-          SemaRef.ActOnIntegerConstant(LoopHelper.Init->getExprLoc(), 0).get(),
-          /*DirectInit=*/false);
-      Decl *CounterDecl = TileCntVars[I];
-      InitStmt = new (Context)
-          DeclStmt(DeclGroupRef::Create(Context, &CounterDecl, 1),
-                   OrigCntVar->getBeginLoc(), OrigCntVar->getEndLoc());
-
-      // For cond-expression: .tile.cnt < DimTileSize  (rectangular bound)
-      CondExpr = SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(),
-                                    BO_LT, MakeTileCntRef(), DimTileSize);
-
-      // For incr-statement: ++.tile.cnt
-      IncrStmt = SemaRef.BuildUnaryOp(CurScope, LoopHelper.Inc->getExprLoc(),
-                                      UO_PreInc, MakeTileCntRef());
-
-      // Compute the logical iteration number:
-      //   .tile.iv = .floor.iv + .tile.cnt
-      ExprResult FloorPlusCnt = SemaRef.BuildBinOp(
-          CurScope, OrigCntVar->getExprLoc(), BO_Add,
-          makeFloorIVRef(SemaRef, FloorIndVars, I, IVTy, OrigCntVar),
-          MakeTileCntRef());
-      if (!FloorPlusCnt.isUsable())
-        return StmtError();
-      ExprResult TileIVAssign =
-          SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(), BO_Assign,
-                             MakeTileIVRef(), FloorPlusCnt.get());
-      if (!TileIVAssign.isUsable())
-        return StmtError();
-      TileIVAssignStmt = TileIVAssign.get();
-    } else {
-      // Min-bounded path: .tile.iv is itself the loop counter, initialized to
-      // .floor.iv and bounded by min(.floor.iv + DimTileSize, NumIterations).
-      SemaRef.AddInitializerToDecl(
-          TileIndVars[I],
-          SemaRef
-              .DefaultLvalueConversion(
-                  makeFloorIVRef(SemaRef, FloorIndVars, I, IVTy, OrigCntVar))
-              .get(),
-          /*DirectInit=*/false);
-      Decl *CounterDecl = TileIndVars[I];
-      InitStmt = new (Context)
-          DeclStmt(DeclGroupRef::Create(Context, &CounterDecl, 1),
-                   OrigCntVar->getBeginLoc(), OrigCntVar->getEndLoc());
-
-      ExprResult EndOfTile = SemaRef.BuildBinOp(
-          CurScope, LoopHelper.Cond->getExprLoc(), BO_Add,
-          makeFloorIVRef(SemaRef, FloorIndVars, I, IVTy, OrigCntVar),
-          DimTileSize);
-      if (!EndOfTile.isUsable())
-        return StmtError();
-      ExprResult IsPartialTile =
-          SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
-                             NumIterations, EndOfTile.get());
-      if (!IsPartialTile.isUsable())
-        return StmtError();
-      ExprResult MinTileAndIterSpace = SemaRef.ActOnConditionalOp(
-          LoopHelper.Cond->getBeginLoc(), LoopHelper.Cond->getEndLoc(),
-          IsPartialTile.get(), NumIterations, EndOfTile.get());
-      if (!MinTileAndIterSpace.isUsable())
-        return StmtError();
-      CondExpr =
-          SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
-                             MakeTileIVRef(), MinTileAndIterSpace.get());
+    // For init-statement: auto .tile.cnt = 0
+    SemaRef.AddInitializerToDecl(
+        TileCntVars[I],
+        SemaRef.ActOnIntegerConstant(LoopHelper.Init->getExprLoc(), 0).get(),
+        /*DirectInit=*/false);
+    Decl *CounterDecl = TileCntVars[I];
+    InitStmt = new (Context)
+        DeclStmt(DeclGroupRef::Create(Context, &CounterDecl, 1),
+                 OrigCntVar->getBeginLoc(), OrigCntVar->getEndLoc());
 
-      IncrStmt = SemaRef.BuildUnaryOp(CurScope, LoopHelper.Inc->getExprLoc(),
-                                      UO_PreInc, MakeTileIVRef());
-    }
+    // For cond-expression: .tile.cnt < DimTileSize  (rectangular bound)
+    CondExpr = SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(),
+                                  BO_LT, MakeTileCntRef(), DimTileSize);
+
+    // For incr-statement: ++.tile.cnt
+    IncrStmt = SemaRef.BuildUnaryOp(CurScope, LoopHelper.Inc->getExprLoc(),
+                                    UO_PreInc, MakeTileCntRef());
+
+    // Compute the logical iteration number:
+    //   .tile.iv = .floor.iv + .tile.cnt
+    ExprResult FloorPlusCnt = SemaRef.BuildBinOp(
+        CurScope, OrigCntVar->getExprLoc(), BO_Add,
+        makeFloorIVRef(SemaRef, FloorIndVars, I, IVTy, OrigCntVar),
+        MakeTileCntRef());
+    if (!FloorPlusCnt.isUsable())
+      return StmtError();
+    ExprResult TileIVAssign =
+        SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(), BO_Assign,
+                           MakeTileIVRef(), FloorPlusCnt.get());
+    if (!TileIVAssign.isUsable())
+      return StmtError();
+    TileIVAssignStmt = TileIVAssign.get();
 
     if (!InitStmt.isUsable() || !CondExpr.isUsable() || !IncrStmt.isUsable())
       return StmtError();

>From 972170e353ed3a4e41c18b3c4b3e7836cbe0316b Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 3 Jun 2026 05:42:14 -0400
Subject: [PATCH 14/18] tests-update

---
 clang/test/OpenMP/interchange_codegen.cpp     |  104 +-
 .../irbuilder_unroll_partial_factor_for.c     |   57 +-
 ...er_unroll_partial_heuristic_constant_for.c |   79 +-
 ...der_unroll_partial_heuristic_runtime_for.c |   79 +-
 .../irbuilder_unroll_unroll_partial_factor.c  |   57 +-
 ...rbuilder_unroll_unroll_partial_heuristic.c |   67 +-
 .../tile_canonical_required_codegen.cpp       |   86 +
 clang/test/OpenMP/tile_codegen.cpp            | 1778 ++++++++---------
 .../OpenMP/tile_codegen_for_dependent.cpp     |   53 +-
 clang/test/OpenMP/tile_codegen_tile_for.cpp   |  110 +-
 clang/test/OpenMP/tile_messages.cpp           |    2 +-
 clang/test/OpenMP/tile_rect_codegen_ir.cpp    |  488 +++--
 clang/test/OpenMP/unroll_codegen_tile_for.cpp |  107 +-
 .../runtime/test/transform/tile/foreach.cpp   |   28 -
 openmp/runtime/test/transform/tile/intfor.c   |   32 -
 .../runtime/test/transform/tile/iterfor.cpp   |   21 -
 .../tile/parallel-wsloop-collapse-foreach.cpp |   84 -
 .../parallel-wsloop-collapse-through-tile.cpp |   44 +
 18 files changed, 1576 insertions(+), 1700 deletions(-)
 create mode 100644 clang/test/OpenMP/tile_canonical_required_codegen.cpp
 create mode 100644 openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile.cpp

diff --git a/clang/test/OpenMP/interchange_codegen.cpp b/clang/test/OpenMP/interchange_codegen.cpp
index b062d42c9f162..67ec7e34c5dfd 100644
--- a/clang/test/OpenMP/interchange_codegen.cpp
+++ b/clang/test/OpenMP/interchange_codegen.cpp
@@ -1290,25 +1290,27 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[MUL279:%.*]] = mul i32 [[TMP144]], [[TMP145]]
 // CHECK1-NEXT:    [[ADD280:%.*]] = add i32 [[TMP143]], [[MUL279]]
 // CHECK1-NEXT:    store i32 [[ADD280]], ptr [[K]], align 4
-// CHECK1-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK1-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK1-NEXT:    [[ADD281:%.*]] = add i32 [[TMP147]], 1
-// CHECK1-NEXT:    [[CMP282:%.*]] = icmp ult i32 [[TMP146]], [[ADD281]]
-// CHECK1-NEXT:    br i1 [[CMP282]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK1-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
+// CHECK1-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
+// CHECK1-NEXT:    [[ADD281:%.*]] = add i32 [[TMP146]], [[TMP147]]
+// CHECK1-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// CHECK1-NEXT:    [[ADD282:%.*]] = add i32 [[TMP148]], 1
+// CHECK1-NEXT:    [[CMP283:%.*]] = icmp ult i32 [[ADD281]], [[ADD282]]
+// CHECK1-NEXT:    br i1 [[CMP283]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
 // CHECK1:       if.then:
-// CHECK1-NEXT:    [[TMP148:%.*]] = load i32, ptr [[I35]], align 4
-// CHECK1-NEXT:    [[TMP149:%.*]] = load i32, ptr [[J36]], align 4
-// CHECK1-NEXT:    [[TMP150:%.*]] = load i32, ptr [[K]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP148]], i32 noundef [[TMP149]], i32 noundef [[TMP150]])
+// CHECK1-NEXT:    [[TMP149:%.*]] = load i32, ptr [[I35]], align 4
+// CHECK1-NEXT:    [[TMP150:%.*]] = load i32, ptr [[J36]], align 4
+// CHECK1-NEXT:    [[TMP151:%.*]] = load i32, ptr [[K]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP149]], i32 noundef [[TMP150]], i32 noundef [[TMP151]])
 // CHECK1-NEXT:    br label [[IF_END]]
 // CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP151:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[ADD283:%.*]] = add nsw i64 [[TMP151]], 1
-// CHECK1-NEXT:    store i64 [[ADD283]], ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP152:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[ADD284:%.*]] = add nsw i64 [[TMP152]], 1
+// CHECK1-NEXT:    store i64 [[ADD284]], ptr [[DOTOMP_IV]], align 8
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -1821,25 +1823,27 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[MUL279:%.*]] = mul i32 [[TMP144]], [[TMP145]]
 // CHECK1-NEXT:    [[ADD280:%.*]] = add i32 [[TMP143]], [[MUL279]]
 // CHECK1-NEXT:    store i32 [[ADD280]], ptr [[K]], align 4
-// CHECK1-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK1-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK1-NEXT:    [[ADD281:%.*]] = add i32 [[TMP147]], 1
-// CHECK1-NEXT:    [[CMP282:%.*]] = icmp ult i32 [[TMP146]], [[ADD281]]
-// CHECK1-NEXT:    br i1 [[CMP282]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK1-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
+// CHECK1-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
+// CHECK1-NEXT:    [[ADD281:%.*]] = add i32 [[TMP146]], [[TMP147]]
+// CHECK1-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// CHECK1-NEXT:    [[ADD282:%.*]] = add i32 [[TMP148]], 1
+// CHECK1-NEXT:    [[CMP283:%.*]] = icmp ult i32 [[ADD281]], [[ADD282]]
+// CHECK1-NEXT:    br i1 [[CMP283]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
 // CHECK1:       if.then:
-// CHECK1-NEXT:    [[TMP148:%.*]] = load i32, ptr [[I35]], align 4
-// CHECK1-NEXT:    [[TMP149:%.*]] = load i32, ptr [[J36]], align 4
-// CHECK1-NEXT:    [[TMP150:%.*]] = load i32, ptr [[K]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP148]], i32 noundef [[TMP149]], i32 noundef [[TMP150]])
+// CHECK1-NEXT:    [[TMP149:%.*]] = load i32, ptr [[I35]], align 4
+// CHECK1-NEXT:    [[TMP150:%.*]] = load i32, ptr [[J36]], align 4
+// CHECK1-NEXT:    [[TMP151:%.*]] = load i32, ptr [[K]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP149]], i32 noundef [[TMP150]], i32 noundef [[TMP151]])
 // CHECK1-NEXT:    br label [[IF_END]]
 // CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP151:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[ADD283:%.*]] = add nsw i64 [[TMP151]], 1
-// CHECK1-NEXT:    store i64 [[ADD283]], ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP152:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[ADD284:%.*]] = add nsw i64 [[TMP152]], 1
+// CHECK1-NEXT:    store i64 [[ADD284]], ptr [[DOTOMP_IV]], align 8
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -4047,25 +4051,27 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[MUL279:%.*]] = mul i32 [[TMP144]], [[TMP145]]
 // CHECK2-NEXT:    [[ADD280:%.*]] = add i32 [[TMP143]], [[MUL279]]
 // CHECK2-NEXT:    store i32 [[ADD280]], ptr [[K]], align 4
-// CHECK2-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK2-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK2-NEXT:    [[ADD281:%.*]] = add i32 [[TMP147]], 1
-// CHECK2-NEXT:    [[CMP282:%.*]] = icmp ult i32 [[TMP146]], [[ADD281]]
-// CHECK2-NEXT:    br i1 [[CMP282]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK2-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
+// CHECK2-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
+// CHECK2-NEXT:    [[ADD281:%.*]] = add i32 [[TMP146]], [[TMP147]]
+// CHECK2-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// CHECK2-NEXT:    [[ADD282:%.*]] = add i32 [[TMP148]], 1
+// CHECK2-NEXT:    [[CMP283:%.*]] = icmp ult i32 [[ADD281]], [[ADD282]]
+// CHECK2-NEXT:    br i1 [[CMP283]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
 // CHECK2:       if.then:
-// CHECK2-NEXT:    [[TMP148:%.*]] = load i32, ptr [[I35]], align 4
-// CHECK2-NEXT:    [[TMP149:%.*]] = load i32, ptr [[J36]], align 4
-// CHECK2-NEXT:    [[TMP150:%.*]] = load i32, ptr [[K]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP148]], i32 noundef [[TMP149]], i32 noundef [[TMP150]])
+// CHECK2-NEXT:    [[TMP149:%.*]] = load i32, ptr [[I35]], align 4
+// CHECK2-NEXT:    [[TMP150:%.*]] = load i32, ptr [[J36]], align 4
+// CHECK2-NEXT:    [[TMP151:%.*]] = load i32, ptr [[K]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP149]], i32 noundef [[TMP150]], i32 noundef [[TMP151]])
 // CHECK2-NEXT:    br label [[IF_END]]
 // CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
-// CHECK2-NEXT:    [[TMP151:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[ADD283:%.*]] = add nsw i64 [[TMP151]], 1
-// CHECK2-NEXT:    store i64 [[ADD283]], ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP152:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[ADD284:%.*]] = add nsw i64 [[TMP152]], 1
+// CHECK2-NEXT:    store i64 [[ADD284]], ptr [[DOTOMP_IV]], align 8
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -4578,25 +4584,27 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[MUL279:%.*]] = mul i32 [[TMP144]], [[TMP145]]
 // CHECK2-NEXT:    [[ADD280:%.*]] = add i32 [[TMP143]], [[MUL279]]
 // CHECK2-NEXT:    store i32 [[ADD280]], ptr [[K]], align 4
-// CHECK2-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK2-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK2-NEXT:    [[ADD281:%.*]] = add i32 [[TMP147]], 1
-// CHECK2-NEXT:    [[CMP282:%.*]] = icmp ult i32 [[TMP146]], [[ADD281]]
-// CHECK2-NEXT:    br i1 [[CMP282]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK2-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
+// CHECK2-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
+// CHECK2-NEXT:    [[ADD281:%.*]] = add i32 [[TMP146]], [[TMP147]]
+// CHECK2-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// CHECK2-NEXT:    [[ADD282:%.*]] = add i32 [[TMP148]], 1
+// CHECK2-NEXT:    [[CMP283:%.*]] = icmp ult i32 [[ADD281]], [[ADD282]]
+// CHECK2-NEXT:    br i1 [[CMP283]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
 // CHECK2:       if.then:
-// CHECK2-NEXT:    [[TMP148:%.*]] = load i32, ptr [[I35]], align 4
-// CHECK2-NEXT:    [[TMP149:%.*]] = load i32, ptr [[J36]], align 4
-// CHECK2-NEXT:    [[TMP150:%.*]] = load i32, ptr [[K]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP148]], i32 noundef [[TMP149]], i32 noundef [[TMP150]])
+// CHECK2-NEXT:    [[TMP149:%.*]] = load i32, ptr [[I35]], align 4
+// CHECK2-NEXT:    [[TMP150:%.*]] = load i32, ptr [[J36]], align 4
+// CHECK2-NEXT:    [[TMP151:%.*]] = load i32, ptr [[K]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP149]], i32 noundef [[TMP150]], i32 noundef [[TMP151]])
 // CHECK2-NEXT:    br label [[IF_END]]
 // CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
-// CHECK2-NEXT:    [[TMP151:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[ADD283:%.*]] = add nsw i64 [[TMP151]], 1
-// CHECK2-NEXT:    store i64 [[ADD283]], ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP152:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[ADD284:%.*]] = add nsw i64 [[TMP152]], 1
+// CHECK2-NEXT:    store i64 [[ADD284]], ptr [[DOTOMP_IV]], align 8
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
diff --git a/clang/test/OpenMP/irbuilder_unroll_partial_factor_for.c b/clang/test/OpenMP/irbuilder_unroll_partial_factor_for.c
index 43711af5f2e33..cf4fa21c9a31c 100644
--- a/clang/test/OpenMP/irbuilder_unroll_partial_factor_for.c
+++ b/clang/test/OpenMP/irbuilder_unroll_partial_factor_for.c
@@ -79,6 +79,8 @@ void unroll_partial_heuristic_for(int n, float *a, float *b, float *c, float *d)
 // CHECK-NEXT:    br i1 [[OMP_FLOOR0_CMP]], label [[OMP_FLOOR0_BODY:%.*]], label [[OMP_FLOOR0_EXIT:%.*]]
 // CHECK:       omp_floor0.body:
 // CHECK-NEXT:    [[TMP13:%.*]] = add i32 [[OMP_FLOOR0_IV]], [[TMP9]]
+// CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP13]], [[TMP4]]
+// CHECK-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i32 [[TMP5]], i32 13
 // CHECK-NEXT:    br label [[OMP_TILE0_PREHEADER:%.*]]
 // CHECK:       omp_tile0.preheader:
 // CHECK-NEXT:    br label [[OMP_TILE0_HEADER:%.*]]
@@ -86,39 +88,36 @@ void unroll_partial_heuristic_for(int n, float *a, float *b, float *c, float *d)
 // CHECK-NEXT:    [[OMP_TILE0_IV:%.*]] = phi i32 [ 0, [[OMP_TILE0_PREHEADER]] ], [ [[OMP_TILE0_NEXT:%.*]], [[OMP_TILE0_INC:%.*]] ]
 // CHECK-NEXT:    br label [[OMP_TILE0_COND:%.*]]
 // CHECK:       omp_tile0.cond:
-// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], 13
+// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], [[TMP15]]
 // CHECK-NEXT:    br i1 [[OMP_TILE0_CMP]], label [[OMP_TILE0_BODY:%.*]], label [[OMP_TILE0_EXIT:%.*]]
 // CHECK:       omp_tile0.body:
-// CHECK-NEXT:    [[TMP14:%.*]] = mul nuw i32 13, [[TMP13]]
-// CHECK-NEXT:    [[TMP15:%.*]] = add nuw i32 [[TMP14]], [[OMP_TILE0_IV]]
-// CHECK-NEXT:    [[OMP_TILE0_INBOUNDS:%.*]] = icmp ult i32 [[TMP15]], [[DOTCOUNT]]
-// CHECK-NEXT:    br i1 [[OMP_TILE0_INBOUNDS]], label [[OMP_LOOP_BODY:%.*]], label [[OMP_TILE_BODY_MERGE:%.*]]
+// CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i32 13, [[TMP13]]
+// CHECK-NEXT:    [[TMP17:%.*]] = add nuw i32 [[TMP16]], [[OMP_TILE0_IV]]
+// CHECK-NEXT:    br label [[OMP_LOOP_BODY:%.*]]
 // CHECK:       omp_loop.body:
-// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP15]], ptr [[AGG_CAPTURED1]])
-// CHECK-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[B_ADDR]], align 8
-// CHECK-NEXT:    [[TMP17:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP17]] to i64
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[IDXPROM]]
-// CHECK-NEXT:    [[TMP18:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-// CHECK-NEXT:    [[TMP19:%.*]] = load ptr, ptr [[C_ADDR]], align 8
-// CHECK-NEXT:    [[TMP20:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP20]] to i64
-// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP19]], i64 [[IDXPROM2]]
-// CHECK-NEXT:    [[TMP21:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
-// CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP18]], [[TMP21]]
-// CHECK-NEXT:    [[TMP22:%.*]] = load ptr, ptr [[D_ADDR]], align 8
-// CHECK-NEXT:    [[TMP23:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[TMP23]] to i64
-// CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[TMP22]], i64 [[IDXPROM4]]
-// CHECK-NEXT:    [[TMP24:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
-// CHECK-NEXT:    [[MUL6:%.*]] = fmul float [[MUL]], [[TMP24]]
-// CHECK-NEXT:    [[TMP25:%.*]] = load ptr, ptr [[A_ADDR]], align 8
-// CHECK-NEXT:    [[TMP26:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM7:%.*]] = sext i32 [[TMP26]] to i64
-// CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[TMP25]], i64 [[IDXPROM7]]
+// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP17]], ptr [[AGG_CAPTURED1]])
+// CHECK-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[B_ADDR]], align 8
+// CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP19]] to i64
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[IDXPROM]]
+// CHECK-NEXT:    [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+// CHECK-NEXT:    [[TMP21:%.*]] = load ptr, ptr [[C_ADDR]], align 8
+// CHECK-NEXT:    [[TMP22:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP22]] to i64
+// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP21]], i64 [[IDXPROM2]]
+// CHECK-NEXT:    [[TMP23:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
+// CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP20]], [[TMP23]]
+// CHECK-NEXT:    [[TMP24:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[TMP25]] to i64
+// CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[TMP24]], i64 [[IDXPROM4]]
+// CHECK-NEXT:    [[TMP26:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
+// CHECK-NEXT:    [[MUL6:%.*]] = fmul float [[MUL]], [[TMP26]]
+// CHECK-NEXT:    [[TMP27:%.*]] = load ptr, ptr [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP28:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM7:%.*]] = sext i32 [[TMP28]] to i64
+// CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[TMP27]], i64 [[IDXPROM7]]
 // CHECK-NEXT:    store float [[MUL6]], ptr [[ARRAYIDX8]], align 4
-// CHECK-NEXT:    br label [[OMP_TILE_BODY_MERGE]]
-// CHECK:       omp_tile.body.merge:
 // CHECK-NEXT:    br label [[OMP_TILE0_INC]]
 // CHECK:       omp_tile0.inc:
 // CHECK-NEXT:    [[OMP_TILE0_NEXT]] = add nuw i32 [[OMP_TILE0_IV]], 1
diff --git a/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_constant_for.c b/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_constant_for.c
index b8a993c9cd8a7..acf0e5164411c 100644
--- a/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_constant_for.c
+++ b/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_constant_for.c
@@ -85,6 +85,8 @@ void unroll_partial_heuristic_constant_for(float *a, float *b, float *c, float *
 // CHECK-NEXT:    br i1 [[OMP_FLOOR0_CMP]], label [[OMP_FLOOR0_BODY:%.*]], label [[OMP_FLOOR0_EXIT:%.*]]
 // CHECK:       omp_floor0.body:
 // CHECK-NEXT:    [[TMP12:%.*]] = add i32 [[OMP_FLOOR0_IV]], [[TMP8]]
+// CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP12]], [[TMP3]]
+// CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP4]], i32 4
 // CHECK-NEXT:    br label [[OMP_TILE0_PREHEADER:%.*]]
 // CHECK:       omp_tile0.preheader:
 // CHECK-NEXT:    br label [[OMP_TILE0_HEADER:%.*]]
@@ -92,57 +94,54 @@ void unroll_partial_heuristic_constant_for(float *a, float *b, float *c, float *
 // CHECK-NEXT:    [[OMP_TILE0_IV:%.*]] = phi i32 [ 0, [[OMP_TILE0_PREHEADER]] ], [ [[OMP_TILE0_NEXT:%.*]], [[OMP_TILE0_INC:%.*]] ]
 // CHECK-NEXT:    br label [[OMP_TILE0_COND:%.*]]
 // CHECK:       omp_tile0.cond:
-// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], 4
+// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], [[TMP14]]
 // CHECK-NEXT:    br i1 [[OMP_TILE0_CMP]], label [[OMP_TILE0_BODY:%.*]], label [[OMP_TILE0_EXIT:%.*]]
 // CHECK:       omp_tile0.body:
-// CHECK-NEXT:    [[TMP13:%.*]] = mul nuw i32 4, [[TMP12]]
-// CHECK-NEXT:    [[TMP14:%.*]] = add nuw i32 [[TMP13]], [[OMP_TILE0_IV]]
-// CHECK-NEXT:    [[OMP_TILE0_INBOUNDS:%.*]] = icmp ult i32 [[TMP14]], [[DOTCOUNT]]
-// CHECK-NEXT:    br i1 [[OMP_TILE0_INBOUNDS]], label [[OMP_LOOP_BODY:%.*]], label [[OMP_TILE_BODY_MERGE:%.*]]
+// CHECK-NEXT:    [[TMP15:%.*]] = mul nuw i32 4, [[TMP12]]
+// CHECK-NEXT:    [[TMP16:%.*]] = add nuw i32 [[TMP15]], [[OMP_TILE0_IV]]
+// CHECK-NEXT:    br label [[OMP_LOOP_BODY:%.*]]
 // CHECK:       omp_loop.body:
-// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP14]], ptr [[AGG_CAPTURED1]])
-// CHECK-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[B_ADDR]], align 8
-// CHECK-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP16]] to i64
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP15]], i64 [[IDXPROM]]
-// CHECK-NEXT:    [[TMP17:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-// CHECK-NEXT:    [[CONV:%.*]] = fpext float [[TMP17]] to double
+// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP16]], ptr [[AGG_CAPTURED1]])
+// CHECK-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[B_ADDR]], align 8
+// CHECK-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP18]] to i64
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[IDXPROM]]
+// CHECK-NEXT:    [[TMP19:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+// CHECK-NEXT:    [[CONV:%.*]] = fpext float [[TMP19]] to double
 // CHECK-NEXT:    [[CALL:%.*]] = call double @sind(double noundef [[CONV]])
-// CHECK-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[C_ADDR]], align 8
-// CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP19]] to i64
-// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[IDXPROM2]]
-// CHECK-NEXT:    [[TMP20:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
-// CHECK-NEXT:    [[CONV4:%.*]] = fpext float [[TMP20]] to double
+// CHECK-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[C_ADDR]], align 8
+// CHECK-NEXT:    [[TMP21:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP21]] to i64
+// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP20]], i64 [[IDXPROM2]]
+// CHECK-NEXT:    [[TMP22:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
+// CHECK-NEXT:    [[CONV4:%.*]] = fpext float [[TMP22]] to double
 // CHECK-NEXT:    [[MUL:%.*]] = fmul double [[CALL]], [[CONV4]]
-// CHECK-NEXT:    [[TMP21:%.*]] = load ptr, ptr [[D_ADDR]], align 8
-// CHECK-NEXT:    [[TMP22:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM5:%.*]] = sext i32 [[TMP22]] to i64
-// CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, ptr [[TMP21]], i64 [[IDXPROM5]]
-// CHECK-NEXT:    [[TMP23:%.*]] = load float, ptr [[ARRAYIDX6]], align 4
-// CHECK-NEXT:    [[CONV7:%.*]] = fpext float [[TMP23]] to double
+// CHECK-NEXT:    [[TMP23:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK-NEXT:    [[TMP24:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM5:%.*]] = sext i32 [[TMP24]] to i64
+// CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, ptr [[TMP23]], i64 [[IDXPROM5]]
+// CHECK-NEXT:    [[TMP25:%.*]] = load float, ptr [[ARRAYIDX6]], align 4
+// CHECK-NEXT:    [[CONV7:%.*]] = fpext float [[TMP25]] to double
 // CHECK-NEXT:    [[MUL8:%.*]] = fmul double [[MUL]], [[CONV7]]
-// CHECK-NEXT:    [[TMP24:%.*]] = load ptr, ptr [[E_ADDR]], align 8
-// CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM9:%.*]] = sext i32 [[TMP25]] to i64
-// CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[TMP24]], i64 [[IDXPROM9]]
-// CHECK-NEXT:    [[TMP26:%.*]] = load float, ptr [[ARRAYIDX10]], align 4
-// CHECK-NEXT:    [[CONV11:%.*]] = fpext float [[TMP26]] to double
+// CHECK-NEXT:    [[TMP26:%.*]] = load ptr, ptr [[E_ADDR]], align 8
+// CHECK-NEXT:    [[TMP27:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM9:%.*]] = sext i32 [[TMP27]] to i64
+// CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[TMP26]], i64 [[IDXPROM9]]
+// CHECK-NEXT:    [[TMP28:%.*]] = load float, ptr [[ARRAYIDX10]], align 4
+// CHECK-NEXT:    [[CONV11:%.*]] = fpext float [[TMP28]] to double
 // CHECK-NEXT:    [[MUL12:%.*]] = fmul double [[MUL8]], [[CONV11]]
-// CHECK-NEXT:    [[TMP27:%.*]] = load float, ptr [[OFFSET_ADDR]], align 4
-// CHECK-NEXT:    [[CONV13:%.*]] = fpext float [[TMP27]] to double
+// CHECK-NEXT:    [[TMP29:%.*]] = load float, ptr [[OFFSET_ADDR]], align 4
+// CHECK-NEXT:    [[CONV13:%.*]] = fpext float [[TMP29]] to double
 // CHECK-NEXT:    [[ADD:%.*]] = fadd double [[MUL12]], [[CONV13]]
-// CHECK-NEXT:    [[TMP28:%.*]] = load ptr, ptr [[A_ADDR]], align 8
-// CHECK-NEXT:    [[TMP29:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM14:%.*]] = sext i32 [[TMP29]] to i64
-// CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds float, ptr [[TMP28]], i64 [[IDXPROM14]]
-// CHECK-NEXT:    [[TMP30:%.*]] = load float, ptr [[ARRAYIDX15]], align 4
-// CHECK-NEXT:    [[CONV16:%.*]] = fpext float [[TMP30]] to double
+// CHECK-NEXT:    [[TMP30:%.*]] = load ptr, ptr [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP31:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM14:%.*]] = sext i32 [[TMP31]] to i64
+// CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds float, ptr [[TMP30]], i64 [[IDXPROM14]]
+// CHECK-NEXT:    [[TMP32:%.*]] = load float, ptr [[ARRAYIDX15]], align 4
+// CHECK-NEXT:    [[CONV16:%.*]] = fpext float [[TMP32]] to double
 // CHECK-NEXT:    [[ADD17:%.*]] = fadd double [[CONV16]], [[ADD]]
 // CHECK-NEXT:    [[CONV18:%.*]] = fptrunc double [[ADD17]] to float
 // CHECK-NEXT:    store float [[CONV18]], ptr [[ARRAYIDX15]], align 4
-// CHECK-NEXT:    br label [[OMP_TILE_BODY_MERGE]]
-// CHECK:       omp_tile.body.merge:
 // CHECK-NEXT:    br label [[OMP_TILE0_INC]]
 // CHECK:       omp_tile0.inc:
 // CHECK-NEXT:    [[OMP_TILE0_NEXT]] = add nuw i32 [[OMP_TILE0_IV]], 1
diff --git a/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_runtime_for.c b/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_runtime_for.c
index 6b48e3ca22888..8fee5773e53b4 100644
--- a/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_runtime_for.c
+++ b/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_runtime_for.c
@@ -87,6 +87,8 @@ void unroll_partial_heuristic_runtime_for(int n, float *a, float *b, float *c, f
 // CHECK-NEXT:    br i1 [[OMP_FLOOR0_CMP]], label [[OMP_FLOOR0_BODY:%.*]], label [[OMP_FLOOR0_EXIT:%.*]]
 // CHECK:       omp_floor0.body:
 // CHECK-NEXT:    [[TMP13:%.*]] = add i32 [[OMP_FLOOR0_IV]], [[TMP9]]
+// CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP13]], [[TMP4]]
+// CHECK-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i32 [[TMP5]], i32 4
 // CHECK-NEXT:    br label [[OMP_TILE0_PREHEADER:%.*]]
 // CHECK:       omp_tile0.preheader:
 // CHECK-NEXT:    br label [[OMP_TILE0_HEADER:%.*]]
@@ -94,57 +96,54 @@ void unroll_partial_heuristic_runtime_for(int n, float *a, float *b, float *c, f
 // CHECK-NEXT:    [[OMP_TILE0_IV:%.*]] = phi i32 [ 0, [[OMP_TILE0_PREHEADER]] ], [ [[OMP_TILE0_NEXT:%.*]], [[OMP_TILE0_INC:%.*]] ]
 // CHECK-NEXT:    br label [[OMP_TILE0_COND:%.*]]
 // CHECK:       omp_tile0.cond:
-// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], 4
+// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], [[TMP15]]
 // CHECK-NEXT:    br i1 [[OMP_TILE0_CMP]], label [[OMP_TILE0_BODY:%.*]], label [[OMP_TILE0_EXIT:%.*]]
 // CHECK:       omp_tile0.body:
-// CHECK-NEXT:    [[TMP14:%.*]] = mul nuw i32 4, [[TMP13]]
-// CHECK-NEXT:    [[TMP15:%.*]] = add nuw i32 [[TMP14]], [[OMP_TILE0_IV]]
-// CHECK-NEXT:    [[OMP_TILE0_INBOUNDS:%.*]] = icmp ult i32 [[TMP15]], [[DOTCOUNT]]
-// CHECK-NEXT:    br i1 [[OMP_TILE0_INBOUNDS]], label [[OMP_LOOP_BODY:%.*]], label [[OMP_TILE_BODY_MERGE:%.*]]
+// CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i32 4, [[TMP13]]
+// CHECK-NEXT:    [[TMP17:%.*]] = add nuw i32 [[TMP16]], [[OMP_TILE0_IV]]
+// CHECK-NEXT:    br label [[OMP_LOOP_BODY:%.*]]
 // CHECK:       omp_loop.body:
-// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP15]], ptr [[AGG_CAPTURED1]])
-// CHECK-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[B_ADDR]], align 8
-// CHECK-NEXT:    [[TMP17:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP17]] to i64
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[IDXPROM]]
-// CHECK-NEXT:    [[TMP18:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-// CHECK-NEXT:    [[CONV:%.*]] = fpext float [[TMP18]] to double
+// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP17]], ptr [[AGG_CAPTURED1]])
+// CHECK-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[B_ADDR]], align 8
+// CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP19]] to i64
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[IDXPROM]]
+// CHECK-NEXT:    [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+// CHECK-NEXT:    [[CONV:%.*]] = fpext float [[TMP20]] to double
 // CHECK-NEXT:    [[CALL:%.*]] = call double @sind(double noundef [[CONV]])
-// CHECK-NEXT:    [[TMP19:%.*]] = load ptr, ptr [[C_ADDR]], align 8
-// CHECK-NEXT:    [[TMP20:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP20]] to i64
-// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP19]], i64 [[IDXPROM2]]
-// CHECK-NEXT:    [[TMP21:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
-// CHECK-NEXT:    [[CONV4:%.*]] = fpext float [[TMP21]] to double
+// CHECK-NEXT:    [[TMP21:%.*]] = load ptr, ptr [[C_ADDR]], align 8
+// CHECK-NEXT:    [[TMP22:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP22]] to i64
+// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP21]], i64 [[IDXPROM2]]
+// CHECK-NEXT:    [[TMP23:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
+// CHECK-NEXT:    [[CONV4:%.*]] = fpext float [[TMP23]] to double
 // CHECK-NEXT:    [[MUL:%.*]] = fmul double [[CALL]], [[CONV4]]
-// CHECK-NEXT:    [[TMP22:%.*]] = load ptr, ptr [[D_ADDR]], align 8
-// CHECK-NEXT:    [[TMP23:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM5:%.*]] = sext i32 [[TMP23]] to i64
-// CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, ptr [[TMP22]], i64 [[IDXPROM5]]
-// CHECK-NEXT:    [[TMP24:%.*]] = load float, ptr [[ARRAYIDX6]], align 4
-// CHECK-NEXT:    [[CONV7:%.*]] = fpext float [[TMP24]] to double
+// CHECK-NEXT:    [[TMP24:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM5:%.*]] = sext i32 [[TMP25]] to i64
+// CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, ptr [[TMP24]], i64 [[IDXPROM5]]
+// CHECK-NEXT:    [[TMP26:%.*]] = load float, ptr [[ARRAYIDX6]], align 4
+// CHECK-NEXT:    [[CONV7:%.*]] = fpext float [[TMP26]] to double
 // CHECK-NEXT:    [[MUL8:%.*]] = fmul double [[MUL]], [[CONV7]]
-// CHECK-NEXT:    [[TMP25:%.*]] = load ptr, ptr [[E_ADDR]], align 8
-// CHECK-NEXT:    [[TMP26:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM9:%.*]] = sext i32 [[TMP26]] to i64
-// CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[TMP25]], i64 [[IDXPROM9]]
-// CHECK-NEXT:    [[TMP27:%.*]] = load float, ptr [[ARRAYIDX10]], align 4
-// CHECK-NEXT:    [[CONV11:%.*]] = fpext float [[TMP27]] to double
+// CHECK-NEXT:    [[TMP27:%.*]] = load ptr, ptr [[E_ADDR]], align 8
+// CHECK-NEXT:    [[TMP28:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM9:%.*]] = sext i32 [[TMP28]] to i64
+// CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[TMP27]], i64 [[IDXPROM9]]
+// CHECK-NEXT:    [[TMP29:%.*]] = load float, ptr [[ARRAYIDX10]], align 4
+// CHECK-NEXT:    [[CONV11:%.*]] = fpext float [[TMP29]] to double
 // CHECK-NEXT:    [[MUL12:%.*]] = fmul double [[MUL8]], [[CONV11]]
-// CHECK-NEXT:    [[TMP28:%.*]] = load float, ptr [[OFFSET_ADDR]], align 4
-// CHECK-NEXT:    [[CONV13:%.*]] = fpext float [[TMP28]] to double
+// CHECK-NEXT:    [[TMP30:%.*]] = load float, ptr [[OFFSET_ADDR]], align 4
+// CHECK-NEXT:    [[CONV13:%.*]] = fpext float [[TMP30]] to double
 // CHECK-NEXT:    [[ADD:%.*]] = fadd double [[MUL12]], [[CONV13]]
-// CHECK-NEXT:    [[TMP29:%.*]] = load ptr, ptr [[A_ADDR]], align 8
-// CHECK-NEXT:    [[TMP30:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM14:%.*]] = sext i32 [[TMP30]] to i64
-// CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds float, ptr [[TMP29]], i64 [[IDXPROM14]]
-// CHECK-NEXT:    [[TMP31:%.*]] = load float, ptr [[ARRAYIDX15]], align 4
-// CHECK-NEXT:    [[CONV16:%.*]] = fpext float [[TMP31]] to double
+// CHECK-NEXT:    [[TMP31:%.*]] = load ptr, ptr [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP32:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM14:%.*]] = sext i32 [[TMP32]] to i64
+// CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds float, ptr [[TMP31]], i64 [[IDXPROM14]]
+// CHECK-NEXT:    [[TMP33:%.*]] = load float, ptr [[ARRAYIDX15]], align 4
+// CHECK-NEXT:    [[CONV16:%.*]] = fpext float [[TMP33]] to double
 // CHECK-NEXT:    [[ADD17:%.*]] = fadd double [[CONV16]], [[ADD]]
 // CHECK-NEXT:    [[CONV18:%.*]] = fptrunc double [[ADD17]] to float
 // CHECK-NEXT:    store float [[CONV18]], ptr [[ARRAYIDX15]], align 4
-// CHECK-NEXT:    br label [[OMP_TILE_BODY_MERGE]]
-// CHECK:       omp_tile.body.merge:
 // CHECK-NEXT:    br label [[OMP_TILE0_INC]]
 // CHECK:       omp_tile0.inc:
 // CHECK-NEXT:    [[OMP_TILE0_NEXT]] = add nuw i32 [[OMP_TILE0_IV]], 1
diff --git a/clang/test/OpenMP/irbuilder_unroll_unroll_partial_factor.c b/clang/test/OpenMP/irbuilder_unroll_unroll_partial_factor.c
index 65bd495ca0426..c4ebcddb097bc 100644
--- a/clang/test/OpenMP/irbuilder_unroll_unroll_partial_factor.c
+++ b/clang/test/OpenMP/irbuilder_unroll_unroll_partial_factor.c
@@ -75,6 +75,8 @@ void unroll_partial_factor_for(float *a, float *b, float *c, float *d) {
 // CHECK-NEXT:    br i1 [[OMP_FLOOR0_CMP]], label [[OMP_FLOOR0_BODY:%.*]], label [[OMP_FLOOR0_EXIT:%.*]]
 // CHECK:       omp_floor0.body:
 // CHECK-NEXT:    [[TMP12:%.*]] = add i32 [[OMP_FLOOR0_IV]], [[TMP8]]
+// CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP12]], [[TMP3]]
+// CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP4]], i32 2
 // CHECK-NEXT:    br label [[OMP_TILE0_PREHEADER:%.*]]
 // CHECK:       omp_tile0.preheader:
 // CHECK-NEXT:    br label [[OMP_TILE0_HEADER:%.*]]
@@ -82,39 +84,36 @@ void unroll_partial_factor_for(float *a, float *b, float *c, float *d) {
 // CHECK-NEXT:    [[OMP_TILE0_IV:%.*]] = phi i32 [ 0, [[OMP_TILE0_PREHEADER]] ], [ [[OMP_TILE0_NEXT:%.*]], [[OMP_TILE0_INC:%.*]] ]
 // CHECK-NEXT:    br label [[OMP_TILE0_COND:%.*]]
 // CHECK:       omp_tile0.cond:
-// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], 2
+// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], [[TMP14]]
 // CHECK-NEXT:    br i1 [[OMP_TILE0_CMP]], label [[OMP_TILE0_BODY:%.*]], label [[OMP_TILE0_EXIT:%.*]]
 // CHECK:       omp_tile0.body:
-// CHECK-NEXT:    [[TMP13:%.*]] = mul nuw i32 2, [[TMP12]]
-// CHECK-NEXT:    [[TMP14:%.*]] = add nuw i32 [[TMP13]], [[OMP_TILE0_IV]]
-// CHECK-NEXT:    [[OMP_TILE0_INBOUNDS:%.*]] = icmp ult i32 [[TMP14]], [[DOTCOUNT]]
-// CHECK-NEXT:    br i1 [[OMP_TILE0_INBOUNDS]], label [[OMP_LOOP_BODY:%.*]], label [[OMP_TILE_BODY_MERGE:%.*]]
+// CHECK-NEXT:    [[TMP15:%.*]] = mul nuw i32 2, [[TMP12]]
+// CHECK-NEXT:    [[TMP16:%.*]] = add nuw i32 [[TMP15]], [[OMP_TILE0_IV]]
+// CHECK-NEXT:    br label [[OMP_LOOP_BODY:%.*]]
 // CHECK:       omp_loop.body:
-// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP14]], ptr [[AGG_CAPTURED1]])
-// CHECK-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[B_ADDR]], align 8
-// CHECK-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP16]] to i64
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP15]], i64 [[IDXPROM]]
-// CHECK-NEXT:    [[TMP17:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-// CHECK-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[C_ADDR]], align 8
-// CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP19]] to i64
-// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[IDXPROM2]]
-// CHECK-NEXT:    [[TMP20:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
-// CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP17]], [[TMP20]]
-// CHECK-NEXT:    [[TMP21:%.*]] = load ptr, ptr [[D_ADDR]], align 8
-// CHECK-NEXT:    [[TMP22:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[TMP22]] to i64
-// CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[TMP21]], i64 [[IDXPROM4]]
-// CHECK-NEXT:    [[TMP23:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
-// CHECK-NEXT:    [[MUL6:%.*]] = fmul float [[MUL]], [[TMP23]]
-// CHECK-NEXT:    [[TMP24:%.*]] = load ptr, ptr [[A_ADDR]], align 8
-// CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM7:%.*]] = sext i32 [[TMP25]] to i64
-// CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[TMP24]], i64 [[IDXPROM7]]
+// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP16]], ptr [[AGG_CAPTURED1]])
+// CHECK-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[B_ADDR]], align 8
+// CHECK-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP18]] to i64
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[IDXPROM]]
+// CHECK-NEXT:    [[TMP19:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+// CHECK-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[C_ADDR]], align 8
+// CHECK-NEXT:    [[TMP21:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP21]] to i64
+// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP20]], i64 [[IDXPROM2]]
+// CHECK-NEXT:    [[TMP22:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
+// CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP19]], [[TMP22]]
+// CHECK-NEXT:    [[TMP23:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK-NEXT:    [[TMP24:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[TMP24]] to i64
+// CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[TMP23]], i64 [[IDXPROM4]]
+// CHECK-NEXT:    [[TMP25:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
+// CHECK-NEXT:    [[MUL6:%.*]] = fmul float [[MUL]], [[TMP25]]
+// CHECK-NEXT:    [[TMP26:%.*]] = load ptr, ptr [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP27:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM7:%.*]] = sext i32 [[TMP27]] to i64
+// CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[TMP26]], i64 [[IDXPROM7]]
 // CHECK-NEXT:    store float [[MUL6]], ptr [[ARRAYIDX8]], align 4
-// CHECK-NEXT:    br label [[OMP_TILE_BODY_MERGE]]
-// CHECK:       omp_tile.body.merge:
 // CHECK-NEXT:    br label [[OMP_TILE0_INC]]
 // CHECK:       omp_tile0.inc:
 // CHECK-NEXT:    [[OMP_TILE0_NEXT]] = add nuw i32 [[OMP_TILE0_IV]], 1
diff --git a/clang/test/OpenMP/irbuilder_unroll_unroll_partial_heuristic.c b/clang/test/OpenMP/irbuilder_unroll_unroll_partial_heuristic.c
index c2b7d46588f66..24d42d265d6a6 100644
--- a/clang/test/OpenMP/irbuilder_unroll_unroll_partial_heuristic.c
+++ b/clang/test/OpenMP/irbuilder_unroll_unroll_partial_heuristic.c
@@ -59,6 +59,8 @@ void unroll_unroll_partial_heuristic(float *a, float *b, float *c, float *d) {
 // CHECK-NEXT:    [[OMP_FLOOR0_CMP:%.*]] = icmp ult i32 [[OMP_FLOOR0_IV]], [[OMP_FLOOR0_TRIPCOUNT]]
 // CHECK-NEXT:    br i1 [[OMP_FLOOR0_CMP]], label [[OMP_FLOOR0_BODY:%.*]], label [[OMP_FLOOR0_EXIT:%.*]]
 // CHECK:       omp_floor0.body:
+// CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[OMP_FLOOR0_IV]], [[TMP3]]
+// CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP4]], i32 8
 // CHECK-NEXT:    br label [[OMP_TILE0_PREHEADER:%.*]]
 // CHECK:       omp_tile0.preheader:
 // CHECK-NEXT:    br label [[OMP_TILE0_HEADER:%.*]]
@@ -66,50 +68,47 @@ void unroll_unroll_partial_heuristic(float *a, float *b, float *c, float *d) {
 // CHECK-NEXT:    [[OMP_TILE0_IV:%.*]] = phi i32 [ 0, [[OMP_TILE0_PREHEADER]] ], [ [[OMP_TILE0_NEXT:%.*]], [[OMP_TILE0_INC:%.*]] ]
 // CHECK-NEXT:    br label [[OMP_TILE0_COND:%.*]]
 // CHECK:       omp_tile0.cond:
-// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], 8
+// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], [[TMP8]]
 // CHECK-NEXT:    br i1 [[OMP_TILE0_CMP]], label [[OMP_TILE0_BODY:%.*]], label [[OMP_TILE0_EXIT:%.*]]
 // CHECK:       omp_tile0.body:
-// CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i32 8, [[OMP_FLOOR0_IV]]
-// CHECK-NEXT:    [[TMP8:%.*]] = add nuw i32 [[TMP7]], [[OMP_TILE0_IV]]
-// CHECK-NEXT:    [[OMP_TILE0_INBOUNDS:%.*]] = icmp ult i32 [[TMP8]], [[DOTCOUNT]]
-// CHECK-NEXT:    br i1 [[OMP_TILE0_INBOUNDS]], label [[OMP_LOOP_BODY:%.*]], label [[OMP_TILE_BODY_MERGE:%.*]]
+// CHECK-NEXT:    [[TMP9:%.*]] = mul nuw i32 8, [[OMP_FLOOR0_IV]]
+// CHECK-NEXT:    [[TMP10:%.*]] = add nuw i32 [[TMP9]], [[OMP_TILE0_IV]]
+// CHECK-NEXT:    br label [[OMP_LOOP_BODY:%.*]]
 // CHECK:       omp_loop.body:
-// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP8]], ptr [[AGG_CAPTURED1]])
-// CHECK-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[B_ADDR]], align 8
-// CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP10]] to i64
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP9]], i64 [[IDXPROM]]
-// CHECK-NEXT:    [[TMP11:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-// CHECK-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[C_ADDR]], align 8
-// CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP13]] to i64
-// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP12]], i64 [[IDXPROM2]]
-// CHECK-NEXT:    [[TMP14:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
-// CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP11]], [[TMP14]]
-// CHECK-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[D_ADDR]], align 8
-// CHECK-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[TMP16]] to i64
-// CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[TMP15]], i64 [[IDXPROM4]]
-// CHECK-NEXT:    [[TMP17:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
-// CHECK-NEXT:    [[MUL6:%.*]] = fmul float [[MUL]], [[TMP17]]
-// CHECK-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[A_ADDR]], align 8
-// CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM7:%.*]] = sext i32 [[TMP19]] to i64
-// CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[IDXPROM7]]
+// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP10]], ptr [[AGG_CAPTURED1]])
+// CHECK-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[B_ADDR]], align 8
+// CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP12]] to i64
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP11]], i64 [[IDXPROM]]
+// CHECK-NEXT:    [[TMP13:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+// CHECK-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[C_ADDR]], align 8
+// CHECK-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP15]] to i64
+// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP14]], i64 [[IDXPROM2]]
+// CHECK-NEXT:    [[TMP16:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
+// CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP13]], [[TMP16]]
+// CHECK-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[TMP18]] to i64
+// CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[IDXPROM4]]
+// CHECK-NEXT:    [[TMP19:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
+// CHECK-NEXT:    [[MUL6:%.*]] = fmul float [[MUL]], [[TMP19]]
+// CHECK-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP21:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM7:%.*]] = sext i32 [[TMP21]] to i64
+// CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[TMP20]], i64 [[IDXPROM7]]
 // CHECK-NEXT:    store float [[MUL6]], ptr [[ARRAYIDX8]], align 4
-// CHECK-NEXT:    br label [[OMP_TILE_BODY_MERGE]]
-// CHECK:       omp_tile.body.merge:
 // CHECK-NEXT:    br label [[OMP_TILE0_INC]]
 // CHECK:       omp_tile0.inc:
 // CHECK-NEXT:    [[OMP_TILE0_NEXT]] = add nuw i32 [[OMP_TILE0_IV]], 1
-// CHECK-NEXT:    br label [[OMP_TILE0_HEADER]], !llvm.loop [[LOOP2:![0-9]+]]
+// CHECK-NEXT:    br label [[OMP_TILE0_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
 // CHECK:       omp_tile0.exit:
 // CHECK-NEXT:    br label [[OMP_TILE0_AFTER:%.*]]
 // CHECK:       omp_tile0.after:
 // CHECK-NEXT:    br label [[OMP_FLOOR0_INC]]
 // CHECK:       omp_floor0.inc:
 // CHECK-NEXT:    [[OMP_FLOOR0_NEXT]] = add nuw i32 [[OMP_FLOOR0_IV]], 1
-// CHECK-NEXT:    br label [[OMP_FLOOR0_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
+// CHECK-NEXT:    br label [[OMP_FLOOR0_HEADER]], !llvm.loop [[LOOP6:![0-9]+]]
 // CHECK:       omp_floor0.exit:
 // CHECK-NEXT:    br label [[OMP_FLOOR0_AFTER:%.*]]
 // CHECK:       omp_floor0.after:
@@ -130,7 +129,7 @@ void unroll_unroll_partial_heuristic(float *a, float *b, float *c, float *d) {
 // CHECK-NEXT:    store ptr [[__CONTEXT]], ptr [[__CONTEXT_ADDR]], align 8
 // CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__CONTEXT_ADDR]], align 8
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw [[STRUCT_ANON:%.*]], ptr [[TMP0]], i32 0, i32 0
-// CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP1]], align 8, !nonnull [[META6:![0-9]+]], !align [[META7:![0-9]+]]
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP1]], align 8
 // CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
 // CHECK-NEXT:    store i32 [[TMP3]], ptr [[DOTSTART]], align 4
 // CHECK-NEXT:    store i32 2, ptr [[DOTSTOP]], align 4
@@ -153,7 +152,7 @@ void unroll_unroll_partial_heuristic(float *a, float *b, float *c, float *d) {
 // CHECK-NEXT:    br label [[COND_END]]
 // CHECK:       cond.end:
 // CHECK-NEXT:    [[COND:%.*]] = phi i32 [ [[DIV]], [[COND_TRUE]] ], [ 0, [[COND_FALSE]] ]
-// CHECK-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[DISTANCE_ADDR]], align 8, !nonnull [[META6]], !align [[META7]]
+// CHECK-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[DISTANCE_ADDR]], align 8
 // CHECK-NEXT:    store i32 [[COND]], ptr [[TMP10]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -173,7 +172,7 @@ void unroll_unroll_partial_heuristic(float *a, float *b, float *c, float *d) {
 // CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[LOGICAL_ADDR]], align 4
 // CHECK-NEXT:    [[MUL:%.*]] = mul i32 1, [[TMP3]]
 // CHECK-NEXT:    [[ADD:%.*]] = add i32 [[TMP2]], [[MUL]]
-// CHECK-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[LOOPVAR_ADDR]], align 8, !nonnull [[META6]], !align [[META7]]
+// CHECK-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[LOOPVAR_ADDR]], align 8
 // CHECK-NEXT:    store i32 [[ADD]], ptr [[TMP4]], align 4
 // CHECK-NEXT:    ret void
 //
diff --git a/clang/test/OpenMP/tile_canonical_required_codegen.cpp b/clang/test/OpenMP/tile_canonical_required_codegen.cpp
new file mode 100644
index 0000000000000..ceeaed4cb666a
--- /dev/null
+++ b/clang/test/OpenMP/tile_canonical_required_codegen.cpp
@@ -0,0 +1,86 @@
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fclang-abi-compat=latest -std=c++20 -fopenmp -ast-print %s | FileCheck %s --check-prefix=PRINT
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fclang-abi-compat=latest -std=c++20 -fopenmp -emit-llvm %s -o - | FileCheck %s --check-prefix=IR
+// expected-no-diagnostics
+
+// Positive tests for canonical intra-tile loop shape.
+
+extern "C" void body(...) {}
+
+// PRINT-LABEL: void collapse2_tile1d(
+// IR-LABEL: define {{.*}} @collapse2_tile1d(
+// collapse(2) consumes floor+tile; predicate stays as a body guard.
+// IR should have one flattened loop body and no nested floor/tile loop.
+// IR: %.tile.cnt.0.iv.i
+// IR: omp.inner.for.body:
+// IR: br i1 %{{.*}}, label %if.then, label %if.end
+// IR: if.then:
+// IR: call void (...) @body
+// IR-NEXT: br label %if.end
+// IR: if.end:
+extern "C" void collapse2_tile1d(int n) {
+#pragma omp parallel for collapse(2)
+#pragma omp tile sizes(2)
+  for (int i = 0; i < n; ++i)
+    body(i);
+}
+
+// PRINT-LABEL: void collapse3_tile1d_nested(
+// IR-LABEL: define {{.*}} @collapse3_tile1d_nested(
+// collapse(3) reaches through the tile guard to include `j`.
+// IR: br i1 %{{.*}}, label %omp_tile.pred.then, label %omp_tile.pred.end
+// IR: omp_tile.pred.then:
+// IR: call void (...) @body
+// IR-NEXT: br label %omp_tile.pred.end
+// IR: omp_tile.pred.end:
+extern "C" void collapse3_tile1d_nested(int n) {
+#pragma omp parallel for collapse(3)
+#pragma omp tile sizes(5)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+}
+
+// PRINT-LABEL: void tile2d_inside_tile1d(
+// IR-LABEL: define {{.*}} @tile2d_inside_tile1d(
+// Nested tiling: both intra-tile loops stay canonical and hoist their predicates.
+// IR: %.tile.cnt.0.iv..floor_0.iv.i = alloca i32
+// IR: %.tile.cnt.0.iv.i = alloca i32
+// IR: %omp_tile.invariant_pred_bound = and i1
+// IR: %omp_tile.invariant_pred_bound{{[0-9]+}} = and i1
+extern "C" void tile2d_inside_tile1d(int n) {
+#pragma omp tile sizes(3)
+#pragma omp tile sizes(2)
+  for (int i = 0; i < n; ++i)
+    body(i);
+}
+
+// PRINT-LABEL: void collapse3_tile2d(
+// IR-LABEL: define {{.*}} @collapse3_tile2d(
+// 2-D tile + collapse(3): dim checks combine and are hoisted into the surviving loop.
+// IR: %.tile.cnt.0.iv.i
+// IR: %.tile.cnt.1.iv.j
+// IR: land.rhs:
+// IR: land.end:
+// IR: %omp_tile.invariant_pred_bound = and i1
+extern "C" void collapse3_tile2d(int n) {
+#pragma omp parallel for collapse(3)
+#pragma omp tile sizes(2, 3)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+}
+
+// PRINT-LABEL: void tile1d_constant_div_no_predicate(
+// IR-LABEL: define {{.*}} @tile1d_constant_div_no_predicate(
+// Exact-divisible case: canonical tile-count loop, no predicate.
+// IR: %.tile.cnt.0.iv.i
+// IR: for.cond1:
+// IR: %[[TC:.*]] = icmp slt i32 %{{.*}}, 5
+// IR-NEXT: br i1 %[[TC]], label %for.body3, label %for.end
+// IR-NOT: omp_tile.invariant_pred_bound
+// IR-NOT: omp_tile.pred
+extern "C" void tile1d_constant_div_no_predicate() {
+#pragma omp tile sizes(5)
+  for (int i = 0; i < 10; ++i)
+    body(i);
+}
diff --git a/clang/test/OpenMP/tile_codegen.cpp b/clang/test/OpenMP/tile_codegen.cpp
index 65fd3a327ad64..a76e46bddfb8a 100644
--- a/clang/test/OpenMP/tile_codegen.cpp
+++ b/clang/test/OpenMP/tile_codegen.cpp
@@ -56,7 +56,7 @@ extern "C" void foo4() {
 
 
 extern "C" void foo5() {
-#pragma omp for collapse(2)
+#pragma omp for collapse(3)
 #pragma omp tile sizes(5)
   for (int i = 7; i < 17; i += 3)
     for (int j = 7; j < 17; j += 3)
@@ -136,8 +136,9 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[I2:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[I:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
@@ -151,45 +152,41 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
 // CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END12:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK1:       for.cond4:
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 5
-// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
-// CHECK1-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK1:       cond.true:
-// CHECK1-NEXT:    br label [[COND_END:%.*]]
-// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP1]], 5
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
+// CHECK1-NEXT:    [[CMP6:%.*]] = icmp slt i32 [[ADD]], 4
+// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP5]], [[CMP6]]
+// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY7:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body7:
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP4]], 5
-// CHECK1-NEXT:    br label [[COND_END]]
-// CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
-// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body8:
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
+// CHECK1-NEXT:    store i32 [[ADD8]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
 // CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
-// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[TMP6]], align 4
-// CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2]], !align [[META3]]
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP8]])
+// CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
+// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[TMP7]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2]], !align [[META3]]
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP9]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP9]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP4:![0-9]+]]
 // CHECK1:       for.end:
 // CHECK1-NEXT:    br label [[FOR_INC10:%.*]]
 // CHECK1:       for.inc10:
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP10]], 5
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP11]], 5
 // CHECK1-NEXT:    store i32 [[ADD11]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
 // CHECK1:       for.end12:
@@ -207,8 +204,9 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -237,54 +235,48 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK1-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP8]], [[ADD5]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK1:       for.cond6:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[ADD7:%.*]] = add i32 [[TMP12]], 1
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD8:%.*]] = add i32 [[TMP13]], 5
-// CHECK1-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
-// CHECK1-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK1:       cond.true:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[ADD10:%.*]] = add i32 [[TMP14]], 1
-// CHECK1-NEXT:    br label [[COND_END:%.*]]
-// CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD11:%.*]] = add i32 [[TMP15]], 5
-// CHECK1-NEXT:    br label [[COND_END]]
-// CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP11]], [[COND]]
-// CHECK1-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body13:
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP10]], 5
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD8:%.*]] = add i32 [[TMP11]], [[TMP12]]
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD9:%.*]] = add i32 [[TMP13]], 1
+// CHECK1-NEXT:    [[CMP10:%.*]] = icmp ult i32 [[ADD8]], [[ADD9]]
+// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP7]], [[CMP10]]
+// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY11:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body11:
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD12:%.*]] = add i32 [[TMP14]], [[TMP15]]
+// CHECK1-NEXT:    store i32 [[ADD12]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
 // CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
 // CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP17]], [[TMP18]]
-// CHECK1-NEXT:    [[ADD14:%.*]] = add i32 [[TMP16]], [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[ADD13:%.*]] = add i32 [[TMP16]], [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[I]], align 4
 // CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP19]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP20]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP7:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC15:%.*]]
-// CHECK1:       for.inc15:
+// CHECK1-NEXT:    br label [[FOR_INC14:%.*]]
+// CHECK1:       for.inc14:
 // CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD16:%.*]] = add i32 [[TMP21]], 5
-// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD15:%.*]] = add i32 [[TMP21]], 5
+// CHECK1-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
-// CHECK1:       for.end17:
+// CHECK1:       for.end16:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -295,11 +287,13 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -310,94 +304,90 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END30:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END26:%.*]]
 // CHECK1:       for.body:
 // CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK1:       for.cond1:
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 4
-// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END27:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END23:%.*]]
 // CHECK1:       for.body3:
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 [[TMP2]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK1:       for.cond4:
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], 5
-// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
-// CHECK1-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK1:       cond.true:
-// CHECK1-NEXT:    br label [[COND_END:%.*]]
-// CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP5]], 5
-// CHECK1-NEXT:    br label [[COND_END]]
-// CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP3]], [[COND]]
-// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END24:%.*]]
-// CHECK1:       for.body8:
-// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
-// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND10:%.*]]
-// CHECK1:       for.cond10:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP2]], 5
+// CHECK1-NEXT:    br i1 [[CMP5]], label [[FOR_BODY6:%.*]], label [[FOR_END20:%.*]]
+// CHECK1:       for.body6:
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[TMP4]]
+// CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
+// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND8:%.*]]
+// CHECK1:       for.cond8:
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP6]], 5
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP7]], [[TMP8]]
+// CHECK1-NEXT:    [[CMP11:%.*]] = icmp slt i32 [[ADD10]], 4
+// CHECK1-NEXT:    br i1 [[CMP11]], label [[LAND_RHS:%.*]], label [[LAND_END:%.*]]
+// CHECK1:       land.rhs:
 // CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP9]], 5
-// CHECK1-NEXT:    [[CMP12:%.*]] = icmp slt i32 4, [[ADD11]]
-// CHECK1-NEXT:    br i1 [[CMP12]], label [[COND_TRUE13:%.*]], label [[COND_FALSE14:%.*]]
-// CHECK1:       cond.true13:
-// CHECK1-NEXT:    br label [[COND_END16:%.*]]
-// CHECK1:       cond.false14:
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP10]], 5
-// CHECK1-NEXT:    br label [[COND_END16]]
-// CHECK1:       cond.end16:
-// CHECK1-NEXT:    [[COND17:%.*]] = phi i32 [ 4, [[COND_TRUE13]] ], [ [[ADD15]], [[COND_FALSE14]] ]
-// CHECK1-NEXT:    [[CMP18:%.*]] = icmp slt i32 [[TMP8]], [[COND17]]
-// CHECK1-NEXT:    br i1 [[CMP18]], label [[FOR_BODY19:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body19:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[TMP11]], 3
-// CHECK1-NEXT:    [[ADD21:%.*]] = add nsw i32 7, [[MUL20]]
-// CHECK1-NEXT:    store i32 [[ADD21]], ptr [[J]], align 4
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP12]], i32 noundef [[TMP13]])
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
+// CHECK1-NEXT:    [[CMP13:%.*]] = icmp slt i32 [[ADD12]], 4
+// CHECK1-NEXT:    br label [[LAND_END]]
+// CHECK1:       land.end:
+// CHECK1-NEXT:    [[TMP11:%.*]] = phi i1 [ false, [[FOR_COND8]] ], [ [[CMP13]], [[LAND_RHS]] ]
+// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP9]], [[TMP11]]
+// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body14:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP12]], [[TMP13]]
+// CHECK1-NEXT:    store i32 [[ADD15]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[MUL16:%.*]] = mul nsw i32 [[TMP14]], 3
+// CHECK1-NEXT:    [[ADD17:%.*]] = add nsw i32 7, [[MUL16]]
+// CHECK1-NEXT:    store i32 [[ADD17]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP15]], i32 noundef [[TMP16]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND10]], !llvm.loop [[LOOP9:![0-9]+]]
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP17]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP9:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC22:%.*]]
-// CHECK1:       for.inc22:
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC23:%.*]] = add nsw i32 [[TMP15]], 1
-// CHECK1-NEXT:    store i32 [[INC23]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_INC18:%.*]]
+// CHECK1:       for.inc18:
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC19:%.*]] = add nsw i32 [[TMP18]], 1
+// CHECK1-NEXT:    store i32 [[INC19]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP10:![0-9]+]]
-// CHECK1:       for.end24:
-// CHECK1-NEXT:    br label [[FOR_INC25:%.*]]
-// CHECK1:       for.inc25:
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD26:%.*]] = add nsw i32 [[TMP16]], 5
-// CHECK1-NEXT:    store i32 [[ADD26]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1:       for.end20:
+// CHECK1-NEXT:    br label [[FOR_INC21:%.*]]
+// CHECK1:       for.inc21:
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD22:%.*]] = add nsw i32 [[TMP19]], 5
+// CHECK1-NEXT:    store i32 [[ADD22]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP11:![0-9]+]]
-// CHECK1:       for.end27:
-// CHECK1-NEXT:    br label [[FOR_INC28:%.*]]
-// CHECK1:       for.inc28:
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD29:%.*]] = add nsw i32 [[TMP17]], 5
-// CHECK1-NEXT:    store i32 [[ADD29]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1:       for.end23:
+// CHECK1-NEXT:    br label [[FOR_INC24:%.*]]
+// CHECK1:       for.inc24:
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD25:%.*]] = add nsw i32 [[TMP20]], 5
+// CHECK1-NEXT:    store i32 [[ADD25]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP12:![0-9]+]]
-// CHECK1:       for.end30:
+// CHECK1:       for.end26:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -455,14 +445,14 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP7]], 4
-// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END22:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
 // CHECK1:       for.body:
 // CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND3:%.*]]
 // CHECK1:       for.cond3:
 // CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP8]], 5
-// CHECK1-NEXT:    br i1 [[CMP4]], label [[FOR_BODY5:%.*]], label [[FOR_END19:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP4]], label [[FOR_BODY5:%.*]], label [[FOR_END21:%.*]]
 // CHECK1:       for.body5:
 // CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
@@ -477,57 +467,61 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond9:
 // CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
 // CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i32 [[TMP12]], 5
-// CHECK1-NEXT:    br i1 [[CMP10]], label [[FOR_BODY11:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body11:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP13]], [[TMP14]]
-// CHECK1-NEXT:    store i32 [[ADD12]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[MUL13:%.*]] = mul nsw i32 [[TMP15]], 3
-// CHECK1-NEXT:    [[ADD14:%.*]] = add nsw i32 7, [[MUL13]]
-// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[J]], align 4
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP15:%.*]] = icmp slt i32 [[TMP16]], 4
-// CHECK1-NEXT:    br i1 [[CMP15]], label [[LAND_LHS_TRUE:%.*]], label [[IF_END:%.*]]
-// CHECK1:       land.lhs.true:
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[CMP16:%.*]] = icmp slt i32 [[TMP17]], 4
-// CHECK1-NEXT:    br i1 [[CMP16]], label [[IF_THEN:%.*]], label [[IF_END]]
-// CHECK1:       if.then:
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[J]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP18]], i32 noundef [[TMP19]])
-// CHECK1-NEXT:    br label [[IF_END]]
-// CHECK1:       if.end:
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP13]], [[TMP14]]
+// CHECK1-NEXT:    [[CMP12:%.*]] = icmp slt i32 [[ADD11]], 4
+// CHECK1-NEXT:    br i1 [[CMP12]], label [[LAND_RHS:%.*]], label [[LAND_END:%.*]]
+// CHECK1:       land.rhs:
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP15]], [[TMP16]]
+// CHECK1-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[ADD13]], 4
+// CHECK1-NEXT:    br label [[LAND_END]]
+// CHECK1:       land.end:
+// CHECK1-NEXT:    [[TMP17:%.*]] = phi i1 [ false, [[FOR_COND9]] ], [ [[CMP14]], [[LAND_RHS]] ]
+// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP10]], [[TMP17]]
+// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY15:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body15:
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP18]], [[TMP19]]
+// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[MUL17:%.*]] = mul nsw i32 [[TMP20]], 3
+// CHECK1-NEXT:    [[ADD18:%.*]] = add nsw i32 7, [[MUL17]]
+// CHECK1-NEXT:    store i32 [[ADD18]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP21]], i32 noundef [[TMP22]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP20]], 1
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP23]], 1
 // CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND9]], !llvm.loop [[LOOP13:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC17:%.*]]
-// CHECK1:       for.inc17:
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC18:%.*]] = add nsw i32 [[TMP21]], 1
-// CHECK1-NEXT:    store i32 [[INC18]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_INC19:%.*]]
+// CHECK1:       for.inc19:
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC20:%.*]] = add nsw i32 [[TMP24]], 1
+// CHECK1-NEXT:    store i32 [[INC20]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND3]], !llvm.loop [[LOOP14:![0-9]+]]
-// CHECK1:       for.end19:
-// CHECK1-NEXT:    br label [[FOR_INC20:%.*]]
-// CHECK1:       for.inc20:
-// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD21:%.*]] = add nsw i32 [[TMP22]], 5
-// CHECK1-NEXT:    store i32 [[ADD21]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1:       for.end21:
+// CHECK1-NEXT:    br label [[FOR_INC22:%.*]]
+// CHECK1:       for.inc22:
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP25]], 5
+// CHECK1-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP15:![0-9]+]]
-// CHECK1:       for.end22:
+// CHECK1:       for.end24:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP23]], 1
-// CHECK1-NEXT:    store i32 [[ADD23]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[ADD25:%.*]] = add nsw i32 [[TMP26]], 1
+// CHECK1-NEXT:    store i32 [[ADD25]], ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -602,14 +596,14 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP9]], 4
-// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY:%.*]], label [[FOR_END27:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY:%.*]], label [[FOR_END29:%.*]]
 // CHECK1:       for.body:
 // CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND8:%.*]]
 // CHECK1:       for.cond8:
 // CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP10]], 5
-// CHECK1-NEXT:    br i1 [[CMP9]], label [[FOR_BODY10:%.*]], label [[FOR_END24:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP9]], label [[FOR_BODY10:%.*]], label [[FOR_END26:%.*]]
 // CHECK1:       for.body10:
 // CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
@@ -624,57 +618,61 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond14:
 // CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
 // CHECK1-NEXT:    [[CMP15:%.*]] = icmp slt i32 [[TMP14]], 5
-// CHECK1-NEXT:    br i1 [[CMP15]], label [[FOR_BODY16:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body16:
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD17:%.*]] = add nsw i32 [[TMP15]], [[TMP16]]
-// CHECK1-NEXT:    store i32 [[ADD17]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[MUL18:%.*]] = mul nsw i32 [[TMP17]], 3
-// CHECK1-NEXT:    [[ADD19:%.*]] = add nsw i32 7, [[MUL18]]
-// CHECK1-NEXT:    store i32 [[ADD19]], ptr [[J]], align 4
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP20:%.*]] = icmp slt i32 [[TMP18]], 4
-// CHECK1-NEXT:    br i1 [[CMP20]], label [[LAND_LHS_TRUE:%.*]], label [[IF_END:%.*]]
-// CHECK1:       land.lhs.true:
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[CMP21:%.*]] = icmp slt i32 [[TMP19]], 4
-// CHECK1-NEXT:    br i1 [[CMP21]], label [[IF_THEN:%.*]], label [[IF_END]]
-// CHECK1:       if.then:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[J]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP20]], i32 noundef [[TMP21]])
-// CHECK1-NEXT:    br label [[IF_END]]
-// CHECK1:       if.end:
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP15]], [[TMP16]]
+// CHECK1-NEXT:    [[CMP17:%.*]] = icmp slt i32 [[ADD16]], 4
+// CHECK1-NEXT:    br i1 [[CMP17]], label [[LAND_RHS:%.*]], label [[LAND_END:%.*]]
+// CHECK1:       land.rhs:
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD18:%.*]] = add nsw i32 [[TMP17]], [[TMP18]]
+// CHECK1-NEXT:    [[CMP19:%.*]] = icmp slt i32 [[ADD18]], 4
+// CHECK1-NEXT:    br label [[LAND_END]]
+// CHECK1:       land.end:
+// CHECK1-NEXT:    [[TMP19:%.*]] = phi i1 [ false, [[FOR_COND14]] ], [ [[CMP19]], [[LAND_RHS]] ]
+// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP15]], [[TMP19]]
+// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY20:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body20:
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD21:%.*]] = add nsw i32 [[TMP20]], [[TMP21]]
+// CHECK1-NEXT:    store i32 [[ADD21]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[MUL22:%.*]] = mul nsw i32 [[TMP22]], 3
+// CHECK1-NEXT:    [[ADD23:%.*]] = add nsw i32 7, [[MUL22]]
+// CHECK1-NEXT:    store i32 [[ADD23]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP23]], i32 noundef [[TMP24]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP22]], 1
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP25]], 1
 // CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND14]], !llvm.loop [[LOOP16:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC22:%.*]]
-// CHECK1:       for.inc22:
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC23:%.*]] = add nsw i32 [[TMP23]], 1
-// CHECK1-NEXT:    store i32 [[INC23]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_INC24:%.*]]
+// CHECK1:       for.inc24:
+// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC25:%.*]] = add nsw i32 [[TMP26]], 1
+// CHECK1-NEXT:    store i32 [[INC25]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP17:![0-9]+]]
-// CHECK1:       for.end24:
-// CHECK1-NEXT:    br label [[FOR_INC25:%.*]]
-// CHECK1:       for.inc25:
-// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD26:%.*]] = add nsw i32 [[TMP24]], 5
-// CHECK1-NEXT:    store i32 [[ADD26]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1:       for.end26:
+// CHECK1-NEXT:    br label [[FOR_INC27:%.*]]
+// CHECK1:       for.inc27:
+// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD28:%.*]] = add nsw i32 [[TMP27]], 5
+// CHECK1-NEXT:    store i32 [[ADD28]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP18:![0-9]+]]
-// CHECK1:       for.end27:
+// CHECK1:       for.end29:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[ADD28:%.*]] = add nsw i32 [[TMP25]], 1
-// CHECK1-NEXT:    store i32 [[ADD28]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[ADD30:%.*]] = add nsw i32 [[TMP28]], 1
+// CHECK1-NEXT:    store i32 [[ADD30]], ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -690,6 +688,7 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[_TMP1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[_TMP2:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
@@ -702,12 +701,12 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK1-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
-// CHECK1-NEXT:    store i32 4, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    store i32 19, ptr [[DOTOMP_UB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
 // CHECK1-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// CHECK1-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], 19
 // CHECK1-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK1:       cond.true:
 // CHECK1-NEXT:    br label [[COND_END:%.*]]
@@ -715,7 +714,7 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
 // CHECK1-NEXT:    br label [[COND_END]]
 // CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[TMP2]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 19, [[COND_TRUE]] ], [ [[TMP2]], [[COND_FALSE]] ]
 // CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[DOTOMP_IV]], align 4
@@ -723,60 +722,65 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       omp.inner.for.cond:
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// CHECK1-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], [[TMP5]]
-// CHECK1-NEXT:    br i1 [[CMP2]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK1-NEXT:    [[CMP3:%.*]] = icmp sle i32 [[TMP4]], [[TMP5]]
+// CHECK1-NEXT:    br i1 [[CMP3]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
 // CHECK1:       omp.inner.for.body:
 // CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP6]], 5
+// CHECK1-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP6]], 20
 // CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[DIV]], 5
 // CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[DIV3:%.*]] = sdiv i32 [[TMP8]], 5
-// CHECK1-NEXT:    [[MUL4:%.*]] = mul nsw i32 [[DIV3]], 5
-// CHECK1-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP7]], [[MUL4]]
-// CHECK1-NEXT:    [[MUL5:%.*]] = mul nsw i32 [[SUB]], 1
-// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL5]]
-// CHECK1-NEXT:    store i32 [[ADD6]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
-// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL8:%.*]] = mul nsw i32 [[TMP11]], 3
-// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL8]]
-// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i32 [[TMP12]], 4
-// CHECK1-NEXT:    br i1 [[CMP10]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK1:       if.then:
-// CHECK1-NEXT:    store i32 7, ptr [[J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND:%.*]]
-// CHECK1:       for.cond:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
-// CHECK1-NEXT:    [[CMP11:%.*]] = icmp slt i32 [[TMP13]], 17
-// CHECK1-NEXT:    br i1 [[CMP11]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[J]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP14]], i32 noundef [[TMP15]])
-// CHECK1-NEXT:    br label [[FOR_INC:%.*]]
-// CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[J]], align 4
-// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP16]], 3
-// CHECK1-NEXT:    store i32 [[ADD12]], ptr [[J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP19:![0-9]+]]
-// CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[IF_END]]
-// CHECK1:       if.end:
+// CHECK1-NEXT:    [[DIV4:%.*]] = sdiv i32 [[TMP8]], 20
+// CHECK1-NEXT:    [[MUL5:%.*]] = mul nsw i32 [[DIV4]], 20
+// CHECK1-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP7]], [[MUL5]]
+// CHECK1-NEXT:    [[DIV6:%.*]] = sdiv i32 [[SUB]], 4
+// CHECK1-NEXT:    [[MUL7:%.*]] = mul nsw i32 [[DIV6]], 1
+// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i32 0, [[MUL7]]
+// CHECK1-NEXT:    store i32 [[ADD8]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[DIV9:%.*]] = sdiv i32 [[TMP10]], 20
+// CHECK1-NEXT:    [[MUL10:%.*]] = mul nsw i32 [[DIV9]], 20
+// CHECK1-NEXT:    [[SUB11:%.*]] = sub nsw i32 [[TMP9]], [[MUL10]]
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[DIV12:%.*]] = sdiv i32 [[TMP12]], 20
+// CHECK1-NEXT:    [[MUL13:%.*]] = mul nsw i32 [[DIV12]], 20
+// CHECK1-NEXT:    [[SUB14:%.*]] = sub nsw i32 [[TMP11]], [[MUL13]]
+// CHECK1-NEXT:    [[DIV15:%.*]] = sdiv i32 [[SUB14]], 4
+// CHECK1-NEXT:    [[MUL16:%.*]] = mul nsw i32 [[DIV15]], 4
+// CHECK1-NEXT:    [[SUB17:%.*]] = sub nsw i32 [[SUB11]], [[MUL16]]
+// CHECK1-NEXT:    [[MUL18:%.*]] = mul nsw i32 [[SUB17]], 3
+// CHECK1-NEXT:    [[ADD19:%.*]] = add nsw i32 7, [[MUL18]]
+// CHECK1-NEXT:    store i32 [[ADD19]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP13]], [[TMP14]]
+// CHECK1-NEXT:    store i32 [[ADD20]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL21:%.*]] = mul nsw i32 [[TMP15]], 3
+// CHECK1-NEXT:    [[ADD22:%.*]] = add nsw i32 7, [[MUL21]]
+// CHECK1-NEXT:    store i32 [[ADD22]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP16]], [[TMP17]]
+// CHECK1-NEXT:    [[CMP24:%.*]] = icmp slt i32 [[ADD23]], 4
+// CHECK1-NEXT:    br i1 [[CMP24]], label [[OMP_TILE_PRED_THEN:%.*]], label [[OMP_TILE_PRED_END:%.*]]
+// CHECK1:       omp_tile.pred.then:
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP18]], i32 noundef [[TMP19]])
+// CHECK1-NEXT:    br label [[OMP_TILE_PRED_END]]
+// CHECK1:       omp_tile.pred.end:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP17]], 1
-// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[ADD25:%.*]] = add nsw i32 [[TMP20]], 1
+// CHECK1-NEXT:    store i32 [[ADD25]], ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -847,38 +851,37 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP8]], 5
-// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body:
 // CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
-// CHECK1-NEXT:    store i32 [[ADD3]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL4:%.*]] = mul nsw i32 [[TMP11]], 3
-// CHECK1-NEXT:    [[ADD5:%.*]] = add nsw i32 7, [[MUL4]]
-// CHECK1-NEXT:    store i32 [[ADD5]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP6:%.*]] = icmp slt i32 [[TMP12]], 4
-// CHECK1-NEXT:    br i1 [[CMP6]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK1:       if.then:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP13]])
-// CHECK1-NEXT:    br label [[IF_END]]
-// CHECK1:       if.end:
+// CHECK1-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[ADD3]], 4
+// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP2]], [[CMP4]]
+// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body:
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD5:%.*]] = add nsw i32 [[TMP11]], [[TMP12]]
+// CHECK1-NEXT:    store i32 [[ADD5]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL6:%.*]] = mul nsw i32 [[TMP13]], 3
+// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL6]]
+// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP14]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP15]], 1
 // CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP20:![0-9]+]]
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP19:![0-9]+]]
 // CHECK1:       for.end:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 [[TMP15]], 1
-// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP16]], 1
+// CHECK1-NEXT:    store i32 [[ADD8]], ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -903,8 +906,9 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[START_ADDR]], align 4
@@ -928,53 +932,47 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK1-NEXT:    [[ADD5:%.*]] = add i32 [[TMP6]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP5]], [[ADD5]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK1:       for.cond6:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[ADD7:%.*]] = add i32 [[TMP9]], 1
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD8:%.*]] = add i32 [[TMP10]], 5
-// CHECK1-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
-// CHECK1-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK1:       cond.true:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[ADD10:%.*]] = add i32 [[TMP11]], 1
-// CHECK1-NEXT:    br label [[COND_END:%.*]]
-// CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD11:%.*]] = add i32 [[TMP12]], 5
-// CHECK1-NEXT:    br label [[COND_END]]
-// CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP8]], [[COND]]
-// CHECK1-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body13:
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP7]], 5
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD8:%.*]] = add i32 [[TMP8]], [[TMP9]]
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD9:%.*]] = add i32 [[TMP10]], 1
+// CHECK1-NEXT:    [[CMP10:%.*]] = icmp ult i32 [[ADD8]], [[ADD9]]
+// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP7]], [[CMP10]]
+// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY11:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body11:
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD12:%.*]] = add i32 [[TMP11]], [[TMP12]]
+// CHECK1-NEXT:    store i32 [[ADD12]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
 // CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP14]], 3
-// CHECK1-NEXT:    [[ADD14:%.*]] = add i32 [[TMP13]], [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[ADD13:%.*]] = add i32 [[TMP13]], [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[I]], align 4
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP15]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP16]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP23:![0-9]+]]
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP22:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC15:%.*]]
-// CHECK1:       for.inc15:
+// CHECK1-NEXT:    br label [[FOR_INC14:%.*]]
+// CHECK1:       for.inc14:
 // CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD16:%.*]] = add i32 [[TMP17]], 5
-// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP24:![0-9]+]]
-// CHECK1:       for.end17:
+// CHECK1-NEXT:    [[ADD15:%.*]] = add i32 [[TMP17]], 5
+// CHECK1-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP23:![0-9]+]]
+// CHECK1:       for.end16:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -983,8 +981,9 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
 // CHECK1-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
@@ -992,78 +991,64 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END15:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK1:       for.cond1:
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], 0
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP2]], 0
 // CHECK1-NEXT:    br i1 [[CMP2]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK1:       cond.true:
 // CHECK1-NEXT:    br label [[COND_END:%.*]]
 // CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[A_ADDR]], align 4
 // CHECK1-NEXT:    br label [[COND_END]]
 // CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP5]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[COND]]
-// CHECK1-NEXT:    [[CMP3:%.*]] = icmp slt i32 4, [[ADD]]
-// CHECK1-NEXT:    br i1 [[CMP3]], label [[COND_TRUE4:%.*]], label [[COND_FALSE5:%.*]]
-// CHECK1:       cond.true4:
-// CHECK1-NEXT:    br label [[COND_END12:%.*]]
-// CHECK1:       cond.false5:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP3]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[TMP1]], [[COND]]
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
+// CHECK1-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[ADD]], 4
+// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP3]], [[CMP4]]
+// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY5:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body5:
 // CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    [[CMP6:%.*]] = icmp sle i32 [[TMP7]], 0
-// CHECK1-NEXT:    br i1 [[CMP6]], label [[COND_TRUE7:%.*]], label [[COND_FALSE8:%.*]]
-// CHECK1:       cond.true7:
-// CHECK1-NEXT:    br label [[COND_END9:%.*]]
-// CHECK1:       cond.false8:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    br label [[COND_END9]]
-// CHECK1:       cond.end9:
-// CHECK1-NEXT:    [[COND10:%.*]] = phi i32 [ 1, [[COND_TRUE7]] ], [ [[TMP8]], [[COND_FALSE8]] ]
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP6]], [[COND10]]
-// CHECK1-NEXT:    br label [[COND_END12]]
-// CHECK1:       cond.end12:
-// CHECK1-NEXT:    [[COND13:%.*]] = phi i32 [ 4, [[COND_TRUE4]] ], [ [[ADD11]], [[COND_END9]] ]
-// CHECK1-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP2]], [[COND13]]
-// CHECK1-NEXT:    br i1 [[CMP14]], label [[FOR_BODY15:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body15:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP9]], 3
-// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP10]])
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP6]], [[TMP7]]
+// CHECK1-NEXT:    store i32 [[ADD6]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP8]], 3
+// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP9]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP25:![0-9]+]]
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP24:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC17:%.*]]
-// CHECK1:       for.inc17:
+// CHECK1-NEXT:    br label [[FOR_INC8:%.*]]
+// CHECK1:       for.inc8:
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    [[CMP9:%.*]] = icmp sle i32 [[TMP11]], 0
+// CHECK1-NEXT:    br i1 [[CMP9]], label [[COND_TRUE10:%.*]], label [[COND_FALSE11:%.*]]
+// CHECK1:       cond.true10:
+// CHECK1-NEXT:    br label [[COND_END12:%.*]]
+// CHECK1:       cond.false11:
 // CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    [[CMP18:%.*]] = icmp sle i32 [[TMP12]], 0
-// CHECK1-NEXT:    br i1 [[CMP18]], label [[COND_TRUE19:%.*]], label [[COND_FALSE20:%.*]]
-// CHECK1:       cond.true19:
-// CHECK1-NEXT:    br label [[COND_END21:%.*]]
-// CHECK1:       cond.false20:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    br label [[COND_END21]]
-// CHECK1:       cond.end21:
-// CHECK1-NEXT:    [[COND22:%.*]] = phi i32 [ 1, [[COND_TRUE19]] ], [ [[TMP13]], [[COND_FALSE20]] ]
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP14]], [[COND22]]
-// CHECK1-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP26:![0-9]+]]
-// CHECK1:       for.end24:
+// CHECK1-NEXT:    br label [[COND_END12]]
+// CHECK1:       cond.end12:
+// CHECK1-NEXT:    [[COND13:%.*]] = phi i32 [ 1, [[COND_TRUE10]] ], [ [[TMP12]], [[COND_FALSE11]] ]
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD14:%.*]] = add nsw i32 [[TMP13]], [[COND13]]
+// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP25:![0-9]+]]
+// CHECK1:       for.end15:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -1076,19 +1061,20 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK1-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
 // CHECK1-NEXT:    store ptr [[ARRAY]], ptr [[__RANGE2]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META27:![0-9]+]]
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META26:![0-9]+]]
 // CHECK1-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP0]], i64 0, i64 0
 // CHECK1-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 12
 // CHECK1-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META27]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META26]]
 // CHECK1-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP1]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[ARRAYDECAY1]], ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META27]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META26]]
 // CHECK1-NEXT:    [[ARRAYDECAY2:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP2]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[ARRAYDECAY2]], ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__END2]], align 8
@@ -1111,37 +1097,31 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK1:       for.cond7:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
-// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
-// CHECK1-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK1:       cond.true:
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
-// CHECK1-NEXT:    br label [[COND_END:%.*]]
-// CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
-// CHECK1-NEXT:    br label [[COND_END]]
-// CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
-// CHECK1-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body14:
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK1-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP11]], 1
+// CHECK1-NEXT:    [[CMP11:%.*]] = icmp slt i64 [[ADD9]], [[ADD10]]
+// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP8]], [[CMP11]]
+// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY12:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body12:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i64 [[TMP12]], [[TMP13]]
+// CHECK1-NEXT:    store i64 [[ADD13]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
-// CHECK1-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
-// CHECK1-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD_PTR14:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
+// CHECK1-NEXT:    store ptr [[ADD_PTR14]], ptr [[__BEGIN2]], align 8
 // CHECK1-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
 // CHECK1-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
 // CHECK1-NEXT:    store double [[TMP17]], ptr [[V]], align 8
@@ -1149,18 +1129,18 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    call void (...) @body(double noundef [[TMP18]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP19]], 1
-// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP28:![0-9]+]]
+// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP27:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC16:%.*]]
-// CHECK1:       for.inc16:
+// CHECK1-NEXT:    br label [[FOR_INC15:%.*]]
+// CHECK1:       for.inc15:
 // CHECK1-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP20]], 5
-// CHECK1-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP29:![0-9]+]]
-// CHECK1:       for.end18:
+// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i64 [[TMP20]], 5
+// CHECK1-NEXT:    store i64 [[ADD16]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP28:![0-9]+]]
+// CHECK1:       for.end17:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -1174,20 +1154,21 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK1-NEXT:    store double 4.200000e+01, ptr [[C]], align 8
 // CHECK1-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
 // CHECK1-NEXT:    store ptr [[ARRAY]], ptr [[__RANGE2]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META27]]
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META26]]
 // CHECK1-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP0]], i64 0, i64 0
 // CHECK1-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 12
 // CHECK1-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META27]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META26]]
 // CHECK1-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP1]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[ARRAYDECAY1]], ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META27]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META26]]
 // CHECK1-NEXT:    [[ARRAYDECAY2:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP2]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[ARRAYDECAY2]], ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__END2]], align 8
@@ -1210,37 +1191,31 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK1:       for.cond7:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
-// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
-// CHECK1-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK1:       cond.true:
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
-// CHECK1-NEXT:    br label [[COND_END:%.*]]
-// CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
-// CHECK1-NEXT:    br label [[COND_END]]
-// CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
-// CHECK1-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body14:
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK1-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP11]], 1
+// CHECK1-NEXT:    [[CMP11:%.*]] = icmp slt i64 [[ADD9]], [[ADD10]]
+// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP8]], [[CMP11]]
+// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY12:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body12:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i64 [[TMP12]], [[TMP13]]
+// CHECK1-NEXT:    store i64 [[ADD13]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
-// CHECK1-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
-// CHECK1-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD_PTR14:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
+// CHECK1-NEXT:    store ptr [[ADD_PTR14]], ptr [[__BEGIN2]], align 8
 // CHECK1-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
 // CHECK1-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
 // CHECK1-NEXT:    store double [[TMP17]], ptr [[V]], align 8
@@ -1249,18 +1224,18 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    call void (...) @body(double noundef [[TMP18]], double noundef [[TMP19]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP20]], 1
-// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP30:![0-9]+]]
+// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP29:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC16:%.*]]
-// CHECK1:       for.inc16:
+// CHECK1-NEXT:    br label [[FOR_INC15:%.*]]
+// CHECK1:       for.inc15:
 // CHECK1-NEXT:    [[TMP21:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP21]], 5
-// CHECK1-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP31:![0-9]+]]
-// CHECK1:       for.end18:
+// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i64 [[TMP21]], 5
+// CHECK1-NEXT:    store i64 [[ADD16]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP30:![0-9]+]]
+// CHECK1:       for.end17:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -1293,8 +1268,9 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:  entry:
 // CHECK2-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[I2:%.*]] = alloca ptr, align 8
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK2-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK2-NEXT:    [[I:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
@@ -1308,45 +1284,41 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
 // CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END12:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK2:       for.cond4:
-// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 5
-// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
-// CHECK2-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK2:       cond.true:
-// CHECK2-NEXT:    br label [[COND_END:%.*]]
-// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP1]], 5
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
+// CHECK2-NEXT:    [[CMP6:%.*]] = icmp slt i32 [[ADD]], 4
+// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP5]], [[CMP6]]
+// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY7:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body7:
 // CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP4]], 5
-// CHECK2-NEXT:    br label [[COND_END]]
-// CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
-// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body8:
-// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
+// CHECK2-NEXT:    store i32 [[ADD8]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
 // CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK2-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
-// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[TMP6]], align 4
-// CHECK2-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2]], !align [[META3]]
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP8]])
+// CHECK2-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
+// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[TMP7]], align 4
+// CHECK2-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2]], !align [[META3]]
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP9]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP9]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP4:![0-9]+]]
 // CHECK2:       for.end:
 // CHECK2-NEXT:    br label [[FOR_INC10:%.*]]
 // CHECK2:       for.inc10:
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP10]], 5
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP11]], 5
 // CHECK2-NEXT:    store i32 [[ADD11]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
 // CHECK2:       for.end12:
@@ -1370,8 +1342,9 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -1400,54 +1373,48 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK2-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP8]], [[ADD5]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 [[TMP10]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK2:       for.cond6:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[ADD7:%.*]] = add i32 [[TMP12]], 1
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD8:%.*]] = add i32 [[TMP13]], 5
-// CHECK2-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
-// CHECK2-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK2:       cond.true:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[ADD10:%.*]] = add i32 [[TMP14]], 1
-// CHECK2-NEXT:    br label [[COND_END:%.*]]
-// CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD11:%.*]] = add i32 [[TMP15]], 5
-// CHECK2-NEXT:    br label [[COND_END]]
-// CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP11]], [[COND]]
-// CHECK2-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body13:
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP10]], 5
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD8:%.*]] = add i32 [[TMP11]], [[TMP12]]
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD9:%.*]] = add i32 [[TMP13]], 1
+// CHECK2-NEXT:    [[CMP10:%.*]] = icmp ult i32 [[ADD8]], [[ADD9]]
+// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP7]], [[CMP10]]
+// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY11:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body11:
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD12:%.*]] = add i32 [[TMP14]], [[TMP15]]
+// CHECK2-NEXT:    store i32 [[ADD12]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
 // CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
 // CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP17]], [[TMP18]]
-// CHECK2-NEXT:    [[ADD14:%.*]] = add i32 [[TMP16]], [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[ADD13:%.*]] = add i32 [[TMP16]], [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD13]], ptr [[I]], align 4
 // CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
 // CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP19]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP20]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP7:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC15:%.*]]
-// CHECK2:       for.inc15:
+// CHECK2-NEXT:    br label [[FOR_INC14:%.*]]
+// CHECK2:       for.inc14:
 // CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD16:%.*]] = add i32 [[TMP21]], 5
-// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD15:%.*]] = add i32 [[TMP21]], 5
+// CHECK2-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
-// CHECK2:       for.end17:
+// CHECK2:       for.end16:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -1461,8 +1428,9 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK2-NEXT:    store double 4.200000e+01, ptr [[C]], align 8
 // CHECK2-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
@@ -1497,37 +1465,31 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK2:       for.cond7:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
-// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
-// CHECK2-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK2:       cond.true:
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
-// CHECK2-NEXT:    br label [[COND_END:%.*]]
-// CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
-// CHECK2-NEXT:    br label [[COND_END]]
-// CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
-// CHECK2-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body14:
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK2-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP11]], 1
+// CHECK2-NEXT:    [[CMP11:%.*]] = icmp slt i64 [[ADD9]], [[ADD10]]
+// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP8]], [[CMP11]]
+// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY12:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body12:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i64 [[TMP12]], [[TMP13]]
+// CHECK2-NEXT:    store i64 [[ADD13]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
-// CHECK2-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
-// CHECK2-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD_PTR14:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
+// CHECK2-NEXT:    store ptr [[ADD_PTR14]], ptr [[__BEGIN2]], align 8
 // CHECK2-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
 // CHECK2-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
 // CHECK2-NEXT:    store double [[TMP17]], ptr [[V]], align 8
@@ -1536,18 +1498,18 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    call void (...) @body(double noundef [[TMP18]], double noundef [[TMP19]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP20]], 1
-// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP10:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC16:%.*]]
-// CHECK2:       for.inc16:
+// CHECK2-NEXT:    br label [[FOR_INC15:%.*]]
+// CHECK2:       for.inc15:
 // CHECK2-NEXT:    [[TMP21:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP21]], 5
-// CHECK2-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i64 [[TMP21]], 5
+// CHECK2-NEXT:    store i64 [[ADD16]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP11:![0-9]+]]
-// CHECK2:       for.end18:
+// CHECK2:       for.end17:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -1558,11 +1520,13 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -1573,94 +1537,90 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END30:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END26:%.*]]
 // CHECK2:       for.body:
 // CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK2:       for.cond1:
 // CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 4
-// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END27:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END23:%.*]]
 // CHECK2:       for.body3:
-// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 [[TMP2]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK2:       for.cond4:
-// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], 5
-// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
-// CHECK2-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK2:       cond.true:
-// CHECK2-NEXT:    br label [[COND_END:%.*]]
-// CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP5]], 5
-// CHECK2-NEXT:    br label [[COND_END]]
-// CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP3]], [[COND]]
-// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END24:%.*]]
-// CHECK2:       for.body8:
-// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
-// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND10:%.*]]
-// CHECK2:       for.cond10:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP2]], 5
+// CHECK2-NEXT:    br i1 [[CMP5]], label [[FOR_BODY6:%.*]], label [[FOR_END20:%.*]]
+// CHECK2:       for.body6:
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[TMP4]]
+// CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
+// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND8:%.*]]
+// CHECK2:       for.cond8:
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP6]], 5
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP7]], [[TMP8]]
+// CHECK2-NEXT:    [[CMP11:%.*]] = icmp slt i32 [[ADD10]], 4
+// CHECK2-NEXT:    br i1 [[CMP11]], label [[LAND_RHS:%.*]], label [[LAND_END:%.*]]
+// CHECK2:       land.rhs:
 // CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP9]], 5
-// CHECK2-NEXT:    [[CMP12:%.*]] = icmp slt i32 4, [[ADD11]]
-// CHECK2-NEXT:    br i1 [[CMP12]], label [[COND_TRUE13:%.*]], label [[COND_FALSE14:%.*]]
-// CHECK2:       cond.true13:
-// CHECK2-NEXT:    br label [[COND_END16:%.*]]
-// CHECK2:       cond.false14:
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP10]], 5
-// CHECK2-NEXT:    br label [[COND_END16]]
-// CHECK2:       cond.end16:
-// CHECK2-NEXT:    [[COND17:%.*]] = phi i32 [ 4, [[COND_TRUE13]] ], [ [[ADD15]], [[COND_FALSE14]] ]
-// CHECK2-NEXT:    [[CMP18:%.*]] = icmp slt i32 [[TMP8]], [[COND17]]
-// CHECK2-NEXT:    br i1 [[CMP18]], label [[FOR_BODY19:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body19:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[TMP11]], 3
-// CHECK2-NEXT:    [[ADD21:%.*]] = add nsw i32 7, [[MUL20]]
-// CHECK2-NEXT:    store i32 [[ADD21]], ptr [[J]], align 4
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP12]], i32 noundef [[TMP13]])
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
+// CHECK2-NEXT:    [[CMP13:%.*]] = icmp slt i32 [[ADD12]], 4
+// CHECK2-NEXT:    br label [[LAND_END]]
+// CHECK2:       land.end:
+// CHECK2-NEXT:    [[TMP11:%.*]] = phi i1 [ false, [[FOR_COND8]] ], [ [[CMP13]], [[LAND_RHS]] ]
+// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP9]], [[TMP11]]
+// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body14:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP12]], [[TMP13]]
+// CHECK2-NEXT:    store i32 [[ADD15]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[MUL16:%.*]] = mul nsw i32 [[TMP14]], 3
+// CHECK2-NEXT:    [[ADD17:%.*]] = add nsw i32 7, [[MUL16]]
+// CHECK2-NEXT:    store i32 [[ADD17]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP15]], i32 noundef [[TMP16]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND10]], !llvm.loop [[LOOP12:![0-9]+]]
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP17]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP12:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC22:%.*]]
-// CHECK2:       for.inc22:
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC23:%.*]] = add nsw i32 [[TMP15]], 1
-// CHECK2-NEXT:    store i32 [[INC23]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_INC18:%.*]]
+// CHECK2:       for.inc18:
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC19:%.*]] = add nsw i32 [[TMP18]], 1
+// CHECK2-NEXT:    store i32 [[INC19]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP13:![0-9]+]]
-// CHECK2:       for.end24:
-// CHECK2-NEXT:    br label [[FOR_INC25:%.*]]
-// CHECK2:       for.inc25:
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD26:%.*]] = add nsw i32 [[TMP16]], 5
-// CHECK2-NEXT:    store i32 [[ADD26]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2:       for.end20:
+// CHECK2-NEXT:    br label [[FOR_INC21:%.*]]
+// CHECK2:       for.inc21:
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD22:%.*]] = add nsw i32 [[TMP19]], 5
+// CHECK2-NEXT:    store i32 [[ADD22]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP14:![0-9]+]]
-// CHECK2:       for.end27:
-// CHECK2-NEXT:    br label [[FOR_INC28:%.*]]
-// CHECK2:       for.inc28:
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD29:%.*]] = add nsw i32 [[TMP17]], 5
-// CHECK2-NEXT:    store i32 [[ADD29]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2:       for.end23:
+// CHECK2-NEXT:    br label [[FOR_INC24:%.*]]
+// CHECK2:       for.inc24:
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD25:%.*]] = add nsw i32 [[TMP20]], 5
+// CHECK2-NEXT:    store i32 [[ADD25]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP15:![0-9]+]]
-// CHECK2:       for.end30:
+// CHECK2:       for.end26:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -1718,14 +1678,14 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP7]], 4
-// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END22:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
 // CHECK2:       for.body:
 // CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND3:%.*]]
 // CHECK2:       for.cond3:
 // CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP8]], 5
-// CHECK2-NEXT:    br i1 [[CMP4]], label [[FOR_BODY5:%.*]], label [[FOR_END19:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP4]], label [[FOR_BODY5:%.*]], label [[FOR_END21:%.*]]
 // CHECK2:       for.body5:
 // CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
@@ -1740,57 +1700,61 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond9:
 // CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
 // CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i32 [[TMP12]], 5
-// CHECK2-NEXT:    br i1 [[CMP10]], label [[FOR_BODY11:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body11:
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP13]], [[TMP14]]
-// CHECK2-NEXT:    store i32 [[ADD12]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[MUL13:%.*]] = mul nsw i32 [[TMP15]], 3
-// CHECK2-NEXT:    [[ADD14:%.*]] = add nsw i32 7, [[MUL13]]
-// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[J]], align 4
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP15:%.*]] = icmp slt i32 [[TMP16]], 4
-// CHECK2-NEXT:    br i1 [[CMP15]], label [[LAND_LHS_TRUE:%.*]], label [[IF_END:%.*]]
-// CHECK2:       land.lhs.true:
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[CMP16:%.*]] = icmp slt i32 [[TMP17]], 4
-// CHECK2-NEXT:    br i1 [[CMP16]], label [[IF_THEN:%.*]], label [[IF_END]]
-// CHECK2:       if.then:
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[J]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP18]], i32 noundef [[TMP19]])
-// CHECK2-NEXT:    br label [[IF_END]]
-// CHECK2:       if.end:
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP13]], [[TMP14]]
+// CHECK2-NEXT:    [[CMP12:%.*]] = icmp slt i32 [[ADD11]], 4
+// CHECK2-NEXT:    br i1 [[CMP12]], label [[LAND_RHS:%.*]], label [[LAND_END:%.*]]
+// CHECK2:       land.rhs:
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP15]], [[TMP16]]
+// CHECK2-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[ADD13]], 4
+// CHECK2-NEXT:    br label [[LAND_END]]
+// CHECK2:       land.end:
+// CHECK2-NEXT:    [[TMP17:%.*]] = phi i1 [ false, [[FOR_COND9]] ], [ [[CMP14]], [[LAND_RHS]] ]
+// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP10]], [[TMP17]]
+// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY15:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body15:
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP18]], [[TMP19]]
+// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[MUL17:%.*]] = mul nsw i32 [[TMP20]], 3
+// CHECK2-NEXT:    [[ADD18:%.*]] = add nsw i32 7, [[MUL17]]
+// CHECK2-NEXT:    store i32 [[ADD18]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP21]], i32 noundef [[TMP22]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP20]], 1
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP23]], 1
 // CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND9]], !llvm.loop [[LOOP16:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC17:%.*]]
-// CHECK2:       for.inc17:
-// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC18:%.*]] = add nsw i32 [[TMP21]], 1
-// CHECK2-NEXT:    store i32 [[INC18]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_INC19:%.*]]
+// CHECK2:       for.inc19:
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC20:%.*]] = add nsw i32 [[TMP24]], 1
+// CHECK2-NEXT:    store i32 [[INC20]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND3]], !llvm.loop [[LOOP17:![0-9]+]]
-// CHECK2:       for.end19:
-// CHECK2-NEXT:    br label [[FOR_INC20:%.*]]
-// CHECK2:       for.inc20:
-// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD21:%.*]] = add nsw i32 [[TMP22]], 5
-// CHECK2-NEXT:    store i32 [[ADD21]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2:       for.end21:
+// CHECK2-NEXT:    br label [[FOR_INC22:%.*]]
+// CHECK2:       for.inc22:
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP25]], 5
+// CHECK2-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP18:![0-9]+]]
-// CHECK2:       for.end22:
+// CHECK2:       for.end24:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
-// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP23]], 1
-// CHECK2-NEXT:    store i32 [[ADD23]], ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[ADD25:%.*]] = add nsw i32 [[TMP26]], 1
+// CHECK2-NEXT:    store i32 [[ADD25]], ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -1865,14 +1829,14 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP9]], 4
-// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY:%.*]], label [[FOR_END27:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY:%.*]], label [[FOR_END29:%.*]]
 // CHECK2:       for.body:
 // CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND8:%.*]]
 // CHECK2:       for.cond8:
 // CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP10]], 5
-// CHECK2-NEXT:    br i1 [[CMP9]], label [[FOR_BODY10:%.*]], label [[FOR_END24:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP9]], label [[FOR_BODY10:%.*]], label [[FOR_END26:%.*]]
 // CHECK2:       for.body10:
 // CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
@@ -1887,57 +1851,61 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond14:
 // CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
 // CHECK2-NEXT:    [[CMP15:%.*]] = icmp slt i32 [[TMP14]], 5
-// CHECK2-NEXT:    br i1 [[CMP15]], label [[FOR_BODY16:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body16:
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD17:%.*]] = add nsw i32 [[TMP15]], [[TMP16]]
-// CHECK2-NEXT:    store i32 [[ADD17]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[MUL18:%.*]] = mul nsw i32 [[TMP17]], 3
-// CHECK2-NEXT:    [[ADD19:%.*]] = add nsw i32 7, [[MUL18]]
-// CHECK2-NEXT:    store i32 [[ADD19]], ptr [[J]], align 4
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP20:%.*]] = icmp slt i32 [[TMP18]], 4
-// CHECK2-NEXT:    br i1 [[CMP20]], label [[LAND_LHS_TRUE:%.*]], label [[IF_END:%.*]]
-// CHECK2:       land.lhs.true:
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[CMP21:%.*]] = icmp slt i32 [[TMP19]], 4
-// CHECK2-NEXT:    br i1 [[CMP21]], label [[IF_THEN:%.*]], label [[IF_END]]
-// CHECK2:       if.then:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[J]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP20]], i32 noundef [[TMP21]])
-// CHECK2-NEXT:    br label [[IF_END]]
-// CHECK2:       if.end:
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP15]], [[TMP16]]
+// CHECK2-NEXT:    [[CMP17:%.*]] = icmp slt i32 [[ADD16]], 4
+// CHECK2-NEXT:    br i1 [[CMP17]], label [[LAND_RHS:%.*]], label [[LAND_END:%.*]]
+// CHECK2:       land.rhs:
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD18:%.*]] = add nsw i32 [[TMP17]], [[TMP18]]
+// CHECK2-NEXT:    [[CMP19:%.*]] = icmp slt i32 [[ADD18]], 4
+// CHECK2-NEXT:    br label [[LAND_END]]
+// CHECK2:       land.end:
+// CHECK2-NEXT:    [[TMP19:%.*]] = phi i1 [ false, [[FOR_COND14]] ], [ [[CMP19]], [[LAND_RHS]] ]
+// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP15]], [[TMP19]]
+// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY20:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body20:
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD21:%.*]] = add nsw i32 [[TMP20]], [[TMP21]]
+// CHECK2-NEXT:    store i32 [[ADD21]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[MUL22:%.*]] = mul nsw i32 [[TMP22]], 3
+// CHECK2-NEXT:    [[ADD23:%.*]] = add nsw i32 7, [[MUL22]]
+// CHECK2-NEXT:    store i32 [[ADD23]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP23]], i32 noundef [[TMP24]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP22]], 1
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP25]], 1
 // CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND14]], !llvm.loop [[LOOP19:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC22:%.*]]
-// CHECK2:       for.inc22:
-// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC23:%.*]] = add nsw i32 [[TMP23]], 1
-// CHECK2-NEXT:    store i32 [[INC23]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_INC24:%.*]]
+// CHECK2:       for.inc24:
+// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC25:%.*]] = add nsw i32 [[TMP26]], 1
+// CHECK2-NEXT:    store i32 [[INC25]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP20:![0-9]+]]
-// CHECK2:       for.end24:
-// CHECK2-NEXT:    br label [[FOR_INC25:%.*]]
-// CHECK2:       for.inc25:
-// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD26:%.*]] = add nsw i32 [[TMP24]], 5
-// CHECK2-NEXT:    store i32 [[ADD26]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2:       for.end26:
+// CHECK2-NEXT:    br label [[FOR_INC27:%.*]]
+// CHECK2:       for.inc27:
+// CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD28:%.*]] = add nsw i32 [[TMP27]], 5
+// CHECK2-NEXT:    store i32 [[ADD28]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP21:![0-9]+]]
-// CHECK2:       for.end27:
+// CHECK2:       for.end29:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
-// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[ADD28:%.*]] = add nsw i32 [[TMP25]], 1
-// CHECK2-NEXT:    store i32 [[ADD28]], ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[ADD30:%.*]] = add nsw i32 [[TMP28]], 1
+// CHECK2-NEXT:    store i32 [[ADD30]], ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -1953,6 +1921,7 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[_TMP1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[_TMP2:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
@@ -1965,12 +1934,12 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK2-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
-// CHECK2-NEXT:    store i32 4, ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    store i32 19, ptr [[DOTOMP_UB]], align 4
 // CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
 // CHECK2-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
 // CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// CHECK2-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], 19
 // CHECK2-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK2:       cond.true:
 // CHECK2-NEXT:    br label [[COND_END:%.*]]
@@ -1978,7 +1947,7 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
 // CHECK2-NEXT:    br label [[COND_END]]
 // CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[TMP2]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 19, [[COND_TRUE]] ], [ [[TMP2]], [[COND_FALSE]] ]
 // CHECK2-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
 // CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
 // CHECK2-NEXT:    store i32 [[TMP3]], ptr [[DOTOMP_IV]], align 4
@@ -1986,60 +1955,65 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       omp.inner.for.cond:
 // CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// CHECK2-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], [[TMP5]]
-// CHECK2-NEXT:    br i1 [[CMP2]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK2-NEXT:    [[CMP3:%.*]] = icmp sle i32 [[TMP4]], [[TMP5]]
+// CHECK2-NEXT:    br i1 [[CMP3]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
 // CHECK2:       omp.inner.for.body:
 // CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP6]], 5
+// CHECK2-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP6]], 20
 // CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[DIV]], 5
 // CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[DIV3:%.*]] = sdiv i32 [[TMP8]], 5
-// CHECK2-NEXT:    [[MUL4:%.*]] = mul nsw i32 [[DIV3]], 5
-// CHECK2-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP7]], [[MUL4]]
-// CHECK2-NEXT:    [[MUL5:%.*]] = mul nsw i32 [[SUB]], 1
-// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL5]]
-// CHECK2-NEXT:    store i32 [[ADD6]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
-// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL8:%.*]] = mul nsw i32 [[TMP11]], 3
-// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL8]]
-// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i32 [[TMP12]], 4
-// CHECK2-NEXT:    br i1 [[CMP10]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK2:       if.then:
-// CHECK2-NEXT:    store i32 7, ptr [[J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND:%.*]]
-// CHECK2:       for.cond:
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
-// CHECK2-NEXT:    [[CMP11:%.*]] = icmp slt i32 [[TMP13]], 17
-// CHECK2-NEXT:    br i1 [[CMP11]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[J]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP14]], i32 noundef [[TMP15]])
-// CHECK2-NEXT:    br label [[FOR_INC:%.*]]
-// CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[J]], align 4
-// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP16]], 3
-// CHECK2-NEXT:    store i32 [[ADD12]], ptr [[J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP22:![0-9]+]]
-// CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[IF_END]]
-// CHECK2:       if.end:
+// CHECK2-NEXT:    [[DIV4:%.*]] = sdiv i32 [[TMP8]], 20
+// CHECK2-NEXT:    [[MUL5:%.*]] = mul nsw i32 [[DIV4]], 20
+// CHECK2-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP7]], [[MUL5]]
+// CHECK2-NEXT:    [[DIV6:%.*]] = sdiv i32 [[SUB]], 4
+// CHECK2-NEXT:    [[MUL7:%.*]] = mul nsw i32 [[DIV6]], 1
+// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i32 0, [[MUL7]]
+// CHECK2-NEXT:    store i32 [[ADD8]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[DIV9:%.*]] = sdiv i32 [[TMP10]], 20
+// CHECK2-NEXT:    [[MUL10:%.*]] = mul nsw i32 [[DIV9]], 20
+// CHECK2-NEXT:    [[SUB11:%.*]] = sub nsw i32 [[TMP9]], [[MUL10]]
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[DIV12:%.*]] = sdiv i32 [[TMP12]], 20
+// CHECK2-NEXT:    [[MUL13:%.*]] = mul nsw i32 [[DIV12]], 20
+// CHECK2-NEXT:    [[SUB14:%.*]] = sub nsw i32 [[TMP11]], [[MUL13]]
+// CHECK2-NEXT:    [[DIV15:%.*]] = sdiv i32 [[SUB14]], 4
+// CHECK2-NEXT:    [[MUL16:%.*]] = mul nsw i32 [[DIV15]], 4
+// CHECK2-NEXT:    [[SUB17:%.*]] = sub nsw i32 [[SUB11]], [[MUL16]]
+// CHECK2-NEXT:    [[MUL18:%.*]] = mul nsw i32 [[SUB17]], 3
+// CHECK2-NEXT:    [[ADD19:%.*]] = add nsw i32 7, [[MUL18]]
+// CHECK2-NEXT:    store i32 [[ADD19]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP13]], [[TMP14]]
+// CHECK2-NEXT:    store i32 [[ADD20]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL21:%.*]] = mul nsw i32 [[TMP15]], 3
+// CHECK2-NEXT:    [[ADD22:%.*]] = add nsw i32 7, [[MUL21]]
+// CHECK2-NEXT:    store i32 [[ADD22]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP16]], [[TMP17]]
+// CHECK2-NEXT:    [[CMP24:%.*]] = icmp slt i32 [[ADD23]], 4
+// CHECK2-NEXT:    br i1 [[CMP24]], label [[OMP_TILE_PRED_THEN:%.*]], label [[OMP_TILE_PRED_END:%.*]]
+// CHECK2:       omp_tile.pred.then:
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP18]], i32 noundef [[TMP19]])
+// CHECK2-NEXT:    br label [[OMP_TILE_PRED_END]]
+// CHECK2:       omp_tile.pred.end:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP17]], 1
-// CHECK2-NEXT:    store i32 [[ADD13]], ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[ADD25:%.*]] = add nsw i32 [[TMP20]], 1
+// CHECK2-NEXT:    store i32 [[ADD25]], ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -2110,38 +2084,37 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP8]], 5
-// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body:
 // CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
-// CHECK2-NEXT:    store i32 [[ADD3]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL4:%.*]] = mul nsw i32 [[TMP11]], 3
-// CHECK2-NEXT:    [[ADD5:%.*]] = add nsw i32 7, [[MUL4]]
-// CHECK2-NEXT:    store i32 [[ADD5]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP6:%.*]] = icmp slt i32 [[TMP12]], 4
-// CHECK2-NEXT:    br i1 [[CMP6]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK2:       if.then:
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP13]])
-// CHECK2-NEXT:    br label [[IF_END]]
-// CHECK2:       if.end:
+// CHECK2-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[ADD3]], 4
+// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP2]], [[CMP4]]
+// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body:
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD5:%.*]] = add nsw i32 [[TMP11]], [[TMP12]]
+// CHECK2-NEXT:    store i32 [[ADD5]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL6:%.*]] = mul nsw i32 [[TMP13]], 3
+// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL6]]
+// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP14]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP15]], 1
 // CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP23:![0-9]+]]
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP22:![0-9]+]]
 // CHECK2:       for.end:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 [[TMP15]], 1
-// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP16]], 1
+// CHECK2-NEXT:    store i32 [[ADD8]], ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -2155,8 +2128,9 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:  entry:
 // CHECK2-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
 // CHECK2-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
@@ -2164,78 +2138,64 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END15:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK2:       for.cond1:
-// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], 0
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP2]], 0
 // CHECK2-NEXT:    br i1 [[CMP2]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK2:       cond.true:
 // CHECK2-NEXT:    br label [[COND_END:%.*]]
 // CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[A_ADDR]], align 4
 // CHECK2-NEXT:    br label [[COND_END]]
 // CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP5]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[COND]]
-// CHECK2-NEXT:    [[CMP3:%.*]] = icmp slt i32 4, [[ADD]]
-// CHECK2-NEXT:    br i1 [[CMP3]], label [[COND_TRUE4:%.*]], label [[COND_FALSE5:%.*]]
-// CHECK2:       cond.true4:
-// CHECK2-NEXT:    br label [[COND_END12:%.*]]
-// CHECK2:       cond.false5:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP3]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[TMP1]], [[COND]]
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
+// CHECK2-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[ADD]], 4
+// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP3]], [[CMP4]]
+// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY5:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body5:
 // CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    [[CMP6:%.*]] = icmp sle i32 [[TMP7]], 0
-// CHECK2-NEXT:    br i1 [[CMP6]], label [[COND_TRUE7:%.*]], label [[COND_FALSE8:%.*]]
-// CHECK2:       cond.true7:
-// CHECK2-NEXT:    br label [[COND_END9:%.*]]
-// CHECK2:       cond.false8:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    br label [[COND_END9]]
-// CHECK2:       cond.end9:
-// CHECK2-NEXT:    [[COND10:%.*]] = phi i32 [ 1, [[COND_TRUE7]] ], [ [[TMP8]], [[COND_FALSE8]] ]
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP6]], [[COND10]]
-// CHECK2-NEXT:    br label [[COND_END12]]
-// CHECK2:       cond.end12:
-// CHECK2-NEXT:    [[COND13:%.*]] = phi i32 [ 4, [[COND_TRUE4]] ], [ [[ADD11]], [[COND_END9]] ]
-// CHECK2-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP2]], [[COND13]]
-// CHECK2-NEXT:    br i1 [[CMP14]], label [[FOR_BODY15:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body15:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP9]], 3
-// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP10]])
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP6]], [[TMP7]]
+// CHECK2-NEXT:    store i32 [[ADD6]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP8]], 3
+// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP9]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP26:![0-9]+]]
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP25:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC17:%.*]]
-// CHECK2:       for.inc17:
+// CHECK2-NEXT:    br label [[FOR_INC8:%.*]]
+// CHECK2:       for.inc8:
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    [[CMP9:%.*]] = icmp sle i32 [[TMP11]], 0
+// CHECK2-NEXT:    br i1 [[CMP9]], label [[COND_TRUE10:%.*]], label [[COND_FALSE11:%.*]]
+// CHECK2:       cond.true10:
+// CHECK2-NEXT:    br label [[COND_END12:%.*]]
+// CHECK2:       cond.false11:
 // CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    [[CMP18:%.*]] = icmp sle i32 [[TMP12]], 0
-// CHECK2-NEXT:    br i1 [[CMP18]], label [[COND_TRUE19:%.*]], label [[COND_FALSE20:%.*]]
-// CHECK2:       cond.true19:
-// CHECK2-NEXT:    br label [[COND_END21:%.*]]
-// CHECK2:       cond.false20:
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    br label [[COND_END21]]
-// CHECK2:       cond.end21:
-// CHECK2-NEXT:    [[COND22:%.*]] = phi i32 [ 1, [[COND_TRUE19]] ], [ [[TMP13]], [[COND_FALSE20]] ]
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP14]], [[COND22]]
-// CHECK2-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP27:![0-9]+]]
-// CHECK2:       for.end24:
+// CHECK2-NEXT:    br label [[COND_END12]]
+// CHECK2:       cond.end12:
+// CHECK2-NEXT:    [[COND13:%.*]] = phi i32 [ 1, [[COND_TRUE10]] ], [ [[TMP12]], [[COND_FALSE11]] ]
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD14:%.*]] = add nsw i32 [[TMP13]], [[COND13]]
+// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP26:![0-9]+]]
+// CHECK2:       for.end15:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -2248,8 +2208,9 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK2-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
 // CHECK2-NEXT:    store ptr [[ARRAY]], ptr [[__RANGE2]], align 8
@@ -2283,37 +2244,31 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK2:       for.cond7:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
-// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
-// CHECK2-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK2:       cond.true:
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
-// CHECK2-NEXT:    br label [[COND_END:%.*]]
-// CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
-// CHECK2-NEXT:    br label [[COND_END]]
-// CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
-// CHECK2-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body14:
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK2-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP11]], 1
+// CHECK2-NEXT:    [[CMP11:%.*]] = icmp slt i64 [[ADD9]], [[ADD10]]
+// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP8]], [[CMP11]]
+// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY12:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body12:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i64 [[TMP12]], [[TMP13]]
+// CHECK2-NEXT:    store i64 [[ADD13]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
-// CHECK2-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
-// CHECK2-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD_PTR14:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
+// CHECK2-NEXT:    store ptr [[ADD_PTR14]], ptr [[__BEGIN2]], align 8
 // CHECK2-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
 // CHECK2-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
 // CHECK2-NEXT:    store double [[TMP17]], ptr [[V]], align 8
@@ -2321,18 +2276,18 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    call void (...) @body(double noundef [[TMP18]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP19]], 1
-// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP28:![0-9]+]]
+// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP27:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC16:%.*]]
-// CHECK2:       for.inc16:
+// CHECK2-NEXT:    br label [[FOR_INC15:%.*]]
+// CHECK2:       for.inc15:
 // CHECK2-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP20]], 5
-// CHECK2-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP29:![0-9]+]]
-// CHECK2:       for.end18:
+// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i64 [[TMP20]], 5
+// CHECK2-NEXT:    store i64 [[ADD16]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP28:![0-9]+]]
+// CHECK2:       for.end17:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -2352,8 +2307,9 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[START_ADDR]], align 4
@@ -2377,53 +2333,47 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK2-NEXT:    [[ADD5:%.*]] = add i32 [[TMP6]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP5]], [[ADD5]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK2:       for.cond6:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[ADD7:%.*]] = add i32 [[TMP9]], 1
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD8:%.*]] = add i32 [[TMP10]], 5
-// CHECK2-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
-// CHECK2-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK2:       cond.true:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[ADD10:%.*]] = add i32 [[TMP11]], 1
-// CHECK2-NEXT:    br label [[COND_END:%.*]]
-// CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD11:%.*]] = add i32 [[TMP12]], 5
-// CHECK2-NEXT:    br label [[COND_END]]
-// CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP8]], [[COND]]
-// CHECK2-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body13:
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP7]], 5
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD8:%.*]] = add i32 [[TMP8]], [[TMP9]]
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD9:%.*]] = add i32 [[TMP10]], 1
+// CHECK2-NEXT:    [[CMP10:%.*]] = icmp ult i32 [[ADD8]], [[ADD9]]
+// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP7]], [[CMP10]]
+// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY11:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body11:
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD12:%.*]] = add i32 [[TMP11]], [[TMP12]]
+// CHECK2-NEXT:    store i32 [[ADD12]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
 // CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP14]], 3
-// CHECK2-NEXT:    [[ADD14:%.*]] = add i32 [[TMP13]], [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[ADD13:%.*]] = add i32 [[TMP13]], [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD13]], ptr [[I]], align 4
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
 // CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP15]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP16]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP30:![0-9]+]]
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP29:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC15:%.*]]
-// CHECK2:       for.inc15:
+// CHECK2-NEXT:    br label [[FOR_INC14:%.*]]
+// CHECK2:       for.inc14:
 // CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD16:%.*]] = add i32 [[TMP17]], 5
-// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP31:![0-9]+]]
-// CHECK2:       for.end17:
+// CHECK2-NEXT:    [[ADD15:%.*]] = add i32 [[TMP17]], 5
+// CHECK2-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP30:![0-9]+]]
+// CHECK2:       for.end16:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -2452,16 +2402,15 @@ extern "C" void foo10(data_t data) {
 // CHECK1: [[LOOP17]] = distinct !{[[LOOP17]], [[META5]]}
 // CHECK1: [[LOOP18]] = distinct !{[[LOOP18]], [[META5]]}
 // CHECK1: [[LOOP19]] = distinct !{[[LOOP19]], [[META5]]}
-// CHECK1: [[LOOP20]] = distinct !{[[LOOP20]], [[META5]]}
+// CHECK1: [[LOOP22]] = distinct !{[[LOOP22]], [[META5]]}
 // CHECK1: [[LOOP23]] = distinct !{[[LOOP23]], [[META5]]}
 // CHECK1: [[LOOP24]] = distinct !{[[LOOP24]], [[META5]]}
 // CHECK1: [[LOOP25]] = distinct !{[[LOOP25]], [[META5]]}
-// CHECK1: [[LOOP26]] = distinct !{[[LOOP26]], [[META5]]}
-// CHECK1: [[META27]] = !{i64 8}
+// CHECK1: [[META26]] = !{i64 8}
+// CHECK1: [[LOOP27]] = distinct !{[[LOOP27]], [[META5]]}
 // CHECK1: [[LOOP28]] = distinct !{[[LOOP28]], [[META5]]}
 // CHECK1: [[LOOP29]] = distinct !{[[LOOP29]], [[META5]]}
 // CHECK1: [[LOOP30]] = distinct !{[[LOOP30]], [[META5]]}
-// CHECK1: [[LOOP31]] = distinct !{[[LOOP31]], [[META5]]}
 //.
 // CHECK2: [[META2]] = !{}
 // CHECK2: [[META3]] = !{i64 4}
@@ -2484,11 +2433,10 @@ extern "C" void foo10(data_t data) {
 // CHECK2: [[LOOP20]] = distinct !{[[LOOP20]], [[META5]]}
 // CHECK2: [[LOOP21]] = distinct !{[[LOOP21]], [[META5]]}
 // CHECK2: [[LOOP22]] = distinct !{[[LOOP22]], [[META5]]}
-// CHECK2: [[LOOP23]] = distinct !{[[LOOP23]], [[META5]]}
+// CHECK2: [[LOOP25]] = distinct !{[[LOOP25]], [[META5]]}
 // CHECK2: [[LOOP26]] = distinct !{[[LOOP26]], [[META5]]}
 // CHECK2: [[LOOP27]] = distinct !{[[LOOP27]], [[META5]]}
 // CHECK2: [[LOOP28]] = distinct !{[[LOOP28]], [[META5]]}
 // CHECK2: [[LOOP29]] = distinct !{[[LOOP29]], [[META5]]}
 // CHECK2: [[LOOP30]] = distinct !{[[LOOP30]], [[META5]]}
-// CHECK2: [[LOOP31]] = distinct !{[[LOOP31]], [[META5]]}
 //.
diff --git a/clang/test/OpenMP/tile_codegen_for_dependent.cpp b/clang/test/OpenMP/tile_codegen_for_dependent.cpp
index aadc8e58bebfc..808f0e71dc3b0 100644
--- a/clang/test/OpenMP/tile_codegen_for_dependent.cpp
+++ b/clang/test/OpenMP/tile_codegen_for_dependent.cpp
@@ -114,35 +114,34 @@ extern "C" void body(...) {}
 // IR:       [[FOR_COND]]:
 // IR-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    [[CMP16:%.*]] = icmp ult i32 [[TMP21]], 4
-// IR-NEXT:    br i1 [[CMP16]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY]]:
 // IR-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I11]], align 4
 // IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    [[ADD17:%.*]] = add i32 [[TMP22]], [[TMP23]]
-// IR-NEXT:    store i32 [[ADD17]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[MUL18:%.*]] = mul i32 [[TMP25]], [[TMP26]]
-// IR-NEXT:    [[ADD19:%.*]] = add i32 [[TMP24]], [[MUL18]]
-// IR-NEXT:    store i32 [[ADD19]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[ADD20:%.*]] = add i32 [[TMP28]], 1
-// IR-NEXT:    [[CMP21:%.*]] = icmp ult i32 [[TMP27]], [[ADD20]]
-// IR-NEXT:    br i1 [[CMP21]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
-// IR:       [[IF_THEN]]:
-// IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void (...) @body(i32 noundef [[TMP29]], i32 noundef [[TMP30]], i32 noundef [[TMP31]], i32 noundef [[TMP32]])
-// IR-NEXT:    br label %[[IF_END]]
-// IR:       [[IF_END]]:
+// IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    [[ADD18:%.*]] = add i32 [[TMP24]], 1
+// IR-NEXT:    [[CMP19:%.*]] = icmp ult i32 [[ADD17]], [[ADD18]]
+// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP16]], [[CMP19]]
+// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY]]:
+// IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I11]], align 4
+// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD20:%.*]] = add i32 [[TMP25]], [[TMP26]]
+// IR-NEXT:    store i32 [[ADD20]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[MUL21:%.*]] = mul i32 [[TMP28]], [[TMP29]]
+// IR-NEXT:    [[ADD22:%.*]] = add i32 [[TMP27]], [[MUL21]]
+// IR-NEXT:    store i32 [[ADD22]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void (...) @body(i32 noundef [[TMP30]], i32 noundef [[TMP31]], i32 noundef [[TMP32]], i32 noundef [[TMP33]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add i32 [[TMP33]], 1
+// IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add i32 [[TMP34]], 1
 // IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP2:![0-9]+]]
 // IR:       [[FOR_END]]:
@@ -150,9 +149,9 @@ extern "C" void body(...) {}
 // IR:       [[OMP_BODY_CONTINUE]]:
 // IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.*]]
 // IR:       [[OMP_INNER_FOR_INC]]:
-// IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[ADD22:%.*]] = add i32 [[TMP34]], 1
-// IR-NEXT:    store i32 [[ADD22]], ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[ADD23:%.*]] = add i32 [[TMP35]], 1
+// IR-NEXT:    store i32 [[ADD23]], ptr [[DOTOMP_IV]], align 4
 // IR-NEXT:    br label %[[OMP_INNER_FOR_COND]]
 // IR:       [[OMP_INNER_FOR_END]]:
 // IR-NEXT:    br label %[[OMP_LOOP_EXIT:.*]]
diff --git a/clang/test/OpenMP/tile_codegen_tile_for.cpp b/clang/test/OpenMP/tile_codegen_tile_for.cpp
index 8153f76b7e1a9..9ef96834fcfc0 100644
--- a/clang/test/OpenMP/tile_codegen_tile_for.cpp
+++ b/clang/test/OpenMP/tile_codegen_tile_for.cpp
@@ -129,75 +129,73 @@ extern "C" void body(...) {}
 // IR:       [[FOR_COND]]:
 // IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
 // IR-NEXT:    [[CMP22:%.*]] = icmp ult i32 [[TMP23]], 3
-// IR-NEXT:    br i1 [[CMP22]], label %[[FOR_BODY:.*]], label %[[FOR_END40:.*]]
-// IR:       [[FOR_BODY]]:
 // IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
 // IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
 // IR-NEXT:    [[ADD23:%.*]] = add i32 [[TMP24]], [[TMP25]]
-// IR-NEXT:    store i32 [[ADD23]], ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[MUL24:%.*]] = mul i32 [[TMP26]], 4
-// IR-NEXT:    [[ADD25:%.*]] = add i32 0, [[MUL24]]
-// IR-NEXT:    store i32 [[ADD25]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    [[ADD26:%.*]] = add i32 [[TMP28]], 1
-// IR-NEXT:    [[CMP27:%.*]] = icmp ult i32 [[TMP27]], [[ADD26]]
-// IR-NEXT:    br i1 [[CMP27]], label %[[IF_THEN:.*]], label %[[IF_END37:.*]]
-// IR:       [[IF_THEN]]:
+// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    [[ADD24:%.*]] = add i32 [[TMP26]], 1
+// IR-NEXT:    [[CMP25:%.*]] = icmp ult i32 [[ADD23]], [[ADD24]]
+// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP22]], [[CMP25]]
+// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label %[[FOR_BODY:.*]], label %[[FOR_END41:.*]]
+// IR:       [[FOR_BODY]]:
+// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
+// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD26:%.*]] = add i32 [[TMP27]], [[TMP28]]
+// IR-NEXT:    store i32 [[ADD26]], ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[MUL27:%.*]] = mul i32 [[TMP29]], 4
+// IR-NEXT:    [[ADD28:%.*]] = add i32 0, [[MUL27]]
+// IR-NEXT:    store i32 [[ADD28]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND28:.*]]
-// IR:       [[FOR_COND28]]:
-// IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[CMP29:%.*]] = icmp ult i32 [[TMP29]], 4
-// IR-NEXT:    br i1 [[CMP29]], label %[[FOR_BODY30:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY30]]:
-// IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD31:%.*]] = add i32 [[TMP30]], [[TMP31]]
-// IR-NEXT:    store i32 [[ADD31]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[MUL32:%.*]] = mul i32 [[TMP33]], [[TMP34]]
-// IR-NEXT:    [[ADD33:%.*]] = add i32 [[TMP32]], [[MUL32]]
-// IR-NEXT:    store i32 [[ADD33]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[ADD34:%.*]] = add i32 [[TMP36]], 1
-// IR-NEXT:    [[CMP35:%.*]] = icmp ult i32 [[TMP35]], [[ADD34]]
-// IR-NEXT:    br i1 [[CMP35]], label %[[IF_THEN36:.*]], label %[[IF_END:.*]]
-// IR:       [[IF_THEN36]]:
-// IR-NEXT:    [[TMP37:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// IR-NEXT:    [[TMP38:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// IR-NEXT:    [[TMP39:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// IR-NEXT:    [[TMP40:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void (...) @body(i32 noundef [[TMP37]], i32 noundef [[TMP38]], i32 noundef [[TMP39]], i32 noundef [[TMP40]])
-// IR-NEXT:    br label %[[IF_END]]
-// IR:       [[IF_END]]:
+// IR-NEXT:    br label %[[FOR_COND29:.*]]
+// IR:       [[FOR_COND29]]:
+// IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[CMP30:%.*]] = icmp ult i32 [[TMP30]], 4
+// IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD31:%.*]] = add i32 [[TMP31]], [[TMP32]]
+// IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    [[ADD32:%.*]] = add i32 [[TMP33]], 1
+// IR-NEXT:    [[CMP33:%.*]] = icmp ult i32 [[ADD31]], [[ADD32]]
+// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND34:%.*]] = and i1 [[CMP30]], [[CMP33]]
+// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND34]], label %[[FOR_BODY35:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY35]]:
+// IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD36:%.*]] = add i32 [[TMP34]], [[TMP35]]
+// IR-NEXT:    store i32 [[ADD36]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[MUL37:%.*]] = mul i32 [[TMP37]], [[TMP38]]
+// IR-NEXT:    [[ADD38:%.*]] = add i32 [[TMP36]], [[MUL37]]
+// IR-NEXT:    store i32 [[ADD38]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP39:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// IR-NEXT:    [[TMP40:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// IR-NEXT:    [[TMP41:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// IR-NEXT:    [[TMP42:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void (...) @body(i32 noundef [[TMP39]], i32 noundef [[TMP40]], i32 noundef [[TMP41]], i32 noundef [[TMP42]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add i32 [[TMP41]], 1
+// IR-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add i32 [[TMP43]], 1
 // IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND28]], !llvm.loop [[LOOP2:![0-9]+]]
+// IR-NEXT:    br label %[[FOR_COND29]], !llvm.loop [[LOOP2:![0-9]+]]
 // IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[IF_END37]]
-// IR:       [[IF_END37]]:
-// IR-NEXT:    br label %[[FOR_INC38:.*]]
-// IR:       [[FOR_INC38]]:
-// IR-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[INC39:%.*]] = add i32 [[TMP42]], 1
-// IR-NEXT:    store i32 [[INC39]], ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_INC39:.*]]
+// IR:       [[FOR_INC39]]:
+// IR-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[INC40:%.*]] = add i32 [[TMP44]], 1
+// IR-NEXT:    store i32 [[INC40]], ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP4:![0-9]+]]
-// IR:       [[FOR_END40]]:
+// IR:       [[FOR_END41]]:
 // IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.*]]
 // IR:       [[OMP_BODY_CONTINUE]]:
 // IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.*]]
 // IR:       [[OMP_INNER_FOR_INC]]:
-// IR-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[ADD41:%.*]] = add i32 [[TMP43]], 1
-// IR-NEXT:    store i32 [[ADD41]], ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[ADD42:%.*]] = add i32 [[TMP45]], 1
+// IR-NEXT:    store i32 [[ADD42]], ptr [[DOTOMP_IV]], align 4
 // IR-NEXT:    br label %[[OMP_INNER_FOR_COND]]
 // IR:       [[OMP_INNER_FOR_END]]:
 // IR-NEXT:    br label %[[OMP_LOOP_EXIT:.*]]
diff --git a/clang/test/OpenMP/tile_messages.cpp b/clang/test/OpenMP/tile_messages.cpp
index 8915d1bb3c546..e1f5155c924e5 100644
--- a/clang/test/OpenMP/tile_messages.cpp
+++ b/clang/test/OpenMP/tile_messages.cpp
@@ -101,7 +101,7 @@ void func() {
     for (int j = 0; j < i; ++j)
       ;
 
-  // expected-error@* {{expected 3 for loops after '#pragma omp for', but found only 2}}
+  // expected-error at +5 {{expected 3 for loops after '#pragma omp for', but found only 2}}
   // expected-note at +1 {{as specified in 'collapse' clause}}
   #pragma omp for collapse(3)
   #pragma omp tile sizes(5)
diff --git a/clang/test/OpenMP/tile_rect_codegen_ir.cpp b/clang/test/OpenMP/tile_rect_codegen_ir.cpp
index 808f1cf0a3d0c..f2b64ced7c46b 100644
--- a/clang/test/OpenMP/tile_rect_codegen_ir.cpp
+++ b/clang/test/OpenMP/tile_rect_codegen_ir.cpp
@@ -1,3 +1,4 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
 // Regression tests for rectangular #pragma omp tile lowering:
 // - Inner loop counter .tile.cnt.* with bound == tile size (constant).
 // - Validity guard (icmp + branch) when remainder iterations exist or trip count
@@ -13,8 +14,9 @@ extern "C" void body(int);
 // IR-SAME: ) #[[ATTR0:[0-9]+]] {
 // IR-NEXT:  [[ENTRY:.*:]]
 // IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    store i32 0, ptr [[I]], align 4
 // IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND:.*]]
@@ -23,43 +25,39 @@ extern "C" void body(int);
 // IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 6
 // IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END9:.*]]
 // IR:       [[FOR_BODY]]:
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1:.*]]
 // IR:       [[FOR_COND1]]:
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 4
-// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 6, [[ADD]]
-// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
-// IR:       [[COND_TRUE]]:
-// IR-NEXT:    br label %[[COND_END:.*]]
-// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 4
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
+// IR-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[ADD]], 6
+// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP2]], [[CMP3]]
+// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label %[[FOR_BODY4:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY4]]:
 // IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 4
-// IR-NEXT:    br label %[[COND_END]]
-// IR:       [[COND_END]]:
-// IR-NEXT:    [[COND:%.*]] = phi i32 [ 6, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
-// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
-// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY5]]:
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD5:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
+// IR-NEXT:    store i32 [[ADD5]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 1
 // IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
 // IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void @body(i32 noundef [[TMP6]])
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void @body(i32 noundef [[TMP7]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP7]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP8]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP2:![0-9]+]]
 // IR:       [[FOR_END]]:
 // IR-NEXT:    br label %[[FOR_INC7:.*]]
 // IR:       [[FOR_INC7]]:
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP8]], 4
+// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP9]], 4
 // IR-NEXT:    store i32 [[ADD8]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP4:![0-9]+]]
 // IR:       [[FOR_END9]]:
@@ -77,56 +75,48 @@ void remainder_6_tile_4(void) {
 // IR-SAME: ) #[[ATTR0]] {
 // IR-NEXT:  [[ENTRY:.*:]]
 // IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    store i32 0, ptr [[I]], align 4
 // IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND:.*]]
 // IR:       [[FOR_COND]]:
 // IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 10
-// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END9:.*]]
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END7:.*]]
 // IR:       [[FOR_BODY]]:
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1:.*]]
 // IR:       [[FOR_COND1]]:
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 5
-// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 10, [[ADD]]
-// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
-// IR:       [[COND_TRUE]]:
-// IR-NEXT:    br label %[[COND_END:.*]]
-// IR:       [[COND_FALSE]]:
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 5
-// IR-NEXT:    br label %[[COND_END]]
-// IR:       [[COND_END]]:
-// IR-NEXT:    [[COND:%.*]] = phi i32 [ 10, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
-// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
-// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY5]]:
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
-// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void @body(i32 noundef [[TMP6]])
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 5
+// IR-NEXT:    br i1 [[CMP2]], label %[[FOR_BODY3:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY3]]:
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
+// IR-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP4]], 1
+// IR-NEXT:    [[ADD4:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD4]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void @body(i32 noundef [[TMP5]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP7]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP6]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP5:![0-9]+]]
 // IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC7:.*]]
-// IR:       [[FOR_INC7]]:
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP8]], 5
-// IR-NEXT:    store i32 [[ADD8]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_INC5:.*]]
+// IR:       [[FOR_INC5]]:
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP7]], 5
+// IR-NEXT:    store i32 [[ADD6]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
-// IR:       [[FOR_END9]]:
+// IR:       [[FOR_END7]]:
 // IR-NEXT:    ret void
 //
 void full_tiles_10_5(void) {
@@ -141,8 +131,9 @@ void full_tiles_10_5(void) {
 // IR-NEXT:  [[ENTRY:.*:]]
 // IR-NEXT:    [[TS_ADDR:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    store i32 [[TS]], ptr [[TS_ADDR]], align 4
 // IR-NEXT:    store i32 0, ptr [[I]], align 4
 // IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
@@ -150,78 +141,64 @@ void full_tiles_10_5(void) {
 // IR:       [[FOR_COND]]:
 // IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 12
-// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END24:.*]]
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END15:.*]]
 // IR:       [[FOR_BODY]]:
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1:.*]]
 // IR:       [[FOR_COND1]]:
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TS_ADDR]], align 4
-// IR-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], 0
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TS_ADDR]], align 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP2]], 0
 // IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
 // IR:       [[COND_TRUE]]:
 // IR-NEXT:    br label %[[COND_END:.*]]
 // IR:       [[COND_FALSE]]:
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TS_ADDR]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TS_ADDR]], align 4
 // IR-NEXT:    br label %[[COND_END]]
 // IR:       [[COND_END]]:
-// IR-NEXT:    [[COND:%.*]] = phi i32 [ 1, %[[COND_TRUE]] ], [ [[TMP5]], %[[COND_FALSE]] ]
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[COND]]
-// IR-NEXT:    [[CMP3:%.*]] = icmp slt i32 12, [[ADD]]
-// IR-NEXT:    br i1 [[CMP3]], label %[[COND_TRUE4:.*]], label %[[COND_FALSE5:.*]]
-// IR:       [[COND_TRUE4]]:
-// IR-NEXT:    br label %[[COND_END12:.*]]
-// IR:       [[COND_FALSE5]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ 1, %[[COND_TRUE]] ], [ [[TMP3]], %[[COND_FALSE]] ]
+// IR-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[TMP1]], [[COND]]
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
+// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[ADD]], 12
+// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP3]], [[CMP4]]
+// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label %[[FOR_BODY5:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY5]]:
 // IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TS_ADDR]], align 4
-// IR-NEXT:    [[CMP6:%.*]] = icmp sle i32 [[TMP7]], 0
-// IR-NEXT:    br i1 [[CMP6]], label %[[COND_TRUE7:.*]], label %[[COND_FALSE8:.*]]
-// IR:       [[COND_TRUE7]]:
-// IR-NEXT:    br label %[[COND_END9:.*]]
-// IR:       [[COND_FALSE8]]:
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TS_ADDR]], align 4
-// IR-NEXT:    br label %[[COND_END9]]
-// IR:       [[COND_END9]]:
-// IR-NEXT:    [[COND10:%.*]] = phi i32 [ 1, %[[COND_TRUE7]] ], [ [[TMP8]], %[[COND_FALSE8]] ]
-// IR-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP6]], [[COND10]]
-// IR-NEXT:    br label %[[COND_END12]]
-// IR:       [[COND_END12]]:
-// IR-NEXT:    [[COND13:%.*]] = phi i32 [ 12, %[[COND_TRUE4]] ], [ [[ADD11]], %[[COND_END9]] ]
-// IR-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP2]], [[COND13]]
-// IR-NEXT:    br i1 [[CMP14]], label %[[FOR_BODY15:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY15]]:
-// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP9]], 1
-// IR-NEXT:    [[ADD16:%.*]] = add nsw i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD16]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void @body(i32 noundef [[TMP10]])
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP6]], [[TMP7]]
+// IR-NEXT:    store i32 [[ADD6]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP8]], 1
+// IR-NEXT:    [[ADD7:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void @body(i32 noundef [[TMP9]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP7:![0-9]+]]
 // IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC17:.*]]
-// IR:       [[FOR_INC17]]:
+// IR-NEXT:    br label %[[FOR_INC8:.*]]
+// IR:       [[FOR_INC8]]:
+// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TS_ADDR]], align 4
+// IR-NEXT:    [[CMP9:%.*]] = icmp sle i32 [[TMP11]], 0
+// IR-NEXT:    br i1 [[CMP9]], label %[[COND_TRUE10:.*]], label %[[COND_FALSE11:.*]]
+// IR:       [[COND_TRUE10]]:
+// IR-NEXT:    br label %[[COND_END12:.*]]
+// IR:       [[COND_FALSE11]]:
 // IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TS_ADDR]], align 4
-// IR-NEXT:    [[CMP18:%.*]] = icmp sle i32 [[TMP12]], 0
-// IR-NEXT:    br i1 [[CMP18]], label %[[COND_TRUE19:.*]], label %[[COND_FALSE20:.*]]
-// IR:       [[COND_TRUE19]]:
-// IR-NEXT:    br label %[[COND_END21:.*]]
-// IR:       [[COND_FALSE20]]:
-// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TS_ADDR]], align 4
-// IR-NEXT:    br label %[[COND_END21]]
-// IR:       [[COND_END21]]:
-// IR-NEXT:    [[COND22:%.*]] = phi i32 [ 1, %[[COND_TRUE19]] ], [ [[TMP13]], %[[COND_FALSE20]] ]
-// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP14]], [[COND22]]
-// IR-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[COND_END12]]
+// IR:       [[COND_END12]]:
+// IR-NEXT:    [[COND13:%.*]] = phi i32 [ 1, %[[COND_TRUE10]] ], [ [[TMP12]], %[[COND_FALSE11]] ]
+// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD14:%.*]] = add nsw i32 [[TMP13]], [[COND13]]
+// IR-NEXT:    store i32 [[ADD14]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
-// IR:       [[FOR_END24]]:
+// IR:       [[FOR_END15]]:
 // IR-NEXT:    ret void
 //
 void var_tile(int ts) {
@@ -235,11 +212,13 @@ void var_tile(int ts) {
 // IR-SAME: ) #[[ATTR0]] {
 // IR-NEXT:  [[ENTRY:.*:]]
 // IR-NEXT:    [[I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[J:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
 // IR-NEXT:    store i32 0, ptr [[I]], align 4
 // IR-NEXT:    store i32 0, ptr [[J]], align 4
 // IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
@@ -247,95 +226,91 @@ void var_tile(int ts) {
 // IR:       [[FOR_COND]]:
 // IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 3
-// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END31:.*]]
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END27:.*]]
 // IR:       [[FOR_BODY]]:
 // IR-NEXT:    store i32 0, ptr [[DOTFLOOR_1_IV_J]], align 4
 // IR-NEXT:    br label %[[FOR_COND1:.*]]
 // IR:       [[FOR_COND1]]:
 // IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 3
-// IR-NEXT:    br i1 [[CMP2]], label %[[FOR_BODY3:.*]], label %[[FOR_END28:.*]]
+// IR-NEXT:    br i1 [[CMP2]], label %[[FOR_BODY3:.*]], label %[[FOR_END24:.*]]
 // IR:       [[FOR_BODY3]]:
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    store i32 [[TMP2]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND4:.*]]
 // IR:       [[FOR_COND4]]:
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], 2
-// IR-NEXT:    [[CMP5:%.*]] = icmp slt i32 3, [[ADD]]
-// IR-NEXT:    br i1 [[CMP5]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
-// IR:       [[COND_TRUE]]:
-// IR-NEXT:    br label %[[COND_END:.*]]
-// IR:       [[COND_FALSE]]:
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP5]], 2
-// IR-NEXT:    br label %[[COND_END]]
-// IR:       [[COND_END]]:
-// IR-NEXT:    [[COND:%.*]] = phi i32 [ 3, %[[COND_TRUE]] ], [ [[ADD6]], %[[COND_FALSE]] ]
-// IR-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP3]], [[COND]]
-// IR-NEXT:    br i1 [[CMP7]], label %[[FOR_BODY8:.*]], label %[[FOR_END25:.*]]
-// IR:       [[FOR_BODY8]]:
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 1
-// IR-NEXT:    [[ADD9:%.*]] = add nsw i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// IR-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_1_IV_J]], align 4
-// IR-NEXT:    br label %[[FOR_COND10:.*]]
-// IR:       [[FOR_COND10]]:
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP2]], 2
+// IR-NEXT:    br i1 [[CMP5]], label %[[FOR_BODY6:.*]], label %[[FOR_END21:.*]]
+// IR:       [[FOR_BODY6]]:
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[TMP4]]
+// IR-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
+// IR-NEXT:    [[ADD7:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// IR-NEXT:    br label %[[FOR_COND8:.*]]
+// IR:       [[FOR_COND8]]:
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// IR-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP6]], 2
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP7]], [[TMP8]]
+// IR-NEXT:    [[CMP11:%.*]] = icmp slt i32 [[ADD10]], 3
+// IR-NEXT:    br i1 [[CMP11]], label %[[LAND_RHS:.*]], label %[[LAND_END:.*]]
+// IR:       [[LAND_RHS]]:
 // IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// IR-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP9]], 2
-// IR-NEXT:    [[CMP12:%.*]] = icmp slt i32 3, [[ADD11]]
-// IR-NEXT:    br i1 [[CMP12]], label %[[COND_TRUE13:.*]], label %[[COND_FALSE14:.*]]
-// IR:       [[COND_TRUE13]]:
-// IR-NEXT:    br label %[[COND_END16:.*]]
-// IR:       [[COND_FALSE14]]:
-// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// IR-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP10]], 2
-// IR-NEXT:    br label %[[COND_END16]]
-// IR:       [[COND_END16]]:
-// IR-NEXT:    [[COND17:%.*]] = phi i32 [ 3, %[[COND_TRUE13]] ], [ [[ADD15]], %[[COND_FALSE14]] ]
-// IR-NEXT:    [[CMP18:%.*]] = icmp slt i32 [[TMP8]], [[COND17]]
-// IR-NEXT:    br i1 [[CMP18]], label %[[FOR_BODY19:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY19]]:
-// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// IR-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[TMP11]], 1
-// IR-NEXT:    [[ADD21:%.*]] = add nsw i32 0, [[MUL20]]
-// IR-NEXT:    store i32 [[ADD21]], ptr [[J]], align 4
-// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
-// IR-NEXT:    [[ADD22:%.*]] = add nsw i32 [[TMP12]], [[TMP13]]
-// IR-NEXT:    call void @body(i32 noundef [[ADD22]])
+// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// IR-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
+// IR-NEXT:    [[CMP13:%.*]] = icmp slt i32 [[ADD12]], 3
+// IR-NEXT:    br label %[[LAND_END]]
+// IR:       [[LAND_END]]:
+// IR-NEXT:    [[TMP11:%.*]] = phi i1 [ false, %[[FOR_COND8]] ], [ [[CMP13]], %[[LAND_RHS]] ]
+// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP9]], [[TMP11]]
+// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label %[[FOR_BODY14:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY14]]:
+// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// IR-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP12]], [[TMP13]]
+// IR-NEXT:    store i32 [[ADD15]], ptr [[DOTTILE_1_IV_J]], align 4
+// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// IR-NEXT:    [[MUL16:%.*]] = mul nsw i32 [[TMP14]], 1
+// IR-NEXT:    [[ADD17:%.*]] = add nsw i32 0, [[MUL16]]
+// IR-NEXT:    store i32 [[ADD17]], ptr [[J]], align 4
+// IR-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    [[TMP16:%.*]] = load i32, ptr [[J]], align 4
+// IR-NEXT:    [[ADD18:%.*]] = add nsw i32 [[TMP15]], [[TMP16]]
+// IR-NEXT:    call void @body(i32 noundef [[ADD18]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
-// IR-NEXT:    br label %[[FOR_COND10]], !llvm.loop [[LOOP9:![0-9]+]]
+// IR-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP17]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// IR-NEXT:    br label %[[FOR_COND8]], !llvm.loop [[LOOP9:![0-9]+]]
 // IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC23:.*]]
-// IR:       [[FOR_INC23]]:
-// IR-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[INC24:%.*]] = add nsw i32 [[TMP15]], 1
-// IR-NEXT:    store i32 [[INC24]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_INC19:.*]]
+// IR:       [[FOR_INC19]]:
+// IR-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[INC20:%.*]] = add nsw i32 [[TMP18]], 1
+// IR-NEXT:    store i32 [[INC20]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND4]], !llvm.loop [[LOOP10:![0-9]+]]
-// IR:       [[FOR_END25]]:
-// IR-NEXT:    br label %[[FOR_INC26:.*]]
-// IR:       [[FOR_INC26]]:
-// IR-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// IR-NEXT:    [[ADD27:%.*]] = add nsw i32 [[TMP16]], 2
-// IR-NEXT:    store i32 [[ADD27]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// IR:       [[FOR_END21]]:
+// IR-NEXT:    br label %[[FOR_INC22:.*]]
+// IR:       [[FOR_INC22]]:
+// IR-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// IR-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP19]], 2
+// IR-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP11:![0-9]+]]
-// IR:       [[FOR_END28]]:
-// IR-NEXT:    br label %[[FOR_INC29:.*]]
-// IR:       [[FOR_INC29]]:
-// IR-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD30:%.*]] = add nsw i32 [[TMP17]], 2
-// IR-NEXT:    store i32 [[ADD30]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR:       [[FOR_END24]]:
+// IR-NEXT:    br label %[[FOR_INC25:.*]]
+// IR:       [[FOR_INC25]]:
+// IR-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD26:%.*]] = add nsw i32 [[TMP20]], 2
+// IR-NEXT:    store i32 [[ADD26]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP12:![0-9]+]]
-// IR:       [[FOR_END31]]:
+// IR:       [[FOR_END27]]:
 // IR-NEXT:    ret void
 //
 void two_dim(void) {
@@ -351,8 +326,9 @@ void two_dim(void) {
 // IR-NEXT:  [[ENTRY:.*:]]
 // IR-NEXT:    [[N_ADDR:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[J:%.*]] = alloca i32, align 4
 // IR-NEXT:    store i32 [[N]], ptr [[N_ADDR]], align 4
 // IR-NEXT:    store i32 0, ptr [[I]], align 4
@@ -363,60 +339,56 @@ void two_dim(void) {
 // IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 6
 // IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END16:.*]]
 // IR:       [[FOR_BODY]]:
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1:.*]]
 // IR:       [[FOR_COND1]]:
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 4
-// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 6, [[ADD]]
-// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
-// IR:       [[COND_TRUE]]:
-// IR-NEXT:    br label %[[COND_END:.*]]
-// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 4
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
+// IR-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[ADD]], 6
+// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP2]], [[CMP3]]
+// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label %[[FOR_BODY4:.*]], label %[[FOR_END13:.*]]
+// IR:       [[FOR_BODY4]]:
 // IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 4
-// IR-NEXT:    br label %[[COND_END]]
-// IR:       [[COND_END]]:
-// IR-NEXT:    [[COND:%.*]] = phi i32 [ 6, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
-// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
-// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END13:.*]]
-// IR:       [[FOR_BODY5]]:
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD5:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
+// IR-NEXT:    store i32 [[ADD5]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 1
 // IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
 // IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
 // IR-NEXT:    store i32 0, ptr [[J]], align 4
 // IR-NEXT:    br label %[[FOR_COND7:.*]]
 // IR:       [[FOR_COND7]]:
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[J]], align 4
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[N_ADDR]], align 4
-// IR-NEXT:    [[CMP8:%.*]] = icmp slt i32 [[TMP6]], [[TMP7]]
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[J]], align 4
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[N_ADDR]], align 4
+// IR-NEXT:    [[CMP8:%.*]] = icmp slt i32 [[TMP7]], [[TMP8]]
 // IR-NEXT:    br i1 [[CMP8]], label %[[FOR_BODY9:.*]], label %[[FOR_END:.*]]
 // IR:       [[FOR_BODY9]]:
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[J]], align 4
-// IR-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP8]], [[TMP9]]
+// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[J]], align 4
+// IR-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
 // IR-NEXT:    call void @body(i32 noundef [[ADD10]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[J]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
+// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[J]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
 // IR-NEXT:    store i32 [[INC]], ptr [[J]], align 4
 // IR-NEXT:    br label %[[FOR_COND7]], !llvm.loop [[LOOP13:![0-9]+]]
 // IR:       [[FOR_END]]:
 // IR-NEXT:    br label %[[FOR_INC11:.*]]
 // IR:       [[FOR_INC11]]:
-// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[INC12:%.*]] = add nsw i32 [[TMP11]], 1
-// IR-NEXT:    store i32 [[INC12]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[INC12:%.*]] = add nsw i32 [[TMP12]], 1
+// IR-NEXT:    store i32 [[INC12]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP14:![0-9]+]]
 // IR:       [[FOR_END13]]:
 // IR-NEXT:    br label %[[FOR_INC14:.*]]
 // IR:       [[FOR_INC14]]:
-// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP12]], 4
+// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP13]], 4
 // IR-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP15:![0-9]+]]
 // IR:       [[FOR_END16]]:
@@ -435,56 +407,48 @@ void nested_inner_j(int n) {
 // IR-SAME: ) #[[ATTR0]] {
 // IR-NEXT:  [[ENTRY:.*:]]
 // IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    store i32 0, ptr [[I]], align 4
 // IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND:.*]]
 // IR:       [[FOR_COND]]:
 // IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 12
-// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END9:.*]]
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END7:.*]]
 // IR:       [[FOR_BODY]]:
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1:.*]]
 // IR:       [[FOR_COND1]]:
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 4
-// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 12, [[ADD]]
-// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
-// IR:       [[COND_TRUE]]:
-// IR-NEXT:    br label %[[COND_END:.*]]
-// IR:       [[COND_FALSE]]:
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 4
-// IR-NEXT:    br label %[[COND_END]]
-// IR:       [[COND_END]]:
-// IR-NEXT:    [[COND:%.*]] = phi i32 [ 12, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
-// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
-// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY5]]:
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
-// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void @body(i32 noundef [[TMP6]])
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 4
+// IR-NEXT:    br i1 [[CMP2]], label %[[FOR_BODY3:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY3]]:
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
+// IR-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP4]], 1
+// IR-NEXT:    [[ADD4:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD4]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void @body(i32 noundef [[TMP5]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP7]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP6]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP16:![0-9]+]]
 // IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC7:.*]]
-// IR:       [[FOR_INC7]]:
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP8]], 4
-// IR-NEXT:    store i32 [[ADD8]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_INC5:.*]]
+// IR:       [[FOR_INC5]]:
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP7]], 4
+// IR-NEXT:    store i32 [[ADD6]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP17:![0-9]+]]
-// IR:       [[FOR_END9]]:
+// IR:       [[FOR_END7]]:
 // IR-NEXT:    ret void
 //
 void elided_predicate(void) {
diff --git a/clang/test/OpenMP/unroll_codegen_tile_for.cpp b/clang/test/OpenMP/unroll_codegen_tile_for.cpp
index 6f13f6b67c8cd..cd28ff73a11ec 100644
--- a/clang/test/OpenMP/unroll_codegen_tile_for.cpp
+++ b/clang/test/OpenMP/unroll_codegen_tile_for.cpp
@@ -123,75 +123,74 @@ extern "C" void body(...) {}
 // IR:       [[FOR_COND]]:
 // IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
 // IR-NEXT:    [[CMP22:%.*]] = icmp ult i32 [[TMP23]], 4
-// IR-NEXT:    br i1 [[CMP22]], label %[[FOR_BODY:.*]], label %[[FOR_END38:.*]]
-// IR:       [[FOR_BODY]]:
 // IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I17]], align 4
 // IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
 // IR-NEXT:    [[ADD23:%.*]] = add i32 [[TMP24]], [[TMP25]]
-// IR-NEXT:    store i32 [[ADD23]], ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    [[MUL24:%.*]] = mul i32 [[TMP26]], 2
-// IR-NEXT:    [[ADD25:%.*]] = add i32 0, [[MUL24]]
-// IR-NEXT:    store i32 [[ADD25]], ptr [[DOTUNROLLED_IV_I]], align 4
-// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    [[ADD26:%.*]] = add i32 [[TMP28]], 1
-// IR-NEXT:    [[CMP27:%.*]] = icmp ult i32 [[TMP27]], [[ADD26]]
-// IR-NEXT:    br i1 [[CMP27]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
-// IR:       [[IF_THEN]]:
-// IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTUNROLLED_IV_I]], align 4
-// IR-NEXT:    store i32 [[TMP29]], ptr [[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND28:.*]]
-// IR:       [[FOR_COND28]]:
-// IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTUNROLLED_IV_I]], align 4
-// IR-NEXT:    [[ADD29:%.*]] = add i32 [[TMP31]], 2
-// IR-NEXT:    [[CMP30:%.*]] = icmp ult i32 [[TMP30]], [[ADD29]]
-// IR-NEXT:    br i1 [[CMP30]], label %[[LAND_RHS:.*]], label %[[LAND_END:.*]]
+// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    [[ADD24:%.*]] = add i32 [[TMP26]], 1
+// IR-NEXT:    [[CMP25:%.*]] = icmp ult i32 [[ADD23]], [[ADD24]]
+// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP22]], [[CMP25]]
+// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label %[[FOR_BODY:.*]], label %[[FOR_END39:.*]]
+// IR:       [[FOR_BODY]]:
+// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I17]], align 4
+// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    [[ADD26:%.*]] = add i32 [[TMP27]], [[TMP28]]
+// IR-NEXT:    store i32 [[ADD26]], ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    [[MUL27:%.*]] = mul i32 [[TMP29]], 2
+// IR-NEXT:    [[ADD28:%.*]] = add i32 0, [[MUL27]]
+// IR-NEXT:    store i32 [[ADD28]], ptr [[DOTUNROLLED_IV_I]], align 4
+// IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTUNROLLED_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP30]], ptr [[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND29:.*]]
+// IR:       [[FOR_COND29]]:
+// IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTUNROLLED_IV_I]], align 4
+// IR-NEXT:    [[ADD30:%.*]] = add i32 [[TMP32]], 2
+// IR-NEXT:    [[CMP31:%.*]] = icmp ult i32 [[TMP31]], [[ADD30]]
+// IR-NEXT:    br i1 [[CMP31]], label %[[LAND_RHS:.*]], label %[[LAND_END:.*]]
 // IR:       [[LAND_RHS]]:
-// IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[ADD31:%.*]] = add i32 [[TMP33]], 1
-// IR-NEXT:    [[CMP32:%.*]] = icmp ult i32 [[TMP32]], [[ADD31]]
+// IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    [[ADD32:%.*]] = add i32 [[TMP34]], 1
+// IR-NEXT:    [[CMP33:%.*]] = icmp ult i32 [[TMP33]], [[ADD32]]
 // IR-NEXT:    br label %[[LAND_END]]
 // IR:       [[LAND_END]]:
-// IR-NEXT:    [[TMP34:%.*]] = phi i1 [ false, %[[FOR_COND28]] ], [ [[CMP32]], %[[LAND_RHS]] ]
-// IR-NEXT:    br i1 [[TMP34]], label %[[FOR_BODY33:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY33]]:
-// IR-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[MUL34:%.*]] = mul i32 [[TMP36]], [[TMP37]]
-// IR-NEXT:    [[ADD35:%.*]] = add i32 [[TMP35]], [[MUL34]]
-// IR-NEXT:    store i32 [[ADD35]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP38:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// IR-NEXT:    [[TMP39:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// IR-NEXT:    [[TMP40:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// IR-NEXT:    [[TMP41:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void (...) @body(i32 noundef [[TMP38]], i32 noundef [[TMP39]], i32 noundef [[TMP40]], i32 noundef [[TMP41]])
+// IR-NEXT:    [[TMP35:%.*]] = phi i1 [ false, %[[FOR_COND29]] ], [ [[CMP33]], %[[LAND_RHS]] ]
+// IR-NEXT:    br i1 [[TMP35]], label %[[FOR_BODY34:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY34]]:
+// IR-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[MUL35:%.*]] = mul i32 [[TMP37]], [[TMP38]]
+// IR-NEXT:    [[ADD36:%.*]] = add i32 [[TMP36]], [[MUL35]]
+// IR-NEXT:    store i32 [[ADD36]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP39:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// IR-NEXT:    [[TMP40:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// IR-NEXT:    [[TMP41:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// IR-NEXT:    [[TMP42:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void (...) @body(i32 noundef [[TMP39]], i32 noundef [[TMP40]], i32 noundef [[TMP41]], i32 noundef [[TMP42]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add i32 [[TMP42]], 1
+// IR-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add i32 [[TMP43]], 1
 // IR-NEXT:    store i32 [[INC]], ptr [[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND28]], !llvm.loop [[LOOP2:![0-9]+]]
+// IR-NEXT:    br label %[[FOR_COND29]], !llvm.loop [[LOOP2:![0-9]+]]
 // IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[IF_END]]
-// IR:       [[IF_END]]:
-// IR-NEXT:    br label %[[FOR_INC36:.*]]
-// IR:       [[FOR_INC36]]:
-// IR-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    [[INC37:%.*]] = add i32 [[TMP43]], 1
-// IR-NEXT:    store i32 [[INC37]], ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_INC37:.*]]
+// IR:       [[FOR_INC37]]:
+// IR-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    [[INC38:%.*]] = add i32 [[TMP44]], 1
+// IR-NEXT:    store i32 [[INC38]], ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP5:![0-9]+]]
-// IR:       [[FOR_END38]]:
+// IR:       [[FOR_END39]]:
 // IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.*]]
 // IR:       [[OMP_BODY_CONTINUE]]:
 // IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.*]]
 // IR:       [[OMP_INNER_FOR_INC]]:
-// IR-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[ADD39:%.*]] = add i32 [[TMP44]], 1
-// IR-NEXT:    store i32 [[ADD39]], ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[ADD40:%.*]] = add i32 [[TMP45]], 1
+// IR-NEXT:    store i32 [[ADD40]], ptr [[DOTOMP_IV]], align 4
 // IR-NEXT:    br label %[[OMP_INNER_FOR_COND]]
 // IR:       [[OMP_INNER_FOR_END]]:
 // IR-NEXT:    br label %[[OMP_LOOP_EXIT:.*]]
diff --git a/openmp/runtime/test/transform/tile/foreach.cpp b/openmp/runtime/test/transform/tile/foreach.cpp
index 17031a4acf187..608a1c896fcb4 100644
--- a/openmp/runtime/test/transform/tile/foreach.cpp
+++ b/openmp/runtime/test/transform/tile/foreach.cpp
@@ -182,10 +182,6 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: v=0 w=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 1
 // CHECK-NEXT: [A] iterator move assign
@@ -195,10 +191,6 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: v=1 w=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -217,14 +209,6 @@ int main() {
 // CHECK-NEXT: [A] iterator advance: 0 += 3
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [B] iterator advance: 0 += 0
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 0
-// CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 1
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 1
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -234,22 +218,10 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: v=2 w=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 3
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [B] iterator advance: 0 += 2
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 2
-// CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
diff --git a/openmp/runtime/test/transform/tile/intfor.c b/openmp/runtime/test/transform/tile/intfor.c
index e980165277b79..c5ceddd83da70 100644
--- a/openmp/runtime/test/transform/tile/intfor.c
+++ b/openmp/runtime/test/transform/tile/intfor.c
@@ -81,8 +81,6 @@ int main() {
 // CHECK-NEXT: i=7 j=19
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
@@ -93,8 +91,6 @@ int main() {
 // CHECK-NEXT: i=10 j=19
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
@@ -105,8 +101,6 @@ int main() {
 // CHECK-NEXT: i=13 j=19
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
@@ -130,22 +124,10 @@ int main() {
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
@@ -160,28 +142,14 @@ int main() {
 // CHECK-NEXT: i=16 j=19
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
diff --git a/openmp/runtime/test/transform/tile/iterfor.cpp b/openmp/runtime/test/transform/tile/iterfor.cpp
index d07b10b3e841a..4b9bf6f41ada6 100644
--- a/openmp/runtime/test/transform/tile/iterfor.cpp
+++ b/openmp/runtime/test/transform/tile/iterfor.cpp
@@ -190,9 +190,6 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=0 j=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 1
 // CHECK-NEXT: [A] iterator move assign
@@ -202,9 +199,6 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=1 j=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -223,12 +217,6 @@ int main() {
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 3
 // CHECK-NEXT: [A] iterator move assign
-// CHECK-NEXT: [B] iterator advance: 0 += 0
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 1
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -238,18 +226,9 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=2 j=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 3
 // CHECK-NEXT: [A] iterator move assign
-// CHECK-NEXT: [B] iterator advance: 0 += 2
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
diff --git a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-foreach.cpp b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-foreach.cpp
index 966ff7f84f548..a007a369453d4 100644
--- a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-foreach.cpp
+++ b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-foreach.cpp
@@ -182,10 +182,6 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=0 v=0 w=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 1
 // CHECK-NEXT: [A] iterator move assign
@@ -195,10 +191,6 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=0 v=1 w=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -217,14 +209,6 @@ int main() {
 // CHECK-NEXT: [A] iterator advance: 0 += 3
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [B] iterator advance: 0 += 0
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 0
-// CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 1
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 1
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -234,22 +218,10 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=0 v=2 w=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 3
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [B] iterator advance: 0 += 2
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 2
-// CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 0
 // CHECK-NEXT: [A] iterator move assign
@@ -287,10 +259,6 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=1 v=0 w=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 1
 // CHECK-NEXT: [A] iterator move assign
@@ -300,10 +268,6 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=1 v=1 w=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -322,14 +286,6 @@ int main() {
 // CHECK-NEXT: [A] iterator advance: 0 += 3
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [B] iterator advance: 0 += 0
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 0
-// CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 1
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 1
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -339,22 +295,10 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=1 v=2 w=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 3
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [B] iterator advance: 0 += 2
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 2
-// CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 0
 // CHECK-NEXT: [A] iterator move assign
@@ -392,10 +336,6 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=2 v=0 w=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 1
 // CHECK-NEXT: [A] iterator move assign
@@ -405,10 +345,6 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=2 v=1 w=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -427,14 +363,6 @@ int main() {
 // CHECK-NEXT: [A] iterator advance: 0 += 3
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [B] iterator advance: 0 += 0
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 0
-// CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 1
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 1
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -444,22 +372,10 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=2 v=2 w=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 3
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [B] iterator advance: 0 += 2
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 2
-// CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
diff --git a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile.cpp b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile.cpp
new file mode 100644
index 0000000000000..c7f7e143f7983
--- /dev/null
+++ b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile.cpp
@@ -0,0 +1,44 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+
+// After tiling `i`, the logical loops are `floor_i`, `tile_i`, and `j`, so
+// `collapse(3)` must reach through the tile guard to include `j`. CodeGen must
+// then keep that guard around the collapsed body once, instead of emitting `j`
+// again as a normal nested loop.
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdio>
+
+int main() {
+  printf("do\n");
+#pragma omp parallel for collapse(3) num_threads(1)
+#pragma omp tile sizes(4)
+  for (int i = 0; i < 6; ++i)
+    for (int j = 0; j < 2; ++j)
+      printf("i=%d j=%d\n", i, j);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+
+// Full tile (.floor.i=0): i=0..3
+// CHECK-NEXT: i=0 j=0
+// CHECK-NEXT: i=0 j=1
+// CHECK-NEXT: i=1 j=0
+// CHECK-NEXT: i=1 j=1
+// CHECK-NEXT: i=2 j=0
+// CHECK-NEXT: i=2 j=1
+// CHECK-NEXT: i=3 j=0
+// CHECK-NEXT: i=3 j=1
+
+// Partial tile (.floor.i=4): i=4,5 in bounds; i=6,7 skipped by the predicate
+// CHECK-NEXT: i=4 j=0
+// CHECK-NEXT: i=4 j=1
+// CHECK-NEXT: i=5 j=0
+// CHECK-NEXT: i=5 j=1
+// CHECK-NEXT: done

>From c42d747d0677fd0059f0bcff09699e965bae91e2 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 15 Jul 2026 10:05:50 -0400
Subject: [PATCH 15/18] preserve vectorized-form through reinterpretation hint

---
 clang/include/clang/Basic/Attr.td  |   2 +
 clang/lib/AST/StmtOpenMP.cpp       |  66 +++---
 clang/lib/CodeGen/CGStmt.cpp       |  48 +----
 clang/lib/CodeGen/CGStmtOpenMP.cpp |  56 ++---
 clang/lib/Sema/SemaOpenMP.cpp      | 325 ++++++++++++++++-------------
 5 files changed, 234 insertions(+), 263 deletions(-)

diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 22f5e0fd33433..7d90c40c5ecf9 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -4881,6 +4881,8 @@ def OMPInvariantPredicateBound : StmtAttr {
   let Spellings = [];
   let SemaHandler = 0;
   let Subjects = SubjectList<[Stmt], ErrorDiag>;
+  let Args = [ExprArgument<"RectCond">, ExprArgument<"Predicate">,
+              ExprArgument<"TileSize">];
   let Documentation = [InternalOnly];
 }
 
diff --git a/clang/lib/AST/StmtOpenMP.cpp b/clang/lib/AST/StmtOpenMP.cpp
index 94eeddc01d1a3..26000198518a8 100644
--- a/clang/lib/AST/StmtOpenMP.cpp
+++ b/clang/lib/AST/StmtOpenMP.cpp
@@ -18,8 +18,8 @@
 using namespace clang;
 using namespace llvm::omp;
 
-// True if the attributed statement carries the internal `omp tile` intra-tile
-// body-guard marker.
+/// Returns true if the attributed statement carries the internal `omp tile`
+/// intra-tile reinterpretation hint (see OMPInvariantPredicateBoundAttr).
 static bool hasOMPInvariantPredicateBound(const AttributedStmt *AS) {
   for (const Attr *A : AS->getAttrs())
     if (isa<OMPInvariantPredicateBoundAttr>(A))
@@ -27,17 +27,26 @@ static bool hasOMPInvariantPredicateBound(const AttributedStmt *AS) {
   return false;
 }
 
-// Like `Stmt::IgnoreContainers`, but also treats the `omp tile` body-guard
-static Stmt *IgnoreContainersAndOMPTileBodyGuard(Stmt *S) {
+/// Peeks through an intra-tile hint wrapper to return the underlying loop;
+/// otherwise returns \p S unchanged. Used by the loop walker once the hint has
+/// already been delivered to the analysis callback and we need to look through
+/// the wrapper to reach the loop body.
+static Stmt *ignoreOMPIntraTileHint(Stmt *S) {
+  if (auto *AS = dyn_cast_or_null<AttributedStmt>(S))
+    if (hasOMPInvariantPredicateBound(AS))
+      return AS->getSubStmt();
+  return S;
+}
+
+/// Like `Stmt::IgnoreContainers`, but stops at (and preserves) an intra-tile
+/// hint wrapper so that the enclosing loop walker can deliver the hint to
+/// loop-associated analysis (e.g. `collapse`). Non-hint attributed statements
+/// and single-statement compounds are still skipped.
+static Stmt *ignoreContainersKeepingIntraTileHint(Stmt *S) {
   while (true) {
     if (auto *AS = dyn_cast_or_null<AttributedStmt>(S)) {
-      if (hasOMPInvariantPredicateBound(AS)) {
-        if (auto *If = dyn_cast<IfStmt>(AS->getSubStmt());
-            If && !If->getElse()) {
-          S = If->getThen();
-          continue;
-        }
-      }
+      if (hasOMPInvariantPredicateBound(AS))
+        break;
       S = AS->getSubStmt();
       continue;
     }
@@ -114,7 +123,7 @@ Stmt *
 OMPLoopBasedDirective::tryToFindNextInnerLoop(Stmt *CurStmt,
                                               bool TryImperfectlyNestedLoops) {
   Stmt *OrigStmt = CurStmt;
-  CurStmt = IgnoreContainersAndOMPTileBodyGuard(CurStmt);
+  CurStmt = ignoreContainersKeepingIntraTileHint(CurStmt);
   // Additional work for imperfectly nested loops, introduced in OpenMP 5.0.
   if (TryImperfectlyNestedLoops) {
     if (auto *CS = dyn_cast<CompoundStmt>(CurStmt)) {
@@ -128,14 +137,17 @@ OMPLoopBasedDirective::tryToFindNextInnerLoop(Stmt *CurStmt,
         for (Stmt *S : CS->body()) {
           if (!S)
             continue;
-          if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(S))
-            S = CanonLoop->getLoopStmt();
-          // Check for the `omp tile` body-guard marker.
-          if (auto *AS = dyn_cast<AttributedStmt>(S);
-              AS && hasOMPInvariantPredicateBound(AS))
-            S = IgnoreContainersAndOMPTileBodyGuard(S);
-          if (isa<ForStmt>(S) || isa<CXXForRangeStmt>(S) ||
-              (isa<OMPLoopBasedDirective>(S) && !isa<OMPLoopDirective>(S))) {
+          // Peek past an OMPCanonicalLoop wrapper and/or an intra-tile hint to
+          // check whether this child is loop-like; keep the original (wrapped)
+          // node in CurStmt so the hint still reaches the loop-analysis
+          // callback.
+          Stmt *Inner = S;
+          if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(Inner))
+            Inner = CanonLoop->getLoopStmt();
+          Inner = ignoreOMPIntraTileHint(Inner);
+          if (isa<ForStmt>(Inner) || isa<CXXForRangeStmt>(Inner) ||
+              (isa<OMPLoopBasedDirective>(Inner) &&
+               !isa<OMPLoopDirective>(Inner))) {
             // Only single loop construct is allowed.
             if (CurStmt) {
               CurStmt = OrigStmt;
@@ -167,7 +179,7 @@ bool OMPLoopBasedDirective::doForAllLoops(
     llvm::function_ref<bool(unsigned, Stmt *)> Callback,
     llvm::function_ref<void(OMPLoopTransformationDirective *)>
         OnTransformationCallback) {
-  CurStmt = IgnoreContainersAndOMPTileBodyGuard(CurStmt);
+  CurStmt = ignoreContainersKeepingIntraTileHint(CurStmt);
   for (unsigned Cnt = 0; Cnt < NumLoops; ++Cnt) {
     while (true) {
       auto *Dir = dyn_cast<OMPLoopTransformationDirective>(CurStmt);
@@ -197,19 +209,23 @@ bool OMPLoopBasedDirective::doForAllLoops(
     }
     if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(CurStmt))
       CurStmt = CanonLoop->getLoopStmt();
+    // Deliver the (possibly hint-wrapped) loop node to the callback so it can
+    // read the intra-tile reinterpretation hint, then look through the wrapper
+    // to reach the loop body.
     if (Callback(Cnt, CurStmt))
       return false;
+    Stmt *LoopStmt = ignoreOMPIntraTileHint(CurStmt);
     // Move on to the next nested for loop, or to the loop body.
     // OpenMP [2.8.1, simd construct, Restrictions]
     // All loops associated with the construct must be perfectly nested; that
     // is, there must be no intervening code nor any OpenMP directive between
     // any two loops.
-    if (auto *For = dyn_cast<ForStmt>(CurStmt)) {
+    if (auto *For = dyn_cast<ForStmt>(LoopStmt)) {
       CurStmt = For->getBody();
     } else {
-      assert(isa<CXXForRangeStmt>(CurStmt) &&
+      assert(isa<CXXForRangeStmt>(LoopStmt) &&
              "Expected canonical for or range-based for loops.");
-      CurStmt = cast<CXXForRangeStmt>(CurStmt)->getBody();
+      CurStmt = cast<CXXForRangeStmt>(LoopStmt)->getBody();
     }
     CurStmt = OMPLoopBasedDirective::tryToFindNextInnerLoop(
         CurStmt, TryImperfectlyNestedLoops);
@@ -223,6 +239,8 @@ void OMPLoopBasedDirective::doForAllLoopsBodies(
   bool Res = OMPLoopBasedDirective::doForAllLoops(
       CurStmt, TryImperfectlyNestedLoops, NumLoops,
       [Callback](unsigned Cnt, Stmt *Loop) {
+        // Look through an intra-tile hint wrapper to reach the loop itself.
+        Loop = ignoreOMPIntraTileHint(Loop);
         Stmt *Body = nullptr;
         if (auto *For = dyn_cast<ForStmt>(Loop)) {
           Body = For->getBody();
diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index da7a1b561b207..27e74d966eca1 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -1271,35 +1271,9 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S,
     ConvergenceTokenStack.pop_back();
 }
 
-// Helper function to find and return the omp-tile body-guard marker on the
-// for-body's last item.
-static std::pair<const AttributedStmt *, const IfStmt *>
-findOMPTileBodyGuard(const Stmt *Body) {
-  const auto *CS = dyn_cast_or_null<CompoundStmt>(Body);
-  if (!CS || CS->body().empty())
-    return {nullptr, nullptr};
-  const auto *AS = dyn_cast<AttributedStmt>(CS->body_back());
-  if (!AS)
-    return {nullptr, nullptr};
-  bool IsMarker = false;
-  for (const Attr *A : AS->getAttrs()) {
-    if (!isa<OMPInvariantPredicateBoundAttr>(A))
-      continue;
-    IsMarker = true;
-    break;
-  }
-  if (!IsMarker)
-    return {nullptr, nullptr};
-  const auto *If = dyn_cast<IfStmt>(AS->getSubStmt());
-  if (!If || If->getElse())
-    return {nullptr, nullptr};
-  return {AS, If};
-}
-
 void CodeGenFunction::EmitForStmt(const ForStmt &S,
                                   ArrayRef<const Attr *> ForAttrs) {
   JumpDest LoopExit = getJumpDestInCurrentScope("for.end");
-  const auto [MarkerAS, MarkerIf] = findOMPTileBodyGuard(S.getBody());
 
   std::optional<LexicalScope> ForScope;
   if (getLangOpts().C99 || getLangOpts().CPlusPlus)
@@ -1367,14 +1341,6 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S,
     // compares unequal to 0.  The condition must be a scalar type.
     llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
 
-    // If the for-body has a marker, evaluate the predicate and AND it into the
-    // loop condition.
-    if (MarkerIf) {
-      llvm::Value *PredVal = EvaluateExprAsBool(MarkerIf->getCond());
-      BoolCondVal = Builder.CreateAnd(BoolCondVal, PredVal,
-                                      "omp_tile.invariant_pred_bound");
-    }
-
     MaybeEmitDeferredVarDeclInit(S.getConditionVariable());
 
     llvm::MDNode *Weights =
@@ -1411,19 +1377,7 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S,
     // Create a separate cleanup scope for the body, in case it is not
     // a compound statement.
     RunCleanupsScope BodyScope(*this);
-    if (MarkerAS) {
-      // If the for-body has a marker, walk the body and emit the then-block of
-      // the marker's predicate.
-      const auto *CS = cast<CompoundStmt>(S.getBody());
-      for (const Stmt *Sub : CS->body()) {
-        if (Sub == MarkerAS)
-          EmitStmt(MarkerIf->getThen());
-        else
-          EmitStmt(Sub);
-      }
-    } else {
-      EmitStmt(S.getBody());
-    }
+    EmitStmt(S.getBody());
   }
 
   // The last block in the loop's body (which unconditionally branches to the
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 93805f7b54432..509d4f3d35815 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -2231,32 +2231,15 @@ class OMPTransformDirectiveScopeRAII {
 };
 } // namespace
 
-// Helper function to look through the statement and return the special marked
-// `omp tile` guard `if`, or nullptr if this is not that shape.
-static const IfStmt *getOMPTileBodyGuardIf(const Stmt *S) {
-  while (S) {
-    if (const auto *AS = dyn_cast<AttributedStmt>(S)) {
-      bool IsMarker = llvm::any_of(AS->getAttrs(), [](const Attr *A) {
-        return isa<OMPInvariantPredicateBoundAttr>(A);
-      });
-      if (!IsMarker) {
-        S = AS->getSubStmt();
-        continue;
-      }
-      const auto *If = dyn_cast<IfStmt>(AS->getSubStmt());
-      if (!If || If->getElse())
-        return nullptr;
-      return If;
-    }
-    if (const auto *CS = dyn_cast<CompoundStmt>(S)) {
-      if (CS->size() != 1)
-        return nullptr;
-      S = CS->body_back();
-      continue;
-    }
-    return nullptr;
-  }
-  return nullptr;
+/// Helper function to look through the droppable intra-tile hint wrapper
+/// (OMPInvariantPredicateBoundAttr) to the loop statement it annotates.
+static const Stmt *lookThroughOMPIntraTileHint(const Stmt *S) {
+  if (const auto *AS = dyn_cast_or_null<AttributedStmt>(S))
+    if (llvm::any_of(AS->getAttrs(), [](const Attr *A) {
+          return isa<OMPInvariantPredicateBoundAttr>(A);
+        }))
+      return AS->getSubStmt();
+  return S;
 }
 
 static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop,
@@ -2275,23 +2258,10 @@ static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop,
     return;
   }
 
-  // If this is the marked `omp tile` guard, emit its predicate as a runtime
-  // body guard and recurse into `Then` so consumed loops are still walked.
-  if (const IfStmt *Guard = getOMPTileBodyGuardIf(S)) {
-    assert(!Guard->getInit() && !Guard->getConditionVariable() &&
-           "omp tile body-guard must be a plain `if (P) Then`");
-    llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_tile.pred.then");
-    llvm::BasicBlock *EndBlock = CGF.createBasicBlock("omp_tile.pred.end");
-    CGF.EmitBranchOnBoolExpr(
-        Guard->getCond(), ThenBlock, EndBlock, /*TrueCount=*/0,
-        Stmt::getLikelihood(Guard->getThen(), Guard->getElse()));
-    CGF.EmitBlock(ThenBlock);
-    emitBody(CGF, Guard->getThen(), NextLoop, MaxLevel, Level);
-    CGF.EmitBranch(EndBlock);
-    CGF.EmitBlock(EndBlock);
-    return;
-  }
-  if (SimplifiedS == NextLoop) {
+  // The loop walker may hand back the intra-tile hint wrapper as `NextLoop`;
+  // match against the loop it annotates. The tile overshoot guard is emitted
+  // separately via EmitOMPLoopBody's finals-conditions handling.
+  if (SimplifiedS == lookThroughOMPIntraTileHint(NextLoop)) {
     if (auto *Dir = dyn_cast<OMPLoopTransformationDirective>(SimplifiedS))
       SimplifiedS = Dir->getTransformedStmt();
     if (const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(SimplifiedS))
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 20b8c99a1def2..701ba2d9f127d 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -8018,6 +8018,9 @@ struct LoopIterationSpace final {
   /// check that the number of iterations for this particular counter must be
   /// finished.
   Expr *FinalCondition = nullptr;
+  /// True if this is a reinterpreted intra-tile loop whose lower bound is a
+  /// live floor dependence.
+  bool ReinterpretTileLB = false;
 };
 
 /// Scan an AST subtree, checking that no decls in the CollapsedLoopVarDecls
@@ -9577,6 +9580,23 @@ static bool checkOpenMPIterationSpace(
     const llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopVarDecls,
     llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopInductionVars) {
   bool SupportsNonRectangular = !isOpenMPLoopTransformationDirective(DKind);
+  // See the tile reinterpretation design note in ActOnOpenMPTileDirective.
+  // If the loop carries the hint, analyze its rectangular form instead.
+  Expr *TileRectCond = nullptr;
+  Expr *TileBodyPredicate = nullptr;
+  Expr *TileTripCount = nullptr;
+  if (auto *AS = dyn_cast_or_null<AttributedStmt>(S)) {
+    for (const Attr *A : AS->getAttrs()) {
+      if (const auto *H = dyn_cast<OMPInvariantPredicateBoundAttr>(A)) {
+        TileRectCond = H->getRectCond();
+        TileBodyPredicate = H->getPredicate();
+        TileTripCount = H->getTileSize();
+        break;
+      }
+    }
+    if (TileRectCond)
+      S = AS->getSubStmt();
+  }
   // OpenMP [2.9.1, Canonical Loop Form]
   //   for (init-expr; test-expr; incr-expr) structured-block
   //   for (range-decl: range-expr) structured-block
@@ -9626,6 +9646,11 @@ static bool checkOpenMPIterationSpace(
 
   bool HasErrors = false;
 
+  // Condition used for iteration-space analysis: the rectangular hint when
+  // reinterpreting an intra-tile loop, otherwise the loop's own condition.
+  Expr *EffectiveCond =
+      TileRectCond ? TileRectCond : (For ? For->getCond() : CXXFor->getCond());
+
   // Check loop variable's type.
   if (ValueDecl *LCDecl = ISC.getLoopDecl()) {
     // OpenMP [2.6, Canonical Loop Form]
@@ -9658,7 +9683,7 @@ static bool checkOpenMPIterationSpace(
            "DSA for non-loop vars");
 
     // Check test-expr.
-    HasErrors |= ISC.checkAndSetCond(For ? For->getCond() : CXXFor->getCond());
+    HasErrors |= ISC.checkAndSetCond(EffectiveCond);
 
     // Check incr-expr.
     HasErrors |= ISC.checkAndSetInc(For ? For->getInc() : CXXFor->getInc());
@@ -9668,8 +9693,18 @@ static bool checkOpenMPIterationSpace(
     return HasErrors;
 
   // Build the loop's iteration space representation.
-  ResultIterSpaces[CurrentNestedLoopCount].PreCond = ISC.buildPreCond(
-      DSA.getCurScope(), For ? For->getCond() : CXXFor->getCond(), Captures);
+  ResultIterSpaces[CurrentNestedLoopCount].PreCond =
+      ISC.buildPreCond(DSA.getCurScope(), EffectiveCond, Captures);
+  // Reinterpreted intra-tile: precondition is trivially true (.floor.iv
+  // unreadable here); overshoot handled by Predicate.
+  if (TileRectCond)
+    ResultIterSpaces[CurrentNestedLoopCount].PreCond =
+        SemaRef
+            .PerformImplicitConversion(
+                SemaRef.ActOnIntegerConstant(SourceLocation(), 1).get(),
+                SemaRef.Context.BoolTy, AssignmentAction::Casting,
+                /*AllowExplicit=*/true)
+            .get();
   ResultIterSpaces[CurrentNestedLoopCount].NumIterations =
       ISC.buildNumIterations(DSA.getCurScope(), ResultIterSpaces,
                              (isOpenMPWorksharingDirective(DKind) ||
@@ -9678,6 +9713,9 @@ static bool checkOpenMPIterationSpace(
                               isOpenMPDistributeDirective(DKind) ||
                               isOpenMPLoopTransformationDirective(DKind)),
                              Captures);
+  // Reinterpreted intra-tile: trip count is the constant TileSize.
+  if (TileTripCount)
+    ResultIterSpaces[CurrentNestedLoopCount].NumIterations = TileTripCount;
   ResultIterSpaces[CurrentNestedLoopCount].CounterVar =
       ISC.buildCounterVar(Captures, DSA);
   ResultIterSpaces[CurrentNestedLoopCount].PrivateCounterVar =
@@ -9697,6 +9735,13 @@ static bool checkOpenMPIterationSpace(
       ISC.buildMinMaxValues(DSA.getCurScope(), Captures);
   ResultIterSpaces[CurrentNestedLoopCount].FinalCondition =
       ISC.buildFinalCondition(DSA.getCurScope());
+  // Reinterpreted intra-tile: body guard is the overshoot Predicate.
+  if (TileBodyPredicate)
+    ResultIterSpaces[CurrentNestedLoopCount].FinalCondition = TileBodyPredicate;
+  // Lower bound is the live .floor.iv; wire as a floor dependence for collapse
+  // (see design note in ActOnOpenMPTileDirective).
+  if (TileRectCond)
+    ResultIterSpaces[CurrentNestedLoopCount].ReinterpretTileLB = true;
   ResultIterSpaces[CurrentNestedLoopCount].IsNonRectangularLB =
       ISC.doesInitDependOnLC();
   ResultIterSpaces[CurrentNestedLoopCount].IsNonRectangularUB =
@@ -10621,6 +10666,23 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
       SourceLocation UpdLoc = IS.IncSrcRange.getBegin();
       ExprResult Iter;
 
+      // Wire the reinterpreted lower bound to its floor dependence (see design
+      // note in ActOnOpenMPTileDirective).
+      if (IS.ReinterpretTileLB && IS.CounterInit) {
+        if (const auto *LBRef =
+                dyn_cast<DeclRefExpr>(IS.CounterInit->IgnoreParenImpCasts())) {
+          const Decl *FloorDecl = LBRef->getDecl()->getCanonicalDecl();
+          for (unsigned K = 0; K < Cnt; ++K) {
+            const auto *CV = dyn_cast_or_null<DeclRefExpr>(Built.Counters[K]);
+            if (CV && CV->getDecl()->getCanonicalDecl() == FloorDecl) {
+              IS.IsNonRectangularLB = true;
+              IS.LoopDependentIdx = K + 1;
+              break;
+            }
+          }
+        }
+      }
+
       // Compute prod
       ExprResult Prod = SemaRef.ActOnIntegerConstant(SourceLocation(), 1).get();
       for (unsigned int K = Cnt + 1; K < NestedLoopCount; ++K)
@@ -10693,11 +10755,13 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
       Built.Finals[Cnt] = Final.get();
       Built.DependentCounters[Cnt] = nullptr;
       Built.DependentInits[Cnt] = nullptr;
-      Built.FinalsConditions[Cnt] = nullptr;
+      // Transfer the body-guard condition: the loop condition for
+      // non-rectangular loops, the overshoot predicate for reinterpreted tiles,
+      // null otherwise.
+      Built.FinalsConditions[Cnt] = IS.FinalCondition;
       if (IS.IsNonRectangularLB || IS.IsNonRectangularUB) {
         Built.DependentCounters[Cnt] = Built.Counters[IS.LoopDependentIdx - 1];
         Built.DependentInits[Cnt] = Built.Inits[IS.LoopDependentIdx - 1];
-        Built.FinalsConditions[Cnt] = IS.FinalCondition;
       }
     }
   }
@@ -15081,10 +15145,8 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
   // Create iteration variables for the generated loops.
   SmallVector<VarDecl *, 4> FloorIndVars;
   SmallVector<VarDecl *, 4> TileIndVars;
-  SmallVector<VarDecl *, 4> TileCntVars;
   FloorIndVars.resize(NumLoops);
   TileIndVars.resize(NumLoops);
-  TileCntVars.resize(NumLoops);
   for (unsigned I = 0; I < NumLoops; ++I) {
     OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
 
@@ -15104,110 +15166,27 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
       FloorIndVars[I] = FloorCntDecl;
     }
 
-    // Logical iteration variable for the tile loop. Retains the meaning of
-    // the original logical iteration number (floor_iv + tile_cnt) so that
-    // LoopHelper.Updates can derive the original loop variable unchanged.
+    // Iteration variable for the tile (i.e. inner) loop.
     {
-      std::string TileIVName =
+      std::string TileCntName =
           (Twine(".tile_") + llvm::utostr(I) + ".iv." + OrigVarName).str();
 
-      auto *TileIVDecl = cast<VarDecl>(IterVarRef->getDecl());
-      TileIVDecl->setDeclName(&SemaRef.PP.getIdentifierTable().get(TileIVName));
-      TileIndVars[I] = TileIVDecl;
-    }
-
-    // Loop counter for the rectangular tile loop [0, TileSize).
-    {
-      std::string TileCntName =
-          (Twine(".tile.cnt.") + llvm::utostr(I) + ".iv." + OrigVarName).str();
-      VarDecl *TileCntDecl =
-          buildVarDecl(SemaRef, {}, CntTy, TileCntName, nullptr, OrigCntVar);
-      TileCntVars[I] = TileCntDecl;
+      // Reuse the iteration variable created by checkOpenMPLoop. It is also
+      // used by the expressions to derive the original iteration variable's
+      // value from the logical iteration number.
+      auto *TileCntDecl = cast<VarDecl>(IterVarRef->getDecl());
+      TileCntDecl->setDeclName(
+          &SemaRef.PP.getIdentifierTable().get(TileCntName));
+      TileIndVars[I] = TileCntDecl;
     }
 
     addLoopPreInits(Context, LoopHelper, LoopStmts[I], OriginalInits[I],
                     PreInits);
-
-    // Declare the logical tile IV in PreInits so it is in scope for the
-    // entire loop nest (it will be assigned in each tile loop body).
-    {
-      Decl *TileIVDeclPtr = TileIndVars[I];
-      PreInits.push_back(new (Context) DeclStmt(
-          DeclGroupRef::Create(Context, &TileIVDeclPtr, 1), {}, {}));
-    }
   }
 
   // Once the original iteration values are set, append the innermost body.
   Stmt *Inner = Body;
 
-  // Build a combined validity predicate that guards the innermost body.
-  // For each tiled dimension, check that the logical iteration number
-  // (.tile.iv) is within the original trip count. This is required because the
-  // tile loop now has rectangular (constant) bounds and may overshoot on the
-  // remainder tile. The predicate is: .tile.iv.0 < N0 && .tile.iv.1 < N1 ...
-  //
-  // Optimization: if every dimension's trip count is a compile-time constant
-  // that is evenly divisible by the corresponding tile size (also a constant),
-  // then the remainder tile is empty and the predicate is trivially true.
-  //
-  bool PredicateNeeded = false;
-  for (unsigned I = 0; I < NumLoops; ++I) {
-    Expr *TSExpr = SizesClause->getSizesRefs()[I];
-    Expr *NExpr = LoopHelpers[I].NumIterations;
-    Expr::EvalResult TSResult;
-    Expr::EvalResult NResult;
-    if (TSExpr->EvaluateAsInt(TSResult, Context) &&
-        NExpr->EvaluateAsInt(NResult, Context)) {
-      llvm::APSInt TileVal = TSResult.Val.getInt();
-      llvm::APSInt TripVal = NResult.Val.getInt();
-      if (TileVal.isStrictlyPositive() && TripVal.srem(TileVal).isZero())
-        continue;
-    }
-    PredicateNeeded = true;
-    break;
-  }
-
-  if (PredicateNeeded) {
-    Expr *CombinedPred = nullptr;
-    for (unsigned I = 0; I < NumLoops; ++I) {
-      auto *OrigCntVar = cast<DeclRefExpr>(LoopHelpers[I].Counters[0]);
-      QualType IVTy = LoopHelpers[I].NumIterations->getType();
-      Expr *FloorRef = buildDeclRefExpr(SemaRef, FloorIndVars[I], IVTy,
-                                        OrigCntVar->getExprLoc());
-      Expr *TileCntRef = buildDeclRefExpr(SemaRef, TileCntVars[I], IVTy,
-                                          OrigCntVar->getExprLoc());
-      ExprResult Sum = SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(),
-                                          BO_Add, FloorRef, TileCntRef);
-      if (!Sum.isUsable())
-        return StmtError();
-      ExprResult DimPred =
-          SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(), BO_LT,
-                             Sum.get(), LoopHelpers[I].NumIterations);
-      if (!DimPred.isUsable())
-        return StmtError();
-      if (CombinedPred) {
-        ExprResult Combined =
-            SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(), BO_LAnd,
-                               CombinedPred, DimPred.get());
-        if (!Combined.isUsable())
-          return StmtError();
-        CombinedPred = Combined.get();
-      } else {
-        CombinedPred = DimPred.get();
-      }
-    }
-    // Use the source body's location for the synthetic if-statement and
-    // AttributedStmt so any later diagnostic points at the original body
-    // location.
-    SourceLocation SyntheticLoc =
-        Body ? Body->getBeginLoc() : Inner->getBeginLoc();
-    Stmt *PredIf = IfStmt::Create(
-        Context, SyntheticLoc, IfStatementKind::Ordinary, nullptr, nullptr,
-        CombinedPred, SyntheticLoc, SyntheticLoc, Inner);
-    auto *Marker = OMPInvariantPredicateBoundAttr::CreateImplicit(Context);
-    Inner = AttributedStmt::Create(Context, SyntheticLoc, {Marker}, PredIf);
-  }
-
   auto MakeDimTileSize = [&SemaRef = this->SemaRef, &CopyTransformer, &Context,
                           SizesClause, CurScope](int I) -> Expr * {
     Expr *DimTileSizeExpr = SizesClause->getSizesRefs()[I];
@@ -15215,7 +15194,7 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     if (DimTileSizeExpr->containsErrors())
       return nullptr;
 
-    if (DimTileSizeExpr->isIntegerConstantExpr(Context))
+    if (isa<ConstantExpr>(DimTileSizeExpr))
       return AssertSuccess(CopyTransformer.TransformExpr(DimTileSizeExpr));
 
     // When the tile size is not a constant but a variable, it is possible to
@@ -15251,11 +15230,26 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
   };
 
   // Create tile loops from the inside to the outside.
-  // Each tile loop uses .tile.cnt as its counter with rectangular bounds
-  // [0, TileSize), and computes .tile.iv = .floor.iv + .tile.cnt to set
-  // the logical iteration number for LoopHelper.Updates.
-  // The min-bounded path instead uses .tile.iv directly, initialized to
-  // .floor.iv and bounded by min(.floor.iv + DimTileSize, NumIterations).
+  //
+  // Each intra-tile loop is emitted in its natural min-bounded form, which has
+  // no per-iteration body predicate and vectorizes well when run directly:
+  //   for (.tile.iv = .floor.iv; .tile.iv < min(.floor.iv + T, N); ++.tile.iv)
+  //
+  // A loop-associated directive that *consumes* this loop (e.g. `collapse`)
+  // instead needs a constant, floor-independent trip count to linearize the
+  // nest, which the min() bound cannot give it. Rather than change the
+  // emitted loop, we attach a droppable OMPInvariantPredicateBoundAttr hint
+  // below carrying an equivalent rectangular reinterpretation:
+  //   RectCond  : .tile.iv < .floor.iv + T   -- rectangular bound, analyzed by
+  //               checkOpenMPIterationSpace in place of the stored condition
+  //   Predicate : .tile.iv < N               -- remainder-tile overshoot,
+  //               applied as a body guard instead of shortening the trip count
+  //   TileSize  : T                          -- constant per-tile trip count
+  // Because the reinterpreted lower bound is still `.floor.iv`, checkOpenMPLoop
+  // wires it as a live dependence on the matching floor counter, so a
+  // collapsed `.tile.iv` re-reads the floor's current value instead of a
+  // stale preheader snapshot. Collapsing through stacked tiles (tile-of-tile)
+  // is not supported: the inner floor is not itself a collapsed counter.
   for (int I = NumLoops - 1; I >= 0; --I) {
     OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
     Expr *NumIterations = LoopHelper.NumIterations;
@@ -15266,68 +15260,96 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     // Commonly used variables. One of the constraints of an AST is that every
     // node object must appear at most once, hence we define a lambda that
     // creates a new AST node at every use.
-    auto MakeTileCntRef = [&SemaRef = this->SemaRef, &TileCntVars, I, IVTy,
-                           OrigCntVar]() {
-      return buildDeclRefExpr(SemaRef, TileCntVars[I], IVTy,
-                              OrigCntVar->getExprLoc());
-    };
     auto MakeTileIVRef = [&SemaRef = this->SemaRef, &TileIndVars, I, IVTy,
                           OrigCntVar]() {
       return buildDeclRefExpr(SemaRef, TileIndVars[I], IVTy,
                               OrigCntVar->getExprLoc());
     };
 
-    Expr *DimTileSize = MakeDimTileSize(I);
-    if (!DimTileSize)
-      return StmtError();
-
-    StmtResult InitStmt;
-    ExprResult CondExpr;
-    ExprResult IncrStmt;
-    Stmt *TileIVAssignStmt = nullptr;
-
-    // For init-statement: auto .tile.cnt = 0
+    // For init-statement: auto .tile.iv = .floor.iv
     SemaRef.AddInitializerToDecl(
-        TileCntVars[I],
-        SemaRef.ActOnIntegerConstant(LoopHelper.Init->getExprLoc(), 0).get(),
+        TileIndVars[I],
+        SemaRef
+            .DefaultLvalueConversion(
+                makeFloorIVRef(SemaRef, FloorIndVars, I, IVTy, OrigCntVar))
+            .get(),
         /*DirectInit=*/false);
-    Decl *CounterDecl = TileCntVars[I];
-    InitStmt = new (Context)
+    Decl *CounterDecl = TileIndVars[I];
+    StmtResult InitStmt = new (Context)
         DeclStmt(DeclGroupRef::Create(Context, &CounterDecl, 1),
                  OrigCntVar->getBeginLoc(), OrigCntVar->getEndLoc());
+    if (!InitStmt.isUsable())
+      return StmtError();
 
-    // For cond-expression: .tile.cnt < DimTileSize  (rectangular bound)
-    CondExpr = SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(),
-                                  BO_LT, MakeTileCntRef(), DimTileSize);
+    // For cond-expression:
+    //   .tile.iv < min(.floor.iv + DimTileSize, NumIterations)
+    Expr *DimTileSize = MakeDimTileSize(I);
+    if (!DimTileSize)
+      return StmtError();
+    ExprResult EndOfTile = SemaRef.BuildBinOp(
+        CurScope, LoopHelper.Cond->getExprLoc(), BO_Add,
+        makeFloorIVRef(SemaRef, FloorIndVars, I, IVTy, OrigCntVar),
+        DimTileSize);
+    if (!EndOfTile.isUsable())
+      return StmtError();
+    ExprResult IsPartialTile =
+        SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
+                           NumIterations, EndOfTile.get());
+    if (!IsPartialTile.isUsable())
+      return StmtError();
+    ExprResult MinTileAndIterSpace = SemaRef.ActOnConditionalOp(
+        LoopHelper.Cond->getBeginLoc(), LoopHelper.Cond->getEndLoc(),
+        IsPartialTile.get(), NumIterations, EndOfTile.get());
+    if (!MinTileAndIterSpace.isUsable())
+      return StmtError();
+    ExprResult CondExpr =
+        SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
+                           MakeTileIVRef(), MinTileAndIterSpace.get());
+    if (!CondExpr.isUsable())
+      return StmtError();
 
-    // For incr-statement: ++.tile.cnt
-    IncrStmt = SemaRef.BuildUnaryOp(CurScope, LoopHelper.Inc->getExprLoc(),
-                                    UO_PreInc, MakeTileCntRef());
+    // For incr-statement: ++.tile.iv
+    ExprResult IncrStmt = SemaRef.BuildUnaryOp(
+        CurScope, LoopHelper.Inc->getExprLoc(), UO_PreInc, MakeTileIVRef());
+    if (!IncrStmt.isUsable())
+      return StmtError();
 
-    // Compute the logical iteration number:
-    //   .tile.iv = .floor.iv + .tile.cnt
-    ExprResult FloorPlusCnt = SemaRef.BuildBinOp(
-        CurScope, OrigCntVar->getExprLoc(), BO_Add,
+    // Build OMPInvariantPredicateBoundAttr hint (RectCond, Predicate,
+    // TileSize).
+    Expr *RectDimTileSize = MakeDimTileSize(I);
+    if (!RectDimTileSize)
+      return StmtError();
+    ExprResult RectEndOfTile = SemaRef.BuildBinOp(
+        CurScope, LoopHelper.Cond->getExprLoc(), BO_Add,
         makeFloorIVRef(SemaRef, FloorIndVars, I, IVTy, OrigCntVar),
-        MakeTileCntRef());
-    if (!FloorPlusCnt.isUsable())
+        RectDimTileSize);
+    if (!RectEndOfTile.isUsable())
       return StmtError();
-    ExprResult TileIVAssign =
-        SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(), BO_Assign,
-                           MakeTileIVRef(), FloorPlusCnt.get());
-    if (!TileIVAssign.isUsable())
+    ExprResult RectCond =
+        SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
+                           MakeTileIVRef(), RectEndOfTile.get());
+    ExprResult Predicate =
+        SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
+                           MakeTileIVRef(), NumIterations);
+    Expr *TileSizeExpr = MakeDimTileSize(I);
+    if (!TileSizeExpr)
       return StmtError();
-    TileIVAssignStmt = TileIVAssign.get();
-
-    if (!InitStmt.isUsable() || !CondExpr.isUsable() || !IncrStmt.isUsable())
+    ExprResult TileSize = SemaRef.PerformImplicitConversion(
+        TileSizeExpr, IVTy, AssignmentAction::Converting,
+        /*AllowExplicit=*/true);
+    if (!RectCond.isUsable() || !Predicate.isUsable() || !TileSize.isUsable())
       return StmtError();
 
+    // Statements to set the original iteration variable's value from the
+    // logical iteration number.
     // Generated for loop is:
     // \code
-    // for (auto .tile.cnt = 0; .tile.cnt < DimTileSize; ++.tile.cnt) {
-    //   .tile.iv = .floor.iv + .tile.cnt;
-    //   Original_counter_update;  // derives orig var from .tile.iv
-    //   Inner;                    // predicated body or inner tile loops
+    // Original_for_init;
+    // for (auto .tile.iv = .floor.iv;
+    //      .tile.iv < min(.floor.iv + DimTileSize, NumIterations);
+    //      ++.tile.iv) {
+    //   Original_Body;
+    //   Original_counter_update;
     // }
     // \endcode
     // FIXME: If the innermost body is an loop itself, inserting these
@@ -15335,8 +15357,6 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     // for applying tiling again). If this is the case, sink the expressions
     // further into the inner loop.
     SmallVector<Stmt *, 4> BodyParts;
-    if (TileIVAssignStmt)
-      BodyParts.push_back(TileIVAssignStmt);
     BodyParts.append(LoopHelper.Updates.begin(), LoopHelper.Updates.end());
     if (auto *SourceCXXFor = dyn_cast<CXXForRangeStmt>(LoopStmt))
       BodyParts.push_back(SourceCXXFor->getLoopVarStmt());
@@ -15347,6 +15367,13 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
         ForStmt(Context, InitStmt.get(), CondExpr.get(), nullptr,
                 IncrStmt.get(), Inner, LoopHelper.Init->getBeginLoc(),
                 LoopHelper.Init->getBeginLoc(), LoopHelper.Inc->getEndLoc());
+
+    // Attach the droppable reinterpretation attribute to the intra-tile loop.
+    auto *Hint = OMPInvariantPredicateBoundAttr::CreateImplicit(
+        Context, RectCond.get(), Predicate.get(), TileSize.get(),
+        Inner->getSourceRange());
+    Inner =
+        AttributedStmt::Create(Context, Inner->getBeginLoc(), {Hint}, Inner);
   }
 
   // Create floor loops from the inside to the outside.

>From 7933f9d84e4cccce35c67db56a9cc002a9cb3a25 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 15 Jul 2026 10:12:35 -0400
Subject: [PATCH 16/18] tests-updated

---
 clang/test/OpenMP/interchange_codegen.cpp     | 3444 +++++++++--------
 .../tile_canonical_required_codegen.cpp       |   86 -
 clang/test/OpenMP/tile_codegen.cpp            | 1980 +++++-----
 .../OpenMP/tile_codegen_for_dependent.cpp     |   53 +-
 clang/test/OpenMP/tile_codegen_tile_for.cpp   |  118 +-
 .../tile_collapse_reinterpret_codegen.cpp     |  108 +
 clang/test/OpenMP/tile_rect_codegen.cpp       |   15 +-
 clang/test/OpenMP/tile_rect_codegen_ir.cpp    |  510 +--
 clang/test/OpenMP/tile_vectorize_codegen.cpp  |   14 +
 clang/test/OpenMP/unroll_codegen_tile_for.cpp |   89 +-
 .../runtime/test/transform/tile/foreach.cpp   |    8 -
 openmp/runtime/test/transform/tile/intfor.c   |   58 +-
 .../runtime/test/transform/tile/iterfor.cpp   |    6 -
 .../tile/parallel-wsloop-collapse-foreach.cpp |   24 -
 ...llel-wsloop-collapse-stacked-tile-deep.cpp |   32 +
 .../parallel-wsloop-collapse-stacked-tile.cpp |   31 +
 ...rallel-wsloop-collapse-through-tile-2d.cpp |   56 +
 ...l-wsloop-collapse-through-tile-compose.cpp |   51 +
 ...-wsloop-collapse-through-tile-iterator.cpp |   74 +
 .../parallel-wsloop-collapse-through-tile.cpp |   10 +-
 20 files changed, 3600 insertions(+), 3167 deletions(-)
 delete mode 100644 clang/test/OpenMP/tile_canonical_required_codegen.cpp
 create mode 100644 clang/test/OpenMP/tile_collapse_reinterpret_codegen.cpp
 create mode 100644 clang/test/OpenMP/tile_vectorize_codegen.cpp
 create mode 100644 openmp/runtime/test/transform/tile/parallel-wsloop-collapse-stacked-tile-deep.cpp
 create mode 100644 openmp/runtime/test/transform/tile/parallel-wsloop-collapse-stacked-tile.cpp
 create mode 100644 openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile-2d.cpp
 create mode 100644 openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile-compose.cpp
 create mode 100644 openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile-iterator.cpp

diff --git a/clang/test/OpenMP/interchange_codegen.cpp b/clang/test/OpenMP/interchange_codegen.cpp
index 67ec7e34c5dfd..cc6e644ac70ac 100644
--- a/clang/test/OpenMP/interchange_codegen.cpp
+++ b/clang/test/OpenMP/interchange_codegen.cpp
@@ -811,21 +811,23 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTNEW_STEP10:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_14:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_17:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_19:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_K:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_K:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[_TMP35:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[I35:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[J36:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV_K37:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_K38:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[I39:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[J40:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV_K41:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_K42:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
@@ -863,454 +865,455 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
 // CHECK1-NEXT:    [[ADD15:%.*]] = add i32 [[TMP15]], 1
 // CHECK1-NEXT:    store i32 [[ADD15]], ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[SUB17:%.*]] = sub i32 [[TMP16]], [[TMP17]]
-// CHECK1-NEXT:    [[SUB18:%.*]] = sub i32 [[SUB17]], 1
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[ADD19:%.*]] = add i32 [[SUB18]], [[TMP18]]
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[DIV20:%.*]] = udiv i32 [[ADD19]], [[TMP19]]
-// CHECK1-NEXT:    [[CONV:%.*]] = zext i32 [[DIV20]] to i64
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB21:%.*]] = sub i32 [[TMP20]], [[TMP21]]
-// CHECK1-NEXT:    [[SUB22:%.*]] = sub i32 [[SUB21]], 1
-// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD23:%.*]] = add i32 [[SUB22]], [[TMP22]]
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV24:%.*]] = udiv i32 [[ADD23]], [[TMP23]]
-// CHECK1-NEXT:    [[CONV25:%.*]] = zext i32 [[DIV24]] to i64
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV25]]
-// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB26:%.*]] = sub i32 [[TMP24]], -31
-// CHECK1-NEXT:    [[DIV27:%.*]] = udiv i32 [[SUB26]], 32
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[_TMP2]], align 4
+// CHECK1-NEXT:    store i32 [[TMP16]], ptr [[DOTCAPTURE_EXPR_16]], align 4
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[_TMP2]], align 4
+// CHECK1-NEXT:    [[ADD18:%.*]] = add i32 [[TMP17]], 32
+// CHECK1-NEXT:    store i32 [[ADD18]], ptr [[DOTCAPTURE_EXPR_17]], align 4
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[SUB20:%.*]] = sub i32 [[TMP18]], [[TMP19]]
+// CHECK1-NEXT:    [[SUB21:%.*]] = sub i32 [[SUB20]], 1
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[ADD22:%.*]] = add i32 [[SUB21]], [[TMP20]]
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[DIV23:%.*]] = udiv i32 [[ADD22]], [[TMP21]]
+// CHECK1-NEXT:    [[CONV:%.*]] = zext i32 [[DIV23]] to i64
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB24:%.*]] = sub i32 [[TMP22]], [[TMP23]]
+// CHECK1-NEXT:    [[SUB25:%.*]] = sub i32 [[SUB24]], 1
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD26:%.*]] = add i32 [[SUB25]], [[TMP24]]
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV27:%.*]] = udiv i32 [[ADD26]], [[TMP25]]
 // CHECK1-NEXT:    [[CONV28:%.*]] = zext i32 [[DIV27]] to i64
-// CHECK1-NEXT:    [[MUL29:%.*]] = mul nsw i64 [[MUL]], [[CONV28]]
-// CHECK1-NEXT:    [[MUL30:%.*]] = mul nsw i64 [[MUL29]], 32
-// CHECK1-NEXT:    [[SUB31:%.*]] = sub nsw i64 [[MUL30]], 1
-// CHECK1-NEXT:    store i64 [[SUB31]], ptr [[DOTCAPTURE_EXPR_16]], align 8
-// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    store i32 [[TMP25]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    store i32 [[TMP26]], ptr [[J]], align 4
-// CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_K]], align 4
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_K]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV28]]
+// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB29:%.*]] = sub i32 [[TMP26]], -31
+// CHECK1-NEXT:    [[DIV30:%.*]] = udiv i32 [[SUB29]], 32
+// CHECK1-NEXT:    [[CONV31:%.*]] = zext i32 [[DIV30]] to i64
+// CHECK1-NEXT:    [[MUL32:%.*]] = mul nsw i64 [[MUL]], [[CONV31]]
+// CHECK1-NEXT:    [[MUL33:%.*]] = mul nsw i64 [[MUL32]], 32
+// CHECK1-NEXT:    [[SUB34:%.*]] = sub nsw i64 [[MUL33]], 1
+// CHECK1-NEXT:    store i64 [[SUB34]], ptr [[DOTCAPTURE_EXPR_19]], align 8
 // CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP27]], [[TMP28]]
+// CHECK1-NEXT:    store i32 [[TMP27]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    store i32 [[TMP28]], ptr [[J]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_K]], align 4
+// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K]], align 4
+// CHECK1-NEXT:    store i32 [[TMP29]], ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[_TMP35]], align 4
+// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP30]], [[TMP31]]
 // CHECK1-NEXT:    br i1 [[CMP]], label [[LAND_LHS_TRUE:%.*]], label [[OMP_PRECOND_END:%.*]]
 // CHECK1:       land.lhs.true:
-// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[CMP32:%.*]] = icmp slt i32 [[TMP29]], [[TMP30]]
-// CHECK1-NEXT:    br i1 [[CMP32]], label [[LAND_LHS_TRUE33:%.*]], label [[OMP_PRECOND_END]]
-// CHECK1:       land.lhs.true33:
-// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[CMP34:%.*]] = icmp ult i32 0, [[TMP31]]
-// CHECK1-NEXT:    br i1 [[CMP34]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
+// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[CMP36:%.*]] = icmp slt i32 [[TMP32]], [[TMP33]]
+// CHECK1-NEXT:    br i1 [[CMP36]], label [[LAND_LHS_TRUE37:%.*]], label [[OMP_PRECOND_END]]
+// CHECK1:       land.lhs.true37:
+// CHECK1-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[CMP38:%.*]] = icmp ult i32 0, [[TMP34]]
+// CHECK1-NEXT:    br i1 [[CMP38]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
 // CHECK1:       omp.precond.then:
 // CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
-// CHECK1-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
-// CHECK1-NEXT:    store i64 [[TMP32]], ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_19]], align 8
+// CHECK1-NEXT:    store i64 [[TMP35]], ptr [[DOTOMP_UB]], align 8
 // CHECK1-NEXT:    store i64 1, ptr [[DOTOMP_STRIDE]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
 // CHECK1-NEXT:    call void @__kmpc_for_static_init_8(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i64 1, i64 1)
-// CHECK1-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
-// CHECK1-NEXT:    [[CMP39:%.*]] = icmp sgt i64 [[TMP33]], [[TMP34]]
-// CHECK1-NEXT:    br i1 [[CMP39]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_19]], align 8
+// CHECK1-NEXT:    [[CMP43:%.*]] = icmp sgt i64 [[TMP36]], [[TMP37]]
+// CHECK1-NEXT:    br i1 [[CMP43]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK1:       cond.true:
-// CHECK1-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK1-NEXT:    [[TMP38:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_19]], align 8
 // CHECK1-NEXT:    br label [[COND_END:%.*]]
 // CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[TMP39:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
 // CHECK1-NEXT:    br label [[COND_END]]
 // CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i64 [ [[TMP35]], [[COND_TRUE]] ], [ [[TMP36]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[COND:%.*]] = phi i64 [ [[TMP38]], [[COND_TRUE]] ], [ [[TMP39]], [[COND_FALSE]] ]
 // CHECK1-NEXT:    store i64 [[COND]], ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
-// CHECK1-NEXT:    store i64 [[TMP37]], ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP40:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
+// CHECK1-NEXT:    store i64 [[TMP40]], ptr [[DOTOMP_IV]], align 8
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK1:       omp.inner.for.cond:
-// CHECK1-NEXT:    [[TMP38:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP39:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[CMP40:%.*]] = icmp sle i64 [[TMP38]], [[TMP39]]
-// CHECK1-NEXT:    br i1 [[CMP40]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
-// CHECK1:       omp.inner.for.body:
-// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[CONV41:%.*]] = sext i32 [[TMP40]] to i64
 // CHECK1-NEXT:    [[TMP41:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB42:%.*]] = sub i32 [[TMP42]], [[TMP43]]
-// CHECK1-NEXT:    [[SUB43:%.*]] = sub i32 [[SUB42]], 1
-// CHECK1-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD44:%.*]] = add i32 [[SUB43]], [[TMP44]]
-// CHECK1-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV45:%.*]] = udiv i32 [[ADD44]], [[TMP45]]
-// CHECK1-NEXT:    [[MUL46:%.*]] = mul i32 1, [[DIV45]]
-// CHECK1-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB47:%.*]] = sub i32 [[TMP46]], -31
-// CHECK1-NEXT:    [[DIV48:%.*]] = udiv i32 [[SUB47]], 32
-// CHECK1-NEXT:    [[MUL49:%.*]] = mul i32 [[MUL46]], [[DIV48]]
-// CHECK1-NEXT:    [[MUL50:%.*]] = mul i32 [[MUL49]], 32
-// CHECK1-NEXT:    [[CONV51:%.*]] = zext i32 [[MUL50]] to i64
-// CHECK1-NEXT:    [[DIV52:%.*]] = sdiv i64 [[TMP41]], [[CONV51]]
-// CHECK1-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[CONV53:%.*]] = sext i32 [[TMP47]] to i64
-// CHECK1-NEXT:    [[MUL54:%.*]] = mul nsw i64 [[DIV52]], [[CONV53]]
-// CHECK1-NEXT:    [[ADD55:%.*]] = add nsw i64 [[CONV41]], [[MUL54]]
-// CHECK1-NEXT:    [[CONV56:%.*]] = trunc i64 [[ADD55]] to i32
-// CHECK1-NEXT:    store i32 [[CONV56]], ptr [[I35]], align 4
-// CHECK1-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP48]] to i64
-// CHECK1-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP50:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB58:%.*]] = sub i32 [[TMP51]], [[TMP52]]
-// CHECK1-NEXT:    [[SUB59:%.*]] = sub i32 [[SUB58]], 1
-// CHECK1-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD60:%.*]] = add i32 [[SUB59]], [[TMP53]]
-// CHECK1-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV61:%.*]] = udiv i32 [[ADD60]], [[TMP54]]
-// CHECK1-NEXT:    [[MUL62:%.*]] = mul i32 1, [[DIV61]]
-// CHECK1-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB63:%.*]] = sub i32 [[TMP55]], -31
-// CHECK1-NEXT:    [[DIV64:%.*]] = udiv i32 [[SUB63]], 32
-// CHECK1-NEXT:    [[MUL65:%.*]] = mul i32 [[MUL62]], [[DIV64]]
-// CHECK1-NEXT:    [[MUL66:%.*]] = mul i32 [[MUL65]], 32
-// CHECK1-NEXT:    [[CONV67:%.*]] = zext i32 [[MUL66]] to i64
-// CHECK1-NEXT:    [[DIV68:%.*]] = sdiv i64 [[TMP50]], [[CONV67]]
-// CHECK1-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB69:%.*]] = sub i32 [[TMP56]], [[TMP57]]
-// CHECK1-NEXT:    [[SUB70:%.*]] = sub i32 [[SUB69]], 1
-// CHECK1-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD71:%.*]] = add i32 [[SUB70]], [[TMP58]]
-// CHECK1-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV72:%.*]] = udiv i32 [[ADD71]], [[TMP59]]
-// CHECK1-NEXT:    [[MUL73:%.*]] = mul i32 1, [[DIV72]]
-// CHECK1-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB74:%.*]] = sub i32 [[TMP60]], -31
-// CHECK1-NEXT:    [[DIV75:%.*]] = udiv i32 [[SUB74]], 32
-// CHECK1-NEXT:    [[MUL76:%.*]] = mul i32 [[MUL73]], [[DIV75]]
-// CHECK1-NEXT:    [[MUL77:%.*]] = mul i32 [[MUL76]], 32
-// CHECK1-NEXT:    [[CONV78:%.*]] = zext i32 [[MUL77]] to i64
-// CHECK1-NEXT:    [[MUL79:%.*]] = mul nsw i64 [[DIV68]], [[CONV78]]
-// CHECK1-NEXT:    [[SUB80:%.*]] = sub nsw i64 [[TMP49]], [[MUL79]]
-// CHECK1-NEXT:    [[TMP61:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB81:%.*]] = sub i32 [[TMP61]], -31
-// CHECK1-NEXT:    [[DIV82:%.*]] = udiv i32 [[SUB81]], 32
-// CHECK1-NEXT:    [[MUL83:%.*]] = mul i32 1, [[DIV82]]
-// CHECK1-NEXT:    [[MUL84:%.*]] = mul i32 [[MUL83]], 32
-// CHECK1-NEXT:    [[CONV85:%.*]] = zext i32 [[MUL84]] to i64
-// CHECK1-NEXT:    [[DIV86:%.*]] = sdiv i64 [[SUB80]], [[CONV85]]
+// CHECK1-NEXT:    [[TMP42:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[CMP44:%.*]] = icmp sle i64 [[TMP41]], [[TMP42]]
+// CHECK1-NEXT:    br i1 [[CMP44]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK1:       omp.inner.for.body:
+// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[CONV45:%.*]] = sext i32 [[TMP43]] to i64
+// CHECK1-NEXT:    [[TMP44:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB46:%.*]] = sub i32 [[TMP45]], [[TMP46]]
+// CHECK1-NEXT:    [[SUB47:%.*]] = sub i32 [[SUB46]], 1
+// CHECK1-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD48:%.*]] = add i32 [[SUB47]], [[TMP47]]
+// CHECK1-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV49:%.*]] = udiv i32 [[ADD48]], [[TMP48]]
+// CHECK1-NEXT:    [[MUL50:%.*]] = mul i32 1, [[DIV49]]
+// CHECK1-NEXT:    [[TMP49:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB51:%.*]] = sub i32 [[TMP49]], -31
+// CHECK1-NEXT:    [[DIV52:%.*]] = udiv i32 [[SUB51]], 32
+// CHECK1-NEXT:    [[MUL53:%.*]] = mul i32 [[MUL50]], [[DIV52]]
+// CHECK1-NEXT:    [[MUL54:%.*]] = mul i32 [[MUL53]], 32
+// CHECK1-NEXT:    [[CONV55:%.*]] = zext i32 [[MUL54]] to i64
+// CHECK1-NEXT:    [[DIV56:%.*]] = sdiv i64 [[TMP44]], [[CONV55]]
+// CHECK1-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP50]] to i64
+// CHECK1-NEXT:    [[MUL58:%.*]] = mul nsw i64 [[DIV56]], [[CONV57]]
+// CHECK1-NEXT:    [[ADD59:%.*]] = add nsw i64 [[CONV45]], [[MUL58]]
+// CHECK1-NEXT:    [[CONV60:%.*]] = trunc i64 [[ADD59]] to i32
+// CHECK1-NEXT:    store i32 [[CONV60]], ptr [[I39]], align 4
+// CHECK1-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[CONV61:%.*]] = sext i32 [[TMP51]] to i64
+// CHECK1-NEXT:    [[TMP52:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP53:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB62:%.*]] = sub i32 [[TMP54]], [[TMP55]]
+// CHECK1-NEXT:    [[SUB63:%.*]] = sub i32 [[SUB62]], 1
+// CHECK1-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD64:%.*]] = add i32 [[SUB63]], [[TMP56]]
+// CHECK1-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV65:%.*]] = udiv i32 [[ADD64]], [[TMP57]]
+// CHECK1-NEXT:    [[MUL66:%.*]] = mul i32 1, [[DIV65]]
+// CHECK1-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB67:%.*]] = sub i32 [[TMP58]], -31
+// CHECK1-NEXT:    [[DIV68:%.*]] = udiv i32 [[SUB67]], 32
+// CHECK1-NEXT:    [[MUL69:%.*]] = mul i32 [[MUL66]], [[DIV68]]
+// CHECK1-NEXT:    [[MUL70:%.*]] = mul i32 [[MUL69]], 32
+// CHECK1-NEXT:    [[CONV71:%.*]] = zext i32 [[MUL70]] to i64
+// CHECK1-NEXT:    [[DIV72:%.*]] = sdiv i64 [[TMP53]], [[CONV71]]
+// CHECK1-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB73:%.*]] = sub i32 [[TMP59]], [[TMP60]]
+// CHECK1-NEXT:    [[SUB74:%.*]] = sub i32 [[SUB73]], 1
+// CHECK1-NEXT:    [[TMP61:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD75:%.*]] = add i32 [[SUB74]], [[TMP61]]
 // CHECK1-NEXT:    [[TMP62:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[CONV87:%.*]] = sext i32 [[TMP62]] to i64
-// CHECK1-NEXT:    [[MUL88:%.*]] = mul nsw i64 [[DIV86]], [[CONV87]]
-// CHECK1-NEXT:    [[ADD89:%.*]] = add nsw i64 [[CONV57]], [[MUL88]]
-// CHECK1-NEXT:    [[CONV90:%.*]] = trunc i64 [[ADD89]] to i32
-// CHECK1-NEXT:    store i32 [[CONV90]], ptr [[J36]], align 4
-// CHECK1-NEXT:    [[TMP63:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP64:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB91:%.*]] = sub i32 [[TMP65]], [[TMP66]]
-// CHECK1-NEXT:    [[SUB92:%.*]] = sub i32 [[SUB91]], 1
-// CHECK1-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD93:%.*]] = add i32 [[SUB92]], [[TMP67]]
-// CHECK1-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV94:%.*]] = udiv i32 [[ADD93]], [[TMP68]]
-// CHECK1-NEXT:    [[MUL95:%.*]] = mul i32 1, [[DIV94]]
-// CHECK1-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB96:%.*]] = sub i32 [[TMP69]], -31
-// CHECK1-NEXT:    [[DIV97:%.*]] = udiv i32 [[SUB96]], 32
-// CHECK1-NEXT:    [[MUL98:%.*]] = mul i32 [[MUL95]], [[DIV97]]
-// CHECK1-NEXT:    [[MUL99:%.*]] = mul i32 [[MUL98]], 32
-// CHECK1-NEXT:    [[CONV100:%.*]] = zext i32 [[MUL99]] to i64
-// CHECK1-NEXT:    [[DIV101:%.*]] = sdiv i64 [[TMP64]], [[CONV100]]
-// CHECK1-NEXT:    [[TMP70:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB102:%.*]] = sub i32 [[TMP70]], [[TMP71]]
-// CHECK1-NEXT:    [[SUB103:%.*]] = sub i32 [[SUB102]], 1
-// CHECK1-NEXT:    [[TMP72:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD104:%.*]] = add i32 [[SUB103]], [[TMP72]]
-// CHECK1-NEXT:    [[TMP73:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV105:%.*]] = udiv i32 [[ADD104]], [[TMP73]]
-// CHECK1-NEXT:    [[MUL106:%.*]] = mul i32 1, [[DIV105]]
-// CHECK1-NEXT:    [[TMP74:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB107:%.*]] = sub i32 [[TMP74]], -31
-// CHECK1-NEXT:    [[DIV108:%.*]] = udiv i32 [[SUB107]], 32
-// CHECK1-NEXT:    [[MUL109:%.*]] = mul i32 [[MUL106]], [[DIV108]]
-// CHECK1-NEXT:    [[MUL110:%.*]] = mul i32 [[MUL109]], 32
-// CHECK1-NEXT:    [[CONV111:%.*]] = zext i32 [[MUL110]] to i64
-// CHECK1-NEXT:    [[MUL112:%.*]] = mul nsw i64 [[DIV101]], [[CONV111]]
-// CHECK1-NEXT:    [[SUB113:%.*]] = sub nsw i64 [[TMP63]], [[MUL112]]
-// CHECK1-NEXT:    [[TMP75:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP76:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP78:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB114:%.*]] = sub i32 [[TMP77]], [[TMP78]]
-// CHECK1-NEXT:    [[SUB115:%.*]] = sub i32 [[SUB114]], 1
-// CHECK1-NEXT:    [[TMP79:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD116:%.*]] = add i32 [[SUB115]], [[TMP79]]
-// CHECK1-NEXT:    [[TMP80:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV117:%.*]] = udiv i32 [[ADD116]], [[TMP80]]
-// CHECK1-NEXT:    [[MUL118:%.*]] = mul i32 1, [[DIV117]]
-// CHECK1-NEXT:    [[TMP81:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB119:%.*]] = sub i32 [[TMP81]], -31
-// CHECK1-NEXT:    [[DIV120:%.*]] = udiv i32 [[SUB119]], 32
-// CHECK1-NEXT:    [[MUL121:%.*]] = mul i32 [[MUL118]], [[DIV120]]
-// CHECK1-NEXT:    [[MUL122:%.*]] = mul i32 [[MUL121]], 32
-// CHECK1-NEXT:    [[CONV123:%.*]] = zext i32 [[MUL122]] to i64
-// CHECK1-NEXT:    [[DIV124:%.*]] = sdiv i64 [[TMP76]], [[CONV123]]
-// CHECK1-NEXT:    [[TMP82:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB125:%.*]] = sub i32 [[TMP82]], [[TMP83]]
-// CHECK1-NEXT:    [[SUB126:%.*]] = sub i32 [[SUB125]], 1
-// CHECK1-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD127:%.*]] = add i32 [[SUB126]], [[TMP84]]
-// CHECK1-NEXT:    [[TMP85:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV128:%.*]] = udiv i32 [[ADD127]], [[TMP85]]
-// CHECK1-NEXT:    [[MUL129:%.*]] = mul i32 1, [[DIV128]]
-// CHECK1-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB130:%.*]] = sub i32 [[TMP86]], -31
-// CHECK1-NEXT:    [[DIV131:%.*]] = udiv i32 [[SUB130]], 32
-// CHECK1-NEXT:    [[MUL132:%.*]] = mul i32 [[MUL129]], [[DIV131]]
-// CHECK1-NEXT:    [[MUL133:%.*]] = mul i32 [[MUL132]], 32
-// CHECK1-NEXT:    [[CONV134:%.*]] = zext i32 [[MUL133]] to i64
-// CHECK1-NEXT:    [[MUL135:%.*]] = mul nsw i64 [[DIV124]], [[CONV134]]
-// CHECK1-NEXT:    [[SUB136:%.*]] = sub nsw i64 [[TMP75]], [[MUL135]]
-// CHECK1-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB137:%.*]] = sub i32 [[TMP87]], -31
-// CHECK1-NEXT:    [[DIV138:%.*]] = udiv i32 [[SUB137]], 32
-// CHECK1-NEXT:    [[MUL139:%.*]] = mul i32 1, [[DIV138]]
-// CHECK1-NEXT:    [[MUL140:%.*]] = mul i32 [[MUL139]], 32
-// CHECK1-NEXT:    [[CONV141:%.*]] = zext i32 [[MUL140]] to i64
-// CHECK1-NEXT:    [[DIV142:%.*]] = sdiv i64 [[SUB136]], [[CONV141]]
-// CHECK1-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB143:%.*]] = sub i32 [[TMP88]], -31
-// CHECK1-NEXT:    [[DIV144:%.*]] = udiv i32 [[SUB143]], 32
-// CHECK1-NEXT:    [[MUL145:%.*]] = mul i32 1, [[DIV144]]
-// CHECK1-NEXT:    [[MUL146:%.*]] = mul i32 [[MUL145]], 32
-// CHECK1-NEXT:    [[CONV147:%.*]] = zext i32 [[MUL146]] to i64
-// CHECK1-NEXT:    [[MUL148:%.*]] = mul nsw i64 [[DIV142]], [[CONV147]]
-// CHECK1-NEXT:    [[SUB149:%.*]] = sub nsw i64 [[SUB113]], [[MUL148]]
-// CHECK1-NEXT:    [[DIV150:%.*]] = sdiv i64 [[SUB149]], 32
-// CHECK1-NEXT:    [[MUL151:%.*]] = mul nsw i64 [[DIV150]], 32
-// CHECK1-NEXT:    [[ADD152:%.*]] = add nsw i64 0, [[MUL151]]
-// CHECK1-NEXT:    [[CONV153:%.*]] = trunc i64 [[ADD152]] to i32
-// CHECK1-NEXT:    store i32 [[CONV153]], ptr [[DOTFLOOR_0_IV_K37]], align 4
-// CHECK1-NEXT:    [[TMP89:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP90:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB154:%.*]] = sub i32 [[TMP91]], [[TMP92]]
-// CHECK1-NEXT:    [[SUB155:%.*]] = sub i32 [[SUB154]], 1
-// CHECK1-NEXT:    [[TMP93:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD156:%.*]] = add i32 [[SUB155]], [[TMP93]]
-// CHECK1-NEXT:    [[TMP94:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV157:%.*]] = udiv i32 [[ADD156]], [[TMP94]]
-// CHECK1-NEXT:    [[MUL158:%.*]] = mul i32 1, [[DIV157]]
-// CHECK1-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB159:%.*]] = sub i32 [[TMP95]], -31
-// CHECK1-NEXT:    [[DIV160:%.*]] = udiv i32 [[SUB159]], 32
-// CHECK1-NEXT:    [[MUL161:%.*]] = mul i32 [[MUL158]], [[DIV160]]
-// CHECK1-NEXT:    [[MUL162:%.*]] = mul i32 [[MUL161]], 32
-// CHECK1-NEXT:    [[CONV163:%.*]] = zext i32 [[MUL162]] to i64
-// CHECK1-NEXT:    [[DIV164:%.*]] = sdiv i64 [[TMP90]], [[CONV163]]
-// CHECK1-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP97:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB165:%.*]] = sub i32 [[TMP96]], [[TMP97]]
-// CHECK1-NEXT:    [[SUB166:%.*]] = sub i32 [[SUB165]], 1
+// CHECK1-NEXT:    [[DIV76:%.*]] = udiv i32 [[ADD75]], [[TMP62]]
+// CHECK1-NEXT:    [[MUL77:%.*]] = mul i32 1, [[DIV76]]
+// CHECK1-NEXT:    [[TMP63:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB78:%.*]] = sub i32 [[TMP63]], -31
+// CHECK1-NEXT:    [[DIV79:%.*]] = udiv i32 [[SUB78]], 32
+// CHECK1-NEXT:    [[MUL80:%.*]] = mul i32 [[MUL77]], [[DIV79]]
+// CHECK1-NEXT:    [[MUL81:%.*]] = mul i32 [[MUL80]], 32
+// CHECK1-NEXT:    [[CONV82:%.*]] = zext i32 [[MUL81]] to i64
+// CHECK1-NEXT:    [[MUL83:%.*]] = mul nsw i64 [[DIV72]], [[CONV82]]
+// CHECK1-NEXT:    [[SUB84:%.*]] = sub nsw i64 [[TMP52]], [[MUL83]]
+// CHECK1-NEXT:    [[TMP64:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB85:%.*]] = sub i32 [[TMP64]], -31
+// CHECK1-NEXT:    [[DIV86:%.*]] = udiv i32 [[SUB85]], 32
+// CHECK1-NEXT:    [[MUL87:%.*]] = mul i32 1, [[DIV86]]
+// CHECK1-NEXT:    [[MUL88:%.*]] = mul i32 [[MUL87]], 32
+// CHECK1-NEXT:    [[CONV89:%.*]] = zext i32 [[MUL88]] to i64
+// CHECK1-NEXT:    [[DIV90:%.*]] = sdiv i64 [[SUB84]], [[CONV89]]
+// CHECK1-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[CONV91:%.*]] = sext i32 [[TMP65]] to i64
+// CHECK1-NEXT:    [[MUL92:%.*]] = mul nsw i64 [[DIV90]], [[CONV91]]
+// CHECK1-NEXT:    [[ADD93:%.*]] = add nsw i64 [[CONV61]], [[MUL92]]
+// CHECK1-NEXT:    [[CONV94:%.*]] = trunc i64 [[ADD93]] to i32
+// CHECK1-NEXT:    store i32 [[CONV94]], ptr [[J40]], align 4
+// CHECK1-NEXT:    [[TMP66:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP67:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB95:%.*]] = sub i32 [[TMP68]], [[TMP69]]
+// CHECK1-NEXT:    [[SUB96:%.*]] = sub i32 [[SUB95]], 1
+// CHECK1-NEXT:    [[TMP70:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD97:%.*]] = add i32 [[SUB96]], [[TMP70]]
+// CHECK1-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV98:%.*]] = udiv i32 [[ADD97]], [[TMP71]]
+// CHECK1-NEXT:    [[MUL99:%.*]] = mul i32 1, [[DIV98]]
+// CHECK1-NEXT:    [[TMP72:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB100:%.*]] = sub i32 [[TMP72]], -31
+// CHECK1-NEXT:    [[DIV101:%.*]] = udiv i32 [[SUB100]], 32
+// CHECK1-NEXT:    [[MUL102:%.*]] = mul i32 [[MUL99]], [[DIV101]]
+// CHECK1-NEXT:    [[MUL103:%.*]] = mul i32 [[MUL102]], 32
+// CHECK1-NEXT:    [[CONV104:%.*]] = zext i32 [[MUL103]] to i64
+// CHECK1-NEXT:    [[DIV105:%.*]] = sdiv i64 [[TMP67]], [[CONV104]]
+// CHECK1-NEXT:    [[TMP73:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP74:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB106:%.*]] = sub i32 [[TMP73]], [[TMP74]]
+// CHECK1-NEXT:    [[SUB107:%.*]] = sub i32 [[SUB106]], 1
+// CHECK1-NEXT:    [[TMP75:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD108:%.*]] = add i32 [[SUB107]], [[TMP75]]
+// CHECK1-NEXT:    [[TMP76:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV109:%.*]] = udiv i32 [[ADD108]], [[TMP76]]
+// CHECK1-NEXT:    [[MUL110:%.*]] = mul i32 1, [[DIV109]]
+// CHECK1-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB111:%.*]] = sub i32 [[TMP77]], -31
+// CHECK1-NEXT:    [[DIV112:%.*]] = udiv i32 [[SUB111]], 32
+// CHECK1-NEXT:    [[MUL113:%.*]] = mul i32 [[MUL110]], [[DIV112]]
+// CHECK1-NEXT:    [[MUL114:%.*]] = mul i32 [[MUL113]], 32
+// CHECK1-NEXT:    [[CONV115:%.*]] = zext i32 [[MUL114]] to i64
+// CHECK1-NEXT:    [[MUL116:%.*]] = mul nsw i64 [[DIV105]], [[CONV115]]
+// CHECK1-NEXT:    [[SUB117:%.*]] = sub nsw i64 [[TMP66]], [[MUL116]]
+// CHECK1-NEXT:    [[TMP78:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP79:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP80:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP81:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB118:%.*]] = sub i32 [[TMP80]], [[TMP81]]
+// CHECK1-NEXT:    [[SUB119:%.*]] = sub i32 [[SUB118]], 1
+// CHECK1-NEXT:    [[TMP82:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD120:%.*]] = add i32 [[SUB119]], [[TMP82]]
+// CHECK1-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV121:%.*]] = udiv i32 [[ADD120]], [[TMP83]]
+// CHECK1-NEXT:    [[MUL122:%.*]] = mul i32 1, [[DIV121]]
+// CHECK1-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB123:%.*]] = sub i32 [[TMP84]], -31
+// CHECK1-NEXT:    [[DIV124:%.*]] = udiv i32 [[SUB123]], 32
+// CHECK1-NEXT:    [[MUL125:%.*]] = mul i32 [[MUL122]], [[DIV124]]
+// CHECK1-NEXT:    [[MUL126:%.*]] = mul i32 [[MUL125]], 32
+// CHECK1-NEXT:    [[CONV127:%.*]] = zext i32 [[MUL126]] to i64
+// CHECK1-NEXT:    [[DIV128:%.*]] = sdiv i64 [[TMP79]], [[CONV127]]
+// CHECK1-NEXT:    [[TMP85:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB129:%.*]] = sub i32 [[TMP85]], [[TMP86]]
+// CHECK1-NEXT:    [[SUB130:%.*]] = sub i32 [[SUB129]], 1
+// CHECK1-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD131:%.*]] = add i32 [[SUB130]], [[TMP87]]
+// CHECK1-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV132:%.*]] = udiv i32 [[ADD131]], [[TMP88]]
+// CHECK1-NEXT:    [[MUL133:%.*]] = mul i32 1, [[DIV132]]
+// CHECK1-NEXT:    [[TMP89:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB134:%.*]] = sub i32 [[TMP89]], -31
+// CHECK1-NEXT:    [[DIV135:%.*]] = udiv i32 [[SUB134]], 32
+// CHECK1-NEXT:    [[MUL136:%.*]] = mul i32 [[MUL133]], [[DIV135]]
+// CHECK1-NEXT:    [[MUL137:%.*]] = mul i32 [[MUL136]], 32
+// CHECK1-NEXT:    [[CONV138:%.*]] = zext i32 [[MUL137]] to i64
+// CHECK1-NEXT:    [[MUL139:%.*]] = mul nsw i64 [[DIV128]], [[CONV138]]
+// CHECK1-NEXT:    [[SUB140:%.*]] = sub nsw i64 [[TMP78]], [[MUL139]]
+// CHECK1-NEXT:    [[TMP90:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB141:%.*]] = sub i32 [[TMP90]], -31
+// CHECK1-NEXT:    [[DIV142:%.*]] = udiv i32 [[SUB141]], 32
+// CHECK1-NEXT:    [[MUL143:%.*]] = mul i32 1, [[DIV142]]
+// CHECK1-NEXT:    [[MUL144:%.*]] = mul i32 [[MUL143]], 32
+// CHECK1-NEXT:    [[CONV145:%.*]] = zext i32 [[MUL144]] to i64
+// CHECK1-NEXT:    [[DIV146:%.*]] = sdiv i64 [[SUB140]], [[CONV145]]
+// CHECK1-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB147:%.*]] = sub i32 [[TMP91]], -31
+// CHECK1-NEXT:    [[DIV148:%.*]] = udiv i32 [[SUB147]], 32
+// CHECK1-NEXT:    [[MUL149:%.*]] = mul i32 1, [[DIV148]]
+// CHECK1-NEXT:    [[MUL150:%.*]] = mul i32 [[MUL149]], 32
+// CHECK1-NEXT:    [[CONV151:%.*]] = zext i32 [[MUL150]] to i64
+// CHECK1-NEXT:    [[MUL152:%.*]] = mul nsw i64 [[DIV146]], [[CONV151]]
+// CHECK1-NEXT:    [[SUB153:%.*]] = sub nsw i64 [[SUB117]], [[MUL152]]
+// CHECK1-NEXT:    [[DIV154:%.*]] = sdiv i64 [[SUB153]], 32
+// CHECK1-NEXT:    [[MUL155:%.*]] = mul nsw i64 [[DIV154]], 32
+// CHECK1-NEXT:    [[ADD156:%.*]] = add nsw i64 0, [[MUL155]]
+// CHECK1-NEXT:    [[CONV157:%.*]] = trunc i64 [[ADD156]] to i32
+// CHECK1-NEXT:    store i32 [[CONV157]], ptr [[DOTFLOOR_0_IV_K41]], align 4
+// CHECK1-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K41]], align 4
+// CHECK1-NEXT:    [[CONV158:%.*]] = zext i32 [[TMP92]] to i64
+// CHECK1-NEXT:    [[TMP93:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP94:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB159:%.*]] = sub i32 [[TMP95]], [[TMP96]]
+// CHECK1-NEXT:    [[SUB160:%.*]] = sub i32 [[SUB159]], 1
+// CHECK1-NEXT:    [[TMP97:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD161:%.*]] = add i32 [[SUB160]], [[TMP97]]
 // CHECK1-NEXT:    [[TMP98:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD167:%.*]] = add i32 [[SUB166]], [[TMP98]]
-// CHECK1-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV168:%.*]] = udiv i32 [[ADD167]], [[TMP99]]
-// CHECK1-NEXT:    [[MUL169:%.*]] = mul i32 1, [[DIV168]]
-// CHECK1-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB170:%.*]] = sub i32 [[TMP100]], -31
-// CHECK1-NEXT:    [[DIV171:%.*]] = udiv i32 [[SUB170]], 32
-// CHECK1-NEXT:    [[MUL172:%.*]] = mul i32 [[MUL169]], [[DIV171]]
-// CHECK1-NEXT:    [[MUL173:%.*]] = mul i32 [[MUL172]], 32
-// CHECK1-NEXT:    [[CONV174:%.*]] = zext i32 [[MUL173]] to i64
-// CHECK1-NEXT:    [[MUL175:%.*]] = mul nsw i64 [[DIV164]], [[CONV174]]
-// CHECK1-NEXT:    [[SUB176:%.*]] = sub nsw i64 [[TMP89]], [[MUL175]]
-// CHECK1-NEXT:    [[TMP101:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP102:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB177:%.*]] = sub i32 [[TMP103]], [[TMP104]]
-// CHECK1-NEXT:    [[SUB178:%.*]] = sub i32 [[SUB177]], 1
-// CHECK1-NEXT:    [[TMP105:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD179:%.*]] = add i32 [[SUB178]], [[TMP105]]
-// CHECK1-NEXT:    [[TMP106:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV180:%.*]] = udiv i32 [[ADD179]], [[TMP106]]
-// CHECK1-NEXT:    [[MUL181:%.*]] = mul i32 1, [[DIV180]]
-// CHECK1-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB182:%.*]] = sub i32 [[TMP107]], -31
-// CHECK1-NEXT:    [[DIV183:%.*]] = udiv i32 [[SUB182]], 32
-// CHECK1-NEXT:    [[MUL184:%.*]] = mul i32 [[MUL181]], [[DIV183]]
-// CHECK1-NEXT:    [[MUL185:%.*]] = mul i32 [[MUL184]], 32
-// CHECK1-NEXT:    [[CONV186:%.*]] = zext i32 [[MUL185]] to i64
-// CHECK1-NEXT:    [[DIV187:%.*]] = sdiv i64 [[TMP102]], [[CONV186]]
-// CHECK1-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB188:%.*]] = sub i32 [[TMP108]], [[TMP109]]
-// CHECK1-NEXT:    [[SUB189:%.*]] = sub i32 [[SUB188]], 1
+// CHECK1-NEXT:    [[DIV162:%.*]] = udiv i32 [[ADD161]], [[TMP98]]
+// CHECK1-NEXT:    [[MUL163:%.*]] = mul i32 1, [[DIV162]]
+// CHECK1-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB164:%.*]] = sub i32 [[TMP99]], -31
+// CHECK1-NEXT:    [[DIV165:%.*]] = udiv i32 [[SUB164]], 32
+// CHECK1-NEXT:    [[MUL166:%.*]] = mul i32 [[MUL163]], [[DIV165]]
+// CHECK1-NEXT:    [[MUL167:%.*]] = mul i32 [[MUL166]], 32
+// CHECK1-NEXT:    [[CONV168:%.*]] = zext i32 [[MUL167]] to i64
+// CHECK1-NEXT:    [[DIV169:%.*]] = sdiv i64 [[TMP94]], [[CONV168]]
+// CHECK1-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP101:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB170:%.*]] = sub i32 [[TMP100]], [[TMP101]]
+// CHECK1-NEXT:    [[SUB171:%.*]] = sub i32 [[SUB170]], 1
+// CHECK1-NEXT:    [[TMP102:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD172:%.*]] = add i32 [[SUB171]], [[TMP102]]
+// CHECK1-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV173:%.*]] = udiv i32 [[ADD172]], [[TMP103]]
+// CHECK1-NEXT:    [[MUL174:%.*]] = mul i32 1, [[DIV173]]
+// CHECK1-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB175:%.*]] = sub i32 [[TMP104]], -31
+// CHECK1-NEXT:    [[DIV176:%.*]] = udiv i32 [[SUB175]], 32
+// CHECK1-NEXT:    [[MUL177:%.*]] = mul i32 [[MUL174]], [[DIV176]]
+// CHECK1-NEXT:    [[MUL178:%.*]] = mul i32 [[MUL177]], 32
+// CHECK1-NEXT:    [[CONV179:%.*]] = zext i32 [[MUL178]] to i64
+// CHECK1-NEXT:    [[MUL180:%.*]] = mul nsw i64 [[DIV169]], [[CONV179]]
+// CHECK1-NEXT:    [[SUB181:%.*]] = sub nsw i64 [[TMP93]], [[MUL180]]
+// CHECK1-NEXT:    [[TMP105:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP106:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB182:%.*]] = sub i32 [[TMP107]], [[TMP108]]
+// CHECK1-NEXT:    [[SUB183:%.*]] = sub i32 [[SUB182]], 1
+// CHECK1-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD184:%.*]] = add i32 [[SUB183]], [[TMP109]]
 // CHECK1-NEXT:    [[TMP110:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD190:%.*]] = add i32 [[SUB189]], [[TMP110]]
-// CHECK1-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV191:%.*]] = udiv i32 [[ADD190]], [[TMP111]]
-// CHECK1-NEXT:    [[MUL192:%.*]] = mul i32 1, [[DIV191]]
-// CHECK1-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB193:%.*]] = sub i32 [[TMP112]], -31
-// CHECK1-NEXT:    [[DIV194:%.*]] = udiv i32 [[SUB193]], 32
-// CHECK1-NEXT:    [[MUL195:%.*]] = mul i32 [[MUL192]], [[DIV194]]
-// CHECK1-NEXT:    [[MUL196:%.*]] = mul i32 [[MUL195]], 32
-// CHECK1-NEXT:    [[CONV197:%.*]] = zext i32 [[MUL196]] to i64
-// CHECK1-NEXT:    [[MUL198:%.*]] = mul nsw i64 [[DIV187]], [[CONV197]]
-// CHECK1-NEXT:    [[SUB199:%.*]] = sub nsw i64 [[TMP101]], [[MUL198]]
-// CHECK1-NEXT:    [[TMP113:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB200:%.*]] = sub i32 [[TMP113]], -31
-// CHECK1-NEXT:    [[DIV201:%.*]] = udiv i32 [[SUB200]], 32
-// CHECK1-NEXT:    [[MUL202:%.*]] = mul i32 1, [[DIV201]]
-// CHECK1-NEXT:    [[MUL203:%.*]] = mul i32 [[MUL202]], 32
-// CHECK1-NEXT:    [[CONV204:%.*]] = zext i32 [[MUL203]] to i64
-// CHECK1-NEXT:    [[DIV205:%.*]] = sdiv i64 [[SUB199]], [[CONV204]]
-// CHECK1-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB206:%.*]] = sub i32 [[TMP114]], -31
-// CHECK1-NEXT:    [[DIV207:%.*]] = udiv i32 [[SUB206]], 32
-// CHECK1-NEXT:    [[MUL208:%.*]] = mul i32 1, [[DIV207]]
-// CHECK1-NEXT:    [[MUL209:%.*]] = mul i32 [[MUL208]], 32
-// CHECK1-NEXT:    [[CONV210:%.*]] = zext i32 [[MUL209]] to i64
-// CHECK1-NEXT:    [[MUL211:%.*]] = mul nsw i64 [[DIV205]], [[CONV210]]
-// CHECK1-NEXT:    [[SUB212:%.*]] = sub nsw i64 [[SUB176]], [[MUL211]]
-// CHECK1-NEXT:    [[TMP115:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP116:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB213:%.*]] = sub i32 [[TMP117]], [[TMP118]]
-// CHECK1-NEXT:    [[SUB214:%.*]] = sub i32 [[SUB213]], 1
-// CHECK1-NEXT:    [[TMP119:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD215:%.*]] = add i32 [[SUB214]], [[TMP119]]
-// CHECK1-NEXT:    [[TMP120:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV216:%.*]] = udiv i32 [[ADD215]], [[TMP120]]
-// CHECK1-NEXT:    [[MUL217:%.*]] = mul i32 1, [[DIV216]]
-// CHECK1-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB218:%.*]] = sub i32 [[TMP121]], -31
-// CHECK1-NEXT:    [[DIV219:%.*]] = udiv i32 [[SUB218]], 32
-// CHECK1-NEXT:    [[MUL220:%.*]] = mul i32 [[MUL217]], [[DIV219]]
-// CHECK1-NEXT:    [[MUL221:%.*]] = mul i32 [[MUL220]], 32
-// CHECK1-NEXT:    [[CONV222:%.*]] = zext i32 [[MUL221]] to i64
-// CHECK1-NEXT:    [[DIV223:%.*]] = sdiv i64 [[TMP116]], [[CONV222]]
-// CHECK1-NEXT:    [[TMP122:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP123:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB224:%.*]] = sub i32 [[TMP122]], [[TMP123]]
-// CHECK1-NEXT:    [[SUB225:%.*]] = sub i32 [[SUB224]], 1
+// CHECK1-NEXT:    [[DIV185:%.*]] = udiv i32 [[ADD184]], [[TMP110]]
+// CHECK1-NEXT:    [[MUL186:%.*]] = mul i32 1, [[DIV185]]
+// CHECK1-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB187:%.*]] = sub i32 [[TMP111]], -31
+// CHECK1-NEXT:    [[DIV188:%.*]] = udiv i32 [[SUB187]], 32
+// CHECK1-NEXT:    [[MUL189:%.*]] = mul i32 [[MUL186]], [[DIV188]]
+// CHECK1-NEXT:    [[MUL190:%.*]] = mul i32 [[MUL189]], 32
+// CHECK1-NEXT:    [[CONV191:%.*]] = zext i32 [[MUL190]] to i64
+// CHECK1-NEXT:    [[DIV192:%.*]] = sdiv i64 [[TMP106]], [[CONV191]]
+// CHECK1-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP113:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB193:%.*]] = sub i32 [[TMP112]], [[TMP113]]
+// CHECK1-NEXT:    [[SUB194:%.*]] = sub i32 [[SUB193]], 1
+// CHECK1-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD195:%.*]] = add i32 [[SUB194]], [[TMP114]]
+// CHECK1-NEXT:    [[TMP115:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV196:%.*]] = udiv i32 [[ADD195]], [[TMP115]]
+// CHECK1-NEXT:    [[MUL197:%.*]] = mul i32 1, [[DIV196]]
+// CHECK1-NEXT:    [[TMP116:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB198:%.*]] = sub i32 [[TMP116]], -31
+// CHECK1-NEXT:    [[DIV199:%.*]] = udiv i32 [[SUB198]], 32
+// CHECK1-NEXT:    [[MUL200:%.*]] = mul i32 [[MUL197]], [[DIV199]]
+// CHECK1-NEXT:    [[MUL201:%.*]] = mul i32 [[MUL200]], 32
+// CHECK1-NEXT:    [[CONV202:%.*]] = zext i32 [[MUL201]] to i64
+// CHECK1-NEXT:    [[MUL203:%.*]] = mul nsw i64 [[DIV192]], [[CONV202]]
+// CHECK1-NEXT:    [[SUB204:%.*]] = sub nsw i64 [[TMP105]], [[MUL203]]
+// CHECK1-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB205:%.*]] = sub i32 [[TMP117]], -31
+// CHECK1-NEXT:    [[DIV206:%.*]] = udiv i32 [[SUB205]], 32
+// CHECK1-NEXT:    [[MUL207:%.*]] = mul i32 1, [[DIV206]]
+// CHECK1-NEXT:    [[MUL208:%.*]] = mul i32 [[MUL207]], 32
+// CHECK1-NEXT:    [[CONV209:%.*]] = zext i32 [[MUL208]] to i64
+// CHECK1-NEXT:    [[DIV210:%.*]] = sdiv i64 [[SUB204]], [[CONV209]]
+// CHECK1-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB211:%.*]] = sub i32 [[TMP118]], -31
+// CHECK1-NEXT:    [[DIV212:%.*]] = udiv i32 [[SUB211]], 32
+// CHECK1-NEXT:    [[MUL213:%.*]] = mul i32 1, [[DIV212]]
+// CHECK1-NEXT:    [[MUL214:%.*]] = mul i32 [[MUL213]], 32
+// CHECK1-NEXT:    [[CONV215:%.*]] = zext i32 [[MUL214]] to i64
+// CHECK1-NEXT:    [[MUL216:%.*]] = mul nsw i64 [[DIV210]], [[CONV215]]
+// CHECK1-NEXT:    [[SUB217:%.*]] = sub nsw i64 [[SUB181]], [[MUL216]]
+// CHECK1-NEXT:    [[TMP119:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP120:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP122:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB218:%.*]] = sub i32 [[TMP121]], [[TMP122]]
+// CHECK1-NEXT:    [[SUB219:%.*]] = sub i32 [[SUB218]], 1
+// CHECK1-NEXT:    [[TMP123:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD220:%.*]] = add i32 [[SUB219]], [[TMP123]]
 // CHECK1-NEXT:    [[TMP124:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD226:%.*]] = add i32 [[SUB225]], [[TMP124]]
-// CHECK1-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV227:%.*]] = udiv i32 [[ADD226]], [[TMP125]]
-// CHECK1-NEXT:    [[MUL228:%.*]] = mul i32 1, [[DIV227]]
-// CHECK1-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB229:%.*]] = sub i32 [[TMP126]], -31
-// CHECK1-NEXT:    [[DIV230:%.*]] = udiv i32 [[SUB229]], 32
-// CHECK1-NEXT:    [[MUL231:%.*]] = mul i32 [[MUL228]], [[DIV230]]
-// CHECK1-NEXT:    [[MUL232:%.*]] = mul i32 [[MUL231]], 32
-// CHECK1-NEXT:    [[CONV233:%.*]] = zext i32 [[MUL232]] to i64
-// CHECK1-NEXT:    [[MUL234:%.*]] = mul nsw i64 [[DIV223]], [[CONV233]]
-// CHECK1-NEXT:    [[SUB235:%.*]] = sub nsw i64 [[TMP115]], [[MUL234]]
-// CHECK1-NEXT:    [[TMP127:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP128:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB236:%.*]] = sub i32 [[TMP129]], [[TMP130]]
-// CHECK1-NEXT:    [[SUB237:%.*]] = sub i32 [[SUB236]], 1
-// CHECK1-NEXT:    [[TMP131:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD238:%.*]] = add i32 [[SUB237]], [[TMP131]]
-// CHECK1-NEXT:    [[TMP132:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV239:%.*]] = udiv i32 [[ADD238]], [[TMP132]]
-// CHECK1-NEXT:    [[MUL240:%.*]] = mul i32 1, [[DIV239]]
-// CHECK1-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB241:%.*]] = sub i32 [[TMP133]], -31
-// CHECK1-NEXT:    [[DIV242:%.*]] = udiv i32 [[SUB241]], 32
-// CHECK1-NEXT:    [[MUL243:%.*]] = mul i32 [[MUL240]], [[DIV242]]
-// CHECK1-NEXT:    [[MUL244:%.*]] = mul i32 [[MUL243]], 32
-// CHECK1-NEXT:    [[CONV245:%.*]] = zext i32 [[MUL244]] to i64
-// CHECK1-NEXT:    [[DIV246:%.*]] = sdiv i64 [[TMP128]], [[CONV245]]
-// CHECK1-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB247:%.*]] = sub i32 [[TMP134]], [[TMP135]]
-// CHECK1-NEXT:    [[SUB248:%.*]] = sub i32 [[SUB247]], 1
+// CHECK1-NEXT:    [[DIV221:%.*]] = udiv i32 [[ADD220]], [[TMP124]]
+// CHECK1-NEXT:    [[MUL222:%.*]] = mul i32 1, [[DIV221]]
+// CHECK1-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB223:%.*]] = sub i32 [[TMP125]], -31
+// CHECK1-NEXT:    [[DIV224:%.*]] = udiv i32 [[SUB223]], 32
+// CHECK1-NEXT:    [[MUL225:%.*]] = mul i32 [[MUL222]], [[DIV224]]
+// CHECK1-NEXT:    [[MUL226:%.*]] = mul i32 [[MUL225]], 32
+// CHECK1-NEXT:    [[CONV227:%.*]] = zext i32 [[MUL226]] to i64
+// CHECK1-NEXT:    [[DIV228:%.*]] = sdiv i64 [[TMP120]], [[CONV227]]
+// CHECK1-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP127:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB229:%.*]] = sub i32 [[TMP126]], [[TMP127]]
+// CHECK1-NEXT:    [[SUB230:%.*]] = sub i32 [[SUB229]], 1
+// CHECK1-NEXT:    [[TMP128:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD231:%.*]] = add i32 [[SUB230]], [[TMP128]]
+// CHECK1-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV232:%.*]] = udiv i32 [[ADD231]], [[TMP129]]
+// CHECK1-NEXT:    [[MUL233:%.*]] = mul i32 1, [[DIV232]]
+// CHECK1-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB234:%.*]] = sub i32 [[TMP130]], -31
+// CHECK1-NEXT:    [[DIV235:%.*]] = udiv i32 [[SUB234]], 32
+// CHECK1-NEXT:    [[MUL236:%.*]] = mul i32 [[MUL233]], [[DIV235]]
+// CHECK1-NEXT:    [[MUL237:%.*]] = mul i32 [[MUL236]], 32
+// CHECK1-NEXT:    [[CONV238:%.*]] = zext i32 [[MUL237]] to i64
+// CHECK1-NEXT:    [[MUL239:%.*]] = mul nsw i64 [[DIV228]], [[CONV238]]
+// CHECK1-NEXT:    [[SUB240:%.*]] = sub nsw i64 [[TMP119]], [[MUL239]]
+// CHECK1-NEXT:    [[TMP131:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP132:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB241:%.*]] = sub i32 [[TMP133]], [[TMP134]]
+// CHECK1-NEXT:    [[SUB242:%.*]] = sub i32 [[SUB241]], 1
+// CHECK1-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD243:%.*]] = add i32 [[SUB242]], [[TMP135]]
 // CHECK1-NEXT:    [[TMP136:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD249:%.*]] = add i32 [[SUB248]], [[TMP136]]
-// CHECK1-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV250:%.*]] = udiv i32 [[ADD249]], [[TMP137]]
-// CHECK1-NEXT:    [[MUL251:%.*]] = mul i32 1, [[DIV250]]
-// CHECK1-NEXT:    [[TMP138:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB252:%.*]] = sub i32 [[TMP138]], -31
-// CHECK1-NEXT:    [[DIV253:%.*]] = udiv i32 [[SUB252]], 32
-// CHECK1-NEXT:    [[MUL254:%.*]] = mul i32 [[MUL251]], [[DIV253]]
-// CHECK1-NEXT:    [[MUL255:%.*]] = mul i32 [[MUL254]], 32
-// CHECK1-NEXT:    [[CONV256:%.*]] = zext i32 [[MUL255]] to i64
-// CHECK1-NEXT:    [[MUL257:%.*]] = mul nsw i64 [[DIV246]], [[CONV256]]
-// CHECK1-NEXT:    [[SUB258:%.*]] = sub nsw i64 [[TMP127]], [[MUL257]]
-// CHECK1-NEXT:    [[TMP139:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB259:%.*]] = sub i32 [[TMP139]], -31
-// CHECK1-NEXT:    [[DIV260:%.*]] = udiv i32 [[SUB259]], 32
-// CHECK1-NEXT:    [[MUL261:%.*]] = mul i32 1, [[DIV260]]
-// CHECK1-NEXT:    [[MUL262:%.*]] = mul i32 [[MUL261]], 32
-// CHECK1-NEXT:    [[CONV263:%.*]] = zext i32 [[MUL262]] to i64
-// CHECK1-NEXT:    [[DIV264:%.*]] = sdiv i64 [[SUB258]], [[CONV263]]
-// CHECK1-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB265:%.*]] = sub i32 [[TMP140]], -31
-// CHECK1-NEXT:    [[DIV266:%.*]] = udiv i32 [[SUB265]], 32
-// CHECK1-NEXT:    [[MUL267:%.*]] = mul i32 1, [[DIV266]]
-// CHECK1-NEXT:    [[MUL268:%.*]] = mul i32 [[MUL267]], 32
-// CHECK1-NEXT:    [[CONV269:%.*]] = zext i32 [[MUL268]] to i64
-// CHECK1-NEXT:    [[MUL270:%.*]] = mul nsw i64 [[DIV264]], [[CONV269]]
-// CHECK1-NEXT:    [[SUB271:%.*]] = sub nsw i64 [[SUB235]], [[MUL270]]
-// CHECK1-NEXT:    [[DIV272:%.*]] = sdiv i64 [[SUB271]], 32
-// CHECK1-NEXT:    [[MUL273:%.*]] = mul nsw i64 [[DIV272]], 32
-// CHECK1-NEXT:    [[SUB274:%.*]] = sub nsw i64 [[SUB212]], [[MUL273]]
-// CHECK1-NEXT:    [[MUL275:%.*]] = mul nsw i64 [[SUB274]], 1
-// CHECK1-NEXT:    [[ADD276:%.*]] = add nsw i64 0, [[MUL275]]
-// CHECK1-NEXT:    [[CONV277:%.*]] = trunc i64 [[ADD276]] to i32
-// CHECK1-NEXT:    store i32 [[CONV277]], ptr [[DOTTILE_CNT_0_IV_K38]], align 4
-// CHECK1-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
-// CHECK1-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
-// CHECK1-NEXT:    [[ADD278:%.*]] = add i32 [[TMP141]], [[TMP142]]
-// CHECK1-NEXT:    store i32 [[ADD278]], ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK1-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
-// CHECK1-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK1-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
-// CHECK1-NEXT:    [[MUL279:%.*]] = mul i32 [[TMP144]], [[TMP145]]
-// CHECK1-NEXT:    [[ADD280:%.*]] = add i32 [[TMP143]], [[MUL279]]
-// CHECK1-NEXT:    store i32 [[ADD280]], ptr [[K]], align 4
-// CHECK1-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
-// CHECK1-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
-// CHECK1-NEXT:    [[ADD281:%.*]] = add i32 [[TMP146]], [[TMP147]]
-// CHECK1-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK1-NEXT:    [[ADD282:%.*]] = add i32 [[TMP148]], 1
-// CHECK1-NEXT:    [[CMP283:%.*]] = icmp ult i32 [[ADD281]], [[ADD282]]
-// CHECK1-NEXT:    br i1 [[CMP283]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK1:       if.then:
-// CHECK1-NEXT:    [[TMP149:%.*]] = load i32, ptr [[I35]], align 4
-// CHECK1-NEXT:    [[TMP150:%.*]] = load i32, ptr [[J36]], align 4
-// CHECK1-NEXT:    [[TMP151:%.*]] = load i32, ptr [[K]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP149]], i32 noundef [[TMP150]], i32 noundef [[TMP151]])
-// CHECK1-NEXT:    br label [[IF_END]]
-// CHECK1:       if.end:
-// CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK1-NEXT:    [[DIV244:%.*]] = udiv i32 [[ADD243]], [[TMP136]]
+// CHECK1-NEXT:    [[MUL245:%.*]] = mul i32 1, [[DIV244]]
+// CHECK1-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB246:%.*]] = sub i32 [[TMP137]], -31
+// CHECK1-NEXT:    [[DIV247:%.*]] = udiv i32 [[SUB246]], 32
+// CHECK1-NEXT:    [[MUL248:%.*]] = mul i32 [[MUL245]], [[DIV247]]
+// CHECK1-NEXT:    [[MUL249:%.*]] = mul i32 [[MUL248]], 32
+// CHECK1-NEXT:    [[CONV250:%.*]] = zext i32 [[MUL249]] to i64
+// CHECK1-NEXT:    [[DIV251:%.*]] = sdiv i64 [[TMP132]], [[CONV250]]
+// CHECK1-NEXT:    [[TMP138:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP139:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB252:%.*]] = sub i32 [[TMP138]], [[TMP139]]
+// CHECK1-NEXT:    [[SUB253:%.*]] = sub i32 [[SUB252]], 1
+// CHECK1-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD254:%.*]] = add i32 [[SUB253]], [[TMP140]]
+// CHECK1-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV255:%.*]] = udiv i32 [[ADD254]], [[TMP141]]
+// CHECK1-NEXT:    [[MUL256:%.*]] = mul i32 1, [[DIV255]]
+// CHECK1-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB257:%.*]] = sub i32 [[TMP142]], -31
+// CHECK1-NEXT:    [[DIV258:%.*]] = udiv i32 [[SUB257]], 32
+// CHECK1-NEXT:    [[MUL259:%.*]] = mul i32 [[MUL256]], [[DIV258]]
+// CHECK1-NEXT:    [[MUL260:%.*]] = mul i32 [[MUL259]], 32
+// CHECK1-NEXT:    [[CONV261:%.*]] = zext i32 [[MUL260]] to i64
+// CHECK1-NEXT:    [[MUL262:%.*]] = mul nsw i64 [[DIV251]], [[CONV261]]
+// CHECK1-NEXT:    [[SUB263:%.*]] = sub nsw i64 [[TMP131]], [[MUL262]]
+// CHECK1-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB264:%.*]] = sub i32 [[TMP143]], -31
+// CHECK1-NEXT:    [[DIV265:%.*]] = udiv i32 [[SUB264]], 32
+// CHECK1-NEXT:    [[MUL266:%.*]] = mul i32 1, [[DIV265]]
+// CHECK1-NEXT:    [[MUL267:%.*]] = mul i32 [[MUL266]], 32
+// CHECK1-NEXT:    [[CONV268:%.*]] = zext i32 [[MUL267]] to i64
+// CHECK1-NEXT:    [[DIV269:%.*]] = sdiv i64 [[SUB263]], [[CONV268]]
+// CHECK1-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB270:%.*]] = sub i32 [[TMP144]], -31
+// CHECK1-NEXT:    [[DIV271:%.*]] = udiv i32 [[SUB270]], 32
+// CHECK1-NEXT:    [[MUL272:%.*]] = mul i32 1, [[DIV271]]
+// CHECK1-NEXT:    [[MUL273:%.*]] = mul i32 [[MUL272]], 32
+// CHECK1-NEXT:    [[CONV274:%.*]] = zext i32 [[MUL273]] to i64
+// CHECK1-NEXT:    [[MUL275:%.*]] = mul nsw i64 [[DIV269]], [[CONV274]]
+// CHECK1-NEXT:    [[SUB276:%.*]] = sub nsw i64 [[SUB240]], [[MUL275]]
+// CHECK1-NEXT:    [[DIV277:%.*]] = sdiv i64 [[SUB276]], 32
+// CHECK1-NEXT:    [[MUL278:%.*]] = mul nsw i64 [[DIV277]], 32
+// CHECK1-NEXT:    [[SUB279:%.*]] = sub nsw i64 [[SUB217]], [[MUL278]]
+// CHECK1-NEXT:    [[MUL280:%.*]] = mul nsw i64 [[SUB279]], 1
+// CHECK1-NEXT:    [[ADD281:%.*]] = add nsw i64 [[CONV158]], [[MUL280]]
+// CHECK1-NEXT:    [[CONV282:%.*]] = trunc i64 [[ADD281]] to i32
+// CHECK1-NEXT:    store i32 [[CONV282]], ptr [[DOTTILE_0_IV_K42]], align 4
+// CHECK1-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTTILE_0_IV_K42]], align 4
+// CHECK1-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// CHECK1-NEXT:    [[ADD283:%.*]] = add i32 [[TMP146]], 1
+// CHECK1-NEXT:    [[CMP284:%.*]] = icmp ult i32 [[TMP145]], [[ADD283]]
+// CHECK1-NEXT:    br i1 [[CMP284]], label [[OMP_BODY_NEXT:%.*]], label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK1:       omp.body.next:
+// CHECK1-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
+// CHECK1-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTTILE_0_IV_K42]], align 4
+// CHECK1-NEXT:    [[TMP149:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
+// CHECK1-NEXT:    [[MUL285:%.*]] = mul i32 [[TMP148]], [[TMP149]]
+// CHECK1-NEXT:    [[ADD286:%.*]] = add i32 [[TMP147]], [[MUL285]]
+// CHECK1-NEXT:    store i32 [[ADD286]], ptr [[K]], align 4
+// CHECK1-NEXT:    [[TMP150:%.*]] = load i32, ptr [[I39]], align 4
+// CHECK1-NEXT:    [[TMP151:%.*]] = load i32, ptr [[J40]], align 4
+// CHECK1-NEXT:    [[TMP152:%.*]] = load i32, ptr [[K]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP150]], i32 noundef [[TMP151]], i32 noundef [[TMP152]])
+// CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP152:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[ADD284:%.*]] = add nsw i64 [[TMP152]], 1
-// CHECK1-NEXT:    store i64 [[ADD284]], ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP153:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[ADD287:%.*]] = add nsw i64 [[TMP153]], 1
+// CHECK1-NEXT:    store i64 [[ADD287]], ptr [[DOTOMP_IV]], align 8
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -1344,21 +1347,23 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTNEW_STEP10:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_14:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_17:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_19:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_K:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_K:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[_TMP35:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[I35:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[J36:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV_K37:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_K38:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[I39:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[J40:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV_K41:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_K42:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
@@ -1396,454 +1401,455 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
 // CHECK1-NEXT:    [[ADD15:%.*]] = add i32 [[TMP15]], 1
 // CHECK1-NEXT:    store i32 [[ADD15]], ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[SUB17:%.*]] = sub i32 [[TMP16]], [[TMP17]]
-// CHECK1-NEXT:    [[SUB18:%.*]] = sub i32 [[SUB17]], 1
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[ADD19:%.*]] = add i32 [[SUB18]], [[TMP18]]
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[DIV20:%.*]] = udiv i32 [[ADD19]], [[TMP19]]
-// CHECK1-NEXT:    [[CONV:%.*]] = zext i32 [[DIV20]] to i64
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB21:%.*]] = sub i32 [[TMP20]], [[TMP21]]
-// CHECK1-NEXT:    [[SUB22:%.*]] = sub i32 [[SUB21]], 1
-// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD23:%.*]] = add i32 [[SUB22]], [[TMP22]]
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV24:%.*]] = udiv i32 [[ADD23]], [[TMP23]]
-// CHECK1-NEXT:    [[CONV25:%.*]] = zext i32 [[DIV24]] to i64
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV25]]
-// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB26:%.*]] = sub i32 [[TMP24]], -63
-// CHECK1-NEXT:    [[DIV27:%.*]] = udiv i32 [[SUB26]], 64
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[_TMP2]], align 4
+// CHECK1-NEXT:    store i32 [[TMP16]], ptr [[DOTCAPTURE_EXPR_16]], align 4
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[_TMP2]], align 4
+// CHECK1-NEXT:    [[ADD18:%.*]] = add i32 [[TMP17]], 64
+// CHECK1-NEXT:    store i32 [[ADD18]], ptr [[DOTCAPTURE_EXPR_17]], align 4
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[SUB20:%.*]] = sub i32 [[TMP18]], [[TMP19]]
+// CHECK1-NEXT:    [[SUB21:%.*]] = sub i32 [[SUB20]], 1
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[ADD22:%.*]] = add i32 [[SUB21]], [[TMP20]]
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[DIV23:%.*]] = udiv i32 [[ADD22]], [[TMP21]]
+// CHECK1-NEXT:    [[CONV:%.*]] = zext i32 [[DIV23]] to i64
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB24:%.*]] = sub i32 [[TMP22]], [[TMP23]]
+// CHECK1-NEXT:    [[SUB25:%.*]] = sub i32 [[SUB24]], 1
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD26:%.*]] = add i32 [[SUB25]], [[TMP24]]
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV27:%.*]] = udiv i32 [[ADD26]], [[TMP25]]
 // CHECK1-NEXT:    [[CONV28:%.*]] = zext i32 [[DIV27]] to i64
-// CHECK1-NEXT:    [[MUL29:%.*]] = mul nsw i64 [[MUL]], [[CONV28]]
-// CHECK1-NEXT:    [[MUL30:%.*]] = mul nsw i64 [[MUL29]], 64
-// CHECK1-NEXT:    [[SUB31:%.*]] = sub nsw i64 [[MUL30]], 1
-// CHECK1-NEXT:    store i64 [[SUB31]], ptr [[DOTCAPTURE_EXPR_16]], align 8
-// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    store i32 [[TMP25]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    store i32 [[TMP26]], ptr [[J]], align 4
-// CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_K]], align 4
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_K]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV28]]
+// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB29:%.*]] = sub i32 [[TMP26]], -63
+// CHECK1-NEXT:    [[DIV30:%.*]] = udiv i32 [[SUB29]], 64
+// CHECK1-NEXT:    [[CONV31:%.*]] = zext i32 [[DIV30]] to i64
+// CHECK1-NEXT:    [[MUL32:%.*]] = mul nsw i64 [[MUL]], [[CONV31]]
+// CHECK1-NEXT:    [[MUL33:%.*]] = mul nsw i64 [[MUL32]], 64
+// CHECK1-NEXT:    [[SUB34:%.*]] = sub nsw i64 [[MUL33]], 1
+// CHECK1-NEXT:    store i64 [[SUB34]], ptr [[DOTCAPTURE_EXPR_19]], align 8
 // CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP27]], [[TMP28]]
+// CHECK1-NEXT:    store i32 [[TMP27]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    store i32 [[TMP28]], ptr [[J]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_K]], align 4
+// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K]], align 4
+// CHECK1-NEXT:    store i32 [[TMP29]], ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[_TMP35]], align 4
+// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP30]], [[TMP31]]
 // CHECK1-NEXT:    br i1 [[CMP]], label [[LAND_LHS_TRUE:%.*]], label [[OMP_PRECOND_END:%.*]]
 // CHECK1:       land.lhs.true:
-// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[CMP32:%.*]] = icmp slt i32 [[TMP29]], [[TMP30]]
-// CHECK1-NEXT:    br i1 [[CMP32]], label [[LAND_LHS_TRUE33:%.*]], label [[OMP_PRECOND_END]]
-// CHECK1:       land.lhs.true33:
-// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[CMP34:%.*]] = icmp ult i32 0, [[TMP31]]
-// CHECK1-NEXT:    br i1 [[CMP34]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
+// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[CMP36:%.*]] = icmp slt i32 [[TMP32]], [[TMP33]]
+// CHECK1-NEXT:    br i1 [[CMP36]], label [[LAND_LHS_TRUE37:%.*]], label [[OMP_PRECOND_END]]
+// CHECK1:       land.lhs.true37:
+// CHECK1-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[CMP38:%.*]] = icmp ult i32 0, [[TMP34]]
+// CHECK1-NEXT:    br i1 [[CMP38]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
 // CHECK1:       omp.precond.then:
 // CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
-// CHECK1-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
-// CHECK1-NEXT:    store i64 [[TMP32]], ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_19]], align 8
+// CHECK1-NEXT:    store i64 [[TMP35]], ptr [[DOTOMP_UB]], align 8
 // CHECK1-NEXT:    store i64 1, ptr [[DOTOMP_STRIDE]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
 // CHECK1-NEXT:    call void @__kmpc_for_static_init_8(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i64 1, i64 1)
-// CHECK1-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
-// CHECK1-NEXT:    [[CMP39:%.*]] = icmp sgt i64 [[TMP33]], [[TMP34]]
-// CHECK1-NEXT:    br i1 [[CMP39]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_19]], align 8
+// CHECK1-NEXT:    [[CMP43:%.*]] = icmp sgt i64 [[TMP36]], [[TMP37]]
+// CHECK1-NEXT:    br i1 [[CMP43]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK1:       cond.true:
-// CHECK1-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK1-NEXT:    [[TMP38:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_19]], align 8
 // CHECK1-NEXT:    br label [[COND_END:%.*]]
 // CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[TMP39:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
 // CHECK1-NEXT:    br label [[COND_END]]
 // CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i64 [ [[TMP35]], [[COND_TRUE]] ], [ [[TMP36]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[COND:%.*]] = phi i64 [ [[TMP38]], [[COND_TRUE]] ], [ [[TMP39]], [[COND_FALSE]] ]
 // CHECK1-NEXT:    store i64 [[COND]], ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
-// CHECK1-NEXT:    store i64 [[TMP37]], ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP40:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
+// CHECK1-NEXT:    store i64 [[TMP40]], ptr [[DOTOMP_IV]], align 8
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK1:       omp.inner.for.cond:
-// CHECK1-NEXT:    [[TMP38:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP39:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[CMP40:%.*]] = icmp sle i64 [[TMP38]], [[TMP39]]
-// CHECK1-NEXT:    br i1 [[CMP40]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
-// CHECK1:       omp.inner.for.body:
-// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[CONV41:%.*]] = sext i32 [[TMP40]] to i64
 // CHECK1-NEXT:    [[TMP41:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB42:%.*]] = sub i32 [[TMP42]], [[TMP43]]
-// CHECK1-NEXT:    [[SUB43:%.*]] = sub i32 [[SUB42]], 1
-// CHECK1-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD44:%.*]] = add i32 [[SUB43]], [[TMP44]]
-// CHECK1-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV45:%.*]] = udiv i32 [[ADD44]], [[TMP45]]
-// CHECK1-NEXT:    [[MUL46:%.*]] = mul i32 1, [[DIV45]]
-// CHECK1-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB47:%.*]] = sub i32 [[TMP46]], -63
-// CHECK1-NEXT:    [[DIV48:%.*]] = udiv i32 [[SUB47]], 64
-// CHECK1-NEXT:    [[MUL49:%.*]] = mul i32 [[MUL46]], [[DIV48]]
-// CHECK1-NEXT:    [[MUL50:%.*]] = mul i32 [[MUL49]], 64
-// CHECK1-NEXT:    [[CONV51:%.*]] = zext i32 [[MUL50]] to i64
-// CHECK1-NEXT:    [[DIV52:%.*]] = sdiv i64 [[TMP41]], [[CONV51]]
-// CHECK1-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[CONV53:%.*]] = sext i32 [[TMP47]] to i64
-// CHECK1-NEXT:    [[MUL54:%.*]] = mul nsw i64 [[DIV52]], [[CONV53]]
-// CHECK1-NEXT:    [[ADD55:%.*]] = add nsw i64 [[CONV41]], [[MUL54]]
-// CHECK1-NEXT:    [[CONV56:%.*]] = trunc i64 [[ADD55]] to i32
-// CHECK1-NEXT:    store i32 [[CONV56]], ptr [[I35]], align 4
-// CHECK1-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP48]] to i64
-// CHECK1-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP50:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB58:%.*]] = sub i32 [[TMP51]], [[TMP52]]
-// CHECK1-NEXT:    [[SUB59:%.*]] = sub i32 [[SUB58]], 1
-// CHECK1-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD60:%.*]] = add i32 [[SUB59]], [[TMP53]]
-// CHECK1-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV61:%.*]] = udiv i32 [[ADD60]], [[TMP54]]
-// CHECK1-NEXT:    [[MUL62:%.*]] = mul i32 1, [[DIV61]]
-// CHECK1-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB63:%.*]] = sub i32 [[TMP55]], -63
-// CHECK1-NEXT:    [[DIV64:%.*]] = udiv i32 [[SUB63]], 64
-// CHECK1-NEXT:    [[MUL65:%.*]] = mul i32 [[MUL62]], [[DIV64]]
-// CHECK1-NEXT:    [[MUL66:%.*]] = mul i32 [[MUL65]], 64
-// CHECK1-NEXT:    [[CONV67:%.*]] = zext i32 [[MUL66]] to i64
-// CHECK1-NEXT:    [[DIV68:%.*]] = sdiv i64 [[TMP50]], [[CONV67]]
-// CHECK1-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB69:%.*]] = sub i32 [[TMP56]], [[TMP57]]
-// CHECK1-NEXT:    [[SUB70:%.*]] = sub i32 [[SUB69]], 1
-// CHECK1-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD71:%.*]] = add i32 [[SUB70]], [[TMP58]]
-// CHECK1-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV72:%.*]] = udiv i32 [[ADD71]], [[TMP59]]
-// CHECK1-NEXT:    [[MUL73:%.*]] = mul i32 1, [[DIV72]]
-// CHECK1-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB74:%.*]] = sub i32 [[TMP60]], -63
-// CHECK1-NEXT:    [[DIV75:%.*]] = udiv i32 [[SUB74]], 64
-// CHECK1-NEXT:    [[MUL76:%.*]] = mul i32 [[MUL73]], [[DIV75]]
-// CHECK1-NEXT:    [[MUL77:%.*]] = mul i32 [[MUL76]], 64
-// CHECK1-NEXT:    [[CONV78:%.*]] = zext i32 [[MUL77]] to i64
-// CHECK1-NEXT:    [[MUL79:%.*]] = mul nsw i64 [[DIV68]], [[CONV78]]
-// CHECK1-NEXT:    [[SUB80:%.*]] = sub nsw i64 [[TMP49]], [[MUL79]]
-// CHECK1-NEXT:    [[TMP61:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB81:%.*]] = sub i32 [[TMP61]], -63
-// CHECK1-NEXT:    [[DIV82:%.*]] = udiv i32 [[SUB81]], 64
-// CHECK1-NEXT:    [[MUL83:%.*]] = mul i32 1, [[DIV82]]
-// CHECK1-NEXT:    [[MUL84:%.*]] = mul i32 [[MUL83]], 64
-// CHECK1-NEXT:    [[CONV85:%.*]] = zext i32 [[MUL84]] to i64
-// CHECK1-NEXT:    [[DIV86:%.*]] = sdiv i64 [[SUB80]], [[CONV85]]
+// CHECK1-NEXT:    [[TMP42:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[CMP44:%.*]] = icmp sle i64 [[TMP41]], [[TMP42]]
+// CHECK1-NEXT:    br i1 [[CMP44]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK1:       omp.inner.for.body:
+// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[CONV45:%.*]] = sext i32 [[TMP43]] to i64
+// CHECK1-NEXT:    [[TMP44:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB46:%.*]] = sub i32 [[TMP45]], [[TMP46]]
+// CHECK1-NEXT:    [[SUB47:%.*]] = sub i32 [[SUB46]], 1
+// CHECK1-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD48:%.*]] = add i32 [[SUB47]], [[TMP47]]
+// CHECK1-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV49:%.*]] = udiv i32 [[ADD48]], [[TMP48]]
+// CHECK1-NEXT:    [[MUL50:%.*]] = mul i32 1, [[DIV49]]
+// CHECK1-NEXT:    [[TMP49:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB51:%.*]] = sub i32 [[TMP49]], -63
+// CHECK1-NEXT:    [[DIV52:%.*]] = udiv i32 [[SUB51]], 64
+// CHECK1-NEXT:    [[MUL53:%.*]] = mul i32 [[MUL50]], [[DIV52]]
+// CHECK1-NEXT:    [[MUL54:%.*]] = mul i32 [[MUL53]], 64
+// CHECK1-NEXT:    [[CONV55:%.*]] = zext i32 [[MUL54]] to i64
+// CHECK1-NEXT:    [[DIV56:%.*]] = sdiv i64 [[TMP44]], [[CONV55]]
+// CHECK1-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP50]] to i64
+// CHECK1-NEXT:    [[MUL58:%.*]] = mul nsw i64 [[DIV56]], [[CONV57]]
+// CHECK1-NEXT:    [[ADD59:%.*]] = add nsw i64 [[CONV45]], [[MUL58]]
+// CHECK1-NEXT:    [[CONV60:%.*]] = trunc i64 [[ADD59]] to i32
+// CHECK1-NEXT:    store i32 [[CONV60]], ptr [[I39]], align 4
+// CHECK1-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[CONV61:%.*]] = sext i32 [[TMP51]] to i64
+// CHECK1-NEXT:    [[TMP52:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP53:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB62:%.*]] = sub i32 [[TMP54]], [[TMP55]]
+// CHECK1-NEXT:    [[SUB63:%.*]] = sub i32 [[SUB62]], 1
+// CHECK1-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD64:%.*]] = add i32 [[SUB63]], [[TMP56]]
+// CHECK1-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV65:%.*]] = udiv i32 [[ADD64]], [[TMP57]]
+// CHECK1-NEXT:    [[MUL66:%.*]] = mul i32 1, [[DIV65]]
+// CHECK1-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB67:%.*]] = sub i32 [[TMP58]], -63
+// CHECK1-NEXT:    [[DIV68:%.*]] = udiv i32 [[SUB67]], 64
+// CHECK1-NEXT:    [[MUL69:%.*]] = mul i32 [[MUL66]], [[DIV68]]
+// CHECK1-NEXT:    [[MUL70:%.*]] = mul i32 [[MUL69]], 64
+// CHECK1-NEXT:    [[CONV71:%.*]] = zext i32 [[MUL70]] to i64
+// CHECK1-NEXT:    [[DIV72:%.*]] = sdiv i64 [[TMP53]], [[CONV71]]
+// CHECK1-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB73:%.*]] = sub i32 [[TMP59]], [[TMP60]]
+// CHECK1-NEXT:    [[SUB74:%.*]] = sub i32 [[SUB73]], 1
+// CHECK1-NEXT:    [[TMP61:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD75:%.*]] = add i32 [[SUB74]], [[TMP61]]
 // CHECK1-NEXT:    [[TMP62:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[CONV87:%.*]] = sext i32 [[TMP62]] to i64
-// CHECK1-NEXT:    [[MUL88:%.*]] = mul nsw i64 [[DIV86]], [[CONV87]]
-// CHECK1-NEXT:    [[ADD89:%.*]] = add nsw i64 [[CONV57]], [[MUL88]]
-// CHECK1-NEXT:    [[CONV90:%.*]] = trunc i64 [[ADD89]] to i32
-// CHECK1-NEXT:    store i32 [[CONV90]], ptr [[J36]], align 4
-// CHECK1-NEXT:    [[TMP63:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP64:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB91:%.*]] = sub i32 [[TMP65]], [[TMP66]]
-// CHECK1-NEXT:    [[SUB92:%.*]] = sub i32 [[SUB91]], 1
-// CHECK1-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD93:%.*]] = add i32 [[SUB92]], [[TMP67]]
-// CHECK1-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV94:%.*]] = udiv i32 [[ADD93]], [[TMP68]]
-// CHECK1-NEXT:    [[MUL95:%.*]] = mul i32 1, [[DIV94]]
-// CHECK1-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB96:%.*]] = sub i32 [[TMP69]], -63
-// CHECK1-NEXT:    [[DIV97:%.*]] = udiv i32 [[SUB96]], 64
-// CHECK1-NEXT:    [[MUL98:%.*]] = mul i32 [[MUL95]], [[DIV97]]
-// CHECK1-NEXT:    [[MUL99:%.*]] = mul i32 [[MUL98]], 64
-// CHECK1-NEXT:    [[CONV100:%.*]] = zext i32 [[MUL99]] to i64
-// CHECK1-NEXT:    [[DIV101:%.*]] = sdiv i64 [[TMP64]], [[CONV100]]
-// CHECK1-NEXT:    [[TMP70:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB102:%.*]] = sub i32 [[TMP70]], [[TMP71]]
-// CHECK1-NEXT:    [[SUB103:%.*]] = sub i32 [[SUB102]], 1
-// CHECK1-NEXT:    [[TMP72:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD104:%.*]] = add i32 [[SUB103]], [[TMP72]]
-// CHECK1-NEXT:    [[TMP73:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV105:%.*]] = udiv i32 [[ADD104]], [[TMP73]]
-// CHECK1-NEXT:    [[MUL106:%.*]] = mul i32 1, [[DIV105]]
-// CHECK1-NEXT:    [[TMP74:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB107:%.*]] = sub i32 [[TMP74]], -63
-// CHECK1-NEXT:    [[DIV108:%.*]] = udiv i32 [[SUB107]], 64
-// CHECK1-NEXT:    [[MUL109:%.*]] = mul i32 [[MUL106]], [[DIV108]]
-// CHECK1-NEXT:    [[MUL110:%.*]] = mul i32 [[MUL109]], 64
-// CHECK1-NEXT:    [[CONV111:%.*]] = zext i32 [[MUL110]] to i64
-// CHECK1-NEXT:    [[MUL112:%.*]] = mul nsw i64 [[DIV101]], [[CONV111]]
-// CHECK1-NEXT:    [[SUB113:%.*]] = sub nsw i64 [[TMP63]], [[MUL112]]
-// CHECK1-NEXT:    [[TMP75:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP76:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP78:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB114:%.*]] = sub i32 [[TMP77]], [[TMP78]]
-// CHECK1-NEXT:    [[SUB115:%.*]] = sub i32 [[SUB114]], 1
-// CHECK1-NEXT:    [[TMP79:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD116:%.*]] = add i32 [[SUB115]], [[TMP79]]
-// CHECK1-NEXT:    [[TMP80:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV117:%.*]] = udiv i32 [[ADD116]], [[TMP80]]
-// CHECK1-NEXT:    [[MUL118:%.*]] = mul i32 1, [[DIV117]]
-// CHECK1-NEXT:    [[TMP81:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB119:%.*]] = sub i32 [[TMP81]], -63
-// CHECK1-NEXT:    [[DIV120:%.*]] = udiv i32 [[SUB119]], 64
-// CHECK1-NEXT:    [[MUL121:%.*]] = mul i32 [[MUL118]], [[DIV120]]
-// CHECK1-NEXT:    [[MUL122:%.*]] = mul i32 [[MUL121]], 64
-// CHECK1-NEXT:    [[CONV123:%.*]] = zext i32 [[MUL122]] to i64
-// CHECK1-NEXT:    [[DIV124:%.*]] = sdiv i64 [[TMP76]], [[CONV123]]
-// CHECK1-NEXT:    [[TMP82:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB125:%.*]] = sub i32 [[TMP82]], [[TMP83]]
-// CHECK1-NEXT:    [[SUB126:%.*]] = sub i32 [[SUB125]], 1
-// CHECK1-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD127:%.*]] = add i32 [[SUB126]], [[TMP84]]
-// CHECK1-NEXT:    [[TMP85:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV128:%.*]] = udiv i32 [[ADD127]], [[TMP85]]
-// CHECK1-NEXT:    [[MUL129:%.*]] = mul i32 1, [[DIV128]]
-// CHECK1-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB130:%.*]] = sub i32 [[TMP86]], -63
-// CHECK1-NEXT:    [[DIV131:%.*]] = udiv i32 [[SUB130]], 64
-// CHECK1-NEXT:    [[MUL132:%.*]] = mul i32 [[MUL129]], [[DIV131]]
-// CHECK1-NEXT:    [[MUL133:%.*]] = mul i32 [[MUL132]], 64
-// CHECK1-NEXT:    [[CONV134:%.*]] = zext i32 [[MUL133]] to i64
-// CHECK1-NEXT:    [[MUL135:%.*]] = mul nsw i64 [[DIV124]], [[CONV134]]
-// CHECK1-NEXT:    [[SUB136:%.*]] = sub nsw i64 [[TMP75]], [[MUL135]]
-// CHECK1-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB137:%.*]] = sub i32 [[TMP87]], -63
-// CHECK1-NEXT:    [[DIV138:%.*]] = udiv i32 [[SUB137]], 64
-// CHECK1-NEXT:    [[MUL139:%.*]] = mul i32 1, [[DIV138]]
-// CHECK1-NEXT:    [[MUL140:%.*]] = mul i32 [[MUL139]], 64
-// CHECK1-NEXT:    [[CONV141:%.*]] = zext i32 [[MUL140]] to i64
-// CHECK1-NEXT:    [[DIV142:%.*]] = sdiv i64 [[SUB136]], [[CONV141]]
-// CHECK1-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB143:%.*]] = sub i32 [[TMP88]], -63
-// CHECK1-NEXT:    [[DIV144:%.*]] = udiv i32 [[SUB143]], 64
-// CHECK1-NEXT:    [[MUL145:%.*]] = mul i32 1, [[DIV144]]
-// CHECK1-NEXT:    [[MUL146:%.*]] = mul i32 [[MUL145]], 64
-// CHECK1-NEXT:    [[CONV147:%.*]] = zext i32 [[MUL146]] to i64
-// CHECK1-NEXT:    [[MUL148:%.*]] = mul nsw i64 [[DIV142]], [[CONV147]]
-// CHECK1-NEXT:    [[SUB149:%.*]] = sub nsw i64 [[SUB113]], [[MUL148]]
-// CHECK1-NEXT:    [[DIV150:%.*]] = sdiv i64 [[SUB149]], 64
-// CHECK1-NEXT:    [[MUL151:%.*]] = mul nsw i64 [[DIV150]], 64
-// CHECK1-NEXT:    [[ADD152:%.*]] = add nsw i64 0, [[MUL151]]
-// CHECK1-NEXT:    [[CONV153:%.*]] = trunc i64 [[ADD152]] to i32
-// CHECK1-NEXT:    store i32 [[CONV153]], ptr [[DOTFLOOR_0_IV_K37]], align 4
-// CHECK1-NEXT:    [[TMP89:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP90:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB154:%.*]] = sub i32 [[TMP91]], [[TMP92]]
-// CHECK1-NEXT:    [[SUB155:%.*]] = sub i32 [[SUB154]], 1
-// CHECK1-NEXT:    [[TMP93:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD156:%.*]] = add i32 [[SUB155]], [[TMP93]]
-// CHECK1-NEXT:    [[TMP94:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV157:%.*]] = udiv i32 [[ADD156]], [[TMP94]]
-// CHECK1-NEXT:    [[MUL158:%.*]] = mul i32 1, [[DIV157]]
-// CHECK1-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB159:%.*]] = sub i32 [[TMP95]], -63
-// CHECK1-NEXT:    [[DIV160:%.*]] = udiv i32 [[SUB159]], 64
-// CHECK1-NEXT:    [[MUL161:%.*]] = mul i32 [[MUL158]], [[DIV160]]
-// CHECK1-NEXT:    [[MUL162:%.*]] = mul i32 [[MUL161]], 64
-// CHECK1-NEXT:    [[CONV163:%.*]] = zext i32 [[MUL162]] to i64
-// CHECK1-NEXT:    [[DIV164:%.*]] = sdiv i64 [[TMP90]], [[CONV163]]
-// CHECK1-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP97:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB165:%.*]] = sub i32 [[TMP96]], [[TMP97]]
-// CHECK1-NEXT:    [[SUB166:%.*]] = sub i32 [[SUB165]], 1
+// CHECK1-NEXT:    [[DIV76:%.*]] = udiv i32 [[ADD75]], [[TMP62]]
+// CHECK1-NEXT:    [[MUL77:%.*]] = mul i32 1, [[DIV76]]
+// CHECK1-NEXT:    [[TMP63:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB78:%.*]] = sub i32 [[TMP63]], -63
+// CHECK1-NEXT:    [[DIV79:%.*]] = udiv i32 [[SUB78]], 64
+// CHECK1-NEXT:    [[MUL80:%.*]] = mul i32 [[MUL77]], [[DIV79]]
+// CHECK1-NEXT:    [[MUL81:%.*]] = mul i32 [[MUL80]], 64
+// CHECK1-NEXT:    [[CONV82:%.*]] = zext i32 [[MUL81]] to i64
+// CHECK1-NEXT:    [[MUL83:%.*]] = mul nsw i64 [[DIV72]], [[CONV82]]
+// CHECK1-NEXT:    [[SUB84:%.*]] = sub nsw i64 [[TMP52]], [[MUL83]]
+// CHECK1-NEXT:    [[TMP64:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB85:%.*]] = sub i32 [[TMP64]], -63
+// CHECK1-NEXT:    [[DIV86:%.*]] = udiv i32 [[SUB85]], 64
+// CHECK1-NEXT:    [[MUL87:%.*]] = mul i32 1, [[DIV86]]
+// CHECK1-NEXT:    [[MUL88:%.*]] = mul i32 [[MUL87]], 64
+// CHECK1-NEXT:    [[CONV89:%.*]] = zext i32 [[MUL88]] to i64
+// CHECK1-NEXT:    [[DIV90:%.*]] = sdiv i64 [[SUB84]], [[CONV89]]
+// CHECK1-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[CONV91:%.*]] = sext i32 [[TMP65]] to i64
+// CHECK1-NEXT:    [[MUL92:%.*]] = mul nsw i64 [[DIV90]], [[CONV91]]
+// CHECK1-NEXT:    [[ADD93:%.*]] = add nsw i64 [[CONV61]], [[MUL92]]
+// CHECK1-NEXT:    [[CONV94:%.*]] = trunc i64 [[ADD93]] to i32
+// CHECK1-NEXT:    store i32 [[CONV94]], ptr [[J40]], align 4
+// CHECK1-NEXT:    [[TMP66:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP67:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB95:%.*]] = sub i32 [[TMP68]], [[TMP69]]
+// CHECK1-NEXT:    [[SUB96:%.*]] = sub i32 [[SUB95]], 1
+// CHECK1-NEXT:    [[TMP70:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD97:%.*]] = add i32 [[SUB96]], [[TMP70]]
+// CHECK1-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV98:%.*]] = udiv i32 [[ADD97]], [[TMP71]]
+// CHECK1-NEXT:    [[MUL99:%.*]] = mul i32 1, [[DIV98]]
+// CHECK1-NEXT:    [[TMP72:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB100:%.*]] = sub i32 [[TMP72]], -63
+// CHECK1-NEXT:    [[DIV101:%.*]] = udiv i32 [[SUB100]], 64
+// CHECK1-NEXT:    [[MUL102:%.*]] = mul i32 [[MUL99]], [[DIV101]]
+// CHECK1-NEXT:    [[MUL103:%.*]] = mul i32 [[MUL102]], 64
+// CHECK1-NEXT:    [[CONV104:%.*]] = zext i32 [[MUL103]] to i64
+// CHECK1-NEXT:    [[DIV105:%.*]] = sdiv i64 [[TMP67]], [[CONV104]]
+// CHECK1-NEXT:    [[TMP73:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP74:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB106:%.*]] = sub i32 [[TMP73]], [[TMP74]]
+// CHECK1-NEXT:    [[SUB107:%.*]] = sub i32 [[SUB106]], 1
+// CHECK1-NEXT:    [[TMP75:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD108:%.*]] = add i32 [[SUB107]], [[TMP75]]
+// CHECK1-NEXT:    [[TMP76:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV109:%.*]] = udiv i32 [[ADD108]], [[TMP76]]
+// CHECK1-NEXT:    [[MUL110:%.*]] = mul i32 1, [[DIV109]]
+// CHECK1-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB111:%.*]] = sub i32 [[TMP77]], -63
+// CHECK1-NEXT:    [[DIV112:%.*]] = udiv i32 [[SUB111]], 64
+// CHECK1-NEXT:    [[MUL113:%.*]] = mul i32 [[MUL110]], [[DIV112]]
+// CHECK1-NEXT:    [[MUL114:%.*]] = mul i32 [[MUL113]], 64
+// CHECK1-NEXT:    [[CONV115:%.*]] = zext i32 [[MUL114]] to i64
+// CHECK1-NEXT:    [[MUL116:%.*]] = mul nsw i64 [[DIV105]], [[CONV115]]
+// CHECK1-NEXT:    [[SUB117:%.*]] = sub nsw i64 [[TMP66]], [[MUL116]]
+// CHECK1-NEXT:    [[TMP78:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP79:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP80:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP81:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB118:%.*]] = sub i32 [[TMP80]], [[TMP81]]
+// CHECK1-NEXT:    [[SUB119:%.*]] = sub i32 [[SUB118]], 1
+// CHECK1-NEXT:    [[TMP82:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD120:%.*]] = add i32 [[SUB119]], [[TMP82]]
+// CHECK1-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV121:%.*]] = udiv i32 [[ADD120]], [[TMP83]]
+// CHECK1-NEXT:    [[MUL122:%.*]] = mul i32 1, [[DIV121]]
+// CHECK1-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB123:%.*]] = sub i32 [[TMP84]], -63
+// CHECK1-NEXT:    [[DIV124:%.*]] = udiv i32 [[SUB123]], 64
+// CHECK1-NEXT:    [[MUL125:%.*]] = mul i32 [[MUL122]], [[DIV124]]
+// CHECK1-NEXT:    [[MUL126:%.*]] = mul i32 [[MUL125]], 64
+// CHECK1-NEXT:    [[CONV127:%.*]] = zext i32 [[MUL126]] to i64
+// CHECK1-NEXT:    [[DIV128:%.*]] = sdiv i64 [[TMP79]], [[CONV127]]
+// CHECK1-NEXT:    [[TMP85:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB129:%.*]] = sub i32 [[TMP85]], [[TMP86]]
+// CHECK1-NEXT:    [[SUB130:%.*]] = sub i32 [[SUB129]], 1
+// CHECK1-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD131:%.*]] = add i32 [[SUB130]], [[TMP87]]
+// CHECK1-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV132:%.*]] = udiv i32 [[ADD131]], [[TMP88]]
+// CHECK1-NEXT:    [[MUL133:%.*]] = mul i32 1, [[DIV132]]
+// CHECK1-NEXT:    [[TMP89:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB134:%.*]] = sub i32 [[TMP89]], -63
+// CHECK1-NEXT:    [[DIV135:%.*]] = udiv i32 [[SUB134]], 64
+// CHECK1-NEXT:    [[MUL136:%.*]] = mul i32 [[MUL133]], [[DIV135]]
+// CHECK1-NEXT:    [[MUL137:%.*]] = mul i32 [[MUL136]], 64
+// CHECK1-NEXT:    [[CONV138:%.*]] = zext i32 [[MUL137]] to i64
+// CHECK1-NEXT:    [[MUL139:%.*]] = mul nsw i64 [[DIV128]], [[CONV138]]
+// CHECK1-NEXT:    [[SUB140:%.*]] = sub nsw i64 [[TMP78]], [[MUL139]]
+// CHECK1-NEXT:    [[TMP90:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB141:%.*]] = sub i32 [[TMP90]], -63
+// CHECK1-NEXT:    [[DIV142:%.*]] = udiv i32 [[SUB141]], 64
+// CHECK1-NEXT:    [[MUL143:%.*]] = mul i32 1, [[DIV142]]
+// CHECK1-NEXT:    [[MUL144:%.*]] = mul i32 [[MUL143]], 64
+// CHECK1-NEXT:    [[CONV145:%.*]] = zext i32 [[MUL144]] to i64
+// CHECK1-NEXT:    [[DIV146:%.*]] = sdiv i64 [[SUB140]], [[CONV145]]
+// CHECK1-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB147:%.*]] = sub i32 [[TMP91]], -63
+// CHECK1-NEXT:    [[DIV148:%.*]] = udiv i32 [[SUB147]], 64
+// CHECK1-NEXT:    [[MUL149:%.*]] = mul i32 1, [[DIV148]]
+// CHECK1-NEXT:    [[MUL150:%.*]] = mul i32 [[MUL149]], 64
+// CHECK1-NEXT:    [[CONV151:%.*]] = zext i32 [[MUL150]] to i64
+// CHECK1-NEXT:    [[MUL152:%.*]] = mul nsw i64 [[DIV146]], [[CONV151]]
+// CHECK1-NEXT:    [[SUB153:%.*]] = sub nsw i64 [[SUB117]], [[MUL152]]
+// CHECK1-NEXT:    [[DIV154:%.*]] = sdiv i64 [[SUB153]], 64
+// CHECK1-NEXT:    [[MUL155:%.*]] = mul nsw i64 [[DIV154]], 64
+// CHECK1-NEXT:    [[ADD156:%.*]] = add nsw i64 0, [[MUL155]]
+// CHECK1-NEXT:    [[CONV157:%.*]] = trunc i64 [[ADD156]] to i32
+// CHECK1-NEXT:    store i32 [[CONV157]], ptr [[DOTFLOOR_0_IV_K41]], align 4
+// CHECK1-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K41]], align 4
+// CHECK1-NEXT:    [[CONV158:%.*]] = zext i32 [[TMP92]] to i64
+// CHECK1-NEXT:    [[TMP93:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP94:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB159:%.*]] = sub i32 [[TMP95]], [[TMP96]]
+// CHECK1-NEXT:    [[SUB160:%.*]] = sub i32 [[SUB159]], 1
+// CHECK1-NEXT:    [[TMP97:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD161:%.*]] = add i32 [[SUB160]], [[TMP97]]
 // CHECK1-NEXT:    [[TMP98:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD167:%.*]] = add i32 [[SUB166]], [[TMP98]]
-// CHECK1-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV168:%.*]] = udiv i32 [[ADD167]], [[TMP99]]
-// CHECK1-NEXT:    [[MUL169:%.*]] = mul i32 1, [[DIV168]]
-// CHECK1-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB170:%.*]] = sub i32 [[TMP100]], -63
-// CHECK1-NEXT:    [[DIV171:%.*]] = udiv i32 [[SUB170]], 64
-// CHECK1-NEXT:    [[MUL172:%.*]] = mul i32 [[MUL169]], [[DIV171]]
-// CHECK1-NEXT:    [[MUL173:%.*]] = mul i32 [[MUL172]], 64
-// CHECK1-NEXT:    [[CONV174:%.*]] = zext i32 [[MUL173]] to i64
-// CHECK1-NEXT:    [[MUL175:%.*]] = mul nsw i64 [[DIV164]], [[CONV174]]
-// CHECK1-NEXT:    [[SUB176:%.*]] = sub nsw i64 [[TMP89]], [[MUL175]]
-// CHECK1-NEXT:    [[TMP101:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP102:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB177:%.*]] = sub i32 [[TMP103]], [[TMP104]]
-// CHECK1-NEXT:    [[SUB178:%.*]] = sub i32 [[SUB177]], 1
-// CHECK1-NEXT:    [[TMP105:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD179:%.*]] = add i32 [[SUB178]], [[TMP105]]
-// CHECK1-NEXT:    [[TMP106:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV180:%.*]] = udiv i32 [[ADD179]], [[TMP106]]
-// CHECK1-NEXT:    [[MUL181:%.*]] = mul i32 1, [[DIV180]]
-// CHECK1-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB182:%.*]] = sub i32 [[TMP107]], -63
-// CHECK1-NEXT:    [[DIV183:%.*]] = udiv i32 [[SUB182]], 64
-// CHECK1-NEXT:    [[MUL184:%.*]] = mul i32 [[MUL181]], [[DIV183]]
-// CHECK1-NEXT:    [[MUL185:%.*]] = mul i32 [[MUL184]], 64
-// CHECK1-NEXT:    [[CONV186:%.*]] = zext i32 [[MUL185]] to i64
-// CHECK1-NEXT:    [[DIV187:%.*]] = sdiv i64 [[TMP102]], [[CONV186]]
-// CHECK1-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB188:%.*]] = sub i32 [[TMP108]], [[TMP109]]
-// CHECK1-NEXT:    [[SUB189:%.*]] = sub i32 [[SUB188]], 1
+// CHECK1-NEXT:    [[DIV162:%.*]] = udiv i32 [[ADD161]], [[TMP98]]
+// CHECK1-NEXT:    [[MUL163:%.*]] = mul i32 1, [[DIV162]]
+// CHECK1-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB164:%.*]] = sub i32 [[TMP99]], -63
+// CHECK1-NEXT:    [[DIV165:%.*]] = udiv i32 [[SUB164]], 64
+// CHECK1-NEXT:    [[MUL166:%.*]] = mul i32 [[MUL163]], [[DIV165]]
+// CHECK1-NEXT:    [[MUL167:%.*]] = mul i32 [[MUL166]], 64
+// CHECK1-NEXT:    [[CONV168:%.*]] = zext i32 [[MUL167]] to i64
+// CHECK1-NEXT:    [[DIV169:%.*]] = sdiv i64 [[TMP94]], [[CONV168]]
+// CHECK1-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP101:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB170:%.*]] = sub i32 [[TMP100]], [[TMP101]]
+// CHECK1-NEXT:    [[SUB171:%.*]] = sub i32 [[SUB170]], 1
+// CHECK1-NEXT:    [[TMP102:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD172:%.*]] = add i32 [[SUB171]], [[TMP102]]
+// CHECK1-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV173:%.*]] = udiv i32 [[ADD172]], [[TMP103]]
+// CHECK1-NEXT:    [[MUL174:%.*]] = mul i32 1, [[DIV173]]
+// CHECK1-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB175:%.*]] = sub i32 [[TMP104]], -63
+// CHECK1-NEXT:    [[DIV176:%.*]] = udiv i32 [[SUB175]], 64
+// CHECK1-NEXT:    [[MUL177:%.*]] = mul i32 [[MUL174]], [[DIV176]]
+// CHECK1-NEXT:    [[MUL178:%.*]] = mul i32 [[MUL177]], 64
+// CHECK1-NEXT:    [[CONV179:%.*]] = zext i32 [[MUL178]] to i64
+// CHECK1-NEXT:    [[MUL180:%.*]] = mul nsw i64 [[DIV169]], [[CONV179]]
+// CHECK1-NEXT:    [[SUB181:%.*]] = sub nsw i64 [[TMP93]], [[MUL180]]
+// CHECK1-NEXT:    [[TMP105:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP106:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB182:%.*]] = sub i32 [[TMP107]], [[TMP108]]
+// CHECK1-NEXT:    [[SUB183:%.*]] = sub i32 [[SUB182]], 1
+// CHECK1-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD184:%.*]] = add i32 [[SUB183]], [[TMP109]]
 // CHECK1-NEXT:    [[TMP110:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD190:%.*]] = add i32 [[SUB189]], [[TMP110]]
-// CHECK1-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV191:%.*]] = udiv i32 [[ADD190]], [[TMP111]]
-// CHECK1-NEXT:    [[MUL192:%.*]] = mul i32 1, [[DIV191]]
-// CHECK1-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB193:%.*]] = sub i32 [[TMP112]], -63
-// CHECK1-NEXT:    [[DIV194:%.*]] = udiv i32 [[SUB193]], 64
-// CHECK1-NEXT:    [[MUL195:%.*]] = mul i32 [[MUL192]], [[DIV194]]
-// CHECK1-NEXT:    [[MUL196:%.*]] = mul i32 [[MUL195]], 64
-// CHECK1-NEXT:    [[CONV197:%.*]] = zext i32 [[MUL196]] to i64
-// CHECK1-NEXT:    [[MUL198:%.*]] = mul nsw i64 [[DIV187]], [[CONV197]]
-// CHECK1-NEXT:    [[SUB199:%.*]] = sub nsw i64 [[TMP101]], [[MUL198]]
-// CHECK1-NEXT:    [[TMP113:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB200:%.*]] = sub i32 [[TMP113]], -63
-// CHECK1-NEXT:    [[DIV201:%.*]] = udiv i32 [[SUB200]], 64
-// CHECK1-NEXT:    [[MUL202:%.*]] = mul i32 1, [[DIV201]]
-// CHECK1-NEXT:    [[MUL203:%.*]] = mul i32 [[MUL202]], 64
-// CHECK1-NEXT:    [[CONV204:%.*]] = zext i32 [[MUL203]] to i64
-// CHECK1-NEXT:    [[DIV205:%.*]] = sdiv i64 [[SUB199]], [[CONV204]]
-// CHECK1-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB206:%.*]] = sub i32 [[TMP114]], -63
-// CHECK1-NEXT:    [[DIV207:%.*]] = udiv i32 [[SUB206]], 64
-// CHECK1-NEXT:    [[MUL208:%.*]] = mul i32 1, [[DIV207]]
-// CHECK1-NEXT:    [[MUL209:%.*]] = mul i32 [[MUL208]], 64
-// CHECK1-NEXT:    [[CONV210:%.*]] = zext i32 [[MUL209]] to i64
-// CHECK1-NEXT:    [[MUL211:%.*]] = mul nsw i64 [[DIV205]], [[CONV210]]
-// CHECK1-NEXT:    [[SUB212:%.*]] = sub nsw i64 [[SUB176]], [[MUL211]]
-// CHECK1-NEXT:    [[TMP115:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP116:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB213:%.*]] = sub i32 [[TMP117]], [[TMP118]]
-// CHECK1-NEXT:    [[SUB214:%.*]] = sub i32 [[SUB213]], 1
-// CHECK1-NEXT:    [[TMP119:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD215:%.*]] = add i32 [[SUB214]], [[TMP119]]
-// CHECK1-NEXT:    [[TMP120:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV216:%.*]] = udiv i32 [[ADD215]], [[TMP120]]
-// CHECK1-NEXT:    [[MUL217:%.*]] = mul i32 1, [[DIV216]]
-// CHECK1-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB218:%.*]] = sub i32 [[TMP121]], -63
-// CHECK1-NEXT:    [[DIV219:%.*]] = udiv i32 [[SUB218]], 64
-// CHECK1-NEXT:    [[MUL220:%.*]] = mul i32 [[MUL217]], [[DIV219]]
-// CHECK1-NEXT:    [[MUL221:%.*]] = mul i32 [[MUL220]], 64
-// CHECK1-NEXT:    [[CONV222:%.*]] = zext i32 [[MUL221]] to i64
-// CHECK1-NEXT:    [[DIV223:%.*]] = sdiv i64 [[TMP116]], [[CONV222]]
-// CHECK1-NEXT:    [[TMP122:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP123:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB224:%.*]] = sub i32 [[TMP122]], [[TMP123]]
-// CHECK1-NEXT:    [[SUB225:%.*]] = sub i32 [[SUB224]], 1
+// CHECK1-NEXT:    [[DIV185:%.*]] = udiv i32 [[ADD184]], [[TMP110]]
+// CHECK1-NEXT:    [[MUL186:%.*]] = mul i32 1, [[DIV185]]
+// CHECK1-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB187:%.*]] = sub i32 [[TMP111]], -63
+// CHECK1-NEXT:    [[DIV188:%.*]] = udiv i32 [[SUB187]], 64
+// CHECK1-NEXT:    [[MUL189:%.*]] = mul i32 [[MUL186]], [[DIV188]]
+// CHECK1-NEXT:    [[MUL190:%.*]] = mul i32 [[MUL189]], 64
+// CHECK1-NEXT:    [[CONV191:%.*]] = zext i32 [[MUL190]] to i64
+// CHECK1-NEXT:    [[DIV192:%.*]] = sdiv i64 [[TMP106]], [[CONV191]]
+// CHECK1-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP113:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB193:%.*]] = sub i32 [[TMP112]], [[TMP113]]
+// CHECK1-NEXT:    [[SUB194:%.*]] = sub i32 [[SUB193]], 1
+// CHECK1-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD195:%.*]] = add i32 [[SUB194]], [[TMP114]]
+// CHECK1-NEXT:    [[TMP115:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV196:%.*]] = udiv i32 [[ADD195]], [[TMP115]]
+// CHECK1-NEXT:    [[MUL197:%.*]] = mul i32 1, [[DIV196]]
+// CHECK1-NEXT:    [[TMP116:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB198:%.*]] = sub i32 [[TMP116]], -63
+// CHECK1-NEXT:    [[DIV199:%.*]] = udiv i32 [[SUB198]], 64
+// CHECK1-NEXT:    [[MUL200:%.*]] = mul i32 [[MUL197]], [[DIV199]]
+// CHECK1-NEXT:    [[MUL201:%.*]] = mul i32 [[MUL200]], 64
+// CHECK1-NEXT:    [[CONV202:%.*]] = zext i32 [[MUL201]] to i64
+// CHECK1-NEXT:    [[MUL203:%.*]] = mul nsw i64 [[DIV192]], [[CONV202]]
+// CHECK1-NEXT:    [[SUB204:%.*]] = sub nsw i64 [[TMP105]], [[MUL203]]
+// CHECK1-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB205:%.*]] = sub i32 [[TMP117]], -63
+// CHECK1-NEXT:    [[DIV206:%.*]] = udiv i32 [[SUB205]], 64
+// CHECK1-NEXT:    [[MUL207:%.*]] = mul i32 1, [[DIV206]]
+// CHECK1-NEXT:    [[MUL208:%.*]] = mul i32 [[MUL207]], 64
+// CHECK1-NEXT:    [[CONV209:%.*]] = zext i32 [[MUL208]] to i64
+// CHECK1-NEXT:    [[DIV210:%.*]] = sdiv i64 [[SUB204]], [[CONV209]]
+// CHECK1-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB211:%.*]] = sub i32 [[TMP118]], -63
+// CHECK1-NEXT:    [[DIV212:%.*]] = udiv i32 [[SUB211]], 64
+// CHECK1-NEXT:    [[MUL213:%.*]] = mul i32 1, [[DIV212]]
+// CHECK1-NEXT:    [[MUL214:%.*]] = mul i32 [[MUL213]], 64
+// CHECK1-NEXT:    [[CONV215:%.*]] = zext i32 [[MUL214]] to i64
+// CHECK1-NEXT:    [[MUL216:%.*]] = mul nsw i64 [[DIV210]], [[CONV215]]
+// CHECK1-NEXT:    [[SUB217:%.*]] = sub nsw i64 [[SUB181]], [[MUL216]]
+// CHECK1-NEXT:    [[TMP119:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP120:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP122:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB218:%.*]] = sub i32 [[TMP121]], [[TMP122]]
+// CHECK1-NEXT:    [[SUB219:%.*]] = sub i32 [[SUB218]], 1
+// CHECK1-NEXT:    [[TMP123:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD220:%.*]] = add i32 [[SUB219]], [[TMP123]]
 // CHECK1-NEXT:    [[TMP124:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD226:%.*]] = add i32 [[SUB225]], [[TMP124]]
-// CHECK1-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV227:%.*]] = udiv i32 [[ADD226]], [[TMP125]]
-// CHECK1-NEXT:    [[MUL228:%.*]] = mul i32 1, [[DIV227]]
-// CHECK1-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB229:%.*]] = sub i32 [[TMP126]], -63
-// CHECK1-NEXT:    [[DIV230:%.*]] = udiv i32 [[SUB229]], 64
-// CHECK1-NEXT:    [[MUL231:%.*]] = mul i32 [[MUL228]], [[DIV230]]
-// CHECK1-NEXT:    [[MUL232:%.*]] = mul i32 [[MUL231]], 64
-// CHECK1-NEXT:    [[CONV233:%.*]] = zext i32 [[MUL232]] to i64
-// CHECK1-NEXT:    [[MUL234:%.*]] = mul nsw i64 [[DIV223]], [[CONV233]]
-// CHECK1-NEXT:    [[SUB235:%.*]] = sub nsw i64 [[TMP115]], [[MUL234]]
-// CHECK1-NEXT:    [[TMP127:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP128:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB236:%.*]] = sub i32 [[TMP129]], [[TMP130]]
-// CHECK1-NEXT:    [[SUB237:%.*]] = sub i32 [[SUB236]], 1
-// CHECK1-NEXT:    [[TMP131:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD238:%.*]] = add i32 [[SUB237]], [[TMP131]]
-// CHECK1-NEXT:    [[TMP132:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV239:%.*]] = udiv i32 [[ADD238]], [[TMP132]]
-// CHECK1-NEXT:    [[MUL240:%.*]] = mul i32 1, [[DIV239]]
-// CHECK1-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB241:%.*]] = sub i32 [[TMP133]], -63
-// CHECK1-NEXT:    [[DIV242:%.*]] = udiv i32 [[SUB241]], 64
-// CHECK1-NEXT:    [[MUL243:%.*]] = mul i32 [[MUL240]], [[DIV242]]
-// CHECK1-NEXT:    [[MUL244:%.*]] = mul i32 [[MUL243]], 64
-// CHECK1-NEXT:    [[CONV245:%.*]] = zext i32 [[MUL244]] to i64
-// CHECK1-NEXT:    [[DIV246:%.*]] = sdiv i64 [[TMP128]], [[CONV245]]
-// CHECK1-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB247:%.*]] = sub i32 [[TMP134]], [[TMP135]]
-// CHECK1-NEXT:    [[SUB248:%.*]] = sub i32 [[SUB247]], 1
+// CHECK1-NEXT:    [[DIV221:%.*]] = udiv i32 [[ADD220]], [[TMP124]]
+// CHECK1-NEXT:    [[MUL222:%.*]] = mul i32 1, [[DIV221]]
+// CHECK1-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB223:%.*]] = sub i32 [[TMP125]], -63
+// CHECK1-NEXT:    [[DIV224:%.*]] = udiv i32 [[SUB223]], 64
+// CHECK1-NEXT:    [[MUL225:%.*]] = mul i32 [[MUL222]], [[DIV224]]
+// CHECK1-NEXT:    [[MUL226:%.*]] = mul i32 [[MUL225]], 64
+// CHECK1-NEXT:    [[CONV227:%.*]] = zext i32 [[MUL226]] to i64
+// CHECK1-NEXT:    [[DIV228:%.*]] = sdiv i64 [[TMP120]], [[CONV227]]
+// CHECK1-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP127:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB229:%.*]] = sub i32 [[TMP126]], [[TMP127]]
+// CHECK1-NEXT:    [[SUB230:%.*]] = sub i32 [[SUB229]], 1
+// CHECK1-NEXT:    [[TMP128:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD231:%.*]] = add i32 [[SUB230]], [[TMP128]]
+// CHECK1-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV232:%.*]] = udiv i32 [[ADD231]], [[TMP129]]
+// CHECK1-NEXT:    [[MUL233:%.*]] = mul i32 1, [[DIV232]]
+// CHECK1-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB234:%.*]] = sub i32 [[TMP130]], -63
+// CHECK1-NEXT:    [[DIV235:%.*]] = udiv i32 [[SUB234]], 64
+// CHECK1-NEXT:    [[MUL236:%.*]] = mul i32 [[MUL233]], [[DIV235]]
+// CHECK1-NEXT:    [[MUL237:%.*]] = mul i32 [[MUL236]], 64
+// CHECK1-NEXT:    [[CONV238:%.*]] = zext i32 [[MUL237]] to i64
+// CHECK1-NEXT:    [[MUL239:%.*]] = mul nsw i64 [[DIV228]], [[CONV238]]
+// CHECK1-NEXT:    [[SUB240:%.*]] = sub nsw i64 [[TMP119]], [[MUL239]]
+// CHECK1-NEXT:    [[TMP131:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP132:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB241:%.*]] = sub i32 [[TMP133]], [[TMP134]]
+// CHECK1-NEXT:    [[SUB242:%.*]] = sub i32 [[SUB241]], 1
+// CHECK1-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD243:%.*]] = add i32 [[SUB242]], [[TMP135]]
 // CHECK1-NEXT:    [[TMP136:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD249:%.*]] = add i32 [[SUB248]], [[TMP136]]
-// CHECK1-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV250:%.*]] = udiv i32 [[ADD249]], [[TMP137]]
-// CHECK1-NEXT:    [[MUL251:%.*]] = mul i32 1, [[DIV250]]
-// CHECK1-NEXT:    [[TMP138:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB252:%.*]] = sub i32 [[TMP138]], -63
-// CHECK1-NEXT:    [[DIV253:%.*]] = udiv i32 [[SUB252]], 64
-// CHECK1-NEXT:    [[MUL254:%.*]] = mul i32 [[MUL251]], [[DIV253]]
-// CHECK1-NEXT:    [[MUL255:%.*]] = mul i32 [[MUL254]], 64
-// CHECK1-NEXT:    [[CONV256:%.*]] = zext i32 [[MUL255]] to i64
-// CHECK1-NEXT:    [[MUL257:%.*]] = mul nsw i64 [[DIV246]], [[CONV256]]
-// CHECK1-NEXT:    [[SUB258:%.*]] = sub nsw i64 [[TMP127]], [[MUL257]]
-// CHECK1-NEXT:    [[TMP139:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB259:%.*]] = sub i32 [[TMP139]], -63
-// CHECK1-NEXT:    [[DIV260:%.*]] = udiv i32 [[SUB259]], 64
-// CHECK1-NEXT:    [[MUL261:%.*]] = mul i32 1, [[DIV260]]
-// CHECK1-NEXT:    [[MUL262:%.*]] = mul i32 [[MUL261]], 64
-// CHECK1-NEXT:    [[CONV263:%.*]] = zext i32 [[MUL262]] to i64
-// CHECK1-NEXT:    [[DIV264:%.*]] = sdiv i64 [[SUB258]], [[CONV263]]
-// CHECK1-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB265:%.*]] = sub i32 [[TMP140]], -63
-// CHECK1-NEXT:    [[DIV266:%.*]] = udiv i32 [[SUB265]], 64
-// CHECK1-NEXT:    [[MUL267:%.*]] = mul i32 1, [[DIV266]]
-// CHECK1-NEXT:    [[MUL268:%.*]] = mul i32 [[MUL267]], 64
-// CHECK1-NEXT:    [[CONV269:%.*]] = zext i32 [[MUL268]] to i64
-// CHECK1-NEXT:    [[MUL270:%.*]] = mul nsw i64 [[DIV264]], [[CONV269]]
-// CHECK1-NEXT:    [[SUB271:%.*]] = sub nsw i64 [[SUB235]], [[MUL270]]
-// CHECK1-NEXT:    [[DIV272:%.*]] = sdiv i64 [[SUB271]], 64
-// CHECK1-NEXT:    [[MUL273:%.*]] = mul nsw i64 [[DIV272]], 64
-// CHECK1-NEXT:    [[SUB274:%.*]] = sub nsw i64 [[SUB212]], [[MUL273]]
-// CHECK1-NEXT:    [[MUL275:%.*]] = mul nsw i64 [[SUB274]], 1
-// CHECK1-NEXT:    [[ADD276:%.*]] = add nsw i64 0, [[MUL275]]
-// CHECK1-NEXT:    [[CONV277:%.*]] = trunc i64 [[ADD276]] to i32
-// CHECK1-NEXT:    store i32 [[CONV277]], ptr [[DOTTILE_CNT_0_IV_K38]], align 4
-// CHECK1-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
-// CHECK1-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
-// CHECK1-NEXT:    [[ADD278:%.*]] = add i32 [[TMP141]], [[TMP142]]
-// CHECK1-NEXT:    store i32 [[ADD278]], ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK1-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
-// CHECK1-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK1-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
-// CHECK1-NEXT:    [[MUL279:%.*]] = mul i32 [[TMP144]], [[TMP145]]
-// CHECK1-NEXT:    [[ADD280:%.*]] = add i32 [[TMP143]], [[MUL279]]
-// CHECK1-NEXT:    store i32 [[ADD280]], ptr [[K]], align 4
-// CHECK1-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
-// CHECK1-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
-// CHECK1-NEXT:    [[ADD281:%.*]] = add i32 [[TMP146]], [[TMP147]]
-// CHECK1-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK1-NEXT:    [[ADD282:%.*]] = add i32 [[TMP148]], 1
-// CHECK1-NEXT:    [[CMP283:%.*]] = icmp ult i32 [[ADD281]], [[ADD282]]
-// CHECK1-NEXT:    br i1 [[CMP283]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK1:       if.then:
-// CHECK1-NEXT:    [[TMP149:%.*]] = load i32, ptr [[I35]], align 4
-// CHECK1-NEXT:    [[TMP150:%.*]] = load i32, ptr [[J36]], align 4
-// CHECK1-NEXT:    [[TMP151:%.*]] = load i32, ptr [[K]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP149]], i32 noundef [[TMP150]], i32 noundef [[TMP151]])
-// CHECK1-NEXT:    br label [[IF_END]]
-// CHECK1:       if.end:
-// CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK1-NEXT:    [[DIV244:%.*]] = udiv i32 [[ADD243]], [[TMP136]]
+// CHECK1-NEXT:    [[MUL245:%.*]] = mul i32 1, [[DIV244]]
+// CHECK1-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB246:%.*]] = sub i32 [[TMP137]], -63
+// CHECK1-NEXT:    [[DIV247:%.*]] = udiv i32 [[SUB246]], 64
+// CHECK1-NEXT:    [[MUL248:%.*]] = mul i32 [[MUL245]], [[DIV247]]
+// CHECK1-NEXT:    [[MUL249:%.*]] = mul i32 [[MUL248]], 64
+// CHECK1-NEXT:    [[CONV250:%.*]] = zext i32 [[MUL249]] to i64
+// CHECK1-NEXT:    [[DIV251:%.*]] = sdiv i64 [[TMP132]], [[CONV250]]
+// CHECK1-NEXT:    [[TMP138:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP139:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB252:%.*]] = sub i32 [[TMP138]], [[TMP139]]
+// CHECK1-NEXT:    [[SUB253:%.*]] = sub i32 [[SUB252]], 1
+// CHECK1-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD254:%.*]] = add i32 [[SUB253]], [[TMP140]]
+// CHECK1-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV255:%.*]] = udiv i32 [[ADD254]], [[TMP141]]
+// CHECK1-NEXT:    [[MUL256:%.*]] = mul i32 1, [[DIV255]]
+// CHECK1-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB257:%.*]] = sub i32 [[TMP142]], -63
+// CHECK1-NEXT:    [[DIV258:%.*]] = udiv i32 [[SUB257]], 64
+// CHECK1-NEXT:    [[MUL259:%.*]] = mul i32 [[MUL256]], [[DIV258]]
+// CHECK1-NEXT:    [[MUL260:%.*]] = mul i32 [[MUL259]], 64
+// CHECK1-NEXT:    [[CONV261:%.*]] = zext i32 [[MUL260]] to i64
+// CHECK1-NEXT:    [[MUL262:%.*]] = mul nsw i64 [[DIV251]], [[CONV261]]
+// CHECK1-NEXT:    [[SUB263:%.*]] = sub nsw i64 [[TMP131]], [[MUL262]]
+// CHECK1-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB264:%.*]] = sub i32 [[TMP143]], -63
+// CHECK1-NEXT:    [[DIV265:%.*]] = udiv i32 [[SUB264]], 64
+// CHECK1-NEXT:    [[MUL266:%.*]] = mul i32 1, [[DIV265]]
+// CHECK1-NEXT:    [[MUL267:%.*]] = mul i32 [[MUL266]], 64
+// CHECK1-NEXT:    [[CONV268:%.*]] = zext i32 [[MUL267]] to i64
+// CHECK1-NEXT:    [[DIV269:%.*]] = sdiv i64 [[SUB263]], [[CONV268]]
+// CHECK1-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB270:%.*]] = sub i32 [[TMP144]], -63
+// CHECK1-NEXT:    [[DIV271:%.*]] = udiv i32 [[SUB270]], 64
+// CHECK1-NEXT:    [[MUL272:%.*]] = mul i32 1, [[DIV271]]
+// CHECK1-NEXT:    [[MUL273:%.*]] = mul i32 [[MUL272]], 64
+// CHECK1-NEXT:    [[CONV274:%.*]] = zext i32 [[MUL273]] to i64
+// CHECK1-NEXT:    [[MUL275:%.*]] = mul nsw i64 [[DIV269]], [[CONV274]]
+// CHECK1-NEXT:    [[SUB276:%.*]] = sub nsw i64 [[SUB240]], [[MUL275]]
+// CHECK1-NEXT:    [[DIV277:%.*]] = sdiv i64 [[SUB276]], 64
+// CHECK1-NEXT:    [[MUL278:%.*]] = mul nsw i64 [[DIV277]], 64
+// CHECK1-NEXT:    [[SUB279:%.*]] = sub nsw i64 [[SUB217]], [[MUL278]]
+// CHECK1-NEXT:    [[MUL280:%.*]] = mul nsw i64 [[SUB279]], 1
+// CHECK1-NEXT:    [[ADD281:%.*]] = add nsw i64 [[CONV158]], [[MUL280]]
+// CHECK1-NEXT:    [[CONV282:%.*]] = trunc i64 [[ADD281]] to i32
+// CHECK1-NEXT:    store i32 [[CONV282]], ptr [[DOTTILE_0_IV_K42]], align 4
+// CHECK1-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTTILE_0_IV_K42]], align 4
+// CHECK1-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// CHECK1-NEXT:    [[ADD283:%.*]] = add i32 [[TMP146]], 1
+// CHECK1-NEXT:    [[CMP284:%.*]] = icmp ult i32 [[TMP145]], [[ADD283]]
+// CHECK1-NEXT:    br i1 [[CMP284]], label [[OMP_BODY_NEXT:%.*]], label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK1:       omp.body.next:
+// CHECK1-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
+// CHECK1-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTTILE_0_IV_K42]], align 4
+// CHECK1-NEXT:    [[TMP149:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
+// CHECK1-NEXT:    [[MUL285:%.*]] = mul i32 [[TMP148]], [[TMP149]]
+// CHECK1-NEXT:    [[ADD286:%.*]] = add i32 [[TMP147]], [[MUL285]]
+// CHECK1-NEXT:    store i32 [[ADD286]], ptr [[K]], align 4
+// CHECK1-NEXT:    [[TMP150:%.*]] = load i32, ptr [[I39]], align 4
+// CHECK1-NEXT:    [[TMP151:%.*]] = load i32, ptr [[J40]], align 4
+// CHECK1-NEXT:    [[TMP152:%.*]] = load i32, ptr [[K]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP150]], i32 noundef [[TMP151]], i32 noundef [[TMP152]])
+// CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP152:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[ADD284:%.*]] = add nsw i64 [[TMP152]], 1
-// CHECK1-NEXT:    store i64 [[ADD284]], ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP153:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[ADD287:%.*]] = add nsw i64 [[TMP153]], 1
+// CHECK1-NEXT:    store i64 [[ADD287]], ptr [[DOTOMP_IV]], align 8
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -3572,21 +3578,23 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTNEW_STEP10:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_14:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_17:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_19:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_K:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_K:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[_TMP35:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[I35:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[J36:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV_K37:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_K38:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[I39:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[J40:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV_K41:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_K42:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
@@ -3624,454 +3632,455 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
 // CHECK2-NEXT:    [[ADD15:%.*]] = add i32 [[TMP15]], 1
 // CHECK2-NEXT:    store i32 [[ADD15]], ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[SUB17:%.*]] = sub i32 [[TMP16]], [[TMP17]]
-// CHECK2-NEXT:    [[SUB18:%.*]] = sub i32 [[SUB17]], 1
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[ADD19:%.*]] = add i32 [[SUB18]], [[TMP18]]
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[DIV20:%.*]] = udiv i32 [[ADD19]], [[TMP19]]
-// CHECK2-NEXT:    [[CONV:%.*]] = zext i32 [[DIV20]] to i64
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB21:%.*]] = sub i32 [[TMP20]], [[TMP21]]
-// CHECK2-NEXT:    [[SUB22:%.*]] = sub i32 [[SUB21]], 1
-// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD23:%.*]] = add i32 [[SUB22]], [[TMP22]]
-// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV24:%.*]] = udiv i32 [[ADD23]], [[TMP23]]
-// CHECK2-NEXT:    [[CONV25:%.*]] = zext i32 [[DIV24]] to i64
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV25]]
-// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB26:%.*]] = sub i32 [[TMP24]], -31
-// CHECK2-NEXT:    [[DIV27:%.*]] = udiv i32 [[SUB26]], 32
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[_TMP2]], align 4
+// CHECK2-NEXT:    store i32 [[TMP16]], ptr [[DOTCAPTURE_EXPR_16]], align 4
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[_TMP2]], align 4
+// CHECK2-NEXT:    [[ADD18:%.*]] = add i32 [[TMP17]], 32
+// CHECK2-NEXT:    store i32 [[ADD18]], ptr [[DOTCAPTURE_EXPR_17]], align 4
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[SUB20:%.*]] = sub i32 [[TMP18]], [[TMP19]]
+// CHECK2-NEXT:    [[SUB21:%.*]] = sub i32 [[SUB20]], 1
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[ADD22:%.*]] = add i32 [[SUB21]], [[TMP20]]
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[DIV23:%.*]] = udiv i32 [[ADD22]], [[TMP21]]
+// CHECK2-NEXT:    [[CONV:%.*]] = zext i32 [[DIV23]] to i64
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB24:%.*]] = sub i32 [[TMP22]], [[TMP23]]
+// CHECK2-NEXT:    [[SUB25:%.*]] = sub i32 [[SUB24]], 1
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD26:%.*]] = add i32 [[SUB25]], [[TMP24]]
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV27:%.*]] = udiv i32 [[ADD26]], [[TMP25]]
 // CHECK2-NEXT:    [[CONV28:%.*]] = zext i32 [[DIV27]] to i64
-// CHECK2-NEXT:    [[MUL29:%.*]] = mul nsw i64 [[MUL]], [[CONV28]]
-// CHECK2-NEXT:    [[MUL30:%.*]] = mul nsw i64 [[MUL29]], 32
-// CHECK2-NEXT:    [[SUB31:%.*]] = sub nsw i64 [[MUL30]], 1
-// CHECK2-NEXT:    store i64 [[SUB31]], ptr [[DOTCAPTURE_EXPR_16]], align 8
-// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    store i32 [[TMP25]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    store i32 [[TMP26]], ptr [[J]], align 4
-// CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_K]], align 4
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_K]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV28]]
+// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB29:%.*]] = sub i32 [[TMP26]], -31
+// CHECK2-NEXT:    [[DIV30:%.*]] = udiv i32 [[SUB29]], 32
+// CHECK2-NEXT:    [[CONV31:%.*]] = zext i32 [[DIV30]] to i64
+// CHECK2-NEXT:    [[MUL32:%.*]] = mul nsw i64 [[MUL]], [[CONV31]]
+// CHECK2-NEXT:    [[MUL33:%.*]] = mul nsw i64 [[MUL32]], 32
+// CHECK2-NEXT:    [[SUB34:%.*]] = sub nsw i64 [[MUL33]], 1
+// CHECK2-NEXT:    store i64 [[SUB34]], ptr [[DOTCAPTURE_EXPR_19]], align 8
 // CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP27]], [[TMP28]]
+// CHECK2-NEXT:    store i32 [[TMP27]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    store i32 [[TMP28]], ptr [[J]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_K]], align 4
+// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K]], align 4
+// CHECK2-NEXT:    store i32 [[TMP29]], ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[_TMP35]], align 4
+// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP30]], [[TMP31]]
 // CHECK2-NEXT:    br i1 [[CMP]], label [[LAND_LHS_TRUE:%.*]], label [[OMP_PRECOND_END:%.*]]
 // CHECK2:       land.lhs.true:
-// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[CMP32:%.*]] = icmp slt i32 [[TMP29]], [[TMP30]]
-// CHECK2-NEXT:    br i1 [[CMP32]], label [[LAND_LHS_TRUE33:%.*]], label [[OMP_PRECOND_END]]
-// CHECK2:       land.lhs.true33:
-// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[CMP34:%.*]] = icmp ult i32 0, [[TMP31]]
-// CHECK2-NEXT:    br i1 [[CMP34]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
+// CHECK2-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[CMP36:%.*]] = icmp slt i32 [[TMP32]], [[TMP33]]
+// CHECK2-NEXT:    br i1 [[CMP36]], label [[LAND_LHS_TRUE37:%.*]], label [[OMP_PRECOND_END]]
+// CHECK2:       land.lhs.true37:
+// CHECK2-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[CMP38:%.*]] = icmp ult i32 0, [[TMP34]]
+// CHECK2-NEXT:    br i1 [[CMP38]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
 // CHECK2:       omp.precond.then:
 // CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
-// CHECK2-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
-// CHECK2-NEXT:    store i64 [[TMP32]], ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_19]], align 8
+// CHECK2-NEXT:    store i64 [[TMP35]], ptr [[DOTOMP_UB]], align 8
 // CHECK2-NEXT:    store i64 1, ptr [[DOTOMP_STRIDE]], align 8
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
 // CHECK2-NEXT:    call void @__kmpc_for_static_init_8(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i64 1, i64 1)
-// CHECK2-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
-// CHECK2-NEXT:    [[CMP39:%.*]] = icmp sgt i64 [[TMP33]], [[TMP34]]
-// CHECK2-NEXT:    br i1 [[CMP39]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_19]], align 8
+// CHECK2-NEXT:    [[CMP43:%.*]] = icmp sgt i64 [[TMP36]], [[TMP37]]
+// CHECK2-NEXT:    br i1 [[CMP43]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK2:       cond.true:
-// CHECK2-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK2-NEXT:    [[TMP38:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_19]], align 8
 // CHECK2-NEXT:    br label [[COND_END:%.*]]
 // CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[TMP39:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
 // CHECK2-NEXT:    br label [[COND_END]]
 // CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i64 [ [[TMP35]], [[COND_TRUE]] ], [ [[TMP36]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[COND:%.*]] = phi i64 [ [[TMP38]], [[COND_TRUE]] ], [ [[TMP39]], [[COND_FALSE]] ]
 // CHECK2-NEXT:    store i64 [[COND]], ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
-// CHECK2-NEXT:    store i64 [[TMP37]], ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP40:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
+// CHECK2-NEXT:    store i64 [[TMP40]], ptr [[DOTOMP_IV]], align 8
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK2:       omp.inner.for.cond:
-// CHECK2-NEXT:    [[TMP38:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP39:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[CMP40:%.*]] = icmp sle i64 [[TMP38]], [[TMP39]]
-// CHECK2-NEXT:    br i1 [[CMP40]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
-// CHECK2:       omp.inner.for.body:
-// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[CONV41:%.*]] = sext i32 [[TMP40]] to i64
 // CHECK2-NEXT:    [[TMP41:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB42:%.*]] = sub i32 [[TMP42]], [[TMP43]]
-// CHECK2-NEXT:    [[SUB43:%.*]] = sub i32 [[SUB42]], 1
-// CHECK2-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD44:%.*]] = add i32 [[SUB43]], [[TMP44]]
-// CHECK2-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV45:%.*]] = udiv i32 [[ADD44]], [[TMP45]]
-// CHECK2-NEXT:    [[MUL46:%.*]] = mul i32 1, [[DIV45]]
-// CHECK2-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB47:%.*]] = sub i32 [[TMP46]], -31
-// CHECK2-NEXT:    [[DIV48:%.*]] = udiv i32 [[SUB47]], 32
-// CHECK2-NEXT:    [[MUL49:%.*]] = mul i32 [[MUL46]], [[DIV48]]
-// CHECK2-NEXT:    [[MUL50:%.*]] = mul i32 [[MUL49]], 32
-// CHECK2-NEXT:    [[CONV51:%.*]] = zext i32 [[MUL50]] to i64
-// CHECK2-NEXT:    [[DIV52:%.*]] = sdiv i64 [[TMP41]], [[CONV51]]
-// CHECK2-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[CONV53:%.*]] = sext i32 [[TMP47]] to i64
-// CHECK2-NEXT:    [[MUL54:%.*]] = mul nsw i64 [[DIV52]], [[CONV53]]
-// CHECK2-NEXT:    [[ADD55:%.*]] = add nsw i64 [[CONV41]], [[MUL54]]
-// CHECK2-NEXT:    [[CONV56:%.*]] = trunc i64 [[ADD55]] to i32
-// CHECK2-NEXT:    store i32 [[CONV56]], ptr [[I35]], align 4
-// CHECK2-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP48]] to i64
-// CHECK2-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP50:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB58:%.*]] = sub i32 [[TMP51]], [[TMP52]]
-// CHECK2-NEXT:    [[SUB59:%.*]] = sub i32 [[SUB58]], 1
-// CHECK2-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD60:%.*]] = add i32 [[SUB59]], [[TMP53]]
-// CHECK2-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV61:%.*]] = udiv i32 [[ADD60]], [[TMP54]]
-// CHECK2-NEXT:    [[MUL62:%.*]] = mul i32 1, [[DIV61]]
-// CHECK2-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB63:%.*]] = sub i32 [[TMP55]], -31
-// CHECK2-NEXT:    [[DIV64:%.*]] = udiv i32 [[SUB63]], 32
-// CHECK2-NEXT:    [[MUL65:%.*]] = mul i32 [[MUL62]], [[DIV64]]
-// CHECK2-NEXT:    [[MUL66:%.*]] = mul i32 [[MUL65]], 32
-// CHECK2-NEXT:    [[CONV67:%.*]] = zext i32 [[MUL66]] to i64
-// CHECK2-NEXT:    [[DIV68:%.*]] = sdiv i64 [[TMP50]], [[CONV67]]
-// CHECK2-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB69:%.*]] = sub i32 [[TMP56]], [[TMP57]]
-// CHECK2-NEXT:    [[SUB70:%.*]] = sub i32 [[SUB69]], 1
-// CHECK2-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD71:%.*]] = add i32 [[SUB70]], [[TMP58]]
-// CHECK2-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV72:%.*]] = udiv i32 [[ADD71]], [[TMP59]]
-// CHECK2-NEXT:    [[MUL73:%.*]] = mul i32 1, [[DIV72]]
-// CHECK2-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB74:%.*]] = sub i32 [[TMP60]], -31
-// CHECK2-NEXT:    [[DIV75:%.*]] = udiv i32 [[SUB74]], 32
-// CHECK2-NEXT:    [[MUL76:%.*]] = mul i32 [[MUL73]], [[DIV75]]
-// CHECK2-NEXT:    [[MUL77:%.*]] = mul i32 [[MUL76]], 32
-// CHECK2-NEXT:    [[CONV78:%.*]] = zext i32 [[MUL77]] to i64
-// CHECK2-NEXT:    [[MUL79:%.*]] = mul nsw i64 [[DIV68]], [[CONV78]]
-// CHECK2-NEXT:    [[SUB80:%.*]] = sub nsw i64 [[TMP49]], [[MUL79]]
-// CHECK2-NEXT:    [[TMP61:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB81:%.*]] = sub i32 [[TMP61]], -31
-// CHECK2-NEXT:    [[DIV82:%.*]] = udiv i32 [[SUB81]], 32
-// CHECK2-NEXT:    [[MUL83:%.*]] = mul i32 1, [[DIV82]]
-// CHECK2-NEXT:    [[MUL84:%.*]] = mul i32 [[MUL83]], 32
-// CHECK2-NEXT:    [[CONV85:%.*]] = zext i32 [[MUL84]] to i64
-// CHECK2-NEXT:    [[DIV86:%.*]] = sdiv i64 [[SUB80]], [[CONV85]]
+// CHECK2-NEXT:    [[TMP42:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[CMP44:%.*]] = icmp sle i64 [[TMP41]], [[TMP42]]
+// CHECK2-NEXT:    br i1 [[CMP44]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK2:       omp.inner.for.body:
+// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[CONV45:%.*]] = sext i32 [[TMP43]] to i64
+// CHECK2-NEXT:    [[TMP44:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB46:%.*]] = sub i32 [[TMP45]], [[TMP46]]
+// CHECK2-NEXT:    [[SUB47:%.*]] = sub i32 [[SUB46]], 1
+// CHECK2-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD48:%.*]] = add i32 [[SUB47]], [[TMP47]]
+// CHECK2-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV49:%.*]] = udiv i32 [[ADD48]], [[TMP48]]
+// CHECK2-NEXT:    [[MUL50:%.*]] = mul i32 1, [[DIV49]]
+// CHECK2-NEXT:    [[TMP49:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB51:%.*]] = sub i32 [[TMP49]], -31
+// CHECK2-NEXT:    [[DIV52:%.*]] = udiv i32 [[SUB51]], 32
+// CHECK2-NEXT:    [[MUL53:%.*]] = mul i32 [[MUL50]], [[DIV52]]
+// CHECK2-NEXT:    [[MUL54:%.*]] = mul i32 [[MUL53]], 32
+// CHECK2-NEXT:    [[CONV55:%.*]] = zext i32 [[MUL54]] to i64
+// CHECK2-NEXT:    [[DIV56:%.*]] = sdiv i64 [[TMP44]], [[CONV55]]
+// CHECK2-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP50]] to i64
+// CHECK2-NEXT:    [[MUL58:%.*]] = mul nsw i64 [[DIV56]], [[CONV57]]
+// CHECK2-NEXT:    [[ADD59:%.*]] = add nsw i64 [[CONV45]], [[MUL58]]
+// CHECK2-NEXT:    [[CONV60:%.*]] = trunc i64 [[ADD59]] to i32
+// CHECK2-NEXT:    store i32 [[CONV60]], ptr [[I39]], align 4
+// CHECK2-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[CONV61:%.*]] = sext i32 [[TMP51]] to i64
+// CHECK2-NEXT:    [[TMP52:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP53:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB62:%.*]] = sub i32 [[TMP54]], [[TMP55]]
+// CHECK2-NEXT:    [[SUB63:%.*]] = sub i32 [[SUB62]], 1
+// CHECK2-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD64:%.*]] = add i32 [[SUB63]], [[TMP56]]
+// CHECK2-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV65:%.*]] = udiv i32 [[ADD64]], [[TMP57]]
+// CHECK2-NEXT:    [[MUL66:%.*]] = mul i32 1, [[DIV65]]
+// CHECK2-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB67:%.*]] = sub i32 [[TMP58]], -31
+// CHECK2-NEXT:    [[DIV68:%.*]] = udiv i32 [[SUB67]], 32
+// CHECK2-NEXT:    [[MUL69:%.*]] = mul i32 [[MUL66]], [[DIV68]]
+// CHECK2-NEXT:    [[MUL70:%.*]] = mul i32 [[MUL69]], 32
+// CHECK2-NEXT:    [[CONV71:%.*]] = zext i32 [[MUL70]] to i64
+// CHECK2-NEXT:    [[DIV72:%.*]] = sdiv i64 [[TMP53]], [[CONV71]]
+// CHECK2-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB73:%.*]] = sub i32 [[TMP59]], [[TMP60]]
+// CHECK2-NEXT:    [[SUB74:%.*]] = sub i32 [[SUB73]], 1
+// CHECK2-NEXT:    [[TMP61:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD75:%.*]] = add i32 [[SUB74]], [[TMP61]]
 // CHECK2-NEXT:    [[TMP62:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[CONV87:%.*]] = sext i32 [[TMP62]] to i64
-// CHECK2-NEXT:    [[MUL88:%.*]] = mul nsw i64 [[DIV86]], [[CONV87]]
-// CHECK2-NEXT:    [[ADD89:%.*]] = add nsw i64 [[CONV57]], [[MUL88]]
-// CHECK2-NEXT:    [[CONV90:%.*]] = trunc i64 [[ADD89]] to i32
-// CHECK2-NEXT:    store i32 [[CONV90]], ptr [[J36]], align 4
-// CHECK2-NEXT:    [[TMP63:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP64:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB91:%.*]] = sub i32 [[TMP65]], [[TMP66]]
-// CHECK2-NEXT:    [[SUB92:%.*]] = sub i32 [[SUB91]], 1
-// CHECK2-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD93:%.*]] = add i32 [[SUB92]], [[TMP67]]
-// CHECK2-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV94:%.*]] = udiv i32 [[ADD93]], [[TMP68]]
-// CHECK2-NEXT:    [[MUL95:%.*]] = mul i32 1, [[DIV94]]
-// CHECK2-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB96:%.*]] = sub i32 [[TMP69]], -31
-// CHECK2-NEXT:    [[DIV97:%.*]] = udiv i32 [[SUB96]], 32
-// CHECK2-NEXT:    [[MUL98:%.*]] = mul i32 [[MUL95]], [[DIV97]]
-// CHECK2-NEXT:    [[MUL99:%.*]] = mul i32 [[MUL98]], 32
-// CHECK2-NEXT:    [[CONV100:%.*]] = zext i32 [[MUL99]] to i64
-// CHECK2-NEXT:    [[DIV101:%.*]] = sdiv i64 [[TMP64]], [[CONV100]]
-// CHECK2-NEXT:    [[TMP70:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB102:%.*]] = sub i32 [[TMP70]], [[TMP71]]
-// CHECK2-NEXT:    [[SUB103:%.*]] = sub i32 [[SUB102]], 1
-// CHECK2-NEXT:    [[TMP72:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD104:%.*]] = add i32 [[SUB103]], [[TMP72]]
-// CHECK2-NEXT:    [[TMP73:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV105:%.*]] = udiv i32 [[ADD104]], [[TMP73]]
-// CHECK2-NEXT:    [[MUL106:%.*]] = mul i32 1, [[DIV105]]
-// CHECK2-NEXT:    [[TMP74:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB107:%.*]] = sub i32 [[TMP74]], -31
-// CHECK2-NEXT:    [[DIV108:%.*]] = udiv i32 [[SUB107]], 32
-// CHECK2-NEXT:    [[MUL109:%.*]] = mul i32 [[MUL106]], [[DIV108]]
-// CHECK2-NEXT:    [[MUL110:%.*]] = mul i32 [[MUL109]], 32
-// CHECK2-NEXT:    [[CONV111:%.*]] = zext i32 [[MUL110]] to i64
-// CHECK2-NEXT:    [[MUL112:%.*]] = mul nsw i64 [[DIV101]], [[CONV111]]
-// CHECK2-NEXT:    [[SUB113:%.*]] = sub nsw i64 [[TMP63]], [[MUL112]]
-// CHECK2-NEXT:    [[TMP75:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP76:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP78:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB114:%.*]] = sub i32 [[TMP77]], [[TMP78]]
-// CHECK2-NEXT:    [[SUB115:%.*]] = sub i32 [[SUB114]], 1
-// CHECK2-NEXT:    [[TMP79:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD116:%.*]] = add i32 [[SUB115]], [[TMP79]]
-// CHECK2-NEXT:    [[TMP80:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV117:%.*]] = udiv i32 [[ADD116]], [[TMP80]]
-// CHECK2-NEXT:    [[MUL118:%.*]] = mul i32 1, [[DIV117]]
-// CHECK2-NEXT:    [[TMP81:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB119:%.*]] = sub i32 [[TMP81]], -31
-// CHECK2-NEXT:    [[DIV120:%.*]] = udiv i32 [[SUB119]], 32
-// CHECK2-NEXT:    [[MUL121:%.*]] = mul i32 [[MUL118]], [[DIV120]]
-// CHECK2-NEXT:    [[MUL122:%.*]] = mul i32 [[MUL121]], 32
-// CHECK2-NEXT:    [[CONV123:%.*]] = zext i32 [[MUL122]] to i64
-// CHECK2-NEXT:    [[DIV124:%.*]] = sdiv i64 [[TMP76]], [[CONV123]]
-// CHECK2-NEXT:    [[TMP82:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB125:%.*]] = sub i32 [[TMP82]], [[TMP83]]
-// CHECK2-NEXT:    [[SUB126:%.*]] = sub i32 [[SUB125]], 1
-// CHECK2-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD127:%.*]] = add i32 [[SUB126]], [[TMP84]]
-// CHECK2-NEXT:    [[TMP85:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV128:%.*]] = udiv i32 [[ADD127]], [[TMP85]]
-// CHECK2-NEXT:    [[MUL129:%.*]] = mul i32 1, [[DIV128]]
-// CHECK2-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB130:%.*]] = sub i32 [[TMP86]], -31
-// CHECK2-NEXT:    [[DIV131:%.*]] = udiv i32 [[SUB130]], 32
-// CHECK2-NEXT:    [[MUL132:%.*]] = mul i32 [[MUL129]], [[DIV131]]
-// CHECK2-NEXT:    [[MUL133:%.*]] = mul i32 [[MUL132]], 32
-// CHECK2-NEXT:    [[CONV134:%.*]] = zext i32 [[MUL133]] to i64
-// CHECK2-NEXT:    [[MUL135:%.*]] = mul nsw i64 [[DIV124]], [[CONV134]]
-// CHECK2-NEXT:    [[SUB136:%.*]] = sub nsw i64 [[TMP75]], [[MUL135]]
-// CHECK2-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB137:%.*]] = sub i32 [[TMP87]], -31
-// CHECK2-NEXT:    [[DIV138:%.*]] = udiv i32 [[SUB137]], 32
-// CHECK2-NEXT:    [[MUL139:%.*]] = mul i32 1, [[DIV138]]
-// CHECK2-NEXT:    [[MUL140:%.*]] = mul i32 [[MUL139]], 32
-// CHECK2-NEXT:    [[CONV141:%.*]] = zext i32 [[MUL140]] to i64
-// CHECK2-NEXT:    [[DIV142:%.*]] = sdiv i64 [[SUB136]], [[CONV141]]
-// CHECK2-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB143:%.*]] = sub i32 [[TMP88]], -31
-// CHECK2-NEXT:    [[DIV144:%.*]] = udiv i32 [[SUB143]], 32
-// CHECK2-NEXT:    [[MUL145:%.*]] = mul i32 1, [[DIV144]]
-// CHECK2-NEXT:    [[MUL146:%.*]] = mul i32 [[MUL145]], 32
-// CHECK2-NEXT:    [[CONV147:%.*]] = zext i32 [[MUL146]] to i64
-// CHECK2-NEXT:    [[MUL148:%.*]] = mul nsw i64 [[DIV142]], [[CONV147]]
-// CHECK2-NEXT:    [[SUB149:%.*]] = sub nsw i64 [[SUB113]], [[MUL148]]
-// CHECK2-NEXT:    [[DIV150:%.*]] = sdiv i64 [[SUB149]], 32
-// CHECK2-NEXT:    [[MUL151:%.*]] = mul nsw i64 [[DIV150]], 32
-// CHECK2-NEXT:    [[ADD152:%.*]] = add nsw i64 0, [[MUL151]]
-// CHECK2-NEXT:    [[CONV153:%.*]] = trunc i64 [[ADD152]] to i32
-// CHECK2-NEXT:    store i32 [[CONV153]], ptr [[DOTFLOOR_0_IV_K37]], align 4
-// CHECK2-NEXT:    [[TMP89:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP90:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB154:%.*]] = sub i32 [[TMP91]], [[TMP92]]
-// CHECK2-NEXT:    [[SUB155:%.*]] = sub i32 [[SUB154]], 1
-// CHECK2-NEXT:    [[TMP93:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD156:%.*]] = add i32 [[SUB155]], [[TMP93]]
-// CHECK2-NEXT:    [[TMP94:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV157:%.*]] = udiv i32 [[ADD156]], [[TMP94]]
-// CHECK2-NEXT:    [[MUL158:%.*]] = mul i32 1, [[DIV157]]
-// CHECK2-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB159:%.*]] = sub i32 [[TMP95]], -31
-// CHECK2-NEXT:    [[DIV160:%.*]] = udiv i32 [[SUB159]], 32
-// CHECK2-NEXT:    [[MUL161:%.*]] = mul i32 [[MUL158]], [[DIV160]]
-// CHECK2-NEXT:    [[MUL162:%.*]] = mul i32 [[MUL161]], 32
-// CHECK2-NEXT:    [[CONV163:%.*]] = zext i32 [[MUL162]] to i64
-// CHECK2-NEXT:    [[DIV164:%.*]] = sdiv i64 [[TMP90]], [[CONV163]]
-// CHECK2-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP97:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB165:%.*]] = sub i32 [[TMP96]], [[TMP97]]
-// CHECK2-NEXT:    [[SUB166:%.*]] = sub i32 [[SUB165]], 1
+// CHECK2-NEXT:    [[DIV76:%.*]] = udiv i32 [[ADD75]], [[TMP62]]
+// CHECK2-NEXT:    [[MUL77:%.*]] = mul i32 1, [[DIV76]]
+// CHECK2-NEXT:    [[TMP63:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB78:%.*]] = sub i32 [[TMP63]], -31
+// CHECK2-NEXT:    [[DIV79:%.*]] = udiv i32 [[SUB78]], 32
+// CHECK2-NEXT:    [[MUL80:%.*]] = mul i32 [[MUL77]], [[DIV79]]
+// CHECK2-NEXT:    [[MUL81:%.*]] = mul i32 [[MUL80]], 32
+// CHECK2-NEXT:    [[CONV82:%.*]] = zext i32 [[MUL81]] to i64
+// CHECK2-NEXT:    [[MUL83:%.*]] = mul nsw i64 [[DIV72]], [[CONV82]]
+// CHECK2-NEXT:    [[SUB84:%.*]] = sub nsw i64 [[TMP52]], [[MUL83]]
+// CHECK2-NEXT:    [[TMP64:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB85:%.*]] = sub i32 [[TMP64]], -31
+// CHECK2-NEXT:    [[DIV86:%.*]] = udiv i32 [[SUB85]], 32
+// CHECK2-NEXT:    [[MUL87:%.*]] = mul i32 1, [[DIV86]]
+// CHECK2-NEXT:    [[MUL88:%.*]] = mul i32 [[MUL87]], 32
+// CHECK2-NEXT:    [[CONV89:%.*]] = zext i32 [[MUL88]] to i64
+// CHECK2-NEXT:    [[DIV90:%.*]] = sdiv i64 [[SUB84]], [[CONV89]]
+// CHECK2-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[CONV91:%.*]] = sext i32 [[TMP65]] to i64
+// CHECK2-NEXT:    [[MUL92:%.*]] = mul nsw i64 [[DIV90]], [[CONV91]]
+// CHECK2-NEXT:    [[ADD93:%.*]] = add nsw i64 [[CONV61]], [[MUL92]]
+// CHECK2-NEXT:    [[CONV94:%.*]] = trunc i64 [[ADD93]] to i32
+// CHECK2-NEXT:    store i32 [[CONV94]], ptr [[J40]], align 4
+// CHECK2-NEXT:    [[TMP66:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP67:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB95:%.*]] = sub i32 [[TMP68]], [[TMP69]]
+// CHECK2-NEXT:    [[SUB96:%.*]] = sub i32 [[SUB95]], 1
+// CHECK2-NEXT:    [[TMP70:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD97:%.*]] = add i32 [[SUB96]], [[TMP70]]
+// CHECK2-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV98:%.*]] = udiv i32 [[ADD97]], [[TMP71]]
+// CHECK2-NEXT:    [[MUL99:%.*]] = mul i32 1, [[DIV98]]
+// CHECK2-NEXT:    [[TMP72:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB100:%.*]] = sub i32 [[TMP72]], -31
+// CHECK2-NEXT:    [[DIV101:%.*]] = udiv i32 [[SUB100]], 32
+// CHECK2-NEXT:    [[MUL102:%.*]] = mul i32 [[MUL99]], [[DIV101]]
+// CHECK2-NEXT:    [[MUL103:%.*]] = mul i32 [[MUL102]], 32
+// CHECK2-NEXT:    [[CONV104:%.*]] = zext i32 [[MUL103]] to i64
+// CHECK2-NEXT:    [[DIV105:%.*]] = sdiv i64 [[TMP67]], [[CONV104]]
+// CHECK2-NEXT:    [[TMP73:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP74:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB106:%.*]] = sub i32 [[TMP73]], [[TMP74]]
+// CHECK2-NEXT:    [[SUB107:%.*]] = sub i32 [[SUB106]], 1
+// CHECK2-NEXT:    [[TMP75:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD108:%.*]] = add i32 [[SUB107]], [[TMP75]]
+// CHECK2-NEXT:    [[TMP76:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV109:%.*]] = udiv i32 [[ADD108]], [[TMP76]]
+// CHECK2-NEXT:    [[MUL110:%.*]] = mul i32 1, [[DIV109]]
+// CHECK2-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB111:%.*]] = sub i32 [[TMP77]], -31
+// CHECK2-NEXT:    [[DIV112:%.*]] = udiv i32 [[SUB111]], 32
+// CHECK2-NEXT:    [[MUL113:%.*]] = mul i32 [[MUL110]], [[DIV112]]
+// CHECK2-NEXT:    [[MUL114:%.*]] = mul i32 [[MUL113]], 32
+// CHECK2-NEXT:    [[CONV115:%.*]] = zext i32 [[MUL114]] to i64
+// CHECK2-NEXT:    [[MUL116:%.*]] = mul nsw i64 [[DIV105]], [[CONV115]]
+// CHECK2-NEXT:    [[SUB117:%.*]] = sub nsw i64 [[TMP66]], [[MUL116]]
+// CHECK2-NEXT:    [[TMP78:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP79:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP80:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP81:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB118:%.*]] = sub i32 [[TMP80]], [[TMP81]]
+// CHECK2-NEXT:    [[SUB119:%.*]] = sub i32 [[SUB118]], 1
+// CHECK2-NEXT:    [[TMP82:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD120:%.*]] = add i32 [[SUB119]], [[TMP82]]
+// CHECK2-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV121:%.*]] = udiv i32 [[ADD120]], [[TMP83]]
+// CHECK2-NEXT:    [[MUL122:%.*]] = mul i32 1, [[DIV121]]
+// CHECK2-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB123:%.*]] = sub i32 [[TMP84]], -31
+// CHECK2-NEXT:    [[DIV124:%.*]] = udiv i32 [[SUB123]], 32
+// CHECK2-NEXT:    [[MUL125:%.*]] = mul i32 [[MUL122]], [[DIV124]]
+// CHECK2-NEXT:    [[MUL126:%.*]] = mul i32 [[MUL125]], 32
+// CHECK2-NEXT:    [[CONV127:%.*]] = zext i32 [[MUL126]] to i64
+// CHECK2-NEXT:    [[DIV128:%.*]] = sdiv i64 [[TMP79]], [[CONV127]]
+// CHECK2-NEXT:    [[TMP85:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB129:%.*]] = sub i32 [[TMP85]], [[TMP86]]
+// CHECK2-NEXT:    [[SUB130:%.*]] = sub i32 [[SUB129]], 1
+// CHECK2-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD131:%.*]] = add i32 [[SUB130]], [[TMP87]]
+// CHECK2-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV132:%.*]] = udiv i32 [[ADD131]], [[TMP88]]
+// CHECK2-NEXT:    [[MUL133:%.*]] = mul i32 1, [[DIV132]]
+// CHECK2-NEXT:    [[TMP89:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB134:%.*]] = sub i32 [[TMP89]], -31
+// CHECK2-NEXT:    [[DIV135:%.*]] = udiv i32 [[SUB134]], 32
+// CHECK2-NEXT:    [[MUL136:%.*]] = mul i32 [[MUL133]], [[DIV135]]
+// CHECK2-NEXT:    [[MUL137:%.*]] = mul i32 [[MUL136]], 32
+// CHECK2-NEXT:    [[CONV138:%.*]] = zext i32 [[MUL137]] to i64
+// CHECK2-NEXT:    [[MUL139:%.*]] = mul nsw i64 [[DIV128]], [[CONV138]]
+// CHECK2-NEXT:    [[SUB140:%.*]] = sub nsw i64 [[TMP78]], [[MUL139]]
+// CHECK2-NEXT:    [[TMP90:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB141:%.*]] = sub i32 [[TMP90]], -31
+// CHECK2-NEXT:    [[DIV142:%.*]] = udiv i32 [[SUB141]], 32
+// CHECK2-NEXT:    [[MUL143:%.*]] = mul i32 1, [[DIV142]]
+// CHECK2-NEXT:    [[MUL144:%.*]] = mul i32 [[MUL143]], 32
+// CHECK2-NEXT:    [[CONV145:%.*]] = zext i32 [[MUL144]] to i64
+// CHECK2-NEXT:    [[DIV146:%.*]] = sdiv i64 [[SUB140]], [[CONV145]]
+// CHECK2-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB147:%.*]] = sub i32 [[TMP91]], -31
+// CHECK2-NEXT:    [[DIV148:%.*]] = udiv i32 [[SUB147]], 32
+// CHECK2-NEXT:    [[MUL149:%.*]] = mul i32 1, [[DIV148]]
+// CHECK2-NEXT:    [[MUL150:%.*]] = mul i32 [[MUL149]], 32
+// CHECK2-NEXT:    [[CONV151:%.*]] = zext i32 [[MUL150]] to i64
+// CHECK2-NEXT:    [[MUL152:%.*]] = mul nsw i64 [[DIV146]], [[CONV151]]
+// CHECK2-NEXT:    [[SUB153:%.*]] = sub nsw i64 [[SUB117]], [[MUL152]]
+// CHECK2-NEXT:    [[DIV154:%.*]] = sdiv i64 [[SUB153]], 32
+// CHECK2-NEXT:    [[MUL155:%.*]] = mul nsw i64 [[DIV154]], 32
+// CHECK2-NEXT:    [[ADD156:%.*]] = add nsw i64 0, [[MUL155]]
+// CHECK2-NEXT:    [[CONV157:%.*]] = trunc i64 [[ADD156]] to i32
+// CHECK2-NEXT:    store i32 [[CONV157]], ptr [[DOTFLOOR_0_IV_K41]], align 4
+// CHECK2-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K41]], align 4
+// CHECK2-NEXT:    [[CONV158:%.*]] = zext i32 [[TMP92]] to i64
+// CHECK2-NEXT:    [[TMP93:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP94:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB159:%.*]] = sub i32 [[TMP95]], [[TMP96]]
+// CHECK2-NEXT:    [[SUB160:%.*]] = sub i32 [[SUB159]], 1
+// CHECK2-NEXT:    [[TMP97:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD161:%.*]] = add i32 [[SUB160]], [[TMP97]]
 // CHECK2-NEXT:    [[TMP98:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD167:%.*]] = add i32 [[SUB166]], [[TMP98]]
-// CHECK2-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV168:%.*]] = udiv i32 [[ADD167]], [[TMP99]]
-// CHECK2-NEXT:    [[MUL169:%.*]] = mul i32 1, [[DIV168]]
-// CHECK2-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB170:%.*]] = sub i32 [[TMP100]], -31
-// CHECK2-NEXT:    [[DIV171:%.*]] = udiv i32 [[SUB170]], 32
-// CHECK2-NEXT:    [[MUL172:%.*]] = mul i32 [[MUL169]], [[DIV171]]
-// CHECK2-NEXT:    [[MUL173:%.*]] = mul i32 [[MUL172]], 32
-// CHECK2-NEXT:    [[CONV174:%.*]] = zext i32 [[MUL173]] to i64
-// CHECK2-NEXT:    [[MUL175:%.*]] = mul nsw i64 [[DIV164]], [[CONV174]]
-// CHECK2-NEXT:    [[SUB176:%.*]] = sub nsw i64 [[TMP89]], [[MUL175]]
-// CHECK2-NEXT:    [[TMP101:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP102:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB177:%.*]] = sub i32 [[TMP103]], [[TMP104]]
-// CHECK2-NEXT:    [[SUB178:%.*]] = sub i32 [[SUB177]], 1
-// CHECK2-NEXT:    [[TMP105:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD179:%.*]] = add i32 [[SUB178]], [[TMP105]]
-// CHECK2-NEXT:    [[TMP106:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV180:%.*]] = udiv i32 [[ADD179]], [[TMP106]]
-// CHECK2-NEXT:    [[MUL181:%.*]] = mul i32 1, [[DIV180]]
-// CHECK2-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB182:%.*]] = sub i32 [[TMP107]], -31
-// CHECK2-NEXT:    [[DIV183:%.*]] = udiv i32 [[SUB182]], 32
-// CHECK2-NEXT:    [[MUL184:%.*]] = mul i32 [[MUL181]], [[DIV183]]
-// CHECK2-NEXT:    [[MUL185:%.*]] = mul i32 [[MUL184]], 32
-// CHECK2-NEXT:    [[CONV186:%.*]] = zext i32 [[MUL185]] to i64
-// CHECK2-NEXT:    [[DIV187:%.*]] = sdiv i64 [[TMP102]], [[CONV186]]
-// CHECK2-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB188:%.*]] = sub i32 [[TMP108]], [[TMP109]]
-// CHECK2-NEXT:    [[SUB189:%.*]] = sub i32 [[SUB188]], 1
+// CHECK2-NEXT:    [[DIV162:%.*]] = udiv i32 [[ADD161]], [[TMP98]]
+// CHECK2-NEXT:    [[MUL163:%.*]] = mul i32 1, [[DIV162]]
+// CHECK2-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB164:%.*]] = sub i32 [[TMP99]], -31
+// CHECK2-NEXT:    [[DIV165:%.*]] = udiv i32 [[SUB164]], 32
+// CHECK2-NEXT:    [[MUL166:%.*]] = mul i32 [[MUL163]], [[DIV165]]
+// CHECK2-NEXT:    [[MUL167:%.*]] = mul i32 [[MUL166]], 32
+// CHECK2-NEXT:    [[CONV168:%.*]] = zext i32 [[MUL167]] to i64
+// CHECK2-NEXT:    [[DIV169:%.*]] = sdiv i64 [[TMP94]], [[CONV168]]
+// CHECK2-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP101:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB170:%.*]] = sub i32 [[TMP100]], [[TMP101]]
+// CHECK2-NEXT:    [[SUB171:%.*]] = sub i32 [[SUB170]], 1
+// CHECK2-NEXT:    [[TMP102:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD172:%.*]] = add i32 [[SUB171]], [[TMP102]]
+// CHECK2-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV173:%.*]] = udiv i32 [[ADD172]], [[TMP103]]
+// CHECK2-NEXT:    [[MUL174:%.*]] = mul i32 1, [[DIV173]]
+// CHECK2-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB175:%.*]] = sub i32 [[TMP104]], -31
+// CHECK2-NEXT:    [[DIV176:%.*]] = udiv i32 [[SUB175]], 32
+// CHECK2-NEXT:    [[MUL177:%.*]] = mul i32 [[MUL174]], [[DIV176]]
+// CHECK2-NEXT:    [[MUL178:%.*]] = mul i32 [[MUL177]], 32
+// CHECK2-NEXT:    [[CONV179:%.*]] = zext i32 [[MUL178]] to i64
+// CHECK2-NEXT:    [[MUL180:%.*]] = mul nsw i64 [[DIV169]], [[CONV179]]
+// CHECK2-NEXT:    [[SUB181:%.*]] = sub nsw i64 [[TMP93]], [[MUL180]]
+// CHECK2-NEXT:    [[TMP105:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP106:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB182:%.*]] = sub i32 [[TMP107]], [[TMP108]]
+// CHECK2-NEXT:    [[SUB183:%.*]] = sub i32 [[SUB182]], 1
+// CHECK2-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD184:%.*]] = add i32 [[SUB183]], [[TMP109]]
 // CHECK2-NEXT:    [[TMP110:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD190:%.*]] = add i32 [[SUB189]], [[TMP110]]
-// CHECK2-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV191:%.*]] = udiv i32 [[ADD190]], [[TMP111]]
-// CHECK2-NEXT:    [[MUL192:%.*]] = mul i32 1, [[DIV191]]
-// CHECK2-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB193:%.*]] = sub i32 [[TMP112]], -31
-// CHECK2-NEXT:    [[DIV194:%.*]] = udiv i32 [[SUB193]], 32
-// CHECK2-NEXT:    [[MUL195:%.*]] = mul i32 [[MUL192]], [[DIV194]]
-// CHECK2-NEXT:    [[MUL196:%.*]] = mul i32 [[MUL195]], 32
-// CHECK2-NEXT:    [[CONV197:%.*]] = zext i32 [[MUL196]] to i64
-// CHECK2-NEXT:    [[MUL198:%.*]] = mul nsw i64 [[DIV187]], [[CONV197]]
-// CHECK2-NEXT:    [[SUB199:%.*]] = sub nsw i64 [[TMP101]], [[MUL198]]
-// CHECK2-NEXT:    [[TMP113:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB200:%.*]] = sub i32 [[TMP113]], -31
-// CHECK2-NEXT:    [[DIV201:%.*]] = udiv i32 [[SUB200]], 32
-// CHECK2-NEXT:    [[MUL202:%.*]] = mul i32 1, [[DIV201]]
-// CHECK2-NEXT:    [[MUL203:%.*]] = mul i32 [[MUL202]], 32
-// CHECK2-NEXT:    [[CONV204:%.*]] = zext i32 [[MUL203]] to i64
-// CHECK2-NEXT:    [[DIV205:%.*]] = sdiv i64 [[SUB199]], [[CONV204]]
-// CHECK2-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB206:%.*]] = sub i32 [[TMP114]], -31
-// CHECK2-NEXT:    [[DIV207:%.*]] = udiv i32 [[SUB206]], 32
-// CHECK2-NEXT:    [[MUL208:%.*]] = mul i32 1, [[DIV207]]
-// CHECK2-NEXT:    [[MUL209:%.*]] = mul i32 [[MUL208]], 32
-// CHECK2-NEXT:    [[CONV210:%.*]] = zext i32 [[MUL209]] to i64
-// CHECK2-NEXT:    [[MUL211:%.*]] = mul nsw i64 [[DIV205]], [[CONV210]]
-// CHECK2-NEXT:    [[SUB212:%.*]] = sub nsw i64 [[SUB176]], [[MUL211]]
-// CHECK2-NEXT:    [[TMP115:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP116:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB213:%.*]] = sub i32 [[TMP117]], [[TMP118]]
-// CHECK2-NEXT:    [[SUB214:%.*]] = sub i32 [[SUB213]], 1
-// CHECK2-NEXT:    [[TMP119:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD215:%.*]] = add i32 [[SUB214]], [[TMP119]]
-// CHECK2-NEXT:    [[TMP120:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV216:%.*]] = udiv i32 [[ADD215]], [[TMP120]]
-// CHECK2-NEXT:    [[MUL217:%.*]] = mul i32 1, [[DIV216]]
-// CHECK2-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB218:%.*]] = sub i32 [[TMP121]], -31
-// CHECK2-NEXT:    [[DIV219:%.*]] = udiv i32 [[SUB218]], 32
-// CHECK2-NEXT:    [[MUL220:%.*]] = mul i32 [[MUL217]], [[DIV219]]
-// CHECK2-NEXT:    [[MUL221:%.*]] = mul i32 [[MUL220]], 32
-// CHECK2-NEXT:    [[CONV222:%.*]] = zext i32 [[MUL221]] to i64
-// CHECK2-NEXT:    [[DIV223:%.*]] = sdiv i64 [[TMP116]], [[CONV222]]
-// CHECK2-NEXT:    [[TMP122:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP123:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB224:%.*]] = sub i32 [[TMP122]], [[TMP123]]
-// CHECK2-NEXT:    [[SUB225:%.*]] = sub i32 [[SUB224]], 1
+// CHECK2-NEXT:    [[DIV185:%.*]] = udiv i32 [[ADD184]], [[TMP110]]
+// CHECK2-NEXT:    [[MUL186:%.*]] = mul i32 1, [[DIV185]]
+// CHECK2-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB187:%.*]] = sub i32 [[TMP111]], -31
+// CHECK2-NEXT:    [[DIV188:%.*]] = udiv i32 [[SUB187]], 32
+// CHECK2-NEXT:    [[MUL189:%.*]] = mul i32 [[MUL186]], [[DIV188]]
+// CHECK2-NEXT:    [[MUL190:%.*]] = mul i32 [[MUL189]], 32
+// CHECK2-NEXT:    [[CONV191:%.*]] = zext i32 [[MUL190]] to i64
+// CHECK2-NEXT:    [[DIV192:%.*]] = sdiv i64 [[TMP106]], [[CONV191]]
+// CHECK2-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP113:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB193:%.*]] = sub i32 [[TMP112]], [[TMP113]]
+// CHECK2-NEXT:    [[SUB194:%.*]] = sub i32 [[SUB193]], 1
+// CHECK2-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD195:%.*]] = add i32 [[SUB194]], [[TMP114]]
+// CHECK2-NEXT:    [[TMP115:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV196:%.*]] = udiv i32 [[ADD195]], [[TMP115]]
+// CHECK2-NEXT:    [[MUL197:%.*]] = mul i32 1, [[DIV196]]
+// CHECK2-NEXT:    [[TMP116:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB198:%.*]] = sub i32 [[TMP116]], -31
+// CHECK2-NEXT:    [[DIV199:%.*]] = udiv i32 [[SUB198]], 32
+// CHECK2-NEXT:    [[MUL200:%.*]] = mul i32 [[MUL197]], [[DIV199]]
+// CHECK2-NEXT:    [[MUL201:%.*]] = mul i32 [[MUL200]], 32
+// CHECK2-NEXT:    [[CONV202:%.*]] = zext i32 [[MUL201]] to i64
+// CHECK2-NEXT:    [[MUL203:%.*]] = mul nsw i64 [[DIV192]], [[CONV202]]
+// CHECK2-NEXT:    [[SUB204:%.*]] = sub nsw i64 [[TMP105]], [[MUL203]]
+// CHECK2-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB205:%.*]] = sub i32 [[TMP117]], -31
+// CHECK2-NEXT:    [[DIV206:%.*]] = udiv i32 [[SUB205]], 32
+// CHECK2-NEXT:    [[MUL207:%.*]] = mul i32 1, [[DIV206]]
+// CHECK2-NEXT:    [[MUL208:%.*]] = mul i32 [[MUL207]], 32
+// CHECK2-NEXT:    [[CONV209:%.*]] = zext i32 [[MUL208]] to i64
+// CHECK2-NEXT:    [[DIV210:%.*]] = sdiv i64 [[SUB204]], [[CONV209]]
+// CHECK2-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB211:%.*]] = sub i32 [[TMP118]], -31
+// CHECK2-NEXT:    [[DIV212:%.*]] = udiv i32 [[SUB211]], 32
+// CHECK2-NEXT:    [[MUL213:%.*]] = mul i32 1, [[DIV212]]
+// CHECK2-NEXT:    [[MUL214:%.*]] = mul i32 [[MUL213]], 32
+// CHECK2-NEXT:    [[CONV215:%.*]] = zext i32 [[MUL214]] to i64
+// CHECK2-NEXT:    [[MUL216:%.*]] = mul nsw i64 [[DIV210]], [[CONV215]]
+// CHECK2-NEXT:    [[SUB217:%.*]] = sub nsw i64 [[SUB181]], [[MUL216]]
+// CHECK2-NEXT:    [[TMP119:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP120:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP122:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB218:%.*]] = sub i32 [[TMP121]], [[TMP122]]
+// CHECK2-NEXT:    [[SUB219:%.*]] = sub i32 [[SUB218]], 1
+// CHECK2-NEXT:    [[TMP123:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD220:%.*]] = add i32 [[SUB219]], [[TMP123]]
 // CHECK2-NEXT:    [[TMP124:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD226:%.*]] = add i32 [[SUB225]], [[TMP124]]
-// CHECK2-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV227:%.*]] = udiv i32 [[ADD226]], [[TMP125]]
-// CHECK2-NEXT:    [[MUL228:%.*]] = mul i32 1, [[DIV227]]
-// CHECK2-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB229:%.*]] = sub i32 [[TMP126]], -31
-// CHECK2-NEXT:    [[DIV230:%.*]] = udiv i32 [[SUB229]], 32
-// CHECK2-NEXT:    [[MUL231:%.*]] = mul i32 [[MUL228]], [[DIV230]]
-// CHECK2-NEXT:    [[MUL232:%.*]] = mul i32 [[MUL231]], 32
-// CHECK2-NEXT:    [[CONV233:%.*]] = zext i32 [[MUL232]] to i64
-// CHECK2-NEXT:    [[MUL234:%.*]] = mul nsw i64 [[DIV223]], [[CONV233]]
-// CHECK2-NEXT:    [[SUB235:%.*]] = sub nsw i64 [[TMP115]], [[MUL234]]
-// CHECK2-NEXT:    [[TMP127:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP128:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB236:%.*]] = sub i32 [[TMP129]], [[TMP130]]
-// CHECK2-NEXT:    [[SUB237:%.*]] = sub i32 [[SUB236]], 1
-// CHECK2-NEXT:    [[TMP131:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD238:%.*]] = add i32 [[SUB237]], [[TMP131]]
-// CHECK2-NEXT:    [[TMP132:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV239:%.*]] = udiv i32 [[ADD238]], [[TMP132]]
-// CHECK2-NEXT:    [[MUL240:%.*]] = mul i32 1, [[DIV239]]
-// CHECK2-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB241:%.*]] = sub i32 [[TMP133]], -31
-// CHECK2-NEXT:    [[DIV242:%.*]] = udiv i32 [[SUB241]], 32
-// CHECK2-NEXT:    [[MUL243:%.*]] = mul i32 [[MUL240]], [[DIV242]]
-// CHECK2-NEXT:    [[MUL244:%.*]] = mul i32 [[MUL243]], 32
-// CHECK2-NEXT:    [[CONV245:%.*]] = zext i32 [[MUL244]] to i64
-// CHECK2-NEXT:    [[DIV246:%.*]] = sdiv i64 [[TMP128]], [[CONV245]]
-// CHECK2-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB247:%.*]] = sub i32 [[TMP134]], [[TMP135]]
-// CHECK2-NEXT:    [[SUB248:%.*]] = sub i32 [[SUB247]], 1
+// CHECK2-NEXT:    [[DIV221:%.*]] = udiv i32 [[ADD220]], [[TMP124]]
+// CHECK2-NEXT:    [[MUL222:%.*]] = mul i32 1, [[DIV221]]
+// CHECK2-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB223:%.*]] = sub i32 [[TMP125]], -31
+// CHECK2-NEXT:    [[DIV224:%.*]] = udiv i32 [[SUB223]], 32
+// CHECK2-NEXT:    [[MUL225:%.*]] = mul i32 [[MUL222]], [[DIV224]]
+// CHECK2-NEXT:    [[MUL226:%.*]] = mul i32 [[MUL225]], 32
+// CHECK2-NEXT:    [[CONV227:%.*]] = zext i32 [[MUL226]] to i64
+// CHECK2-NEXT:    [[DIV228:%.*]] = sdiv i64 [[TMP120]], [[CONV227]]
+// CHECK2-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP127:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB229:%.*]] = sub i32 [[TMP126]], [[TMP127]]
+// CHECK2-NEXT:    [[SUB230:%.*]] = sub i32 [[SUB229]], 1
+// CHECK2-NEXT:    [[TMP128:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD231:%.*]] = add i32 [[SUB230]], [[TMP128]]
+// CHECK2-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV232:%.*]] = udiv i32 [[ADD231]], [[TMP129]]
+// CHECK2-NEXT:    [[MUL233:%.*]] = mul i32 1, [[DIV232]]
+// CHECK2-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB234:%.*]] = sub i32 [[TMP130]], -31
+// CHECK2-NEXT:    [[DIV235:%.*]] = udiv i32 [[SUB234]], 32
+// CHECK2-NEXT:    [[MUL236:%.*]] = mul i32 [[MUL233]], [[DIV235]]
+// CHECK2-NEXT:    [[MUL237:%.*]] = mul i32 [[MUL236]], 32
+// CHECK2-NEXT:    [[CONV238:%.*]] = zext i32 [[MUL237]] to i64
+// CHECK2-NEXT:    [[MUL239:%.*]] = mul nsw i64 [[DIV228]], [[CONV238]]
+// CHECK2-NEXT:    [[SUB240:%.*]] = sub nsw i64 [[TMP119]], [[MUL239]]
+// CHECK2-NEXT:    [[TMP131:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP132:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB241:%.*]] = sub i32 [[TMP133]], [[TMP134]]
+// CHECK2-NEXT:    [[SUB242:%.*]] = sub i32 [[SUB241]], 1
+// CHECK2-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD243:%.*]] = add i32 [[SUB242]], [[TMP135]]
 // CHECK2-NEXT:    [[TMP136:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD249:%.*]] = add i32 [[SUB248]], [[TMP136]]
-// CHECK2-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV250:%.*]] = udiv i32 [[ADD249]], [[TMP137]]
-// CHECK2-NEXT:    [[MUL251:%.*]] = mul i32 1, [[DIV250]]
-// CHECK2-NEXT:    [[TMP138:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB252:%.*]] = sub i32 [[TMP138]], -31
-// CHECK2-NEXT:    [[DIV253:%.*]] = udiv i32 [[SUB252]], 32
-// CHECK2-NEXT:    [[MUL254:%.*]] = mul i32 [[MUL251]], [[DIV253]]
-// CHECK2-NEXT:    [[MUL255:%.*]] = mul i32 [[MUL254]], 32
-// CHECK2-NEXT:    [[CONV256:%.*]] = zext i32 [[MUL255]] to i64
-// CHECK2-NEXT:    [[MUL257:%.*]] = mul nsw i64 [[DIV246]], [[CONV256]]
-// CHECK2-NEXT:    [[SUB258:%.*]] = sub nsw i64 [[TMP127]], [[MUL257]]
-// CHECK2-NEXT:    [[TMP139:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB259:%.*]] = sub i32 [[TMP139]], -31
-// CHECK2-NEXT:    [[DIV260:%.*]] = udiv i32 [[SUB259]], 32
-// CHECK2-NEXT:    [[MUL261:%.*]] = mul i32 1, [[DIV260]]
-// CHECK2-NEXT:    [[MUL262:%.*]] = mul i32 [[MUL261]], 32
-// CHECK2-NEXT:    [[CONV263:%.*]] = zext i32 [[MUL262]] to i64
-// CHECK2-NEXT:    [[DIV264:%.*]] = sdiv i64 [[SUB258]], [[CONV263]]
-// CHECK2-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB265:%.*]] = sub i32 [[TMP140]], -31
-// CHECK2-NEXT:    [[DIV266:%.*]] = udiv i32 [[SUB265]], 32
-// CHECK2-NEXT:    [[MUL267:%.*]] = mul i32 1, [[DIV266]]
-// CHECK2-NEXT:    [[MUL268:%.*]] = mul i32 [[MUL267]], 32
-// CHECK2-NEXT:    [[CONV269:%.*]] = zext i32 [[MUL268]] to i64
-// CHECK2-NEXT:    [[MUL270:%.*]] = mul nsw i64 [[DIV264]], [[CONV269]]
-// CHECK2-NEXT:    [[SUB271:%.*]] = sub nsw i64 [[SUB235]], [[MUL270]]
-// CHECK2-NEXT:    [[DIV272:%.*]] = sdiv i64 [[SUB271]], 32
-// CHECK2-NEXT:    [[MUL273:%.*]] = mul nsw i64 [[DIV272]], 32
-// CHECK2-NEXT:    [[SUB274:%.*]] = sub nsw i64 [[SUB212]], [[MUL273]]
-// CHECK2-NEXT:    [[MUL275:%.*]] = mul nsw i64 [[SUB274]], 1
-// CHECK2-NEXT:    [[ADD276:%.*]] = add nsw i64 0, [[MUL275]]
-// CHECK2-NEXT:    [[CONV277:%.*]] = trunc i64 [[ADD276]] to i32
-// CHECK2-NEXT:    store i32 [[CONV277]], ptr [[DOTTILE_CNT_0_IV_K38]], align 4
-// CHECK2-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
-// CHECK2-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
-// CHECK2-NEXT:    [[ADD278:%.*]] = add i32 [[TMP141]], [[TMP142]]
-// CHECK2-NEXT:    store i32 [[ADD278]], ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK2-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
-// CHECK2-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK2-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
-// CHECK2-NEXT:    [[MUL279:%.*]] = mul i32 [[TMP144]], [[TMP145]]
-// CHECK2-NEXT:    [[ADD280:%.*]] = add i32 [[TMP143]], [[MUL279]]
-// CHECK2-NEXT:    store i32 [[ADD280]], ptr [[K]], align 4
-// CHECK2-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
-// CHECK2-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
-// CHECK2-NEXT:    [[ADD281:%.*]] = add i32 [[TMP146]], [[TMP147]]
-// CHECK2-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK2-NEXT:    [[ADD282:%.*]] = add i32 [[TMP148]], 1
-// CHECK2-NEXT:    [[CMP283:%.*]] = icmp ult i32 [[ADD281]], [[ADD282]]
-// CHECK2-NEXT:    br i1 [[CMP283]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK2:       if.then:
-// CHECK2-NEXT:    [[TMP149:%.*]] = load i32, ptr [[I35]], align 4
-// CHECK2-NEXT:    [[TMP150:%.*]] = load i32, ptr [[J36]], align 4
-// CHECK2-NEXT:    [[TMP151:%.*]] = load i32, ptr [[K]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP149]], i32 noundef [[TMP150]], i32 noundef [[TMP151]])
-// CHECK2-NEXT:    br label [[IF_END]]
-// CHECK2:       if.end:
-// CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK2-NEXT:    [[DIV244:%.*]] = udiv i32 [[ADD243]], [[TMP136]]
+// CHECK2-NEXT:    [[MUL245:%.*]] = mul i32 1, [[DIV244]]
+// CHECK2-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB246:%.*]] = sub i32 [[TMP137]], -31
+// CHECK2-NEXT:    [[DIV247:%.*]] = udiv i32 [[SUB246]], 32
+// CHECK2-NEXT:    [[MUL248:%.*]] = mul i32 [[MUL245]], [[DIV247]]
+// CHECK2-NEXT:    [[MUL249:%.*]] = mul i32 [[MUL248]], 32
+// CHECK2-NEXT:    [[CONV250:%.*]] = zext i32 [[MUL249]] to i64
+// CHECK2-NEXT:    [[DIV251:%.*]] = sdiv i64 [[TMP132]], [[CONV250]]
+// CHECK2-NEXT:    [[TMP138:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP139:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB252:%.*]] = sub i32 [[TMP138]], [[TMP139]]
+// CHECK2-NEXT:    [[SUB253:%.*]] = sub i32 [[SUB252]], 1
+// CHECK2-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD254:%.*]] = add i32 [[SUB253]], [[TMP140]]
+// CHECK2-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV255:%.*]] = udiv i32 [[ADD254]], [[TMP141]]
+// CHECK2-NEXT:    [[MUL256:%.*]] = mul i32 1, [[DIV255]]
+// CHECK2-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB257:%.*]] = sub i32 [[TMP142]], -31
+// CHECK2-NEXT:    [[DIV258:%.*]] = udiv i32 [[SUB257]], 32
+// CHECK2-NEXT:    [[MUL259:%.*]] = mul i32 [[MUL256]], [[DIV258]]
+// CHECK2-NEXT:    [[MUL260:%.*]] = mul i32 [[MUL259]], 32
+// CHECK2-NEXT:    [[CONV261:%.*]] = zext i32 [[MUL260]] to i64
+// CHECK2-NEXT:    [[MUL262:%.*]] = mul nsw i64 [[DIV251]], [[CONV261]]
+// CHECK2-NEXT:    [[SUB263:%.*]] = sub nsw i64 [[TMP131]], [[MUL262]]
+// CHECK2-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB264:%.*]] = sub i32 [[TMP143]], -31
+// CHECK2-NEXT:    [[DIV265:%.*]] = udiv i32 [[SUB264]], 32
+// CHECK2-NEXT:    [[MUL266:%.*]] = mul i32 1, [[DIV265]]
+// CHECK2-NEXT:    [[MUL267:%.*]] = mul i32 [[MUL266]], 32
+// CHECK2-NEXT:    [[CONV268:%.*]] = zext i32 [[MUL267]] to i64
+// CHECK2-NEXT:    [[DIV269:%.*]] = sdiv i64 [[SUB263]], [[CONV268]]
+// CHECK2-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB270:%.*]] = sub i32 [[TMP144]], -31
+// CHECK2-NEXT:    [[DIV271:%.*]] = udiv i32 [[SUB270]], 32
+// CHECK2-NEXT:    [[MUL272:%.*]] = mul i32 1, [[DIV271]]
+// CHECK2-NEXT:    [[MUL273:%.*]] = mul i32 [[MUL272]], 32
+// CHECK2-NEXT:    [[CONV274:%.*]] = zext i32 [[MUL273]] to i64
+// CHECK2-NEXT:    [[MUL275:%.*]] = mul nsw i64 [[DIV269]], [[CONV274]]
+// CHECK2-NEXT:    [[SUB276:%.*]] = sub nsw i64 [[SUB240]], [[MUL275]]
+// CHECK2-NEXT:    [[DIV277:%.*]] = sdiv i64 [[SUB276]], 32
+// CHECK2-NEXT:    [[MUL278:%.*]] = mul nsw i64 [[DIV277]], 32
+// CHECK2-NEXT:    [[SUB279:%.*]] = sub nsw i64 [[SUB217]], [[MUL278]]
+// CHECK2-NEXT:    [[MUL280:%.*]] = mul nsw i64 [[SUB279]], 1
+// CHECK2-NEXT:    [[ADD281:%.*]] = add nsw i64 [[CONV158]], [[MUL280]]
+// CHECK2-NEXT:    [[CONV282:%.*]] = trunc i64 [[ADD281]] to i32
+// CHECK2-NEXT:    store i32 [[CONV282]], ptr [[DOTTILE_0_IV_K42]], align 4
+// CHECK2-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTTILE_0_IV_K42]], align 4
+// CHECK2-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// CHECK2-NEXT:    [[ADD283:%.*]] = add i32 [[TMP146]], 1
+// CHECK2-NEXT:    [[CMP284:%.*]] = icmp ult i32 [[TMP145]], [[ADD283]]
+// CHECK2-NEXT:    br i1 [[CMP284]], label [[OMP_BODY_NEXT:%.*]], label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK2:       omp.body.next:
+// CHECK2-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
+// CHECK2-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTTILE_0_IV_K42]], align 4
+// CHECK2-NEXT:    [[TMP149:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
+// CHECK2-NEXT:    [[MUL285:%.*]] = mul i32 [[TMP148]], [[TMP149]]
+// CHECK2-NEXT:    [[ADD286:%.*]] = add i32 [[TMP147]], [[MUL285]]
+// CHECK2-NEXT:    store i32 [[ADD286]], ptr [[K]], align 4
+// CHECK2-NEXT:    [[TMP150:%.*]] = load i32, ptr [[I39]], align 4
+// CHECK2-NEXT:    [[TMP151:%.*]] = load i32, ptr [[J40]], align 4
+// CHECK2-NEXT:    [[TMP152:%.*]] = load i32, ptr [[K]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP150]], i32 noundef [[TMP151]], i32 noundef [[TMP152]])
+// CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
-// CHECK2-NEXT:    [[TMP152:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[ADD284:%.*]] = add nsw i64 [[TMP152]], 1
-// CHECK2-NEXT:    store i64 [[ADD284]], ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP153:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[ADD287:%.*]] = add nsw i64 [[TMP153]], 1
+// CHECK2-NEXT:    store i64 [[ADD287]], ptr [[DOTOMP_IV]], align 8
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -4105,21 +4114,23 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTNEW_STEP10:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_14:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_17:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_19:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_K:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_K:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[_TMP35:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[I35:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[J36:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV_K37:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_K38:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[I39:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[J40:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV_K41:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_K42:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
@@ -4157,454 +4168,455 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
 // CHECK2-NEXT:    [[ADD15:%.*]] = add i32 [[TMP15]], 1
 // CHECK2-NEXT:    store i32 [[ADD15]], ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[SUB17:%.*]] = sub i32 [[TMP16]], [[TMP17]]
-// CHECK2-NEXT:    [[SUB18:%.*]] = sub i32 [[SUB17]], 1
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[ADD19:%.*]] = add i32 [[SUB18]], [[TMP18]]
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[DIV20:%.*]] = udiv i32 [[ADD19]], [[TMP19]]
-// CHECK2-NEXT:    [[CONV:%.*]] = zext i32 [[DIV20]] to i64
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB21:%.*]] = sub i32 [[TMP20]], [[TMP21]]
-// CHECK2-NEXT:    [[SUB22:%.*]] = sub i32 [[SUB21]], 1
-// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD23:%.*]] = add i32 [[SUB22]], [[TMP22]]
-// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV24:%.*]] = udiv i32 [[ADD23]], [[TMP23]]
-// CHECK2-NEXT:    [[CONV25:%.*]] = zext i32 [[DIV24]] to i64
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV25]]
-// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB26:%.*]] = sub i32 [[TMP24]], -63
-// CHECK2-NEXT:    [[DIV27:%.*]] = udiv i32 [[SUB26]], 64
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[_TMP2]], align 4
+// CHECK2-NEXT:    store i32 [[TMP16]], ptr [[DOTCAPTURE_EXPR_16]], align 4
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[_TMP2]], align 4
+// CHECK2-NEXT:    [[ADD18:%.*]] = add i32 [[TMP17]], 64
+// CHECK2-NEXT:    store i32 [[ADD18]], ptr [[DOTCAPTURE_EXPR_17]], align 4
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[SUB20:%.*]] = sub i32 [[TMP18]], [[TMP19]]
+// CHECK2-NEXT:    [[SUB21:%.*]] = sub i32 [[SUB20]], 1
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[ADD22:%.*]] = add i32 [[SUB21]], [[TMP20]]
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[DIV23:%.*]] = udiv i32 [[ADD22]], [[TMP21]]
+// CHECK2-NEXT:    [[CONV:%.*]] = zext i32 [[DIV23]] to i64
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB24:%.*]] = sub i32 [[TMP22]], [[TMP23]]
+// CHECK2-NEXT:    [[SUB25:%.*]] = sub i32 [[SUB24]], 1
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD26:%.*]] = add i32 [[SUB25]], [[TMP24]]
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV27:%.*]] = udiv i32 [[ADD26]], [[TMP25]]
 // CHECK2-NEXT:    [[CONV28:%.*]] = zext i32 [[DIV27]] to i64
-// CHECK2-NEXT:    [[MUL29:%.*]] = mul nsw i64 [[MUL]], [[CONV28]]
-// CHECK2-NEXT:    [[MUL30:%.*]] = mul nsw i64 [[MUL29]], 64
-// CHECK2-NEXT:    [[SUB31:%.*]] = sub nsw i64 [[MUL30]], 1
-// CHECK2-NEXT:    store i64 [[SUB31]], ptr [[DOTCAPTURE_EXPR_16]], align 8
-// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    store i32 [[TMP25]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    store i32 [[TMP26]], ptr [[J]], align 4
-// CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_K]], align 4
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_K]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV28]]
+// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB29:%.*]] = sub i32 [[TMP26]], -63
+// CHECK2-NEXT:    [[DIV30:%.*]] = udiv i32 [[SUB29]], 64
+// CHECK2-NEXT:    [[CONV31:%.*]] = zext i32 [[DIV30]] to i64
+// CHECK2-NEXT:    [[MUL32:%.*]] = mul nsw i64 [[MUL]], [[CONV31]]
+// CHECK2-NEXT:    [[MUL33:%.*]] = mul nsw i64 [[MUL32]], 64
+// CHECK2-NEXT:    [[SUB34:%.*]] = sub nsw i64 [[MUL33]], 1
+// CHECK2-NEXT:    store i64 [[SUB34]], ptr [[DOTCAPTURE_EXPR_19]], align 8
 // CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP27]], [[TMP28]]
+// CHECK2-NEXT:    store i32 [[TMP27]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    store i32 [[TMP28]], ptr [[J]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_K]], align 4
+// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K]], align 4
+// CHECK2-NEXT:    store i32 [[TMP29]], ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[_TMP35]], align 4
+// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP30]], [[TMP31]]
 // CHECK2-NEXT:    br i1 [[CMP]], label [[LAND_LHS_TRUE:%.*]], label [[OMP_PRECOND_END:%.*]]
 // CHECK2:       land.lhs.true:
-// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[CMP32:%.*]] = icmp slt i32 [[TMP29]], [[TMP30]]
-// CHECK2-NEXT:    br i1 [[CMP32]], label [[LAND_LHS_TRUE33:%.*]], label [[OMP_PRECOND_END]]
-// CHECK2:       land.lhs.true33:
-// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[CMP34:%.*]] = icmp ult i32 0, [[TMP31]]
-// CHECK2-NEXT:    br i1 [[CMP34]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
+// CHECK2-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[CMP36:%.*]] = icmp slt i32 [[TMP32]], [[TMP33]]
+// CHECK2-NEXT:    br i1 [[CMP36]], label [[LAND_LHS_TRUE37:%.*]], label [[OMP_PRECOND_END]]
+// CHECK2:       land.lhs.true37:
+// CHECK2-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[CMP38:%.*]] = icmp ult i32 0, [[TMP34]]
+// CHECK2-NEXT:    br i1 [[CMP38]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
 // CHECK2:       omp.precond.then:
 // CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
-// CHECK2-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
-// CHECK2-NEXT:    store i64 [[TMP32]], ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_19]], align 8
+// CHECK2-NEXT:    store i64 [[TMP35]], ptr [[DOTOMP_UB]], align 8
 // CHECK2-NEXT:    store i64 1, ptr [[DOTOMP_STRIDE]], align 8
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
 // CHECK2-NEXT:    call void @__kmpc_for_static_init_8(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i64 1, i64 1)
-// CHECK2-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
-// CHECK2-NEXT:    [[CMP39:%.*]] = icmp sgt i64 [[TMP33]], [[TMP34]]
-// CHECK2-NEXT:    br i1 [[CMP39]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_19]], align 8
+// CHECK2-NEXT:    [[CMP43:%.*]] = icmp sgt i64 [[TMP36]], [[TMP37]]
+// CHECK2-NEXT:    br i1 [[CMP43]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK2:       cond.true:
-// CHECK2-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK2-NEXT:    [[TMP38:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_19]], align 8
 // CHECK2-NEXT:    br label [[COND_END:%.*]]
 // CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[TMP39:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
 // CHECK2-NEXT:    br label [[COND_END]]
 // CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i64 [ [[TMP35]], [[COND_TRUE]] ], [ [[TMP36]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[COND:%.*]] = phi i64 [ [[TMP38]], [[COND_TRUE]] ], [ [[TMP39]], [[COND_FALSE]] ]
 // CHECK2-NEXT:    store i64 [[COND]], ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
-// CHECK2-NEXT:    store i64 [[TMP37]], ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP40:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
+// CHECK2-NEXT:    store i64 [[TMP40]], ptr [[DOTOMP_IV]], align 8
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK2:       omp.inner.for.cond:
-// CHECK2-NEXT:    [[TMP38:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP39:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[CMP40:%.*]] = icmp sle i64 [[TMP38]], [[TMP39]]
-// CHECK2-NEXT:    br i1 [[CMP40]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
-// CHECK2:       omp.inner.for.body:
-// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[CONV41:%.*]] = sext i32 [[TMP40]] to i64
 // CHECK2-NEXT:    [[TMP41:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB42:%.*]] = sub i32 [[TMP42]], [[TMP43]]
-// CHECK2-NEXT:    [[SUB43:%.*]] = sub i32 [[SUB42]], 1
-// CHECK2-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD44:%.*]] = add i32 [[SUB43]], [[TMP44]]
-// CHECK2-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV45:%.*]] = udiv i32 [[ADD44]], [[TMP45]]
-// CHECK2-NEXT:    [[MUL46:%.*]] = mul i32 1, [[DIV45]]
-// CHECK2-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB47:%.*]] = sub i32 [[TMP46]], -63
-// CHECK2-NEXT:    [[DIV48:%.*]] = udiv i32 [[SUB47]], 64
-// CHECK2-NEXT:    [[MUL49:%.*]] = mul i32 [[MUL46]], [[DIV48]]
-// CHECK2-NEXT:    [[MUL50:%.*]] = mul i32 [[MUL49]], 64
-// CHECK2-NEXT:    [[CONV51:%.*]] = zext i32 [[MUL50]] to i64
-// CHECK2-NEXT:    [[DIV52:%.*]] = sdiv i64 [[TMP41]], [[CONV51]]
-// CHECK2-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[CONV53:%.*]] = sext i32 [[TMP47]] to i64
-// CHECK2-NEXT:    [[MUL54:%.*]] = mul nsw i64 [[DIV52]], [[CONV53]]
-// CHECK2-NEXT:    [[ADD55:%.*]] = add nsw i64 [[CONV41]], [[MUL54]]
-// CHECK2-NEXT:    [[CONV56:%.*]] = trunc i64 [[ADD55]] to i32
-// CHECK2-NEXT:    store i32 [[CONV56]], ptr [[I35]], align 4
-// CHECK2-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP48]] to i64
-// CHECK2-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP50:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB58:%.*]] = sub i32 [[TMP51]], [[TMP52]]
-// CHECK2-NEXT:    [[SUB59:%.*]] = sub i32 [[SUB58]], 1
-// CHECK2-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD60:%.*]] = add i32 [[SUB59]], [[TMP53]]
-// CHECK2-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV61:%.*]] = udiv i32 [[ADD60]], [[TMP54]]
-// CHECK2-NEXT:    [[MUL62:%.*]] = mul i32 1, [[DIV61]]
-// CHECK2-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB63:%.*]] = sub i32 [[TMP55]], -63
-// CHECK2-NEXT:    [[DIV64:%.*]] = udiv i32 [[SUB63]], 64
-// CHECK2-NEXT:    [[MUL65:%.*]] = mul i32 [[MUL62]], [[DIV64]]
-// CHECK2-NEXT:    [[MUL66:%.*]] = mul i32 [[MUL65]], 64
-// CHECK2-NEXT:    [[CONV67:%.*]] = zext i32 [[MUL66]] to i64
-// CHECK2-NEXT:    [[DIV68:%.*]] = sdiv i64 [[TMP50]], [[CONV67]]
-// CHECK2-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB69:%.*]] = sub i32 [[TMP56]], [[TMP57]]
-// CHECK2-NEXT:    [[SUB70:%.*]] = sub i32 [[SUB69]], 1
-// CHECK2-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD71:%.*]] = add i32 [[SUB70]], [[TMP58]]
-// CHECK2-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV72:%.*]] = udiv i32 [[ADD71]], [[TMP59]]
-// CHECK2-NEXT:    [[MUL73:%.*]] = mul i32 1, [[DIV72]]
-// CHECK2-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB74:%.*]] = sub i32 [[TMP60]], -63
-// CHECK2-NEXT:    [[DIV75:%.*]] = udiv i32 [[SUB74]], 64
-// CHECK2-NEXT:    [[MUL76:%.*]] = mul i32 [[MUL73]], [[DIV75]]
-// CHECK2-NEXT:    [[MUL77:%.*]] = mul i32 [[MUL76]], 64
-// CHECK2-NEXT:    [[CONV78:%.*]] = zext i32 [[MUL77]] to i64
-// CHECK2-NEXT:    [[MUL79:%.*]] = mul nsw i64 [[DIV68]], [[CONV78]]
-// CHECK2-NEXT:    [[SUB80:%.*]] = sub nsw i64 [[TMP49]], [[MUL79]]
-// CHECK2-NEXT:    [[TMP61:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB81:%.*]] = sub i32 [[TMP61]], -63
-// CHECK2-NEXT:    [[DIV82:%.*]] = udiv i32 [[SUB81]], 64
-// CHECK2-NEXT:    [[MUL83:%.*]] = mul i32 1, [[DIV82]]
-// CHECK2-NEXT:    [[MUL84:%.*]] = mul i32 [[MUL83]], 64
-// CHECK2-NEXT:    [[CONV85:%.*]] = zext i32 [[MUL84]] to i64
-// CHECK2-NEXT:    [[DIV86:%.*]] = sdiv i64 [[SUB80]], [[CONV85]]
+// CHECK2-NEXT:    [[TMP42:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[CMP44:%.*]] = icmp sle i64 [[TMP41]], [[TMP42]]
+// CHECK2-NEXT:    br i1 [[CMP44]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK2:       omp.inner.for.body:
+// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[CONV45:%.*]] = sext i32 [[TMP43]] to i64
+// CHECK2-NEXT:    [[TMP44:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB46:%.*]] = sub i32 [[TMP45]], [[TMP46]]
+// CHECK2-NEXT:    [[SUB47:%.*]] = sub i32 [[SUB46]], 1
+// CHECK2-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD48:%.*]] = add i32 [[SUB47]], [[TMP47]]
+// CHECK2-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV49:%.*]] = udiv i32 [[ADD48]], [[TMP48]]
+// CHECK2-NEXT:    [[MUL50:%.*]] = mul i32 1, [[DIV49]]
+// CHECK2-NEXT:    [[TMP49:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB51:%.*]] = sub i32 [[TMP49]], -63
+// CHECK2-NEXT:    [[DIV52:%.*]] = udiv i32 [[SUB51]], 64
+// CHECK2-NEXT:    [[MUL53:%.*]] = mul i32 [[MUL50]], [[DIV52]]
+// CHECK2-NEXT:    [[MUL54:%.*]] = mul i32 [[MUL53]], 64
+// CHECK2-NEXT:    [[CONV55:%.*]] = zext i32 [[MUL54]] to i64
+// CHECK2-NEXT:    [[DIV56:%.*]] = sdiv i64 [[TMP44]], [[CONV55]]
+// CHECK2-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP50]] to i64
+// CHECK2-NEXT:    [[MUL58:%.*]] = mul nsw i64 [[DIV56]], [[CONV57]]
+// CHECK2-NEXT:    [[ADD59:%.*]] = add nsw i64 [[CONV45]], [[MUL58]]
+// CHECK2-NEXT:    [[CONV60:%.*]] = trunc i64 [[ADD59]] to i32
+// CHECK2-NEXT:    store i32 [[CONV60]], ptr [[I39]], align 4
+// CHECK2-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[CONV61:%.*]] = sext i32 [[TMP51]] to i64
+// CHECK2-NEXT:    [[TMP52:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP53:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB62:%.*]] = sub i32 [[TMP54]], [[TMP55]]
+// CHECK2-NEXT:    [[SUB63:%.*]] = sub i32 [[SUB62]], 1
+// CHECK2-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD64:%.*]] = add i32 [[SUB63]], [[TMP56]]
+// CHECK2-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV65:%.*]] = udiv i32 [[ADD64]], [[TMP57]]
+// CHECK2-NEXT:    [[MUL66:%.*]] = mul i32 1, [[DIV65]]
+// CHECK2-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB67:%.*]] = sub i32 [[TMP58]], -63
+// CHECK2-NEXT:    [[DIV68:%.*]] = udiv i32 [[SUB67]], 64
+// CHECK2-NEXT:    [[MUL69:%.*]] = mul i32 [[MUL66]], [[DIV68]]
+// CHECK2-NEXT:    [[MUL70:%.*]] = mul i32 [[MUL69]], 64
+// CHECK2-NEXT:    [[CONV71:%.*]] = zext i32 [[MUL70]] to i64
+// CHECK2-NEXT:    [[DIV72:%.*]] = sdiv i64 [[TMP53]], [[CONV71]]
+// CHECK2-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB73:%.*]] = sub i32 [[TMP59]], [[TMP60]]
+// CHECK2-NEXT:    [[SUB74:%.*]] = sub i32 [[SUB73]], 1
+// CHECK2-NEXT:    [[TMP61:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD75:%.*]] = add i32 [[SUB74]], [[TMP61]]
 // CHECK2-NEXT:    [[TMP62:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[CONV87:%.*]] = sext i32 [[TMP62]] to i64
-// CHECK2-NEXT:    [[MUL88:%.*]] = mul nsw i64 [[DIV86]], [[CONV87]]
-// CHECK2-NEXT:    [[ADD89:%.*]] = add nsw i64 [[CONV57]], [[MUL88]]
-// CHECK2-NEXT:    [[CONV90:%.*]] = trunc i64 [[ADD89]] to i32
-// CHECK2-NEXT:    store i32 [[CONV90]], ptr [[J36]], align 4
-// CHECK2-NEXT:    [[TMP63:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP64:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB91:%.*]] = sub i32 [[TMP65]], [[TMP66]]
-// CHECK2-NEXT:    [[SUB92:%.*]] = sub i32 [[SUB91]], 1
-// CHECK2-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD93:%.*]] = add i32 [[SUB92]], [[TMP67]]
-// CHECK2-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV94:%.*]] = udiv i32 [[ADD93]], [[TMP68]]
-// CHECK2-NEXT:    [[MUL95:%.*]] = mul i32 1, [[DIV94]]
-// CHECK2-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB96:%.*]] = sub i32 [[TMP69]], -63
-// CHECK2-NEXT:    [[DIV97:%.*]] = udiv i32 [[SUB96]], 64
-// CHECK2-NEXT:    [[MUL98:%.*]] = mul i32 [[MUL95]], [[DIV97]]
-// CHECK2-NEXT:    [[MUL99:%.*]] = mul i32 [[MUL98]], 64
-// CHECK2-NEXT:    [[CONV100:%.*]] = zext i32 [[MUL99]] to i64
-// CHECK2-NEXT:    [[DIV101:%.*]] = sdiv i64 [[TMP64]], [[CONV100]]
-// CHECK2-NEXT:    [[TMP70:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB102:%.*]] = sub i32 [[TMP70]], [[TMP71]]
-// CHECK2-NEXT:    [[SUB103:%.*]] = sub i32 [[SUB102]], 1
-// CHECK2-NEXT:    [[TMP72:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD104:%.*]] = add i32 [[SUB103]], [[TMP72]]
-// CHECK2-NEXT:    [[TMP73:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV105:%.*]] = udiv i32 [[ADD104]], [[TMP73]]
-// CHECK2-NEXT:    [[MUL106:%.*]] = mul i32 1, [[DIV105]]
-// CHECK2-NEXT:    [[TMP74:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB107:%.*]] = sub i32 [[TMP74]], -63
-// CHECK2-NEXT:    [[DIV108:%.*]] = udiv i32 [[SUB107]], 64
-// CHECK2-NEXT:    [[MUL109:%.*]] = mul i32 [[MUL106]], [[DIV108]]
-// CHECK2-NEXT:    [[MUL110:%.*]] = mul i32 [[MUL109]], 64
-// CHECK2-NEXT:    [[CONV111:%.*]] = zext i32 [[MUL110]] to i64
-// CHECK2-NEXT:    [[MUL112:%.*]] = mul nsw i64 [[DIV101]], [[CONV111]]
-// CHECK2-NEXT:    [[SUB113:%.*]] = sub nsw i64 [[TMP63]], [[MUL112]]
-// CHECK2-NEXT:    [[TMP75:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP76:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP78:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB114:%.*]] = sub i32 [[TMP77]], [[TMP78]]
-// CHECK2-NEXT:    [[SUB115:%.*]] = sub i32 [[SUB114]], 1
-// CHECK2-NEXT:    [[TMP79:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD116:%.*]] = add i32 [[SUB115]], [[TMP79]]
-// CHECK2-NEXT:    [[TMP80:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV117:%.*]] = udiv i32 [[ADD116]], [[TMP80]]
-// CHECK2-NEXT:    [[MUL118:%.*]] = mul i32 1, [[DIV117]]
-// CHECK2-NEXT:    [[TMP81:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB119:%.*]] = sub i32 [[TMP81]], -63
-// CHECK2-NEXT:    [[DIV120:%.*]] = udiv i32 [[SUB119]], 64
-// CHECK2-NEXT:    [[MUL121:%.*]] = mul i32 [[MUL118]], [[DIV120]]
-// CHECK2-NEXT:    [[MUL122:%.*]] = mul i32 [[MUL121]], 64
-// CHECK2-NEXT:    [[CONV123:%.*]] = zext i32 [[MUL122]] to i64
-// CHECK2-NEXT:    [[DIV124:%.*]] = sdiv i64 [[TMP76]], [[CONV123]]
-// CHECK2-NEXT:    [[TMP82:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB125:%.*]] = sub i32 [[TMP82]], [[TMP83]]
-// CHECK2-NEXT:    [[SUB126:%.*]] = sub i32 [[SUB125]], 1
-// CHECK2-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD127:%.*]] = add i32 [[SUB126]], [[TMP84]]
-// CHECK2-NEXT:    [[TMP85:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV128:%.*]] = udiv i32 [[ADD127]], [[TMP85]]
-// CHECK2-NEXT:    [[MUL129:%.*]] = mul i32 1, [[DIV128]]
-// CHECK2-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB130:%.*]] = sub i32 [[TMP86]], -63
-// CHECK2-NEXT:    [[DIV131:%.*]] = udiv i32 [[SUB130]], 64
-// CHECK2-NEXT:    [[MUL132:%.*]] = mul i32 [[MUL129]], [[DIV131]]
-// CHECK2-NEXT:    [[MUL133:%.*]] = mul i32 [[MUL132]], 64
-// CHECK2-NEXT:    [[CONV134:%.*]] = zext i32 [[MUL133]] to i64
-// CHECK2-NEXT:    [[MUL135:%.*]] = mul nsw i64 [[DIV124]], [[CONV134]]
-// CHECK2-NEXT:    [[SUB136:%.*]] = sub nsw i64 [[TMP75]], [[MUL135]]
-// CHECK2-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB137:%.*]] = sub i32 [[TMP87]], -63
-// CHECK2-NEXT:    [[DIV138:%.*]] = udiv i32 [[SUB137]], 64
-// CHECK2-NEXT:    [[MUL139:%.*]] = mul i32 1, [[DIV138]]
-// CHECK2-NEXT:    [[MUL140:%.*]] = mul i32 [[MUL139]], 64
-// CHECK2-NEXT:    [[CONV141:%.*]] = zext i32 [[MUL140]] to i64
-// CHECK2-NEXT:    [[DIV142:%.*]] = sdiv i64 [[SUB136]], [[CONV141]]
-// CHECK2-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB143:%.*]] = sub i32 [[TMP88]], -63
-// CHECK2-NEXT:    [[DIV144:%.*]] = udiv i32 [[SUB143]], 64
-// CHECK2-NEXT:    [[MUL145:%.*]] = mul i32 1, [[DIV144]]
-// CHECK2-NEXT:    [[MUL146:%.*]] = mul i32 [[MUL145]], 64
-// CHECK2-NEXT:    [[CONV147:%.*]] = zext i32 [[MUL146]] to i64
-// CHECK2-NEXT:    [[MUL148:%.*]] = mul nsw i64 [[DIV142]], [[CONV147]]
-// CHECK2-NEXT:    [[SUB149:%.*]] = sub nsw i64 [[SUB113]], [[MUL148]]
-// CHECK2-NEXT:    [[DIV150:%.*]] = sdiv i64 [[SUB149]], 64
-// CHECK2-NEXT:    [[MUL151:%.*]] = mul nsw i64 [[DIV150]], 64
-// CHECK2-NEXT:    [[ADD152:%.*]] = add nsw i64 0, [[MUL151]]
-// CHECK2-NEXT:    [[CONV153:%.*]] = trunc i64 [[ADD152]] to i32
-// CHECK2-NEXT:    store i32 [[CONV153]], ptr [[DOTFLOOR_0_IV_K37]], align 4
-// CHECK2-NEXT:    [[TMP89:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP90:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB154:%.*]] = sub i32 [[TMP91]], [[TMP92]]
-// CHECK2-NEXT:    [[SUB155:%.*]] = sub i32 [[SUB154]], 1
-// CHECK2-NEXT:    [[TMP93:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD156:%.*]] = add i32 [[SUB155]], [[TMP93]]
-// CHECK2-NEXT:    [[TMP94:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV157:%.*]] = udiv i32 [[ADD156]], [[TMP94]]
-// CHECK2-NEXT:    [[MUL158:%.*]] = mul i32 1, [[DIV157]]
-// CHECK2-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB159:%.*]] = sub i32 [[TMP95]], -63
-// CHECK2-NEXT:    [[DIV160:%.*]] = udiv i32 [[SUB159]], 64
-// CHECK2-NEXT:    [[MUL161:%.*]] = mul i32 [[MUL158]], [[DIV160]]
-// CHECK2-NEXT:    [[MUL162:%.*]] = mul i32 [[MUL161]], 64
-// CHECK2-NEXT:    [[CONV163:%.*]] = zext i32 [[MUL162]] to i64
-// CHECK2-NEXT:    [[DIV164:%.*]] = sdiv i64 [[TMP90]], [[CONV163]]
-// CHECK2-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP97:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB165:%.*]] = sub i32 [[TMP96]], [[TMP97]]
-// CHECK2-NEXT:    [[SUB166:%.*]] = sub i32 [[SUB165]], 1
+// CHECK2-NEXT:    [[DIV76:%.*]] = udiv i32 [[ADD75]], [[TMP62]]
+// CHECK2-NEXT:    [[MUL77:%.*]] = mul i32 1, [[DIV76]]
+// CHECK2-NEXT:    [[TMP63:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB78:%.*]] = sub i32 [[TMP63]], -63
+// CHECK2-NEXT:    [[DIV79:%.*]] = udiv i32 [[SUB78]], 64
+// CHECK2-NEXT:    [[MUL80:%.*]] = mul i32 [[MUL77]], [[DIV79]]
+// CHECK2-NEXT:    [[MUL81:%.*]] = mul i32 [[MUL80]], 64
+// CHECK2-NEXT:    [[CONV82:%.*]] = zext i32 [[MUL81]] to i64
+// CHECK2-NEXT:    [[MUL83:%.*]] = mul nsw i64 [[DIV72]], [[CONV82]]
+// CHECK2-NEXT:    [[SUB84:%.*]] = sub nsw i64 [[TMP52]], [[MUL83]]
+// CHECK2-NEXT:    [[TMP64:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB85:%.*]] = sub i32 [[TMP64]], -63
+// CHECK2-NEXT:    [[DIV86:%.*]] = udiv i32 [[SUB85]], 64
+// CHECK2-NEXT:    [[MUL87:%.*]] = mul i32 1, [[DIV86]]
+// CHECK2-NEXT:    [[MUL88:%.*]] = mul i32 [[MUL87]], 64
+// CHECK2-NEXT:    [[CONV89:%.*]] = zext i32 [[MUL88]] to i64
+// CHECK2-NEXT:    [[DIV90:%.*]] = sdiv i64 [[SUB84]], [[CONV89]]
+// CHECK2-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[CONV91:%.*]] = sext i32 [[TMP65]] to i64
+// CHECK2-NEXT:    [[MUL92:%.*]] = mul nsw i64 [[DIV90]], [[CONV91]]
+// CHECK2-NEXT:    [[ADD93:%.*]] = add nsw i64 [[CONV61]], [[MUL92]]
+// CHECK2-NEXT:    [[CONV94:%.*]] = trunc i64 [[ADD93]] to i32
+// CHECK2-NEXT:    store i32 [[CONV94]], ptr [[J40]], align 4
+// CHECK2-NEXT:    [[TMP66:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP67:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB95:%.*]] = sub i32 [[TMP68]], [[TMP69]]
+// CHECK2-NEXT:    [[SUB96:%.*]] = sub i32 [[SUB95]], 1
+// CHECK2-NEXT:    [[TMP70:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD97:%.*]] = add i32 [[SUB96]], [[TMP70]]
+// CHECK2-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV98:%.*]] = udiv i32 [[ADD97]], [[TMP71]]
+// CHECK2-NEXT:    [[MUL99:%.*]] = mul i32 1, [[DIV98]]
+// CHECK2-NEXT:    [[TMP72:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB100:%.*]] = sub i32 [[TMP72]], -63
+// CHECK2-NEXT:    [[DIV101:%.*]] = udiv i32 [[SUB100]], 64
+// CHECK2-NEXT:    [[MUL102:%.*]] = mul i32 [[MUL99]], [[DIV101]]
+// CHECK2-NEXT:    [[MUL103:%.*]] = mul i32 [[MUL102]], 64
+// CHECK2-NEXT:    [[CONV104:%.*]] = zext i32 [[MUL103]] to i64
+// CHECK2-NEXT:    [[DIV105:%.*]] = sdiv i64 [[TMP67]], [[CONV104]]
+// CHECK2-NEXT:    [[TMP73:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP74:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB106:%.*]] = sub i32 [[TMP73]], [[TMP74]]
+// CHECK2-NEXT:    [[SUB107:%.*]] = sub i32 [[SUB106]], 1
+// CHECK2-NEXT:    [[TMP75:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD108:%.*]] = add i32 [[SUB107]], [[TMP75]]
+// CHECK2-NEXT:    [[TMP76:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV109:%.*]] = udiv i32 [[ADD108]], [[TMP76]]
+// CHECK2-NEXT:    [[MUL110:%.*]] = mul i32 1, [[DIV109]]
+// CHECK2-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB111:%.*]] = sub i32 [[TMP77]], -63
+// CHECK2-NEXT:    [[DIV112:%.*]] = udiv i32 [[SUB111]], 64
+// CHECK2-NEXT:    [[MUL113:%.*]] = mul i32 [[MUL110]], [[DIV112]]
+// CHECK2-NEXT:    [[MUL114:%.*]] = mul i32 [[MUL113]], 64
+// CHECK2-NEXT:    [[CONV115:%.*]] = zext i32 [[MUL114]] to i64
+// CHECK2-NEXT:    [[MUL116:%.*]] = mul nsw i64 [[DIV105]], [[CONV115]]
+// CHECK2-NEXT:    [[SUB117:%.*]] = sub nsw i64 [[TMP66]], [[MUL116]]
+// CHECK2-NEXT:    [[TMP78:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP79:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP80:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP81:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB118:%.*]] = sub i32 [[TMP80]], [[TMP81]]
+// CHECK2-NEXT:    [[SUB119:%.*]] = sub i32 [[SUB118]], 1
+// CHECK2-NEXT:    [[TMP82:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD120:%.*]] = add i32 [[SUB119]], [[TMP82]]
+// CHECK2-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV121:%.*]] = udiv i32 [[ADD120]], [[TMP83]]
+// CHECK2-NEXT:    [[MUL122:%.*]] = mul i32 1, [[DIV121]]
+// CHECK2-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB123:%.*]] = sub i32 [[TMP84]], -63
+// CHECK2-NEXT:    [[DIV124:%.*]] = udiv i32 [[SUB123]], 64
+// CHECK2-NEXT:    [[MUL125:%.*]] = mul i32 [[MUL122]], [[DIV124]]
+// CHECK2-NEXT:    [[MUL126:%.*]] = mul i32 [[MUL125]], 64
+// CHECK2-NEXT:    [[CONV127:%.*]] = zext i32 [[MUL126]] to i64
+// CHECK2-NEXT:    [[DIV128:%.*]] = sdiv i64 [[TMP79]], [[CONV127]]
+// CHECK2-NEXT:    [[TMP85:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB129:%.*]] = sub i32 [[TMP85]], [[TMP86]]
+// CHECK2-NEXT:    [[SUB130:%.*]] = sub i32 [[SUB129]], 1
+// CHECK2-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD131:%.*]] = add i32 [[SUB130]], [[TMP87]]
+// CHECK2-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV132:%.*]] = udiv i32 [[ADD131]], [[TMP88]]
+// CHECK2-NEXT:    [[MUL133:%.*]] = mul i32 1, [[DIV132]]
+// CHECK2-NEXT:    [[TMP89:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB134:%.*]] = sub i32 [[TMP89]], -63
+// CHECK2-NEXT:    [[DIV135:%.*]] = udiv i32 [[SUB134]], 64
+// CHECK2-NEXT:    [[MUL136:%.*]] = mul i32 [[MUL133]], [[DIV135]]
+// CHECK2-NEXT:    [[MUL137:%.*]] = mul i32 [[MUL136]], 64
+// CHECK2-NEXT:    [[CONV138:%.*]] = zext i32 [[MUL137]] to i64
+// CHECK2-NEXT:    [[MUL139:%.*]] = mul nsw i64 [[DIV128]], [[CONV138]]
+// CHECK2-NEXT:    [[SUB140:%.*]] = sub nsw i64 [[TMP78]], [[MUL139]]
+// CHECK2-NEXT:    [[TMP90:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB141:%.*]] = sub i32 [[TMP90]], -63
+// CHECK2-NEXT:    [[DIV142:%.*]] = udiv i32 [[SUB141]], 64
+// CHECK2-NEXT:    [[MUL143:%.*]] = mul i32 1, [[DIV142]]
+// CHECK2-NEXT:    [[MUL144:%.*]] = mul i32 [[MUL143]], 64
+// CHECK2-NEXT:    [[CONV145:%.*]] = zext i32 [[MUL144]] to i64
+// CHECK2-NEXT:    [[DIV146:%.*]] = sdiv i64 [[SUB140]], [[CONV145]]
+// CHECK2-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB147:%.*]] = sub i32 [[TMP91]], -63
+// CHECK2-NEXT:    [[DIV148:%.*]] = udiv i32 [[SUB147]], 64
+// CHECK2-NEXT:    [[MUL149:%.*]] = mul i32 1, [[DIV148]]
+// CHECK2-NEXT:    [[MUL150:%.*]] = mul i32 [[MUL149]], 64
+// CHECK2-NEXT:    [[CONV151:%.*]] = zext i32 [[MUL150]] to i64
+// CHECK2-NEXT:    [[MUL152:%.*]] = mul nsw i64 [[DIV146]], [[CONV151]]
+// CHECK2-NEXT:    [[SUB153:%.*]] = sub nsw i64 [[SUB117]], [[MUL152]]
+// CHECK2-NEXT:    [[DIV154:%.*]] = sdiv i64 [[SUB153]], 64
+// CHECK2-NEXT:    [[MUL155:%.*]] = mul nsw i64 [[DIV154]], 64
+// CHECK2-NEXT:    [[ADD156:%.*]] = add nsw i64 0, [[MUL155]]
+// CHECK2-NEXT:    [[CONV157:%.*]] = trunc i64 [[ADD156]] to i32
+// CHECK2-NEXT:    store i32 [[CONV157]], ptr [[DOTFLOOR_0_IV_K41]], align 4
+// CHECK2-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K41]], align 4
+// CHECK2-NEXT:    [[CONV158:%.*]] = zext i32 [[TMP92]] to i64
+// CHECK2-NEXT:    [[TMP93:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP94:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB159:%.*]] = sub i32 [[TMP95]], [[TMP96]]
+// CHECK2-NEXT:    [[SUB160:%.*]] = sub i32 [[SUB159]], 1
+// CHECK2-NEXT:    [[TMP97:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD161:%.*]] = add i32 [[SUB160]], [[TMP97]]
 // CHECK2-NEXT:    [[TMP98:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD167:%.*]] = add i32 [[SUB166]], [[TMP98]]
-// CHECK2-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV168:%.*]] = udiv i32 [[ADD167]], [[TMP99]]
-// CHECK2-NEXT:    [[MUL169:%.*]] = mul i32 1, [[DIV168]]
-// CHECK2-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB170:%.*]] = sub i32 [[TMP100]], -63
-// CHECK2-NEXT:    [[DIV171:%.*]] = udiv i32 [[SUB170]], 64
-// CHECK2-NEXT:    [[MUL172:%.*]] = mul i32 [[MUL169]], [[DIV171]]
-// CHECK2-NEXT:    [[MUL173:%.*]] = mul i32 [[MUL172]], 64
-// CHECK2-NEXT:    [[CONV174:%.*]] = zext i32 [[MUL173]] to i64
-// CHECK2-NEXT:    [[MUL175:%.*]] = mul nsw i64 [[DIV164]], [[CONV174]]
-// CHECK2-NEXT:    [[SUB176:%.*]] = sub nsw i64 [[TMP89]], [[MUL175]]
-// CHECK2-NEXT:    [[TMP101:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP102:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB177:%.*]] = sub i32 [[TMP103]], [[TMP104]]
-// CHECK2-NEXT:    [[SUB178:%.*]] = sub i32 [[SUB177]], 1
-// CHECK2-NEXT:    [[TMP105:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD179:%.*]] = add i32 [[SUB178]], [[TMP105]]
-// CHECK2-NEXT:    [[TMP106:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV180:%.*]] = udiv i32 [[ADD179]], [[TMP106]]
-// CHECK2-NEXT:    [[MUL181:%.*]] = mul i32 1, [[DIV180]]
-// CHECK2-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB182:%.*]] = sub i32 [[TMP107]], -63
-// CHECK2-NEXT:    [[DIV183:%.*]] = udiv i32 [[SUB182]], 64
-// CHECK2-NEXT:    [[MUL184:%.*]] = mul i32 [[MUL181]], [[DIV183]]
-// CHECK2-NEXT:    [[MUL185:%.*]] = mul i32 [[MUL184]], 64
-// CHECK2-NEXT:    [[CONV186:%.*]] = zext i32 [[MUL185]] to i64
-// CHECK2-NEXT:    [[DIV187:%.*]] = sdiv i64 [[TMP102]], [[CONV186]]
-// CHECK2-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB188:%.*]] = sub i32 [[TMP108]], [[TMP109]]
-// CHECK2-NEXT:    [[SUB189:%.*]] = sub i32 [[SUB188]], 1
+// CHECK2-NEXT:    [[DIV162:%.*]] = udiv i32 [[ADD161]], [[TMP98]]
+// CHECK2-NEXT:    [[MUL163:%.*]] = mul i32 1, [[DIV162]]
+// CHECK2-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB164:%.*]] = sub i32 [[TMP99]], -63
+// CHECK2-NEXT:    [[DIV165:%.*]] = udiv i32 [[SUB164]], 64
+// CHECK2-NEXT:    [[MUL166:%.*]] = mul i32 [[MUL163]], [[DIV165]]
+// CHECK2-NEXT:    [[MUL167:%.*]] = mul i32 [[MUL166]], 64
+// CHECK2-NEXT:    [[CONV168:%.*]] = zext i32 [[MUL167]] to i64
+// CHECK2-NEXT:    [[DIV169:%.*]] = sdiv i64 [[TMP94]], [[CONV168]]
+// CHECK2-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP101:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB170:%.*]] = sub i32 [[TMP100]], [[TMP101]]
+// CHECK2-NEXT:    [[SUB171:%.*]] = sub i32 [[SUB170]], 1
+// CHECK2-NEXT:    [[TMP102:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD172:%.*]] = add i32 [[SUB171]], [[TMP102]]
+// CHECK2-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV173:%.*]] = udiv i32 [[ADD172]], [[TMP103]]
+// CHECK2-NEXT:    [[MUL174:%.*]] = mul i32 1, [[DIV173]]
+// CHECK2-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB175:%.*]] = sub i32 [[TMP104]], -63
+// CHECK2-NEXT:    [[DIV176:%.*]] = udiv i32 [[SUB175]], 64
+// CHECK2-NEXT:    [[MUL177:%.*]] = mul i32 [[MUL174]], [[DIV176]]
+// CHECK2-NEXT:    [[MUL178:%.*]] = mul i32 [[MUL177]], 64
+// CHECK2-NEXT:    [[CONV179:%.*]] = zext i32 [[MUL178]] to i64
+// CHECK2-NEXT:    [[MUL180:%.*]] = mul nsw i64 [[DIV169]], [[CONV179]]
+// CHECK2-NEXT:    [[SUB181:%.*]] = sub nsw i64 [[TMP93]], [[MUL180]]
+// CHECK2-NEXT:    [[TMP105:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP106:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB182:%.*]] = sub i32 [[TMP107]], [[TMP108]]
+// CHECK2-NEXT:    [[SUB183:%.*]] = sub i32 [[SUB182]], 1
+// CHECK2-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD184:%.*]] = add i32 [[SUB183]], [[TMP109]]
 // CHECK2-NEXT:    [[TMP110:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD190:%.*]] = add i32 [[SUB189]], [[TMP110]]
-// CHECK2-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV191:%.*]] = udiv i32 [[ADD190]], [[TMP111]]
-// CHECK2-NEXT:    [[MUL192:%.*]] = mul i32 1, [[DIV191]]
-// CHECK2-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB193:%.*]] = sub i32 [[TMP112]], -63
-// CHECK2-NEXT:    [[DIV194:%.*]] = udiv i32 [[SUB193]], 64
-// CHECK2-NEXT:    [[MUL195:%.*]] = mul i32 [[MUL192]], [[DIV194]]
-// CHECK2-NEXT:    [[MUL196:%.*]] = mul i32 [[MUL195]], 64
-// CHECK2-NEXT:    [[CONV197:%.*]] = zext i32 [[MUL196]] to i64
-// CHECK2-NEXT:    [[MUL198:%.*]] = mul nsw i64 [[DIV187]], [[CONV197]]
-// CHECK2-NEXT:    [[SUB199:%.*]] = sub nsw i64 [[TMP101]], [[MUL198]]
-// CHECK2-NEXT:    [[TMP113:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB200:%.*]] = sub i32 [[TMP113]], -63
-// CHECK2-NEXT:    [[DIV201:%.*]] = udiv i32 [[SUB200]], 64
-// CHECK2-NEXT:    [[MUL202:%.*]] = mul i32 1, [[DIV201]]
-// CHECK2-NEXT:    [[MUL203:%.*]] = mul i32 [[MUL202]], 64
-// CHECK2-NEXT:    [[CONV204:%.*]] = zext i32 [[MUL203]] to i64
-// CHECK2-NEXT:    [[DIV205:%.*]] = sdiv i64 [[SUB199]], [[CONV204]]
-// CHECK2-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB206:%.*]] = sub i32 [[TMP114]], -63
-// CHECK2-NEXT:    [[DIV207:%.*]] = udiv i32 [[SUB206]], 64
-// CHECK2-NEXT:    [[MUL208:%.*]] = mul i32 1, [[DIV207]]
-// CHECK2-NEXT:    [[MUL209:%.*]] = mul i32 [[MUL208]], 64
-// CHECK2-NEXT:    [[CONV210:%.*]] = zext i32 [[MUL209]] to i64
-// CHECK2-NEXT:    [[MUL211:%.*]] = mul nsw i64 [[DIV205]], [[CONV210]]
-// CHECK2-NEXT:    [[SUB212:%.*]] = sub nsw i64 [[SUB176]], [[MUL211]]
-// CHECK2-NEXT:    [[TMP115:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP116:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB213:%.*]] = sub i32 [[TMP117]], [[TMP118]]
-// CHECK2-NEXT:    [[SUB214:%.*]] = sub i32 [[SUB213]], 1
-// CHECK2-NEXT:    [[TMP119:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD215:%.*]] = add i32 [[SUB214]], [[TMP119]]
-// CHECK2-NEXT:    [[TMP120:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV216:%.*]] = udiv i32 [[ADD215]], [[TMP120]]
-// CHECK2-NEXT:    [[MUL217:%.*]] = mul i32 1, [[DIV216]]
-// CHECK2-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB218:%.*]] = sub i32 [[TMP121]], -63
-// CHECK2-NEXT:    [[DIV219:%.*]] = udiv i32 [[SUB218]], 64
-// CHECK2-NEXT:    [[MUL220:%.*]] = mul i32 [[MUL217]], [[DIV219]]
-// CHECK2-NEXT:    [[MUL221:%.*]] = mul i32 [[MUL220]], 64
-// CHECK2-NEXT:    [[CONV222:%.*]] = zext i32 [[MUL221]] to i64
-// CHECK2-NEXT:    [[DIV223:%.*]] = sdiv i64 [[TMP116]], [[CONV222]]
-// CHECK2-NEXT:    [[TMP122:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP123:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB224:%.*]] = sub i32 [[TMP122]], [[TMP123]]
-// CHECK2-NEXT:    [[SUB225:%.*]] = sub i32 [[SUB224]], 1
+// CHECK2-NEXT:    [[DIV185:%.*]] = udiv i32 [[ADD184]], [[TMP110]]
+// CHECK2-NEXT:    [[MUL186:%.*]] = mul i32 1, [[DIV185]]
+// CHECK2-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB187:%.*]] = sub i32 [[TMP111]], -63
+// CHECK2-NEXT:    [[DIV188:%.*]] = udiv i32 [[SUB187]], 64
+// CHECK2-NEXT:    [[MUL189:%.*]] = mul i32 [[MUL186]], [[DIV188]]
+// CHECK2-NEXT:    [[MUL190:%.*]] = mul i32 [[MUL189]], 64
+// CHECK2-NEXT:    [[CONV191:%.*]] = zext i32 [[MUL190]] to i64
+// CHECK2-NEXT:    [[DIV192:%.*]] = sdiv i64 [[TMP106]], [[CONV191]]
+// CHECK2-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP113:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB193:%.*]] = sub i32 [[TMP112]], [[TMP113]]
+// CHECK2-NEXT:    [[SUB194:%.*]] = sub i32 [[SUB193]], 1
+// CHECK2-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD195:%.*]] = add i32 [[SUB194]], [[TMP114]]
+// CHECK2-NEXT:    [[TMP115:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV196:%.*]] = udiv i32 [[ADD195]], [[TMP115]]
+// CHECK2-NEXT:    [[MUL197:%.*]] = mul i32 1, [[DIV196]]
+// CHECK2-NEXT:    [[TMP116:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB198:%.*]] = sub i32 [[TMP116]], -63
+// CHECK2-NEXT:    [[DIV199:%.*]] = udiv i32 [[SUB198]], 64
+// CHECK2-NEXT:    [[MUL200:%.*]] = mul i32 [[MUL197]], [[DIV199]]
+// CHECK2-NEXT:    [[MUL201:%.*]] = mul i32 [[MUL200]], 64
+// CHECK2-NEXT:    [[CONV202:%.*]] = zext i32 [[MUL201]] to i64
+// CHECK2-NEXT:    [[MUL203:%.*]] = mul nsw i64 [[DIV192]], [[CONV202]]
+// CHECK2-NEXT:    [[SUB204:%.*]] = sub nsw i64 [[TMP105]], [[MUL203]]
+// CHECK2-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB205:%.*]] = sub i32 [[TMP117]], -63
+// CHECK2-NEXT:    [[DIV206:%.*]] = udiv i32 [[SUB205]], 64
+// CHECK2-NEXT:    [[MUL207:%.*]] = mul i32 1, [[DIV206]]
+// CHECK2-NEXT:    [[MUL208:%.*]] = mul i32 [[MUL207]], 64
+// CHECK2-NEXT:    [[CONV209:%.*]] = zext i32 [[MUL208]] to i64
+// CHECK2-NEXT:    [[DIV210:%.*]] = sdiv i64 [[SUB204]], [[CONV209]]
+// CHECK2-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB211:%.*]] = sub i32 [[TMP118]], -63
+// CHECK2-NEXT:    [[DIV212:%.*]] = udiv i32 [[SUB211]], 64
+// CHECK2-NEXT:    [[MUL213:%.*]] = mul i32 1, [[DIV212]]
+// CHECK2-NEXT:    [[MUL214:%.*]] = mul i32 [[MUL213]], 64
+// CHECK2-NEXT:    [[CONV215:%.*]] = zext i32 [[MUL214]] to i64
+// CHECK2-NEXT:    [[MUL216:%.*]] = mul nsw i64 [[DIV210]], [[CONV215]]
+// CHECK2-NEXT:    [[SUB217:%.*]] = sub nsw i64 [[SUB181]], [[MUL216]]
+// CHECK2-NEXT:    [[TMP119:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP120:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP122:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB218:%.*]] = sub i32 [[TMP121]], [[TMP122]]
+// CHECK2-NEXT:    [[SUB219:%.*]] = sub i32 [[SUB218]], 1
+// CHECK2-NEXT:    [[TMP123:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD220:%.*]] = add i32 [[SUB219]], [[TMP123]]
 // CHECK2-NEXT:    [[TMP124:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD226:%.*]] = add i32 [[SUB225]], [[TMP124]]
-// CHECK2-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV227:%.*]] = udiv i32 [[ADD226]], [[TMP125]]
-// CHECK2-NEXT:    [[MUL228:%.*]] = mul i32 1, [[DIV227]]
-// CHECK2-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB229:%.*]] = sub i32 [[TMP126]], -63
-// CHECK2-NEXT:    [[DIV230:%.*]] = udiv i32 [[SUB229]], 64
-// CHECK2-NEXT:    [[MUL231:%.*]] = mul i32 [[MUL228]], [[DIV230]]
-// CHECK2-NEXT:    [[MUL232:%.*]] = mul i32 [[MUL231]], 64
-// CHECK2-NEXT:    [[CONV233:%.*]] = zext i32 [[MUL232]] to i64
-// CHECK2-NEXT:    [[MUL234:%.*]] = mul nsw i64 [[DIV223]], [[CONV233]]
-// CHECK2-NEXT:    [[SUB235:%.*]] = sub nsw i64 [[TMP115]], [[MUL234]]
-// CHECK2-NEXT:    [[TMP127:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP128:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB236:%.*]] = sub i32 [[TMP129]], [[TMP130]]
-// CHECK2-NEXT:    [[SUB237:%.*]] = sub i32 [[SUB236]], 1
-// CHECK2-NEXT:    [[TMP131:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD238:%.*]] = add i32 [[SUB237]], [[TMP131]]
-// CHECK2-NEXT:    [[TMP132:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV239:%.*]] = udiv i32 [[ADD238]], [[TMP132]]
-// CHECK2-NEXT:    [[MUL240:%.*]] = mul i32 1, [[DIV239]]
-// CHECK2-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB241:%.*]] = sub i32 [[TMP133]], -63
-// CHECK2-NEXT:    [[DIV242:%.*]] = udiv i32 [[SUB241]], 64
-// CHECK2-NEXT:    [[MUL243:%.*]] = mul i32 [[MUL240]], [[DIV242]]
-// CHECK2-NEXT:    [[MUL244:%.*]] = mul i32 [[MUL243]], 64
-// CHECK2-NEXT:    [[CONV245:%.*]] = zext i32 [[MUL244]] to i64
-// CHECK2-NEXT:    [[DIV246:%.*]] = sdiv i64 [[TMP128]], [[CONV245]]
-// CHECK2-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB247:%.*]] = sub i32 [[TMP134]], [[TMP135]]
-// CHECK2-NEXT:    [[SUB248:%.*]] = sub i32 [[SUB247]], 1
+// CHECK2-NEXT:    [[DIV221:%.*]] = udiv i32 [[ADD220]], [[TMP124]]
+// CHECK2-NEXT:    [[MUL222:%.*]] = mul i32 1, [[DIV221]]
+// CHECK2-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB223:%.*]] = sub i32 [[TMP125]], -63
+// CHECK2-NEXT:    [[DIV224:%.*]] = udiv i32 [[SUB223]], 64
+// CHECK2-NEXT:    [[MUL225:%.*]] = mul i32 [[MUL222]], [[DIV224]]
+// CHECK2-NEXT:    [[MUL226:%.*]] = mul i32 [[MUL225]], 64
+// CHECK2-NEXT:    [[CONV227:%.*]] = zext i32 [[MUL226]] to i64
+// CHECK2-NEXT:    [[DIV228:%.*]] = sdiv i64 [[TMP120]], [[CONV227]]
+// CHECK2-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP127:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB229:%.*]] = sub i32 [[TMP126]], [[TMP127]]
+// CHECK2-NEXT:    [[SUB230:%.*]] = sub i32 [[SUB229]], 1
+// CHECK2-NEXT:    [[TMP128:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD231:%.*]] = add i32 [[SUB230]], [[TMP128]]
+// CHECK2-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV232:%.*]] = udiv i32 [[ADD231]], [[TMP129]]
+// CHECK2-NEXT:    [[MUL233:%.*]] = mul i32 1, [[DIV232]]
+// CHECK2-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB234:%.*]] = sub i32 [[TMP130]], -63
+// CHECK2-NEXT:    [[DIV235:%.*]] = udiv i32 [[SUB234]], 64
+// CHECK2-NEXT:    [[MUL236:%.*]] = mul i32 [[MUL233]], [[DIV235]]
+// CHECK2-NEXT:    [[MUL237:%.*]] = mul i32 [[MUL236]], 64
+// CHECK2-NEXT:    [[CONV238:%.*]] = zext i32 [[MUL237]] to i64
+// CHECK2-NEXT:    [[MUL239:%.*]] = mul nsw i64 [[DIV228]], [[CONV238]]
+// CHECK2-NEXT:    [[SUB240:%.*]] = sub nsw i64 [[TMP119]], [[MUL239]]
+// CHECK2-NEXT:    [[TMP131:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP132:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB241:%.*]] = sub i32 [[TMP133]], [[TMP134]]
+// CHECK2-NEXT:    [[SUB242:%.*]] = sub i32 [[SUB241]], 1
+// CHECK2-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD243:%.*]] = add i32 [[SUB242]], [[TMP135]]
 // CHECK2-NEXT:    [[TMP136:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD249:%.*]] = add i32 [[SUB248]], [[TMP136]]
-// CHECK2-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV250:%.*]] = udiv i32 [[ADD249]], [[TMP137]]
-// CHECK2-NEXT:    [[MUL251:%.*]] = mul i32 1, [[DIV250]]
-// CHECK2-NEXT:    [[TMP138:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB252:%.*]] = sub i32 [[TMP138]], -63
-// CHECK2-NEXT:    [[DIV253:%.*]] = udiv i32 [[SUB252]], 64
-// CHECK2-NEXT:    [[MUL254:%.*]] = mul i32 [[MUL251]], [[DIV253]]
-// CHECK2-NEXT:    [[MUL255:%.*]] = mul i32 [[MUL254]], 64
-// CHECK2-NEXT:    [[CONV256:%.*]] = zext i32 [[MUL255]] to i64
-// CHECK2-NEXT:    [[MUL257:%.*]] = mul nsw i64 [[DIV246]], [[CONV256]]
-// CHECK2-NEXT:    [[SUB258:%.*]] = sub nsw i64 [[TMP127]], [[MUL257]]
-// CHECK2-NEXT:    [[TMP139:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB259:%.*]] = sub i32 [[TMP139]], -63
-// CHECK2-NEXT:    [[DIV260:%.*]] = udiv i32 [[SUB259]], 64
-// CHECK2-NEXT:    [[MUL261:%.*]] = mul i32 1, [[DIV260]]
-// CHECK2-NEXT:    [[MUL262:%.*]] = mul i32 [[MUL261]], 64
-// CHECK2-NEXT:    [[CONV263:%.*]] = zext i32 [[MUL262]] to i64
-// CHECK2-NEXT:    [[DIV264:%.*]] = sdiv i64 [[SUB258]], [[CONV263]]
-// CHECK2-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB265:%.*]] = sub i32 [[TMP140]], -63
-// CHECK2-NEXT:    [[DIV266:%.*]] = udiv i32 [[SUB265]], 64
-// CHECK2-NEXT:    [[MUL267:%.*]] = mul i32 1, [[DIV266]]
-// CHECK2-NEXT:    [[MUL268:%.*]] = mul i32 [[MUL267]], 64
-// CHECK2-NEXT:    [[CONV269:%.*]] = zext i32 [[MUL268]] to i64
-// CHECK2-NEXT:    [[MUL270:%.*]] = mul nsw i64 [[DIV264]], [[CONV269]]
-// CHECK2-NEXT:    [[SUB271:%.*]] = sub nsw i64 [[SUB235]], [[MUL270]]
-// CHECK2-NEXT:    [[DIV272:%.*]] = sdiv i64 [[SUB271]], 64
-// CHECK2-NEXT:    [[MUL273:%.*]] = mul nsw i64 [[DIV272]], 64
-// CHECK2-NEXT:    [[SUB274:%.*]] = sub nsw i64 [[SUB212]], [[MUL273]]
-// CHECK2-NEXT:    [[MUL275:%.*]] = mul nsw i64 [[SUB274]], 1
-// CHECK2-NEXT:    [[ADD276:%.*]] = add nsw i64 0, [[MUL275]]
-// CHECK2-NEXT:    [[CONV277:%.*]] = trunc i64 [[ADD276]] to i32
-// CHECK2-NEXT:    store i32 [[CONV277]], ptr [[DOTTILE_CNT_0_IV_K38]], align 4
-// CHECK2-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
-// CHECK2-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
-// CHECK2-NEXT:    [[ADD278:%.*]] = add i32 [[TMP141]], [[TMP142]]
-// CHECK2-NEXT:    store i32 [[ADD278]], ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK2-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
-// CHECK2-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK2-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
-// CHECK2-NEXT:    [[MUL279:%.*]] = mul i32 [[TMP144]], [[TMP145]]
-// CHECK2-NEXT:    [[ADD280:%.*]] = add i32 [[TMP143]], [[MUL279]]
-// CHECK2-NEXT:    store i32 [[ADD280]], ptr [[K]], align 4
-// CHECK2-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
-// CHECK2-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
-// CHECK2-NEXT:    [[ADD281:%.*]] = add i32 [[TMP146]], [[TMP147]]
-// CHECK2-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK2-NEXT:    [[ADD282:%.*]] = add i32 [[TMP148]], 1
-// CHECK2-NEXT:    [[CMP283:%.*]] = icmp ult i32 [[ADD281]], [[ADD282]]
-// CHECK2-NEXT:    br i1 [[CMP283]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK2:       if.then:
-// CHECK2-NEXT:    [[TMP149:%.*]] = load i32, ptr [[I35]], align 4
-// CHECK2-NEXT:    [[TMP150:%.*]] = load i32, ptr [[J36]], align 4
-// CHECK2-NEXT:    [[TMP151:%.*]] = load i32, ptr [[K]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP149]], i32 noundef [[TMP150]], i32 noundef [[TMP151]])
-// CHECK2-NEXT:    br label [[IF_END]]
-// CHECK2:       if.end:
-// CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK2-NEXT:    [[DIV244:%.*]] = udiv i32 [[ADD243]], [[TMP136]]
+// CHECK2-NEXT:    [[MUL245:%.*]] = mul i32 1, [[DIV244]]
+// CHECK2-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB246:%.*]] = sub i32 [[TMP137]], -63
+// CHECK2-NEXT:    [[DIV247:%.*]] = udiv i32 [[SUB246]], 64
+// CHECK2-NEXT:    [[MUL248:%.*]] = mul i32 [[MUL245]], [[DIV247]]
+// CHECK2-NEXT:    [[MUL249:%.*]] = mul i32 [[MUL248]], 64
+// CHECK2-NEXT:    [[CONV250:%.*]] = zext i32 [[MUL249]] to i64
+// CHECK2-NEXT:    [[DIV251:%.*]] = sdiv i64 [[TMP132]], [[CONV250]]
+// CHECK2-NEXT:    [[TMP138:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP139:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB252:%.*]] = sub i32 [[TMP138]], [[TMP139]]
+// CHECK2-NEXT:    [[SUB253:%.*]] = sub i32 [[SUB252]], 1
+// CHECK2-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD254:%.*]] = add i32 [[SUB253]], [[TMP140]]
+// CHECK2-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV255:%.*]] = udiv i32 [[ADD254]], [[TMP141]]
+// CHECK2-NEXT:    [[MUL256:%.*]] = mul i32 1, [[DIV255]]
+// CHECK2-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB257:%.*]] = sub i32 [[TMP142]], -63
+// CHECK2-NEXT:    [[DIV258:%.*]] = udiv i32 [[SUB257]], 64
+// CHECK2-NEXT:    [[MUL259:%.*]] = mul i32 [[MUL256]], [[DIV258]]
+// CHECK2-NEXT:    [[MUL260:%.*]] = mul i32 [[MUL259]], 64
+// CHECK2-NEXT:    [[CONV261:%.*]] = zext i32 [[MUL260]] to i64
+// CHECK2-NEXT:    [[MUL262:%.*]] = mul nsw i64 [[DIV251]], [[CONV261]]
+// CHECK2-NEXT:    [[SUB263:%.*]] = sub nsw i64 [[TMP131]], [[MUL262]]
+// CHECK2-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB264:%.*]] = sub i32 [[TMP143]], -63
+// CHECK2-NEXT:    [[DIV265:%.*]] = udiv i32 [[SUB264]], 64
+// CHECK2-NEXT:    [[MUL266:%.*]] = mul i32 1, [[DIV265]]
+// CHECK2-NEXT:    [[MUL267:%.*]] = mul i32 [[MUL266]], 64
+// CHECK2-NEXT:    [[CONV268:%.*]] = zext i32 [[MUL267]] to i64
+// CHECK2-NEXT:    [[DIV269:%.*]] = sdiv i64 [[SUB263]], [[CONV268]]
+// CHECK2-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB270:%.*]] = sub i32 [[TMP144]], -63
+// CHECK2-NEXT:    [[DIV271:%.*]] = udiv i32 [[SUB270]], 64
+// CHECK2-NEXT:    [[MUL272:%.*]] = mul i32 1, [[DIV271]]
+// CHECK2-NEXT:    [[MUL273:%.*]] = mul i32 [[MUL272]], 64
+// CHECK2-NEXT:    [[CONV274:%.*]] = zext i32 [[MUL273]] to i64
+// CHECK2-NEXT:    [[MUL275:%.*]] = mul nsw i64 [[DIV269]], [[CONV274]]
+// CHECK2-NEXT:    [[SUB276:%.*]] = sub nsw i64 [[SUB240]], [[MUL275]]
+// CHECK2-NEXT:    [[DIV277:%.*]] = sdiv i64 [[SUB276]], 64
+// CHECK2-NEXT:    [[MUL278:%.*]] = mul nsw i64 [[DIV277]], 64
+// CHECK2-NEXT:    [[SUB279:%.*]] = sub nsw i64 [[SUB217]], [[MUL278]]
+// CHECK2-NEXT:    [[MUL280:%.*]] = mul nsw i64 [[SUB279]], 1
+// CHECK2-NEXT:    [[ADD281:%.*]] = add nsw i64 [[CONV158]], [[MUL280]]
+// CHECK2-NEXT:    [[CONV282:%.*]] = trunc i64 [[ADD281]] to i32
+// CHECK2-NEXT:    store i32 [[CONV282]], ptr [[DOTTILE_0_IV_K42]], align 4
+// CHECK2-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTTILE_0_IV_K42]], align 4
+// CHECK2-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// CHECK2-NEXT:    [[ADD283:%.*]] = add i32 [[TMP146]], 1
+// CHECK2-NEXT:    [[CMP284:%.*]] = icmp ult i32 [[TMP145]], [[ADD283]]
+// CHECK2-NEXT:    br i1 [[CMP284]], label [[OMP_BODY_NEXT:%.*]], label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK2:       omp.body.next:
+// CHECK2-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
+// CHECK2-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTTILE_0_IV_K42]], align 4
+// CHECK2-NEXT:    [[TMP149:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
+// CHECK2-NEXT:    [[MUL285:%.*]] = mul i32 [[TMP148]], [[TMP149]]
+// CHECK2-NEXT:    [[ADD286:%.*]] = add i32 [[TMP147]], [[MUL285]]
+// CHECK2-NEXT:    store i32 [[ADD286]], ptr [[K]], align 4
+// CHECK2-NEXT:    [[TMP150:%.*]] = load i32, ptr [[I39]], align 4
+// CHECK2-NEXT:    [[TMP151:%.*]] = load i32, ptr [[J40]], align 4
+// CHECK2-NEXT:    [[TMP152:%.*]] = load i32, ptr [[K]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP150]], i32 noundef [[TMP151]], i32 noundef [[TMP152]])
+// CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
-// CHECK2-NEXT:    [[TMP152:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[ADD284:%.*]] = add nsw i64 [[TMP152]], 1
-// CHECK2-NEXT:    store i64 [[ADD284]], ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP153:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[ADD287:%.*]] = add nsw i64 [[TMP153]], 1
+// CHECK2-NEXT:    store i64 [[ADD287]], ptr [[DOTOMP_IV]], align 8
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
diff --git a/clang/test/OpenMP/tile_canonical_required_codegen.cpp b/clang/test/OpenMP/tile_canonical_required_codegen.cpp
deleted file mode 100644
index ceeaed4cb666a..0000000000000
--- a/clang/test/OpenMP/tile_canonical_required_codegen.cpp
+++ /dev/null
@@ -1,86 +0,0 @@
-// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fclang-abi-compat=latest -std=c++20 -fopenmp -ast-print %s | FileCheck %s --check-prefix=PRINT
-// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fclang-abi-compat=latest -std=c++20 -fopenmp -emit-llvm %s -o - | FileCheck %s --check-prefix=IR
-// expected-no-diagnostics
-
-// Positive tests for canonical intra-tile loop shape.
-
-extern "C" void body(...) {}
-
-// PRINT-LABEL: void collapse2_tile1d(
-// IR-LABEL: define {{.*}} @collapse2_tile1d(
-// collapse(2) consumes floor+tile; predicate stays as a body guard.
-// IR should have one flattened loop body and no nested floor/tile loop.
-// IR: %.tile.cnt.0.iv.i
-// IR: omp.inner.for.body:
-// IR: br i1 %{{.*}}, label %if.then, label %if.end
-// IR: if.then:
-// IR: call void (...) @body
-// IR-NEXT: br label %if.end
-// IR: if.end:
-extern "C" void collapse2_tile1d(int n) {
-#pragma omp parallel for collapse(2)
-#pragma omp tile sizes(2)
-  for (int i = 0; i < n; ++i)
-    body(i);
-}
-
-// PRINT-LABEL: void collapse3_tile1d_nested(
-// IR-LABEL: define {{.*}} @collapse3_tile1d_nested(
-// collapse(3) reaches through the tile guard to include `j`.
-// IR: br i1 %{{.*}}, label %omp_tile.pred.then, label %omp_tile.pred.end
-// IR: omp_tile.pred.then:
-// IR: call void (...) @body
-// IR-NEXT: br label %omp_tile.pred.end
-// IR: omp_tile.pred.end:
-extern "C" void collapse3_tile1d_nested(int n) {
-#pragma omp parallel for collapse(3)
-#pragma omp tile sizes(5)
-  for (int i = 0; i < n; ++i)
-    for (int j = 0; j < n; ++j)
-      body(i, j);
-}
-
-// PRINT-LABEL: void tile2d_inside_tile1d(
-// IR-LABEL: define {{.*}} @tile2d_inside_tile1d(
-// Nested tiling: both intra-tile loops stay canonical and hoist their predicates.
-// IR: %.tile.cnt.0.iv..floor_0.iv.i = alloca i32
-// IR: %.tile.cnt.0.iv.i = alloca i32
-// IR: %omp_tile.invariant_pred_bound = and i1
-// IR: %omp_tile.invariant_pred_bound{{[0-9]+}} = and i1
-extern "C" void tile2d_inside_tile1d(int n) {
-#pragma omp tile sizes(3)
-#pragma omp tile sizes(2)
-  for (int i = 0; i < n; ++i)
-    body(i);
-}
-
-// PRINT-LABEL: void collapse3_tile2d(
-// IR-LABEL: define {{.*}} @collapse3_tile2d(
-// 2-D tile + collapse(3): dim checks combine and are hoisted into the surviving loop.
-// IR: %.tile.cnt.0.iv.i
-// IR: %.tile.cnt.1.iv.j
-// IR: land.rhs:
-// IR: land.end:
-// IR: %omp_tile.invariant_pred_bound = and i1
-extern "C" void collapse3_tile2d(int n) {
-#pragma omp parallel for collapse(3)
-#pragma omp tile sizes(2, 3)
-  for (int i = 0; i < n; ++i)
-    for (int j = 0; j < n; ++j)
-      body(i, j);
-}
-
-// PRINT-LABEL: void tile1d_constant_div_no_predicate(
-// IR-LABEL: define {{.*}} @tile1d_constant_div_no_predicate(
-// Exact-divisible case: canonical tile-count loop, no predicate.
-// IR: %.tile.cnt.0.iv.i
-// IR: for.cond1:
-// IR: %[[TC:.*]] = icmp slt i32 %{{.*}}, 5
-// IR-NEXT: br i1 %[[TC]], label %for.body3, label %for.end
-// IR-NOT: omp_tile.invariant_pred_bound
-// IR-NOT: omp_tile.pred
-extern "C" void tile1d_constant_div_no_predicate() {
-#pragma omp tile sizes(5)
-  for (int i = 0; i < 10; ++i)
-    body(i);
-}
diff --git a/clang/test/OpenMP/tile_codegen.cpp b/clang/test/OpenMP/tile_codegen.cpp
index a76e46bddfb8a..6ec808d67e389 100644
--- a/clang/test/OpenMP/tile_codegen.cpp
+++ b/clang/test/OpenMP/tile_codegen.cpp
@@ -136,9 +136,8 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[I2:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[I:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
@@ -152,41 +151,45 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
 // CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END12:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK1:       for.cond4:
-// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP1]], 5
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
-// CHECK1-NEXT:    [[CMP6:%.*]] = icmp slt i32 [[ADD]], 4
-// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP5]], [[CMP6]]
-// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY7:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body7:
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 5
+// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK1-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
-// CHECK1-NEXT:    store i32 [[ADD8]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
+// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP4]], 5
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
+// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body8:
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
 // CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
-// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[TMP7]], align 4
-// CHECK1-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2]], !align [[META3]]
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP9]])
+// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
+// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[TMP6]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2]], !align [[META3]]
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP8]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP9]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP4:![0-9]+]]
 // CHECK1:       for.end:
 // CHECK1-NEXT:    br label [[FOR_INC10:%.*]]
 // CHECK1:       for.inc10:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP11]], 5
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP10]], 5
 // CHECK1-NEXT:    store i32 [[ADD11]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
 // CHECK1:       for.end12:
@@ -204,9 +207,8 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -235,48 +237,54 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK1-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP8]], [[ADD5]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK1:       for.cond6:
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP10]], 5
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD8:%.*]] = add i32 [[TMP11]], [[TMP12]]
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[ADD9:%.*]] = add i32 [[TMP13]], 1
-// CHECK1-NEXT:    [[CMP10:%.*]] = icmp ult i32 [[ADD8]], [[ADD9]]
-// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP7]], [[CMP10]]
-// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY11:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body11:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD12:%.*]] = add i32 [[TMP14]], [[TMP15]]
-// CHECK1-NEXT:    store i32 [[ADD12]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD7:%.*]] = add i32 [[TMP12]], 1
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD8:%.*]] = add i32 [[TMP13]], 5
+// CHECK1-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
+// CHECK1-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD10:%.*]] = add i32 [[TMP14]], 1
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD11:%.*]] = add i32 [[TMP15]], 5
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP11]], [[COND]]
+// CHECK1-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body13:
 // CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
 // CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
 // CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP17]], [[TMP18]]
-// CHECK1-NEXT:    [[ADD13:%.*]] = add i32 [[TMP16]], [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[ADD14:%.*]] = add i32 [[TMP16]], [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
 // CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP19]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP20]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP7:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC14:%.*]]
-// CHECK1:       for.inc14:
+// CHECK1-NEXT:    br label [[FOR_INC15:%.*]]
+// CHECK1:       for.inc15:
 // CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD15:%.*]] = add i32 [[TMP21]], 5
-// CHECK1-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD16:%.*]] = add i32 [[TMP21]], 5
+// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
-// CHECK1:       for.end16:
+// CHECK1:       for.end17:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -287,13 +295,11 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -304,90 +310,94 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END26:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END30:%.*]]
 // CHECK1:       for.body:
 // CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK1:       for.cond1:
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 4
-// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END23:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END27:%.*]]
 // CHECK1:       for.body3:
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[TMP2]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK1:       for.cond4:
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP2]], 5
-// CHECK1-NEXT:    br i1 [[CMP5]], label [[FOR_BODY6:%.*]], label [[FOR_END20:%.*]]
-// CHECK1:       for.body6:
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[TMP4]]
-// CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
-// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND8:%.*]]
-// CHECK1:       for.cond8:
-// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP6]], 5
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP7]], [[TMP8]]
-// CHECK1-NEXT:    [[CMP11:%.*]] = icmp slt i32 [[ADD10]], 4
-// CHECK1-NEXT:    br i1 [[CMP11]], label [[LAND_RHS:%.*]], label [[LAND_END:%.*]]
-// CHECK1:       land.rhs:
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], 5
+// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK1-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP5]], 5
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP3]], [[COND]]
+// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END24:%.*]]
+// CHECK1:       for.body8:
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
+// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND10:%.*]]
+// CHECK1:       for.cond10:
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
 // CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
-// CHECK1-NEXT:    [[CMP13:%.*]] = icmp slt i32 [[ADD12]], 4
-// CHECK1-NEXT:    br label [[LAND_END]]
-// CHECK1:       land.end:
-// CHECK1-NEXT:    [[TMP11:%.*]] = phi i1 [ false, [[FOR_COND8]] ], [ [[CMP13]], [[LAND_RHS]] ]
-// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP9]], [[TMP11]]
-// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body14:
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP12]], [[TMP13]]
-// CHECK1-NEXT:    store i32 [[ADD15]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[MUL16:%.*]] = mul nsw i32 [[TMP14]], 3
-// CHECK1-NEXT:    [[ADD17:%.*]] = add nsw i32 7, [[MUL16]]
-// CHECK1-NEXT:    store i32 [[ADD17]], ptr [[J]], align 4
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[J]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP15]], i32 noundef [[TMP16]])
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP9]], 5
+// CHECK1-NEXT:    [[CMP12:%.*]] = icmp slt i32 4, [[ADD11]]
+// CHECK1-NEXT:    br i1 [[CMP12]], label [[COND_TRUE13:%.*]], label [[COND_FALSE14:%.*]]
+// CHECK1:       cond.true13:
+// CHECK1-NEXT:    br label [[COND_END16:%.*]]
+// CHECK1:       cond.false14:
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP10]], 5
+// CHECK1-NEXT:    br label [[COND_END16]]
+// CHECK1:       cond.end16:
+// CHECK1-NEXT:    [[COND17:%.*]] = phi i32 [ 4, [[COND_TRUE13]] ], [ [[ADD15]], [[COND_FALSE14]] ]
+// CHECK1-NEXT:    [[CMP18:%.*]] = icmp slt i32 [[TMP8]], [[COND17]]
+// CHECK1-NEXT:    br i1 [[CMP18]], label [[FOR_BODY19:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body19:
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[TMP11]], 3
+// CHECK1-NEXT:    [[ADD21:%.*]] = add nsw i32 7, [[MUL20]]
+// CHECK1-NEXT:    store i32 [[ADD21]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP12]], i32 noundef [[TMP13]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP17]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP9:![0-9]+]]
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND10]], !llvm.loop [[LOOP9:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC18:%.*]]
-// CHECK1:       for.inc18:
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC19:%.*]] = add nsw i32 [[TMP18]], 1
-// CHECK1-NEXT:    store i32 [[INC19]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_INC22:%.*]]
+// CHECK1:       for.inc22:
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC23:%.*]] = add nsw i32 [[TMP15]], 1
+// CHECK1-NEXT:    store i32 [[INC23]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP10:![0-9]+]]
-// CHECK1:       for.end20:
-// CHECK1-NEXT:    br label [[FOR_INC21:%.*]]
-// CHECK1:       for.inc21:
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD22:%.*]] = add nsw i32 [[TMP19]], 5
-// CHECK1-NEXT:    store i32 [[ADD22]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1:       for.end24:
+// CHECK1-NEXT:    br label [[FOR_INC25:%.*]]
+// CHECK1:       for.inc25:
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD26:%.*]] = add nsw i32 [[TMP16]], 5
+// CHECK1-NEXT:    store i32 [[ADD26]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP11:![0-9]+]]
-// CHECK1:       for.end23:
-// CHECK1-NEXT:    br label [[FOR_INC24:%.*]]
-// CHECK1:       for.inc24:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD25:%.*]] = add nsw i32 [[TMP20]], 5
-// CHECK1-NEXT:    store i32 [[ADD25]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1:       for.end27:
+// CHECK1-NEXT:    br label [[FOR_INC28:%.*]]
+// CHECK1:       for.inc28:
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD29:%.*]] = add nsw i32 [[TMP17]], 5
+// CHECK1-NEXT:    store i32 [[ADD29]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP12:![0-9]+]]
-// CHECK1:       for.end26:
+// CHECK1:       for.end30:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -397,17 +407,15 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
 // CHECK1-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK1-NEXT:    store i32 7, ptr [[J]], align 4
@@ -445,83 +453,87 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP7]], 4
-// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END32:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[TMP8]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND3:%.*]]
 // CHECK1:       for.cond3:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP8]], 5
-// CHECK1-NEXT:    br i1 [[CMP4]], label [[FOR_BODY5:%.*]], label [[FOR_END21:%.*]]
-// CHECK1:       for.body5:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
-// CHECK1-NEXT:    store i32 [[ADD6]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL7:%.*]] = mul nsw i32 [[TMP11]], 3
-// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i32 7, [[MUL7]]
-// CHECK1-NEXT:    store i32 [[ADD8]], ptr [[I]], align 4
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND9:%.*]]
-// CHECK1:       for.cond9:
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i32 [[TMP12]], 5
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP13]], [[TMP14]]
-// CHECK1-NEXT:    [[CMP12:%.*]] = icmp slt i32 [[ADD11]], 4
-// CHECK1-NEXT:    br i1 [[CMP12]], label [[LAND_RHS:%.*]], label [[LAND_END:%.*]]
-// CHECK1:       land.rhs:
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD4:%.*]] = add nsw i32 [[TMP10]], 5
+// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD4]]
+// CHECK1-NEXT:    br i1 [[CMP5]], label [[COND_TRUE6:%.*]], label [[COND_FALSE7:%.*]]
+// CHECK1:       cond.true6:
+// CHECK1-NEXT:    br label [[COND_END9:%.*]]
+// CHECK1:       cond.false7:
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP11]], 5
+// CHECK1-NEXT:    br label [[COND_END9]]
+// CHECK1:       cond.end9:
+// CHECK1-NEXT:    [[COND10:%.*]] = phi i32 [ 4, [[COND_TRUE6]] ], [ [[ADD8]], [[COND_FALSE7]] ]
+// CHECK1-NEXT:    [[CMP11:%.*]] = icmp slt i32 [[TMP9]], [[COND10]]
+// CHECK1-NEXT:    br i1 [[CMP11]], label [[FOR_BODY12:%.*]], label [[FOR_END29:%.*]]
+// CHECK1:       for.body12:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL13:%.*]] = mul nsw i32 [[TMP12]], 3
+// CHECK1-NEXT:    [[ADD14:%.*]] = add nsw i32 7, [[MUL13]]
+// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    store i32 [[TMP13]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND15:%.*]]
+// CHECK1:       for.cond15:
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP15]], [[TMP16]]
-// CHECK1-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[ADD13]], 4
-// CHECK1-NEXT:    br label [[LAND_END]]
-// CHECK1:       land.end:
-// CHECK1-NEXT:    [[TMP17:%.*]] = phi i1 [ false, [[FOR_COND9]] ], [ [[CMP14]], [[LAND_RHS]] ]
-// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP10]], [[TMP17]]
-// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY15:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body15:
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP18]], [[TMP19]]
-// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[MUL17:%.*]] = mul nsw i32 [[TMP20]], 3
-// CHECK1-NEXT:    [[ADD18:%.*]] = add nsw i32 7, [[MUL17]]
-// CHECK1-NEXT:    store i32 [[ADD18]], ptr [[J]], align 4
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[J]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP21]], i32 noundef [[TMP22]])
+// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP15]], 5
+// CHECK1-NEXT:    [[CMP17:%.*]] = icmp slt i32 4, [[ADD16]]
+// CHECK1-NEXT:    br i1 [[CMP17]], label [[COND_TRUE18:%.*]], label [[COND_FALSE19:%.*]]
+// CHECK1:       cond.true18:
+// CHECK1-NEXT:    br label [[COND_END21:%.*]]
+// CHECK1:       cond.false19:
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP16]], 5
+// CHECK1-NEXT:    br label [[COND_END21]]
+// CHECK1:       cond.end21:
+// CHECK1-NEXT:    [[COND22:%.*]] = phi i32 [ 4, [[COND_TRUE18]] ], [ [[ADD20]], [[COND_FALSE19]] ]
+// CHECK1-NEXT:    [[CMP23:%.*]] = icmp slt i32 [[TMP14]], [[COND22]]
+// CHECK1-NEXT:    br i1 [[CMP23]], label [[FOR_BODY24:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body24:
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[MUL25:%.*]] = mul nsw i32 [[TMP17]], 3
+// CHECK1-NEXT:    [[ADD26:%.*]] = add nsw i32 7, [[MUL25]]
+// CHECK1-NEXT:    store i32 [[ADD26]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP18]], i32 noundef [[TMP19]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP23]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND9]], !llvm.loop [[LOOP13:![0-9]+]]
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP20]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND15]], !llvm.loop [[LOOP13:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC19:%.*]]
-// CHECK1:       for.inc19:
-// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC20:%.*]] = add nsw i32 [[TMP24]], 1
-// CHECK1-NEXT:    store i32 [[INC20]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_INC27:%.*]]
+// CHECK1:       for.inc27:
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC28:%.*]] = add nsw i32 [[TMP21]], 1
+// CHECK1-NEXT:    store i32 [[INC28]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND3]], !llvm.loop [[LOOP14:![0-9]+]]
-// CHECK1:       for.end21:
-// CHECK1-NEXT:    br label [[FOR_INC22:%.*]]
-// CHECK1:       for.inc22:
-// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP25]], 5
-// CHECK1-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1:       for.end29:
+// CHECK1-NEXT:    br label [[FOR_INC30:%.*]]
+// CHECK1:       for.inc30:
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD31:%.*]] = add nsw i32 [[TMP22]], 5
+// CHECK1-NEXT:    store i32 [[ADD31]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP15:![0-9]+]]
-// CHECK1:       for.end24:
+// CHECK1:       for.end32:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[ADD25:%.*]] = add nsw i32 [[TMP26]], 1
-// CHECK1-NEXT:    store i32 [[ADD25]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[ADD33:%.*]] = add nsw i32 [[TMP23]], 1
+// CHECK1-NEXT:    store i32 [[ADD33]], ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -538,9 +550,7 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[_TMP1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
@@ -548,8 +558,8 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[K:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK1-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK1-NEXT:    store i32 7, ptr [[J]], align 4
@@ -596,83 +606,87 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP9]], 4
-// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY:%.*]], label [[FOR_END29:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY:%.*]], label [[FOR_END37:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND8:%.*]]
 // CHECK1:       for.cond8:
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP10]], 5
-// CHECK1-NEXT:    br i1 [[CMP9]], label [[FOR_BODY10:%.*]], label [[FOR_END26:%.*]]
-// CHECK1:       for.body10:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP11]], [[TMP12]]
-// CHECK1-NEXT:    store i32 [[ADD11]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL12:%.*]] = mul nsw i32 [[TMP13]], 3
-// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i32 7, [[MUL12]]
-// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[I]], align 4
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND14:%.*]]
-// CHECK1:       for.cond14:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[CMP15:%.*]] = icmp slt i32 [[TMP14]], 5
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP15]], [[TMP16]]
-// CHECK1-NEXT:    [[CMP17:%.*]] = icmp slt i32 [[ADD16]], 4
-// CHECK1-NEXT:    br i1 [[CMP17]], label [[LAND_RHS:%.*]], label [[LAND_END:%.*]]
-// CHECK1:       land.rhs:
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i32 [[TMP12]], 5
+// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i32 4, [[ADD9]]
+// CHECK1-NEXT:    br i1 [[CMP10]], label [[COND_TRUE11:%.*]], label [[COND_FALSE12:%.*]]
+// CHECK1:       cond.true11:
+// CHECK1-NEXT:    br label [[COND_END14:%.*]]
+// CHECK1:       cond.false12:
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP13]], 5
+// CHECK1-NEXT:    br label [[COND_END14]]
+// CHECK1:       cond.end14:
+// CHECK1-NEXT:    [[COND15:%.*]] = phi i32 [ 4, [[COND_TRUE11]] ], [ [[ADD13]], [[COND_FALSE12]] ]
+// CHECK1-NEXT:    [[CMP16:%.*]] = icmp slt i32 [[TMP11]], [[COND15]]
+// CHECK1-NEXT:    br i1 [[CMP16]], label [[FOR_BODY17:%.*]], label [[FOR_END34:%.*]]
+// CHECK1:       for.body17:
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL18:%.*]] = mul nsw i32 [[TMP14]], 3
+// CHECK1-NEXT:    [[ADD19:%.*]] = add nsw i32 7, [[MUL18]]
+// CHECK1-NEXT:    store i32 [[ADD19]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    store i32 [[TMP15]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND20:%.*]]
+// CHECK1:       for.cond20:
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
 // CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD18:%.*]] = add nsw i32 [[TMP17]], [[TMP18]]
-// CHECK1-NEXT:    [[CMP19:%.*]] = icmp slt i32 [[ADD18]], 4
-// CHECK1-NEXT:    br label [[LAND_END]]
-// CHECK1:       land.end:
-// CHECK1-NEXT:    [[TMP19:%.*]] = phi i1 [ false, [[FOR_COND14]] ], [ [[CMP19]], [[LAND_RHS]] ]
-// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP15]], [[TMP19]]
-// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY20:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body20:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD21:%.*]] = add nsw i32 [[TMP20]], [[TMP21]]
-// CHECK1-NEXT:    store i32 [[ADD21]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[MUL22:%.*]] = mul nsw i32 [[TMP22]], 3
-// CHECK1-NEXT:    [[ADD23:%.*]] = add nsw i32 7, [[MUL22]]
-// CHECK1-NEXT:    store i32 [[ADD23]], ptr [[J]], align 4
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[J]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP23]], i32 noundef [[TMP24]])
+// CHECK1-NEXT:    [[ADD21:%.*]] = add nsw i32 [[TMP17]], 5
+// CHECK1-NEXT:    [[CMP22:%.*]] = icmp slt i32 4, [[ADD21]]
+// CHECK1-NEXT:    br i1 [[CMP22]], label [[COND_TRUE23:%.*]], label [[COND_FALSE24:%.*]]
+// CHECK1:       cond.true23:
+// CHECK1-NEXT:    br label [[COND_END26:%.*]]
+// CHECK1:       cond.false24:
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD25:%.*]] = add nsw i32 [[TMP18]], 5
+// CHECK1-NEXT:    br label [[COND_END26]]
+// CHECK1:       cond.end26:
+// CHECK1-NEXT:    [[COND27:%.*]] = phi i32 [ 4, [[COND_TRUE23]] ], [ [[ADD25]], [[COND_FALSE24]] ]
+// CHECK1-NEXT:    [[CMP28:%.*]] = icmp slt i32 [[TMP16]], [[COND27]]
+// CHECK1-NEXT:    br i1 [[CMP28]], label [[FOR_BODY29:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body29:
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[MUL30:%.*]] = mul nsw i32 [[TMP19]], 3
+// CHECK1-NEXT:    [[ADD31:%.*]] = add nsw i32 7, [[MUL30]]
+// CHECK1-NEXT:    store i32 [[ADD31]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP20]], i32 noundef [[TMP21]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP25]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND14]], !llvm.loop [[LOOP16:![0-9]+]]
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP22]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND20]], !llvm.loop [[LOOP16:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC24:%.*]]
-// CHECK1:       for.inc24:
-// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC25:%.*]] = add nsw i32 [[TMP26]], 1
-// CHECK1-NEXT:    store i32 [[INC25]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_INC32:%.*]]
+// CHECK1:       for.inc32:
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC33:%.*]] = add nsw i32 [[TMP23]], 1
+// CHECK1-NEXT:    store i32 [[INC33]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP17:![0-9]+]]
-// CHECK1:       for.end26:
-// CHECK1-NEXT:    br label [[FOR_INC27:%.*]]
-// CHECK1:       for.inc27:
-// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD28:%.*]] = add nsw i32 [[TMP27]], 5
-// CHECK1-NEXT:    store i32 [[ADD28]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1:       for.end34:
+// CHECK1-NEXT:    br label [[FOR_INC35:%.*]]
+// CHECK1:       for.inc35:
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD36:%.*]] = add nsw i32 [[TMP24]], 5
+// CHECK1-NEXT:    store i32 [[ADD36]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP18:![0-9]+]]
-// CHECK1:       for.end29:
+// CHECK1:       for.end37:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[ADD30:%.*]] = add nsw i32 [[TMP28]], 1
-// CHECK1-NEXT:    store i32 [[ADD30]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[ADD38:%.*]] = add nsw i32 [[TMP25]], 1
+// CHECK1-NEXT:    store i32 [[ADD38]], ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -690,91 +704,90 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[_TMP1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[_TMP2:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK1-NEXT:    store i32 7, ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP]], align 4
+// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], 5
+// CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTCAPTURE_EXPR_3]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 19, ptr [[DOTOMP_UB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
 // CHECK1-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
-// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// CHECK1-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], 19
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP3]], 19
 // CHECK1-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK1:       cond.true:
 // CHECK1-NEXT:    br label [[COND_END:%.*]]
 // CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
 // CHECK1-NEXT:    br label [[COND_END]]
 // CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 19, [[COND_TRUE]] ], [ [[TMP2]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 19, [[COND_TRUE]] ], [ [[TMP4]], [[COND_FALSE]] ]
 // CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
-// CHECK1-NEXT:    store i32 [[TMP3]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
+// CHECK1-NEXT:    store i32 [[TMP5]], ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK1:       omp.inner.for.cond:
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// CHECK1-NEXT:    [[CMP3:%.*]] = icmp sle i32 [[TMP4]], [[TMP5]]
-// CHECK1-NEXT:    br i1 [[CMP3]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
-// CHECK1:       omp.inner.for.body:
 // CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP6]], 20
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[DIV]], 5
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[CMP4:%.*]] = icmp sle i32 [[TMP6]], [[TMP7]]
+// CHECK1-NEXT:    br i1 [[CMP4]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK1:       omp.inner.for.body:
 // CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[DIV4:%.*]] = sdiv i32 [[TMP8]], 20
-// CHECK1-NEXT:    [[MUL5:%.*]] = mul nsw i32 [[DIV4]], 20
-// CHECK1-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP7]], [[MUL5]]
-// CHECK1-NEXT:    [[DIV6:%.*]] = sdiv i32 [[SUB]], 4
-// CHECK1-NEXT:    [[MUL7:%.*]] = mul nsw i32 [[DIV6]], 1
-// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i32 0, [[MUL7]]
-// CHECK1-NEXT:    store i32 [[ADD8]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP8]], 20
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[DIV]], 5
+// CHECK1-NEXT:    [[ADD5:%.*]] = add nsw i32 0, [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD5]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[DIV9:%.*]] = sdiv i32 [[TMP10]], 20
-// CHECK1-NEXT:    [[MUL10:%.*]] = mul nsw i32 [[DIV9]], 20
-// CHECK1-NEXT:    [[SUB11:%.*]] = sub nsw i32 [[TMP9]], [[MUL10]]
 // CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[DIV6:%.*]] = sdiv i32 [[TMP11]], 20
+// CHECK1-NEXT:    [[MUL7:%.*]] = mul nsw i32 [[DIV6]], 20
+// CHECK1-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP10]], [[MUL7]]
+// CHECK1-NEXT:    [[DIV8:%.*]] = sdiv i32 [[SUB]], 4
+// CHECK1-NEXT:    [[MUL9:%.*]] = mul nsw i32 [[DIV8]], 1
+// CHECK1-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP9]], [[MUL9]]
+// CHECK1-NEXT:    store i32 [[ADD10]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[DIV12:%.*]] = sdiv i32 [[TMP12]], 20
-// CHECK1-NEXT:    [[MUL13:%.*]] = mul nsw i32 [[DIV12]], 20
-// CHECK1-NEXT:    [[SUB14:%.*]] = sub nsw i32 [[TMP11]], [[MUL13]]
-// CHECK1-NEXT:    [[DIV15:%.*]] = sdiv i32 [[SUB14]], 4
-// CHECK1-NEXT:    [[MUL16:%.*]] = mul nsw i32 [[DIV15]], 4
-// CHECK1-NEXT:    [[SUB17:%.*]] = sub nsw i32 [[SUB11]], [[MUL16]]
-// CHECK1-NEXT:    [[MUL18:%.*]] = mul nsw i32 [[SUB17]], 3
-// CHECK1-NEXT:    [[ADD19:%.*]] = add nsw i32 7, [[MUL18]]
-// CHECK1-NEXT:    store i32 [[ADD19]], ptr [[J]], align 4
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP13]], [[TMP14]]
-// CHECK1-NEXT:    store i32 [[ADD20]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL21:%.*]] = mul nsw i32 [[TMP15]], 3
-// CHECK1-NEXT:    [[ADD22:%.*]] = add nsw i32 7, [[MUL21]]
-// CHECK1-NEXT:    store i32 [[ADD22]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP16]], [[TMP17]]
-// CHECK1-NEXT:    [[CMP24:%.*]] = icmp slt i32 [[ADD23]], 4
-// CHECK1-NEXT:    br i1 [[CMP24]], label [[OMP_TILE_PRED_THEN:%.*]], label [[OMP_TILE_PRED_END:%.*]]
-// CHECK1:       omp_tile.pred.then:
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[DIV11:%.*]] = sdiv i32 [[TMP13]], 20
+// CHECK1-NEXT:    [[MUL12:%.*]] = mul nsw i32 [[DIV11]], 20
+// CHECK1-NEXT:    [[SUB13:%.*]] = sub nsw i32 [[TMP12]], [[MUL12]]
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[DIV14:%.*]] = sdiv i32 [[TMP15]], 20
+// CHECK1-NEXT:    [[MUL15:%.*]] = mul nsw i32 [[DIV14]], 20
+// CHECK1-NEXT:    [[SUB16:%.*]] = sub nsw i32 [[TMP14]], [[MUL15]]
+// CHECK1-NEXT:    [[DIV17:%.*]] = sdiv i32 [[SUB16]], 4
+// CHECK1-NEXT:    [[MUL18:%.*]] = mul nsw i32 [[DIV17]], 4
+// CHECK1-NEXT:    [[SUB19:%.*]] = sub nsw i32 [[SUB13]], [[MUL18]]
+// CHECK1-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[SUB19]], 3
+// CHECK1-NEXT:    [[ADD21:%.*]] = add nsw i32 7, [[MUL20]]
+// CHECK1-NEXT:    store i32 [[ADD21]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP22:%.*]] = icmp slt i32 [[TMP16]], 4
+// CHECK1-NEXT:    br i1 [[CMP22]], label [[OMP_BODY_NEXT:%.*]], label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK1:       omp.body.next:
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL23:%.*]] = mul nsw i32 [[TMP17]], 3
+// CHECK1-NEXT:    [[ADD24:%.*]] = add nsw i32 7, [[MUL23]]
+// CHECK1-NEXT:    store i32 [[ADD24]], ptr [[I]], align 4
 // CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[J]], align 4
 // CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP18]], i32 noundef [[TMP19]])
-// CHECK1-NEXT:    br label [[OMP_TILE_PRED_END]]
-// CHECK1:       omp_tile.pred.end:
-// CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
@@ -805,13 +818,12 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
 // CHECK1-NEXT:    store i32 7, ptr [[I]], align 4
@@ -846,42 +858,46 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP7]], 5
 // CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[TMP8]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND:%.*]]
 // CHECK1:       for.cond:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP8]], 5
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
-// CHECK1-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[ADD3]], 4
-// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP2]], [[CMP4]]
-// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body:
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD2:%.*]] = add nsw i32 [[TMP10]], 5
+// CHECK1-NEXT:    [[CMP3:%.*]] = icmp slt i32 4, [[ADD2]]
+// CHECK1-NEXT:    br i1 [[CMP3]], label [[COND_TRUE4:%.*]], label [[COND_FALSE5:%.*]]
+// CHECK1:       cond.true4:
+// CHECK1-NEXT:    br label [[COND_END7:%.*]]
+// CHECK1:       cond.false5:
 // CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD5:%.*]] = add nsw i32 [[TMP11]], [[TMP12]]
-// CHECK1-NEXT:    store i32 [[ADD5]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL6:%.*]] = mul nsw i32 [[TMP13]], 3
-// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL6]]
-// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP14]])
+// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP11]], 5
+// CHECK1-NEXT:    br label [[COND_END7]]
+// CHECK1:       cond.end7:
+// CHECK1-NEXT:    [[COND8:%.*]] = phi i32 [ 4, [[COND_TRUE4]] ], [ [[ADD6]], [[COND_FALSE5]] ]
+// CHECK1-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP9]], [[COND8]]
+// CHECK1-NEXT:    br i1 [[CMP9]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL10:%.*]] = mul nsw i32 [[TMP12]], 3
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 7, [[MUL10]]
+// CHECK1-NEXT:    store i32 [[ADD11]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP13]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP15]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP19:![0-9]+]]
 // CHECK1:       for.end:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP16]], 1
-// CHECK1-NEXT:    store i32 [[ADD8]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP15]], 1
+// CHECK1-NEXT:    store i32 [[ADD12]], ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -906,9 +922,8 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[START_ADDR]], align 4
@@ -932,47 +947,53 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK1-NEXT:    [[ADD5:%.*]] = add i32 [[TMP6]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP5]], [[ADD5]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK1:       for.cond6:
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP7]], 5
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD8:%.*]] = add i32 [[TMP8]], [[TMP9]]
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[ADD9:%.*]] = add i32 [[TMP10]], 1
-// CHECK1-NEXT:    [[CMP10:%.*]] = icmp ult i32 [[ADD8]], [[ADD9]]
-// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP7]], [[CMP10]]
-// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY11:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body11:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD12:%.*]] = add i32 [[TMP11]], [[TMP12]]
-// CHECK1-NEXT:    store i32 [[ADD12]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD7:%.*]] = add i32 [[TMP9]], 1
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD8:%.*]] = add i32 [[TMP10]], 5
+// CHECK1-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
+// CHECK1-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD10:%.*]] = add i32 [[TMP11]], 1
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD11:%.*]] = add i32 [[TMP12]], 5
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP8]], [[COND]]
+// CHECK1-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body13:
 // CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
 // CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP14]], 3
-// CHECK1-NEXT:    [[ADD13:%.*]] = add i32 [[TMP13]], [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[ADD14:%.*]] = add i32 [[TMP13]], [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP15]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP16]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP22:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC14:%.*]]
-// CHECK1:       for.inc14:
+// CHECK1-NEXT:    br label [[FOR_INC15:%.*]]
+// CHECK1:       for.inc15:
 // CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD15:%.*]] = add i32 [[TMP17]], 5
-// CHECK1-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD16:%.*]] = add i32 [[TMP17]], 5
+// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP23:![0-9]+]]
-// CHECK1:       for.end16:
+// CHECK1:       for.end17:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -981,9 +1002,8 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
 // CHECK1-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
@@ -991,64 +1011,78 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END15:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK1:       for.cond1:
-// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP2]], 0
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], 0
 // CHECK1-NEXT:    br i1 [[CMP2]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK1:       cond.true:
 // CHECK1-NEXT:    br label [[COND_END:%.*]]
 // CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[A_ADDR]], align 4
 // CHECK1-NEXT:    br label [[COND_END]]
 // CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP3]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[TMP1]], [[COND]]
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
-// CHECK1-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[ADD]], 4
-// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP3]], [[CMP4]]
-// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY5:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body5:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP5]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[COND]]
+// CHECK1-NEXT:    [[CMP3:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK1-NEXT:    br i1 [[CMP3]], label [[COND_TRUE4:%.*]], label [[COND_FALSE5:%.*]]
+// CHECK1:       cond.true4:
+// CHECK1-NEXT:    br label [[COND_END12:%.*]]
+// CHECK1:       cond.false5:
 // CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP6]], [[TMP7]]
-// CHECK1-NEXT:    store i32 [[ADD6]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP8]], 3
-// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP9]])
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    [[CMP6:%.*]] = icmp sle i32 [[TMP7]], 0
+// CHECK1-NEXT:    br i1 [[CMP6]], label [[COND_TRUE7:%.*]], label [[COND_FALSE8:%.*]]
+// CHECK1:       cond.true7:
+// CHECK1-NEXT:    br label [[COND_END9:%.*]]
+// CHECK1:       cond.false8:
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    br label [[COND_END9]]
+// CHECK1:       cond.end9:
+// CHECK1-NEXT:    [[COND10:%.*]] = phi i32 [ 1, [[COND_TRUE7]] ], [ [[TMP8]], [[COND_FALSE8]] ]
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP6]], [[COND10]]
+// CHECK1-NEXT:    br label [[COND_END12]]
+// CHECK1:       cond.end12:
+// CHECK1-NEXT:    [[COND13:%.*]] = phi i32 [ 4, [[COND_TRUE4]] ], [ [[ADD11]], [[COND_END9]] ]
+// CHECK1-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP2]], [[COND13]]
+// CHECK1-NEXT:    br i1 [[CMP14]], label [[FOR_BODY15:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body15:
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP9]], 3
+// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP10]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP24:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC8:%.*]]
-// CHECK1:       for.inc8:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    [[CMP9:%.*]] = icmp sle i32 [[TMP11]], 0
-// CHECK1-NEXT:    br i1 [[CMP9]], label [[COND_TRUE10:%.*]], label [[COND_FALSE11:%.*]]
-// CHECK1:       cond.true10:
-// CHECK1-NEXT:    br label [[COND_END12:%.*]]
-// CHECK1:       cond.false11:
+// CHECK1-NEXT:    br label [[FOR_INC17:%.*]]
+// CHECK1:       for.inc17:
 // CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    br label [[COND_END12]]
-// CHECK1:       cond.end12:
-// CHECK1-NEXT:    [[COND13:%.*]] = phi i32 [ 1, [[COND_TRUE10]] ], [ [[TMP12]], [[COND_FALSE11]] ]
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD14:%.*]] = add nsw i32 [[TMP13]], [[COND13]]
-// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP18:%.*]] = icmp sle i32 [[TMP12]], 0
+// CHECK1-NEXT:    br i1 [[CMP18]], label [[COND_TRUE19:%.*]], label [[COND_FALSE20:%.*]]
+// CHECK1:       cond.true19:
+// CHECK1-NEXT:    br label [[COND_END21:%.*]]
+// CHECK1:       cond.false20:
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    br label [[COND_END21]]
+// CHECK1:       cond.end21:
+// CHECK1-NEXT:    [[COND22:%.*]] = phi i32 [ 1, [[COND_TRUE19]] ], [ [[TMP13]], [[COND_FALSE20]] ]
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP14]], [[COND22]]
+// CHECK1-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP25:![0-9]+]]
-// CHECK1:       for.end15:
+// CHECK1:       for.end24:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -1061,9 +1095,8 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK1-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
 // CHECK1-NEXT:    store ptr [[ARRAY]], ptr [[__RANGE2]], align 8
@@ -1097,31 +1130,37 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK1:       for.cond7:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK1-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP11]], 1
-// CHECK1-NEXT:    [[CMP11:%.*]] = icmp slt i64 [[ADD9]], [[ADD10]]
-// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP8]], [[CMP11]]
-// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY12:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body12:
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i64 [[TMP12]], [[TMP13]]
-// CHECK1-NEXT:    store i64 [[ADD13]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
+// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
+// CHECK1-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
+// CHECK1-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body14:
 // CHECK1-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
-// CHECK1-NEXT:    [[ADD_PTR14:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
-// CHECK1-NEXT:    store ptr [[ADD_PTR14]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
+// CHECK1-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
 // CHECK1-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
 // CHECK1-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
 // CHECK1-NEXT:    store double [[TMP17]], ptr [[V]], align 8
@@ -1129,18 +1168,18 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    call void (...) @body(double noundef [[TMP18]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP19]], 1
-// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP27:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC15:%.*]]
-// CHECK1:       for.inc15:
+// CHECK1-NEXT:    br label [[FOR_INC16:%.*]]
+// CHECK1:       for.inc16:
 // CHECK1-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i64 [[TMP20]], 5
-// CHECK1-NEXT:    store i64 [[ADD16]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP20]], 5
+// CHECK1-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP28:![0-9]+]]
-// CHECK1:       for.end17:
+// CHECK1:       for.end18:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -1154,9 +1193,8 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK1-NEXT:    store double 4.200000e+01, ptr [[C]], align 8
 // CHECK1-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
@@ -1191,31 +1229,37 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK1:       for.cond7:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK1-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP11]], 1
-// CHECK1-NEXT:    [[CMP11:%.*]] = icmp slt i64 [[ADD9]], [[ADD10]]
-// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP8]], [[CMP11]]
-// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY12:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body12:
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i64 [[TMP12]], [[TMP13]]
-// CHECK1-NEXT:    store i64 [[ADD13]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
+// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
+// CHECK1-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
+// CHECK1-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body14:
 // CHECK1-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
-// CHECK1-NEXT:    [[ADD_PTR14:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
-// CHECK1-NEXT:    store ptr [[ADD_PTR14]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
+// CHECK1-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
 // CHECK1-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
 // CHECK1-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
 // CHECK1-NEXT:    store double [[TMP17]], ptr [[V]], align 8
@@ -1224,18 +1268,18 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    call void (...) @body(double noundef [[TMP18]], double noundef [[TMP19]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP20]], 1
-// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP29:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC15:%.*]]
-// CHECK1:       for.inc15:
+// CHECK1-NEXT:    br label [[FOR_INC16:%.*]]
+// CHECK1:       for.inc16:
 // CHECK1-NEXT:    [[TMP21:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i64 [[TMP21]], 5
-// CHECK1-NEXT:    store i64 [[ADD16]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP21]], 5
+// CHECK1-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP30:![0-9]+]]
-// CHECK1:       for.end17:
+// CHECK1:       for.end18:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -1268,9 +1312,8 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:  entry:
 // CHECK2-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[I2:%.*]] = alloca ptr, align 8
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK2-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK2-NEXT:    [[I:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
@@ -1284,41 +1327,45 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
 // CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END12:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK2:       for.cond4:
-// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP1]], 5
-// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
-// CHECK2-NEXT:    [[CMP6:%.*]] = icmp slt i32 [[ADD]], 4
-// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP5]], [[CMP6]]
-// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY7:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body7:
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 5
+// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK2-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
 // CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
-// CHECK2-NEXT:    store i32 [[ADD8]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
+// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP4]], 5
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
+// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body8:
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
 // CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK2-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
-// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[TMP7]], align 4
-// CHECK2-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2]], !align [[META3]]
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP9]])
+// CHECK2-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
+// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[TMP6]], align 4
+// CHECK2-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2]], !align [[META3]]
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP8]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP9]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP4:![0-9]+]]
 // CHECK2:       for.end:
 // CHECK2-NEXT:    br label [[FOR_INC10:%.*]]
 // CHECK2:       for.inc10:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP11]], 5
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP10]], 5
 // CHECK2-NEXT:    store i32 [[ADD11]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
 // CHECK2:       for.end12:
@@ -1342,9 +1389,8 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -1373,48 +1419,54 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK2-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP8]], [[ADD5]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[TMP10]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK2:       for.cond6:
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP10]], 5
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD8:%.*]] = add i32 [[TMP11]], [[TMP12]]
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[ADD9:%.*]] = add i32 [[TMP13]], 1
-// CHECK2-NEXT:    [[CMP10:%.*]] = icmp ult i32 [[ADD8]], [[ADD9]]
-// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP7]], [[CMP10]]
-// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY11:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body11:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD12:%.*]] = add i32 [[TMP14]], [[TMP15]]
-// CHECK2-NEXT:    store i32 [[ADD12]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD7:%.*]] = add i32 [[TMP12]], 1
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD8:%.*]] = add i32 [[TMP13]], 5
+// CHECK2-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
+// CHECK2-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD10:%.*]] = add i32 [[TMP14]], 1
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD11:%.*]] = add i32 [[TMP15]], 5
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP11]], [[COND]]
+// CHECK2-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body13:
 // CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
 // CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
 // CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP17]], [[TMP18]]
-// CHECK2-NEXT:    [[ADD13:%.*]] = add i32 [[TMP16]], [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD13]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[ADD14:%.*]] = add i32 [[TMP16]], [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
 // CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
 // CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP19]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP20]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP7:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC14:%.*]]
-// CHECK2:       for.inc14:
+// CHECK2-NEXT:    br label [[FOR_INC15:%.*]]
+// CHECK2:       for.inc15:
 // CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD15:%.*]] = add i32 [[TMP21]], 5
-// CHECK2-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD16:%.*]] = add i32 [[TMP21]], 5
+// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
-// CHECK2:       for.end16:
+// CHECK2:       for.end17:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -1428,9 +1480,8 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK2-NEXT:    store double 4.200000e+01, ptr [[C]], align 8
 // CHECK2-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
@@ -1465,31 +1516,37 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK2:       for.cond7:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK2-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP11]], 1
-// CHECK2-NEXT:    [[CMP11:%.*]] = icmp slt i64 [[ADD9]], [[ADD10]]
-// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP8]], [[CMP11]]
-// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY12:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body12:
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i64 [[TMP12]], [[TMP13]]
-// CHECK2-NEXT:    store i64 [[ADD13]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
+// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
+// CHECK2-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
+// CHECK2-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body14:
 // CHECK2-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
-// CHECK2-NEXT:    [[ADD_PTR14:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
-// CHECK2-NEXT:    store ptr [[ADD_PTR14]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
+// CHECK2-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
 // CHECK2-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
 // CHECK2-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
 // CHECK2-NEXT:    store double [[TMP17]], ptr [[V]], align 8
@@ -1498,18 +1555,18 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    call void (...) @body(double noundef [[TMP18]], double noundef [[TMP19]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP20]], 1
-// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP10:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC15:%.*]]
-// CHECK2:       for.inc15:
+// CHECK2-NEXT:    br label [[FOR_INC16:%.*]]
+// CHECK2:       for.inc16:
 // CHECK2-NEXT:    [[TMP21:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i64 [[TMP21]], 5
-// CHECK2-NEXT:    store i64 [[ADD16]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP21]], 5
+// CHECK2-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP11:![0-9]+]]
-// CHECK2:       for.end17:
+// CHECK2:       for.end18:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -1520,13 +1577,11 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -1537,90 +1592,94 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END26:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END30:%.*]]
 // CHECK2:       for.body:
 // CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK2:       for.cond1:
 // CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 4
-// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END23:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END27:%.*]]
 // CHECK2:       for.body3:
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[TMP2]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK2:       for.cond4:
-// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP2]], 5
-// CHECK2-NEXT:    br i1 [[CMP5]], label [[FOR_BODY6:%.*]], label [[FOR_END20:%.*]]
-// CHECK2:       for.body6:
-// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[TMP4]]
-// CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
-// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND8:%.*]]
-// CHECK2:       for.cond8:
-// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP6]], 5
-// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP7]], [[TMP8]]
-// CHECK2-NEXT:    [[CMP11:%.*]] = icmp slt i32 [[ADD10]], 4
-// CHECK2-NEXT:    br i1 [[CMP11]], label [[LAND_RHS:%.*]], label [[LAND_END:%.*]]
-// CHECK2:       land.rhs:
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], 5
+// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK2-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP5]], 5
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP3]], [[COND]]
+// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END24:%.*]]
+// CHECK2:       for.body8:
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
+// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND10:%.*]]
+// CHECK2:       for.cond10:
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
 // CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
-// CHECK2-NEXT:    [[CMP13:%.*]] = icmp slt i32 [[ADD12]], 4
-// CHECK2-NEXT:    br label [[LAND_END]]
-// CHECK2:       land.end:
-// CHECK2-NEXT:    [[TMP11:%.*]] = phi i1 [ false, [[FOR_COND8]] ], [ [[CMP13]], [[LAND_RHS]] ]
-// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP9]], [[TMP11]]
-// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body14:
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP12]], [[TMP13]]
-// CHECK2-NEXT:    store i32 [[ADD15]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[MUL16:%.*]] = mul nsw i32 [[TMP14]], 3
-// CHECK2-NEXT:    [[ADD17:%.*]] = add nsw i32 7, [[MUL16]]
-// CHECK2-NEXT:    store i32 [[ADD17]], ptr [[J]], align 4
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[J]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP15]], i32 noundef [[TMP16]])
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP9]], 5
+// CHECK2-NEXT:    [[CMP12:%.*]] = icmp slt i32 4, [[ADD11]]
+// CHECK2-NEXT:    br i1 [[CMP12]], label [[COND_TRUE13:%.*]], label [[COND_FALSE14:%.*]]
+// CHECK2:       cond.true13:
+// CHECK2-NEXT:    br label [[COND_END16:%.*]]
+// CHECK2:       cond.false14:
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP10]], 5
+// CHECK2-NEXT:    br label [[COND_END16]]
+// CHECK2:       cond.end16:
+// CHECK2-NEXT:    [[COND17:%.*]] = phi i32 [ 4, [[COND_TRUE13]] ], [ [[ADD15]], [[COND_FALSE14]] ]
+// CHECK2-NEXT:    [[CMP18:%.*]] = icmp slt i32 [[TMP8]], [[COND17]]
+// CHECK2-NEXT:    br i1 [[CMP18]], label [[FOR_BODY19:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body19:
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[TMP11]], 3
+// CHECK2-NEXT:    [[ADD21:%.*]] = add nsw i32 7, [[MUL20]]
+// CHECK2-NEXT:    store i32 [[ADD21]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP12]], i32 noundef [[TMP13]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP17]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP12:![0-9]+]]
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND10]], !llvm.loop [[LOOP12:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC18:%.*]]
-// CHECK2:       for.inc18:
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC19:%.*]] = add nsw i32 [[TMP18]], 1
-// CHECK2-NEXT:    store i32 [[INC19]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_INC22:%.*]]
+// CHECK2:       for.inc22:
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC23:%.*]] = add nsw i32 [[TMP15]], 1
+// CHECK2-NEXT:    store i32 [[INC23]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP13:![0-9]+]]
-// CHECK2:       for.end20:
-// CHECK2-NEXT:    br label [[FOR_INC21:%.*]]
-// CHECK2:       for.inc21:
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD22:%.*]] = add nsw i32 [[TMP19]], 5
-// CHECK2-NEXT:    store i32 [[ADD22]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2:       for.end24:
+// CHECK2-NEXT:    br label [[FOR_INC25:%.*]]
+// CHECK2:       for.inc25:
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD26:%.*]] = add nsw i32 [[TMP16]], 5
+// CHECK2-NEXT:    store i32 [[ADD26]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP14:![0-9]+]]
-// CHECK2:       for.end23:
-// CHECK2-NEXT:    br label [[FOR_INC24:%.*]]
-// CHECK2:       for.inc24:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD25:%.*]] = add nsw i32 [[TMP20]], 5
-// CHECK2-NEXT:    store i32 [[ADD25]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2:       for.end27:
+// CHECK2-NEXT:    br label [[FOR_INC28:%.*]]
+// CHECK2:       for.inc28:
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD29:%.*]] = add nsw i32 [[TMP17]], 5
+// CHECK2-NEXT:    store i32 [[ADD29]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP15:![0-9]+]]
-// CHECK2:       for.end26:
+// CHECK2:       for.end30:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -1630,17 +1689,15 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
 // CHECK2-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK2-NEXT:    store i32 7, ptr [[J]], align 4
@@ -1678,83 +1735,87 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP7]], 4
-// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END32:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[TMP8]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND3:%.*]]
 // CHECK2:       for.cond3:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP8]], 5
-// CHECK2-NEXT:    br i1 [[CMP4]], label [[FOR_BODY5:%.*]], label [[FOR_END21:%.*]]
-// CHECK2:       for.body5:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
-// CHECK2-NEXT:    store i32 [[ADD6]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL7:%.*]] = mul nsw i32 [[TMP11]], 3
-// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i32 7, [[MUL7]]
-// CHECK2-NEXT:    store i32 [[ADD8]], ptr [[I]], align 4
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND9:%.*]]
-// CHECK2:       for.cond9:
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i32 [[TMP12]], 5
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP13]], [[TMP14]]
-// CHECK2-NEXT:    [[CMP12:%.*]] = icmp slt i32 [[ADD11]], 4
-// CHECK2-NEXT:    br i1 [[CMP12]], label [[LAND_RHS:%.*]], label [[LAND_END:%.*]]
-// CHECK2:       land.rhs:
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD4:%.*]] = add nsw i32 [[TMP10]], 5
+// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD4]]
+// CHECK2-NEXT:    br i1 [[CMP5]], label [[COND_TRUE6:%.*]], label [[COND_FALSE7:%.*]]
+// CHECK2:       cond.true6:
+// CHECK2-NEXT:    br label [[COND_END9:%.*]]
+// CHECK2:       cond.false7:
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP11]], 5
+// CHECK2-NEXT:    br label [[COND_END9]]
+// CHECK2:       cond.end9:
+// CHECK2-NEXT:    [[COND10:%.*]] = phi i32 [ 4, [[COND_TRUE6]] ], [ [[ADD8]], [[COND_FALSE7]] ]
+// CHECK2-NEXT:    [[CMP11:%.*]] = icmp slt i32 [[TMP9]], [[COND10]]
+// CHECK2-NEXT:    br i1 [[CMP11]], label [[FOR_BODY12:%.*]], label [[FOR_END29:%.*]]
+// CHECK2:       for.body12:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL13:%.*]] = mul nsw i32 [[TMP12]], 3
+// CHECK2-NEXT:    [[ADD14:%.*]] = add nsw i32 7, [[MUL13]]
+// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    store i32 [[TMP13]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND15:%.*]]
+// CHECK2:       for.cond15:
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP15]], [[TMP16]]
-// CHECK2-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[ADD13]], 4
-// CHECK2-NEXT:    br label [[LAND_END]]
-// CHECK2:       land.end:
-// CHECK2-NEXT:    [[TMP17:%.*]] = phi i1 [ false, [[FOR_COND9]] ], [ [[CMP14]], [[LAND_RHS]] ]
-// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP10]], [[TMP17]]
-// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY15:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body15:
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP18]], [[TMP19]]
-// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[MUL17:%.*]] = mul nsw i32 [[TMP20]], 3
-// CHECK2-NEXT:    [[ADD18:%.*]] = add nsw i32 7, [[MUL17]]
-// CHECK2-NEXT:    store i32 [[ADD18]], ptr [[J]], align 4
-// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[J]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP21]], i32 noundef [[TMP22]])
+// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP15]], 5
+// CHECK2-NEXT:    [[CMP17:%.*]] = icmp slt i32 4, [[ADD16]]
+// CHECK2-NEXT:    br i1 [[CMP17]], label [[COND_TRUE18:%.*]], label [[COND_FALSE19:%.*]]
+// CHECK2:       cond.true18:
+// CHECK2-NEXT:    br label [[COND_END21:%.*]]
+// CHECK2:       cond.false19:
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP16]], 5
+// CHECK2-NEXT:    br label [[COND_END21]]
+// CHECK2:       cond.end21:
+// CHECK2-NEXT:    [[COND22:%.*]] = phi i32 [ 4, [[COND_TRUE18]] ], [ [[ADD20]], [[COND_FALSE19]] ]
+// CHECK2-NEXT:    [[CMP23:%.*]] = icmp slt i32 [[TMP14]], [[COND22]]
+// CHECK2-NEXT:    br i1 [[CMP23]], label [[FOR_BODY24:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body24:
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[MUL25:%.*]] = mul nsw i32 [[TMP17]], 3
+// CHECK2-NEXT:    [[ADD26:%.*]] = add nsw i32 7, [[MUL25]]
+// CHECK2-NEXT:    store i32 [[ADD26]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP18]], i32 noundef [[TMP19]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP23]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND9]], !llvm.loop [[LOOP16:![0-9]+]]
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP20]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND15]], !llvm.loop [[LOOP16:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC19:%.*]]
-// CHECK2:       for.inc19:
-// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC20:%.*]] = add nsw i32 [[TMP24]], 1
-// CHECK2-NEXT:    store i32 [[INC20]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_INC27:%.*]]
+// CHECK2:       for.inc27:
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC28:%.*]] = add nsw i32 [[TMP21]], 1
+// CHECK2-NEXT:    store i32 [[INC28]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND3]], !llvm.loop [[LOOP17:![0-9]+]]
-// CHECK2:       for.end21:
-// CHECK2-NEXT:    br label [[FOR_INC22:%.*]]
-// CHECK2:       for.inc22:
-// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP25]], 5
-// CHECK2-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2:       for.end29:
+// CHECK2-NEXT:    br label [[FOR_INC30:%.*]]
+// CHECK2:       for.inc30:
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD31:%.*]] = add nsw i32 [[TMP22]], 5
+// CHECK2-NEXT:    store i32 [[ADD31]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP18:![0-9]+]]
-// CHECK2:       for.end24:
+// CHECK2:       for.end32:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
-// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[ADD25:%.*]] = add nsw i32 [[TMP26]], 1
-// CHECK2-NEXT:    store i32 [[ADD25]], ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[ADD33:%.*]] = add nsw i32 [[TMP23]], 1
+// CHECK2-NEXT:    store i32 [[ADD33]], ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -1771,9 +1832,7 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[_TMP1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
@@ -1781,8 +1840,8 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[K:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK2-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK2-NEXT:    store i32 7, ptr [[J]], align 4
@@ -1829,83 +1888,87 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP9]], 4
-// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY:%.*]], label [[FOR_END29:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY:%.*]], label [[FOR_END37:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[TMP10]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND8:%.*]]
 // CHECK2:       for.cond8:
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP10]], 5
-// CHECK2-NEXT:    br i1 [[CMP9]], label [[FOR_BODY10:%.*]], label [[FOR_END26:%.*]]
-// CHECK2:       for.body10:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP11]], [[TMP12]]
-// CHECK2-NEXT:    store i32 [[ADD11]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL12:%.*]] = mul nsw i32 [[TMP13]], 3
-// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i32 7, [[MUL12]]
-// CHECK2-NEXT:    store i32 [[ADD13]], ptr [[I]], align 4
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND14:%.*]]
-// CHECK2:       for.cond14:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[CMP15:%.*]] = icmp slt i32 [[TMP14]], 5
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP15]], [[TMP16]]
-// CHECK2-NEXT:    [[CMP17:%.*]] = icmp slt i32 [[ADD16]], 4
-// CHECK2-NEXT:    br i1 [[CMP17]], label [[LAND_RHS:%.*]], label [[LAND_END:%.*]]
-// CHECK2:       land.rhs:
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i32 [[TMP12]], 5
+// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i32 4, [[ADD9]]
+// CHECK2-NEXT:    br i1 [[CMP10]], label [[COND_TRUE11:%.*]], label [[COND_FALSE12:%.*]]
+// CHECK2:       cond.true11:
+// CHECK2-NEXT:    br label [[COND_END14:%.*]]
+// CHECK2:       cond.false12:
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP13]], 5
+// CHECK2-NEXT:    br label [[COND_END14]]
+// CHECK2:       cond.end14:
+// CHECK2-NEXT:    [[COND15:%.*]] = phi i32 [ 4, [[COND_TRUE11]] ], [ [[ADD13]], [[COND_FALSE12]] ]
+// CHECK2-NEXT:    [[CMP16:%.*]] = icmp slt i32 [[TMP11]], [[COND15]]
+// CHECK2-NEXT:    br i1 [[CMP16]], label [[FOR_BODY17:%.*]], label [[FOR_END34:%.*]]
+// CHECK2:       for.body17:
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL18:%.*]] = mul nsw i32 [[TMP14]], 3
+// CHECK2-NEXT:    [[ADD19:%.*]] = add nsw i32 7, [[MUL18]]
+// CHECK2-NEXT:    store i32 [[ADD19]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    store i32 [[TMP15]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND20:%.*]]
+// CHECK2:       for.cond20:
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
 // CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD18:%.*]] = add nsw i32 [[TMP17]], [[TMP18]]
-// CHECK2-NEXT:    [[CMP19:%.*]] = icmp slt i32 [[ADD18]], 4
-// CHECK2-NEXT:    br label [[LAND_END]]
-// CHECK2:       land.end:
-// CHECK2-NEXT:    [[TMP19:%.*]] = phi i1 [ false, [[FOR_COND14]] ], [ [[CMP19]], [[LAND_RHS]] ]
-// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP15]], [[TMP19]]
-// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY20:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body20:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD21:%.*]] = add nsw i32 [[TMP20]], [[TMP21]]
-// CHECK2-NEXT:    store i32 [[ADD21]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[MUL22:%.*]] = mul nsw i32 [[TMP22]], 3
-// CHECK2-NEXT:    [[ADD23:%.*]] = add nsw i32 7, [[MUL22]]
-// CHECK2-NEXT:    store i32 [[ADD23]], ptr [[J]], align 4
-// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[J]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP23]], i32 noundef [[TMP24]])
+// CHECK2-NEXT:    [[ADD21:%.*]] = add nsw i32 [[TMP17]], 5
+// CHECK2-NEXT:    [[CMP22:%.*]] = icmp slt i32 4, [[ADD21]]
+// CHECK2-NEXT:    br i1 [[CMP22]], label [[COND_TRUE23:%.*]], label [[COND_FALSE24:%.*]]
+// CHECK2:       cond.true23:
+// CHECK2-NEXT:    br label [[COND_END26:%.*]]
+// CHECK2:       cond.false24:
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD25:%.*]] = add nsw i32 [[TMP18]], 5
+// CHECK2-NEXT:    br label [[COND_END26]]
+// CHECK2:       cond.end26:
+// CHECK2-NEXT:    [[COND27:%.*]] = phi i32 [ 4, [[COND_TRUE23]] ], [ [[ADD25]], [[COND_FALSE24]] ]
+// CHECK2-NEXT:    [[CMP28:%.*]] = icmp slt i32 [[TMP16]], [[COND27]]
+// CHECK2-NEXT:    br i1 [[CMP28]], label [[FOR_BODY29:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body29:
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[MUL30:%.*]] = mul nsw i32 [[TMP19]], 3
+// CHECK2-NEXT:    [[ADD31:%.*]] = add nsw i32 7, [[MUL30]]
+// CHECK2-NEXT:    store i32 [[ADD31]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP20]], i32 noundef [[TMP21]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP25]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND14]], !llvm.loop [[LOOP19:![0-9]+]]
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP22]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND20]], !llvm.loop [[LOOP19:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC24:%.*]]
-// CHECK2:       for.inc24:
-// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC25:%.*]] = add nsw i32 [[TMP26]], 1
-// CHECK2-NEXT:    store i32 [[INC25]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_INC32:%.*]]
+// CHECK2:       for.inc32:
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC33:%.*]] = add nsw i32 [[TMP23]], 1
+// CHECK2-NEXT:    store i32 [[INC33]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP20:![0-9]+]]
-// CHECK2:       for.end26:
-// CHECK2-NEXT:    br label [[FOR_INC27:%.*]]
-// CHECK2:       for.inc27:
-// CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD28:%.*]] = add nsw i32 [[TMP27]], 5
-// CHECK2-NEXT:    store i32 [[ADD28]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2:       for.end34:
+// CHECK2-NEXT:    br label [[FOR_INC35:%.*]]
+// CHECK2:       for.inc35:
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD36:%.*]] = add nsw i32 [[TMP24]], 5
+// CHECK2-NEXT:    store i32 [[ADD36]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP21:![0-9]+]]
-// CHECK2:       for.end29:
+// CHECK2:       for.end37:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
-// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[ADD30:%.*]] = add nsw i32 [[TMP28]], 1
-// CHECK2-NEXT:    store i32 [[ADD30]], ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[ADD38:%.*]] = add nsw i32 [[TMP25]], 1
+// CHECK2-NEXT:    store i32 [[ADD38]], ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -1923,91 +1986,90 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[_TMP1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[_TMP2:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK2-NEXT:    store i32 7, ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP]], align 4
+// CHECK2-NEXT:    store i32 [[TMP1]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], 5
+// CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTCAPTURE_EXPR_3]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK2-NEXT:    store i32 19, ptr [[DOTOMP_UB]], align 4
 // CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
 // CHECK2-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
-// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// CHECK2-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], 19
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP3]], 19
 // CHECK2-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK2:       cond.true:
 // CHECK2-NEXT:    br label [[COND_END:%.*]]
 // CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
 // CHECK2-NEXT:    br label [[COND_END]]
 // CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 19, [[COND_TRUE]] ], [ [[TMP2]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 19, [[COND_TRUE]] ], [ [[TMP4]], [[COND_FALSE]] ]
 // CHECK2-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
-// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
-// CHECK2-NEXT:    store i32 [[TMP3]], ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
+// CHECK2-NEXT:    store i32 [[TMP5]], ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK2:       omp.inner.for.cond:
-// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// CHECK2-NEXT:    [[CMP3:%.*]] = icmp sle i32 [[TMP4]], [[TMP5]]
-// CHECK2-NEXT:    br i1 [[CMP3]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
-// CHECK2:       omp.inner.for.body:
 // CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP6]], 20
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[DIV]], 5
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    [[CMP4:%.*]] = icmp sle i32 [[TMP6]], [[TMP7]]
+// CHECK2-NEXT:    br i1 [[CMP4]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK2:       omp.inner.for.body:
 // CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[DIV4:%.*]] = sdiv i32 [[TMP8]], 20
-// CHECK2-NEXT:    [[MUL5:%.*]] = mul nsw i32 [[DIV4]], 20
-// CHECK2-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP7]], [[MUL5]]
-// CHECK2-NEXT:    [[DIV6:%.*]] = sdiv i32 [[SUB]], 4
-// CHECK2-NEXT:    [[MUL7:%.*]] = mul nsw i32 [[DIV6]], 1
-// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i32 0, [[MUL7]]
-// CHECK2-NEXT:    store i32 [[ADD8]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP8]], 20
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[DIV]], 5
+// CHECK2-NEXT:    [[ADD5:%.*]] = add nsw i32 0, [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD5]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[DIV9:%.*]] = sdiv i32 [[TMP10]], 20
-// CHECK2-NEXT:    [[MUL10:%.*]] = mul nsw i32 [[DIV9]], 20
-// CHECK2-NEXT:    [[SUB11:%.*]] = sub nsw i32 [[TMP9]], [[MUL10]]
 // CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[DIV6:%.*]] = sdiv i32 [[TMP11]], 20
+// CHECK2-NEXT:    [[MUL7:%.*]] = mul nsw i32 [[DIV6]], 20
+// CHECK2-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP10]], [[MUL7]]
+// CHECK2-NEXT:    [[DIV8:%.*]] = sdiv i32 [[SUB]], 4
+// CHECK2-NEXT:    [[MUL9:%.*]] = mul nsw i32 [[DIV8]], 1
+// CHECK2-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP9]], [[MUL9]]
+// CHECK2-NEXT:    store i32 [[ADD10]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[DIV12:%.*]] = sdiv i32 [[TMP12]], 20
-// CHECK2-NEXT:    [[MUL13:%.*]] = mul nsw i32 [[DIV12]], 20
-// CHECK2-NEXT:    [[SUB14:%.*]] = sub nsw i32 [[TMP11]], [[MUL13]]
-// CHECK2-NEXT:    [[DIV15:%.*]] = sdiv i32 [[SUB14]], 4
-// CHECK2-NEXT:    [[MUL16:%.*]] = mul nsw i32 [[DIV15]], 4
-// CHECK2-NEXT:    [[SUB17:%.*]] = sub nsw i32 [[SUB11]], [[MUL16]]
-// CHECK2-NEXT:    [[MUL18:%.*]] = mul nsw i32 [[SUB17]], 3
-// CHECK2-NEXT:    [[ADD19:%.*]] = add nsw i32 7, [[MUL18]]
-// CHECK2-NEXT:    store i32 [[ADD19]], ptr [[J]], align 4
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP13]], [[TMP14]]
-// CHECK2-NEXT:    store i32 [[ADD20]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL21:%.*]] = mul nsw i32 [[TMP15]], 3
-// CHECK2-NEXT:    [[ADD22:%.*]] = add nsw i32 7, [[MUL21]]
-// CHECK2-NEXT:    store i32 [[ADD22]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP16]], [[TMP17]]
-// CHECK2-NEXT:    [[CMP24:%.*]] = icmp slt i32 [[ADD23]], 4
-// CHECK2-NEXT:    br i1 [[CMP24]], label [[OMP_TILE_PRED_THEN:%.*]], label [[OMP_TILE_PRED_END:%.*]]
-// CHECK2:       omp_tile.pred.then:
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[DIV11:%.*]] = sdiv i32 [[TMP13]], 20
+// CHECK2-NEXT:    [[MUL12:%.*]] = mul nsw i32 [[DIV11]], 20
+// CHECK2-NEXT:    [[SUB13:%.*]] = sub nsw i32 [[TMP12]], [[MUL12]]
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[DIV14:%.*]] = sdiv i32 [[TMP15]], 20
+// CHECK2-NEXT:    [[MUL15:%.*]] = mul nsw i32 [[DIV14]], 20
+// CHECK2-NEXT:    [[SUB16:%.*]] = sub nsw i32 [[TMP14]], [[MUL15]]
+// CHECK2-NEXT:    [[DIV17:%.*]] = sdiv i32 [[SUB16]], 4
+// CHECK2-NEXT:    [[MUL18:%.*]] = mul nsw i32 [[DIV17]], 4
+// CHECK2-NEXT:    [[SUB19:%.*]] = sub nsw i32 [[SUB13]], [[MUL18]]
+// CHECK2-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[SUB19]], 3
+// CHECK2-NEXT:    [[ADD21:%.*]] = add nsw i32 7, [[MUL20]]
+// CHECK2-NEXT:    store i32 [[ADD21]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP22:%.*]] = icmp slt i32 [[TMP16]], 4
+// CHECK2-NEXT:    br i1 [[CMP22]], label [[OMP_BODY_NEXT:%.*]], label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK2:       omp.body.next:
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL23:%.*]] = mul nsw i32 [[TMP17]], 3
+// CHECK2-NEXT:    [[ADD24:%.*]] = add nsw i32 7, [[MUL23]]
+// CHECK2-NEXT:    store i32 [[ADD24]], ptr [[I]], align 4
 // CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
 // CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[J]], align 4
 // CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP18]], i32 noundef [[TMP19]])
-// CHECK2-NEXT:    br label [[OMP_TILE_PRED_END]]
-// CHECK2:       omp_tile.pred.end:
-// CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
@@ -2038,13 +2100,12 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK2-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
 // CHECK2-NEXT:    store i32 7, ptr [[I]], align 4
@@ -2079,42 +2140,46 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP7]], 5
 // CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[TMP8]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND:%.*]]
 // CHECK2:       for.cond:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP8]], 5
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
-// CHECK2-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[ADD3]], 4
-// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP2]], [[CMP4]]
-// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body:
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD2:%.*]] = add nsw i32 [[TMP10]], 5
+// CHECK2-NEXT:    [[CMP3:%.*]] = icmp slt i32 4, [[ADD2]]
+// CHECK2-NEXT:    br i1 [[CMP3]], label [[COND_TRUE4:%.*]], label [[COND_FALSE5:%.*]]
+// CHECK2:       cond.true4:
+// CHECK2-NEXT:    br label [[COND_END7:%.*]]
+// CHECK2:       cond.false5:
 // CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD5:%.*]] = add nsw i32 [[TMP11]], [[TMP12]]
-// CHECK2-NEXT:    store i32 [[ADD5]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL6:%.*]] = mul nsw i32 [[TMP13]], 3
-// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL6]]
-// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP14]])
+// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP11]], 5
+// CHECK2-NEXT:    br label [[COND_END7]]
+// CHECK2:       cond.end7:
+// CHECK2-NEXT:    [[COND8:%.*]] = phi i32 [ 4, [[COND_TRUE4]] ], [ [[ADD6]], [[COND_FALSE5]] ]
+// CHECK2-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP9]], [[COND8]]
+// CHECK2-NEXT:    br i1 [[CMP9]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL10:%.*]] = mul nsw i32 [[TMP12]], 3
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 7, [[MUL10]]
+// CHECK2-NEXT:    store i32 [[ADD11]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP13]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP15]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP22:![0-9]+]]
 // CHECK2:       for.end:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP16]], 1
-// CHECK2-NEXT:    store i32 [[ADD8]], ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP15]], 1
+// CHECK2-NEXT:    store i32 [[ADD12]], ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -2128,9 +2193,8 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:  entry:
 // CHECK2-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
 // CHECK2-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
@@ -2138,64 +2202,78 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END15:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK2:       for.cond1:
-// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP2]], 0
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], 0
 // CHECK2-NEXT:    br i1 [[CMP2]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK2:       cond.true:
 // CHECK2-NEXT:    br label [[COND_END:%.*]]
 // CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[A_ADDR]], align 4
 // CHECK2-NEXT:    br label [[COND_END]]
 // CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP3]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[TMP1]], [[COND]]
-// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
-// CHECK2-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[ADD]], 4
-// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP3]], [[CMP4]]
-// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY5:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body5:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP5]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[COND]]
+// CHECK2-NEXT:    [[CMP3:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK2-NEXT:    br i1 [[CMP3]], label [[COND_TRUE4:%.*]], label [[COND_FALSE5:%.*]]
+// CHECK2:       cond.true4:
+// CHECK2-NEXT:    br label [[COND_END12:%.*]]
+// CHECK2:       cond.false5:
 // CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP6]], [[TMP7]]
-// CHECK2-NEXT:    store i32 [[ADD6]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP8]], 3
-// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP9]])
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    [[CMP6:%.*]] = icmp sle i32 [[TMP7]], 0
+// CHECK2-NEXT:    br i1 [[CMP6]], label [[COND_TRUE7:%.*]], label [[COND_FALSE8:%.*]]
+// CHECK2:       cond.true7:
+// CHECK2-NEXT:    br label [[COND_END9:%.*]]
+// CHECK2:       cond.false8:
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    br label [[COND_END9]]
+// CHECK2:       cond.end9:
+// CHECK2-NEXT:    [[COND10:%.*]] = phi i32 [ 1, [[COND_TRUE7]] ], [ [[TMP8]], [[COND_FALSE8]] ]
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP6]], [[COND10]]
+// CHECK2-NEXT:    br label [[COND_END12]]
+// CHECK2:       cond.end12:
+// CHECK2-NEXT:    [[COND13:%.*]] = phi i32 [ 4, [[COND_TRUE4]] ], [ [[ADD11]], [[COND_END9]] ]
+// CHECK2-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP2]], [[COND13]]
+// CHECK2-NEXT:    br i1 [[CMP14]], label [[FOR_BODY15:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body15:
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP9]], 3
+// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP10]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP25:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC8:%.*]]
-// CHECK2:       for.inc8:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    [[CMP9:%.*]] = icmp sle i32 [[TMP11]], 0
-// CHECK2-NEXT:    br i1 [[CMP9]], label [[COND_TRUE10:%.*]], label [[COND_FALSE11:%.*]]
-// CHECK2:       cond.true10:
-// CHECK2-NEXT:    br label [[COND_END12:%.*]]
-// CHECK2:       cond.false11:
+// CHECK2-NEXT:    br label [[FOR_INC17:%.*]]
+// CHECK2:       for.inc17:
 // CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    br label [[COND_END12]]
-// CHECK2:       cond.end12:
-// CHECK2-NEXT:    [[COND13:%.*]] = phi i32 [ 1, [[COND_TRUE10]] ], [ [[TMP12]], [[COND_FALSE11]] ]
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD14:%.*]] = add nsw i32 [[TMP13]], [[COND13]]
-// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP18:%.*]] = icmp sle i32 [[TMP12]], 0
+// CHECK2-NEXT:    br i1 [[CMP18]], label [[COND_TRUE19:%.*]], label [[COND_FALSE20:%.*]]
+// CHECK2:       cond.true19:
+// CHECK2-NEXT:    br label [[COND_END21:%.*]]
+// CHECK2:       cond.false20:
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    br label [[COND_END21]]
+// CHECK2:       cond.end21:
+// CHECK2-NEXT:    [[COND22:%.*]] = phi i32 [ 1, [[COND_TRUE19]] ], [ [[TMP13]], [[COND_FALSE20]] ]
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP14]], [[COND22]]
+// CHECK2-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP26:![0-9]+]]
-// CHECK2:       for.end15:
+// CHECK2:       for.end24:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -2208,9 +2286,8 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK2-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
 // CHECK2-NEXT:    store ptr [[ARRAY]], ptr [[__RANGE2]], align 8
@@ -2244,31 +2321,37 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK2:       for.cond7:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK2-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP11]], 1
-// CHECK2-NEXT:    [[CMP11:%.*]] = icmp slt i64 [[ADD9]], [[ADD10]]
-// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP8]], [[CMP11]]
-// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY12:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body12:
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i64 [[TMP12]], [[TMP13]]
-// CHECK2-NEXT:    store i64 [[ADD13]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
+// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
+// CHECK2-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
+// CHECK2-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body14:
 // CHECK2-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
-// CHECK2-NEXT:    [[ADD_PTR14:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
-// CHECK2-NEXT:    store ptr [[ADD_PTR14]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
+// CHECK2-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
 // CHECK2-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
 // CHECK2-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
 // CHECK2-NEXT:    store double [[TMP17]], ptr [[V]], align 8
@@ -2276,18 +2359,18 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    call void (...) @body(double noundef [[TMP18]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP19]], 1
-// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP27:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC15:%.*]]
-// CHECK2:       for.inc15:
+// CHECK2-NEXT:    br label [[FOR_INC16:%.*]]
+// CHECK2:       for.inc16:
 // CHECK2-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i64 [[TMP20]], 5
-// CHECK2-NEXT:    store i64 [[ADD16]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP20]], 5
+// CHECK2-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP28:![0-9]+]]
-// CHECK2:       for.end17:
+// CHECK2:       for.end18:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -2307,9 +2390,8 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[START_ADDR]], align 4
@@ -2333,47 +2415,53 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK2-NEXT:    [[ADD5:%.*]] = add i32 [[TMP6]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP5]], [[ADD5]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK2:       for.cond6:
-// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP7]], 5
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD8:%.*]] = add i32 [[TMP8]], [[TMP9]]
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[ADD9:%.*]] = add i32 [[TMP10]], 1
-// CHECK2-NEXT:    [[CMP10:%.*]] = icmp ult i32 [[ADD8]], [[ADD9]]
-// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP7]], [[CMP10]]
-// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY11:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body11:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD12:%.*]] = add i32 [[TMP11]], [[TMP12]]
-// CHECK2-NEXT:    store i32 [[ADD12]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD7:%.*]] = add i32 [[TMP9]], 1
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD8:%.*]] = add i32 [[TMP10]], 5
+// CHECK2-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
+// CHECK2-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD10:%.*]] = add i32 [[TMP11]], 1
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD11:%.*]] = add i32 [[TMP12]], 5
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP8]], [[COND]]
+// CHECK2-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body13:
 // CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
 // CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP14]], 3
-// CHECK2-NEXT:    [[ADD13:%.*]] = add i32 [[TMP13]], [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD13]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[ADD14:%.*]] = add i32 [[TMP13]], [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
 // CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP15]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP16]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP29:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC14:%.*]]
-// CHECK2:       for.inc14:
+// CHECK2-NEXT:    br label [[FOR_INC15:%.*]]
+// CHECK2:       for.inc15:
 // CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD15:%.*]] = add i32 [[TMP17]], 5
-// CHECK2-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD16:%.*]] = add i32 [[TMP17]], 5
+// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP30:![0-9]+]]
-// CHECK2:       for.end16:
+// CHECK2:       for.end17:
 // CHECK2-NEXT:    ret void
 //
 //
diff --git a/clang/test/OpenMP/tile_codegen_for_dependent.cpp b/clang/test/OpenMP/tile_codegen_for_dependent.cpp
index 808f0e71dc3b0..7401ab1a2ded3 100644
--- a/clang/test/OpenMP/tile_codegen_for_dependent.cpp
+++ b/clang/test/OpenMP/tile_codegen_for_dependent.cpp
@@ -31,7 +31,6 @@ extern "C" void body(...) {}
 // IR-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTCAPTURE_EXPR_5:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
@@ -40,7 +39,7 @@ extern "C" void body(...) {}
 // IR-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_0_IV_I11:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
 // IR-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // IR-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
@@ -109,30 +108,36 @@ extern "C" void body(...) {}
 // IR-NEXT:    [[MUL:%.*]] = mul i32 [[TMP20]], 4
 // IR-NEXT:    [[ADD15:%.*]] = add i32 0, [[MUL]]
 // IR-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I11]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I11]], align 4
+// IR-NEXT:    store i32 [[TMP21]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND:.*]]
 // IR:       [[FOR_COND]]:
-// IR-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[CMP16:%.*]] = icmp ult i32 [[TMP21]], 4
-// IR-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I11]], align 4
-// IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD17:%.*]] = add i32 [[TMP22]], [[TMP23]]
-// IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[ADD18:%.*]] = add i32 [[TMP24]], 1
-// IR-NEXT:    [[CMP19:%.*]] = icmp ult i32 [[ADD17]], [[ADD18]]
-// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP16]], [[CMP19]]
-// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// IR-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    [[ADD16:%.*]] = add i32 [[TMP23]], 1
+// IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I11]], align 4
+// IR-NEXT:    [[ADD17:%.*]] = add i32 [[TMP24]], 4
+// IR-NEXT:    [[CMP18:%.*]] = icmp ult i32 [[ADD16]], [[ADD17]]
+// IR-NEXT:    br i1 [[CMP18]], label %[[COND_TRUE19:.*]], label %[[COND_FALSE21:.*]]
+// IR:       [[COND_TRUE19]]:
+// IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    [[ADD20:%.*]] = add i32 [[TMP25]], 1
+// IR-NEXT:    br label %[[COND_END23:.*]]
+// IR:       [[COND_FALSE21]]:
+// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I11]], align 4
+// IR-NEXT:    [[ADD22:%.*]] = add i32 [[TMP26]], 4
+// IR-NEXT:    br label %[[COND_END23]]
+// IR:       [[COND_END23]]:
+// IR-NEXT:    [[COND24:%.*]] = phi i32 [ [[ADD20]], %[[COND_TRUE19]] ], [ [[ADD22]], %[[COND_FALSE21]] ]
+// IR-NEXT:    [[CMP25:%.*]] = icmp ult i32 [[TMP22]], [[COND24]]
+// IR-NEXT:    br i1 [[CMP25]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
 // IR:       [[FOR_BODY]]:
-// IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I11]], align 4
-// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD20:%.*]] = add i32 [[TMP25]], [[TMP26]]
-// IR-NEXT:    store i32 [[ADD20]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
 // IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[MUL21:%.*]] = mul i32 [[TMP28]], [[TMP29]]
-// IR-NEXT:    [[ADD22:%.*]] = add i32 [[TMP27]], [[MUL21]]
-// IR-NEXT:    store i32 [[ADD22]], ptr [[I]], align 4
+// IR-NEXT:    [[MUL26:%.*]] = mul i32 [[TMP28]], [[TMP29]]
+// IR-NEXT:    [[ADD27:%.*]] = add i32 [[TMP27]], [[MUL26]]
+// IR-NEXT:    store i32 [[ADD27]], ptr [[I]], align 4
 // IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[START_ADDR]], align 4
 // IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[END_ADDR]], align 4
 // IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
@@ -140,9 +145,9 @@ extern "C" void body(...) {}
 // IR-NEXT:    call void (...) @body(i32 noundef [[TMP30]], i32 noundef [[TMP31]], i32 noundef [[TMP32]], i32 noundef [[TMP33]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    [[INC:%.*]] = add i32 [[TMP34]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP2:![0-9]+]]
 // IR:       [[FOR_END]]:
 // IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.*]]
@@ -150,8 +155,8 @@ extern "C" void body(...) {}
 // IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.*]]
 // IR:       [[OMP_INNER_FOR_INC]]:
 // IR-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[ADD23:%.*]] = add i32 [[TMP35]], 1
-// IR-NEXT:    store i32 [[ADD23]], ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[ADD28:%.*]] = add i32 [[TMP35]], 1
+// IR-NEXT:    store i32 [[ADD28]], ptr [[DOTOMP_IV]], align 4
 // IR-NEXT:    br label %[[OMP_INNER_FOR_COND]]
 // IR:       [[OMP_INNER_FOR_END]]:
 // IR-NEXT:    br label %[[OMP_LOOP_EXIT:.*]]
diff --git a/clang/test/OpenMP/tile_codegen_tile_for.cpp b/clang/test/OpenMP/tile_codegen_tile_for.cpp
index 9ef96834fcfc0..3d0092042cdfc 100644
--- a/clang/test/OpenMP/tile_codegen_tile_for.cpp
+++ b/clang/test/OpenMP/tile_codegen_tile_for.cpp
@@ -32,11 +32,9 @@ extern "C" void body(...) {}
 // IR-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTCAPTURE_EXPR_5:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV__FLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTCAPTURE_EXPR_13:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_0_IV__FLOOR_0_IV_I:%.*]] = alloca i32, align 4
@@ -45,8 +43,8 @@ extern "C" void body(...) {}
 // IR-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_0_IV__FLOOR_0_IV_I17:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV__FLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
 // IR-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // IR-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
@@ -124,52 +122,64 @@ extern "C" void body(...) {}
 // IR-NEXT:    [[MUL:%.*]] = mul i32 [[TMP22]], 3
 // IR-NEXT:    [[ADD21:%.*]] = add i32 0, [[MUL]]
 // IR-NEXT:    store i32 [[ADD21]], ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
+// IR-NEXT:    store i32 [[TMP23]], ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND:.*]]
 // IR:       [[FOR_COND]]:
-// IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[CMP22:%.*]] = icmp ult i32 [[TMP23]], 3
-// IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
-// IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD23:%.*]] = add i32 [[TMP24]], [[TMP25]]
-// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    [[ADD24:%.*]] = add i32 [[TMP26]], 1
-// IR-NEXT:    [[CMP25:%.*]] = icmp ult i32 [[ADD23]], [[ADD24]]
-// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP22]], [[CMP25]]
-// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label %[[FOR_BODY:.*]], label %[[FOR_END41:.*]]
+// IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    [[ADD22:%.*]] = add i32 [[TMP25]], 1
+// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
+// IR-NEXT:    [[ADD23:%.*]] = add i32 [[TMP26]], 3
+// IR-NEXT:    [[CMP24:%.*]] = icmp ult i32 [[ADD22]], [[ADD23]]
+// IR-NEXT:    br i1 [[CMP24]], label %[[COND_TRUE25:.*]], label %[[COND_FALSE27:.*]]
+// IR:       [[COND_TRUE25]]:
+// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    [[ADD26:%.*]] = add i32 [[TMP27]], 1
+// IR-NEXT:    br label %[[COND_END29:.*]]
+// IR:       [[COND_FALSE27]]:
+// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
+// IR-NEXT:    [[ADD28:%.*]] = add i32 [[TMP28]], 3
+// IR-NEXT:    br label %[[COND_END29]]
+// IR:       [[COND_END29]]:
+// IR-NEXT:    [[COND30:%.*]] = phi i32 [ [[ADD26]], %[[COND_TRUE25]] ], [ [[ADD28]], %[[COND_FALSE27]] ]
+// IR-NEXT:    [[CMP31:%.*]] = icmp ult i32 [[TMP24]], [[COND30]]
+// IR-NEXT:    br i1 [[CMP31]], label %[[FOR_BODY:.*]], label %[[FOR_END50:.*]]
 // IR:       [[FOR_BODY]]:
-// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
-// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD26:%.*]] = add i32 [[TMP27]], [[TMP28]]
-// IR-NEXT:    store i32 [[ADD26]], ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
 // IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[MUL27:%.*]] = mul i32 [[TMP29]], 4
-// IR-NEXT:    [[ADD28:%.*]] = add i32 0, [[MUL27]]
-// IR-NEXT:    store i32 [[ADD28]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND29:.*]]
-// IR:       [[FOR_COND29]]:
-// IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[CMP30:%.*]] = icmp ult i32 [[TMP30]], 4
-// IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD31:%.*]] = add i32 [[TMP31]], [[TMP32]]
-// IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[ADD32:%.*]] = add i32 [[TMP33]], 1
-// IR-NEXT:    [[CMP33:%.*]] = icmp ult i32 [[ADD31]], [[ADD32]]
-// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND34:%.*]] = and i1 [[CMP30]], [[CMP33]]
-// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND34]], label %[[FOR_BODY35:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY35]]:
-// IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD36:%.*]] = add i32 [[TMP34]], [[TMP35]]
-// IR-NEXT:    store i32 [[ADD36]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL32:%.*]] = mul i32 [[TMP29]], 4
+// IR-NEXT:    [[ADD33:%.*]] = add i32 0, [[MUL32]]
+// IR-NEXT:    store i32 [[ADD33]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP30]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND34:.*]]
+// IR:       [[FOR_COND34]]:
+// IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    [[ADD35:%.*]] = add i32 [[TMP32]], 1
+// IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD36:%.*]] = add i32 [[TMP33]], 4
+// IR-NEXT:    [[CMP37:%.*]] = icmp ult i32 [[ADD35]], [[ADD36]]
+// IR-NEXT:    br i1 [[CMP37]], label %[[COND_TRUE38:.*]], label %[[COND_FALSE40:.*]]
+// IR:       [[COND_TRUE38]]:
+// IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    [[ADD39:%.*]] = add i32 [[TMP34]], 1
+// IR-NEXT:    br label %[[COND_END42:.*]]
+// IR:       [[COND_FALSE40]]:
+// IR-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD41:%.*]] = add i32 [[TMP35]], 4
+// IR-NEXT:    br label %[[COND_END42]]
+// IR:       [[COND_END42]]:
+// IR-NEXT:    [[COND43:%.*]] = phi i32 [ [[ADD39]], %[[COND_TRUE38]] ], [ [[ADD41]], %[[COND_FALSE40]] ]
+// IR-NEXT:    [[CMP44:%.*]] = icmp ult i32 [[TMP31]], [[COND43]]
+// IR-NEXT:    br i1 [[CMP44]], label %[[FOR_BODY45:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY45]]:
 // IR-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
 // IR-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[MUL37:%.*]] = mul i32 [[TMP37]], [[TMP38]]
-// IR-NEXT:    [[ADD38:%.*]] = add i32 [[TMP36]], [[MUL37]]
-// IR-NEXT:    store i32 [[ADD38]], ptr [[I]], align 4
+// IR-NEXT:    [[MUL46:%.*]] = mul i32 [[TMP37]], [[TMP38]]
+// IR-NEXT:    [[ADD47:%.*]] = add i32 [[TMP36]], [[MUL46]]
+// IR-NEXT:    store i32 [[ADD47]], ptr [[I]], align 4
 // IR-NEXT:    [[TMP39:%.*]] = load i32, ptr [[START_ADDR]], align 4
 // IR-NEXT:    [[TMP40:%.*]] = load i32, ptr [[END_ADDR]], align 4
 // IR-NEXT:    [[TMP41:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
@@ -177,25 +187,25 @@ extern "C" void body(...) {}
 // IR-NEXT:    call void (...) @body(i32 noundef [[TMP39]], i32 noundef [[TMP40]], i32 noundef [[TMP41]], i32 noundef [[TMP42]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    [[INC:%.*]] = add i32 [[TMP43]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND29]], !llvm.loop [[LOOP2:![0-9]+]]
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND34]], !llvm.loop [[LOOP2:![0-9]+]]
 // IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC39:.*]]
-// IR:       [[FOR_INC39]]:
-// IR-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[INC40:%.*]] = add i32 [[TMP44]], 1
-// IR-NEXT:    store i32 [[INC40]], ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_INC48:.*]]
+// IR:       [[FOR_INC48]]:
+// IR-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[INC49:%.*]] = add i32 [[TMP44]], 1
+// IR-NEXT:    store i32 [[INC49]], ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP4:![0-9]+]]
-// IR:       [[FOR_END41]]:
+// IR:       [[FOR_END50]]:
 // IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.*]]
 // IR:       [[OMP_BODY_CONTINUE]]:
 // IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.*]]
 // IR:       [[OMP_INNER_FOR_INC]]:
 // IR-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[ADD42:%.*]] = add i32 [[TMP45]], 1
-// IR-NEXT:    store i32 [[ADD42]], ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[ADD51:%.*]] = add i32 [[TMP45]], 1
+// IR-NEXT:    store i32 [[ADD51]], ptr [[DOTOMP_IV]], align 4
 // IR-NEXT:    br label %[[OMP_INNER_FOR_COND]]
 // IR:       [[OMP_INNER_FOR_END]]:
 // IR-NEXT:    br label %[[OMP_LOOP_EXIT:.*]]
diff --git a/clang/test/OpenMP/tile_collapse_reinterpret_codegen.cpp b/clang/test/OpenMP/tile_collapse_reinterpret_codegen.cpp
new file mode 100644
index 0000000000000..f88db73778f2b
--- /dev/null
+++ b/clang/test/OpenMP/tile_collapse_reinterpret_codegen.cpp
@@ -0,0 +1,108 @@
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fclang-abi-compat=latest -std=c++20 -fopenmp -ast-print %s | FileCheck %s --check-prefix=PRINT
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fclang-abi-compat=latest -std=c++20 -fopenmp -emit-llvm %s -o - | FileCheck %s --check-prefix=IR
+// expected-no-diagnostics
+
+// Codegen tests for how a loop-associated directive (e.g. 'omp for' with a
+// 'collapse' clause) consumes a '#pragma omp tile' intra-tile loop.
+//
+// These tests pin the key IR invariants of the below design:
+// The intra-tile loop is stored in its min-bounded form
+//   for (.tile.iv = .floor.iv; .tile.iv < min(.floor.iv + tilesize, N); ++.tile.iv)
+// and carries an internal OMPInvariantPredicateBoundAttr hint. When a directive
+// consumes the loop, it is reinterpreted as a rectangular loop with a constant
+// per-tile trip count (the tile size), and the overshoot of a partial last tile
+// is handled by a predicate emitted as a body guard.
+
+extern "C" void body(...) {}
+
+// PRINT-LABEL: void collapse2_tile1d(
+// PRINT:       #pragma omp parallel for collapse(2)
+// PRINT-NEXT:  #pragma omp tile sizes(2)
+// IR-LABEL: define internal void @collapse2_tile1d.omp_outlined
+// IR:       %.floor_0.iv{{[0-9a-z.]*}} = alloca i32
+// IR:       %.tile_0.iv{{[0-9a-z.]*}} = alloca i32
+// IR-NOT:   .tile.cnt
+// IR:       omp.inner.for.body:
+// IR:       store i32 %{{.*}}, ptr %.tile_0.iv
+// IR:       %[[PRED:.*]] = icmp slt i32 %{{.*}}, %{{.*}}
+// IR-NEXT:  br i1 %[[PRED]], label %omp.body.next, label %omp.body.continue
+// IR:       omp.body.next:
+// IR:       call void (...) @body
+// IR-NEXT:  br label %omp.body.continue
+// IR:       omp.body.continue:
+extern "C" void collapse2_tile1d(int n) {
+#pragma omp parallel for collapse(2)
+#pragma omp tile sizes(2)
+  for (int i = 0; i < n; ++i)
+    body(i);
+}
+
+// PRINT-LABEL: void collapse3_tile1d_nested(
+// PRINT:       #pragma omp parallel for collapse(3)
+// PRINT-NEXT:  #pragma omp tile sizes(5)
+// IR-LABEL: define internal void @collapse3_tile1d_nested.omp_outlined
+// IR-NOT:   .tile.cnt
+// IR:       omp.inner.for.body:
+// IR:       store i32 %{{.*}}, ptr %.tile_0.iv
+// IR:       %[[PRED:.*]] = icmp slt i32 %{{.*}}, %{{.*}}
+// IR-NEXT:  br i1 %[[PRED]], label %omp.body.next, label %omp.body.continue
+// IR:       omp.body.next:
+// IR:       call void (...) @body
+// IR-NEXT:  br label %omp.body.continue
+extern "C" void collapse3_tile1d_nested(int n) {
+#pragma omp parallel for collapse(3)
+#pragma omp tile sizes(5)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+}
+
+// PRINT-LABEL: void tile2d_inside_tile1d(
+// PRINT:       #pragma omp tile sizes(3)
+// PRINT-NEXT:  #pragma omp tile sizes(2)
+// IR-LABEL: define {{.*}} @tile2d_inside_tile1d(
+// IR:       %.tile_0.iv{{[0-9a-z.]*}} = alloca i32
+// IR-NOT:   .tile.cnt
+// IR:       cond.true:
+// IR:       cond.false:
+// IR:       cond.end:
+// IR:       %{{.*}} = phi i32
+// IR-NOT:   omp.body.next
+extern "C" void tile2d_inside_tile1d(int n) {
+#pragma omp tile sizes(3)
+#pragma omp tile sizes(2)
+  for (int i = 0; i < n; ++i)
+    body(i);
+}
+
+// PRINT-LABEL: void collapse3_tile2d(
+// PRINT:       #pragma omp parallel for collapse(3)
+// PRINT-NEXT:  #pragma omp tile sizes(2, 3)
+// IR-LABEL: define internal void @collapse3_tile2d.omp_outlined
+// IR-NOT:   .tile.cnt
+// IR:       omp.inner.for.body:
+// IR:       store i32 %{{.*}}, ptr %.tile_0.iv
+// IR:       br i1 %{{.*}}, label %omp.body.next, label %omp.body.continue
+// IR:       omp.body.next:
+// IR:       call void (...) @body
+extern "C" void collapse3_tile2d(int n) {
+#pragma omp parallel for collapse(3)
+#pragma omp tile sizes(2, 3)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+}
+
+// PRINT-LABEL: void tile1d_constant_div_no_predicate(
+// PRINT:       #pragma omp tile sizes(5)
+// IR-LABEL: define {{.*}} @tile1d_constant_div_no_predicate(
+// IR:       %.floor_0.iv{{[0-9a-z.]*}} = alloca i32
+// IR:       %.tile_0.iv{{[0-9a-z.]*}} = alloca i32
+// IR-NOT:   .tile.cnt
+// IR-NOT:   omp.body.next
+// IR:       call void (...) @body
+extern "C" void tile1d_constant_div_no_predicate() {
+#pragma omp tile sizes(5)
+  for (int i = 0; i < 10; ++i)
+    body(i);
+}
diff --git a/clang/test/OpenMP/tile_rect_codegen.cpp b/clang/test/OpenMP/tile_rect_codegen.cpp
index 589914eb9d63e..ab63de9343fd2 100644
--- a/clang/test/OpenMP/tile_rect_codegen.cpp
+++ b/clang/test/OpenMP/tile_rect_codegen.cpp
@@ -1,14 +1,15 @@
 // RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fclang-abi-compat=latest -std=c++20 -fopenmp -ast-print %s | FileCheck %s
 // expected-no-diagnostics
 
+// -ast-print round-trip tests: a `#pragma omp tile` directive and its associated
+// loop must print back as written. This checks parsing/printing only; for the
+// generated IR (min-bounded lowering) see tile_rect_codegen_ir.cpp.
+
 extern "C" void body(...) {}
 
 // CHECK-LABEL: void rect_tile_1d(
 void rect_tile_1d() {
-  // Tile size 5, trip count 4: 4 % 5 != 0, so predicate is needed.
-  // The tile loop should have rectangular bound (the tile size) and
-  // the body should be guarded by a validity predicate.
-  //
+  // Partial last tile (trip count 4, tile 5); -ast-print prints the original loop.
   // CHECK: #pragma omp tile sizes(5)
   // CHECK-NEXT: for (int i = 7; i < 17; i += 3)
   #pragma omp tile sizes(5)
@@ -29,7 +30,7 @@ void rect_tile_2d() {
 
 // CHECK-LABEL: void rect_tile_exact_div(
 void rect_tile_exact_div() {
-  // Tile size 5, trip count 10: 10 % 5 == 0, so predicate is NOT needed.
+  // Exact-dividing trip count (10, tile 5); -ast-print prints the original loop.
   // CHECK: #pragma omp tile sizes(5)
   // CHECK-NEXT: for (int i = 0; i < 10; i += 1)
   #pragma omp tile sizes(5)
@@ -39,8 +40,8 @@ void rect_tile_exact_div() {
 
 // CHECK-LABEL: void rect_tile_nested_body_loop(
 void rect_tile_nested_body_loop(int n) {
-  // After tiling i, the j loop (from the associated body) should be part
-  // of the same canonical nest as the floor and tile loops.
+  // The inner `j` loop lives in the tiled construct's body; -ast-print shows the
+  // directive and the outer `i` loop printed back.
   // CHECK: #pragma omp tile sizes(4)
   // CHECK-NEXT: for (int i = 0; i < 6; i += 1)
   #pragma omp tile sizes(4)
diff --git a/clang/test/OpenMP/tile_rect_codegen_ir.cpp b/clang/test/OpenMP/tile_rect_codegen_ir.cpp
index f2b64ced7c46b..027cd8385044c 100644
--- a/clang/test/OpenMP/tile_rect_codegen_ir.cpp
+++ b/clang/test/OpenMP/tile_rect_codegen_ir.cpp
@@ -1,22 +1,26 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
-// Regression tests for rectangular #pragma omp tile lowering:
-// - Inner loop counter .tile.cnt.* with bound == tile size (constant).
-// - Validity guard (icmp + branch) when remainder iterations exist or trip count
-//   is not proven divisible at compile time.
+// Regression tests for standalone `#pragma omp tile` lowering (no consuming
+// directive). The intra-tile loop is emitted in its min-bounded form
+//   for (.tile.iv = .floor.iv; .tile.iv < min(.floor.iv + tilesize, N); ++.tile.iv)
+// so the upper bound is materialized as a `min()` (cond.true/cond.false/phi over
+// the two `.tile_*.iv`/`.floor_*.iv` counters). There is no `.tile.cnt`
+// rectangular counter and no body-guard predicate here; those only appear when a
+// directive (e.g. `collapse`) consumes the tile -- see
+// tile_collapse_reinterpret_codegen.cpp.
 //
 // RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fopenmp -emit-llvm -o - %s | FileCheck %s --check-prefix=IR
 // expected-no-diagnostics
 
 extern "C" void body(int);
 
-// Trip count 6, tile 4 → remainder; body must be guarded.
+// Trip count 6, tile 4 → the last tile is partial; the min() upper bound stops
+// the tile loop at the trip count (no separate body guard).
 // IR-LABEL: define dso_local void @_Z18remainder_6_tile_4v(
 // IR-SAME: ) #[[ATTR0:[0-9]+]] {
 // IR-NEXT:  [[ENTRY:.*:]]
 // IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    store i32 0, ptr [[I]], align 4
 // IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND:.*]]
@@ -25,39 +29,43 @@ extern "C" void body(int);
 // IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 6
 // IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END9:.*]]
 // IR:       [[FOR_BODY]]:
-// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1:.*]]
 // IR:       [[FOR_COND1]]:
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 4
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
-// IR-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[ADD]], 6
-// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP2]], [[CMP3]]
-// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label %[[FOR_BODY4:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY4]]:
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 6, [[ADD]]
+// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
 // IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD5:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
-// IR-NEXT:    store i32 [[ADD5]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 1
+// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 4
+// IR-NEXT:    br label %[[COND_END]]
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ 6, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
+// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
+// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY5]]:
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
 // IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
 // IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void @body(i32 noundef [[TMP7]])
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void @body(i32 noundef [[TMP6]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP8]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP7]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP2:![0-9]+]]
 // IR:       [[FOR_END]]:
 // IR-NEXT:    br label %[[FOR_INC7:.*]]
 // IR:       [[FOR_INC7]]:
-// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP9]], 4
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP8]], 4
 // IR-NEXT:    store i32 [[ADD8]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP4:![0-9]+]]
 // IR:       [[FOR_END9]]:
@@ -75,48 +83,56 @@ void remainder_6_tile_4(void) {
 // IR-SAME: ) #[[ATTR0]] {
 // IR-NEXT:  [[ENTRY:.*:]]
 // IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    store i32 0, ptr [[I]], align 4
 // IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND:.*]]
 // IR:       [[FOR_COND]]:
 // IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 10
-// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END7:.*]]
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END9:.*]]
 // IR:       [[FOR_BODY]]:
-// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1:.*]]
 // IR:       [[FOR_COND1]]:
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 5
-// IR-NEXT:    br i1 [[CMP2]], label %[[FOR_BODY3:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY3]]:
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
-// IR-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP4]], 1
-// IR-NEXT:    [[ADD4:%.*]] = add nsw i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD4]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void @body(i32 noundef [[TMP5]])
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 5
+// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 10, [[ADD]]
+// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 5
+// IR-NEXT:    br label %[[COND_END]]
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ 10, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
+// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
+// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY5]]:
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
+// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void @body(i32 noundef [[TMP6]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP6]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP7]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP5:![0-9]+]]
 // IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC5:.*]]
-// IR:       [[FOR_INC5]]:
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP7]], 5
-// IR-NEXT:    store i32 [[ADD6]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_INC7:.*]]
+// IR:       [[FOR_INC7]]:
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP8]], 5
+// IR-NEXT:    store i32 [[ADD8]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
-// IR:       [[FOR_END7]]:
+// IR:       [[FOR_END9]]:
 // IR-NEXT:    ret void
 //
 void full_tiles_10_5(void) {
@@ -131,9 +147,8 @@ void full_tiles_10_5(void) {
 // IR-NEXT:  [[ENTRY:.*:]]
 // IR-NEXT:    [[TS_ADDR:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    store i32 [[TS]], ptr [[TS_ADDR]], align 4
 // IR-NEXT:    store i32 0, ptr [[I]], align 4
 // IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
@@ -141,64 +156,78 @@ void full_tiles_10_5(void) {
 // IR:       [[FOR_COND]]:
 // IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 12
-// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END15:.*]]
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END24:.*]]
 // IR:       [[FOR_BODY]]:
-// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1:.*]]
 // IR:       [[FOR_COND1]]:
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TS_ADDR]], align 4
-// IR-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP2]], 0
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TS_ADDR]], align 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], 0
 // IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
 // IR:       [[COND_TRUE]]:
 // IR-NEXT:    br label %[[COND_END:.*]]
 // IR:       [[COND_FALSE]]:
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TS_ADDR]], align 4
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TS_ADDR]], align 4
 // IR-NEXT:    br label %[[COND_END]]
 // IR:       [[COND_END]]:
-// IR-NEXT:    [[COND:%.*]] = phi i32 [ 1, %[[COND_TRUE]] ], [ [[TMP3]], %[[COND_FALSE]] ]
-// IR-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[TMP1]], [[COND]]
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
-// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[ADD]], 12
-// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP3]], [[CMP4]]
-// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label %[[FOR_BODY5:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY5]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ 1, %[[COND_TRUE]] ], [ [[TMP5]], %[[COND_FALSE]] ]
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[COND]]
+// IR-NEXT:    [[CMP3:%.*]] = icmp slt i32 12, [[ADD]]
+// IR-NEXT:    br i1 [[CMP3]], label %[[COND_TRUE4:.*]], label %[[COND_FALSE5:.*]]
+// IR:       [[COND_TRUE4]]:
+// IR-NEXT:    br label %[[COND_END12:.*]]
+// IR:       [[COND_FALSE5]]:
 // IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP6]], [[TMP7]]
-// IR-NEXT:    store i32 [[ADD6]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP8]], 1
-// IR-NEXT:    [[ADD7:%.*]] = add nsw i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void @body(i32 noundef [[TMP9]])
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TS_ADDR]], align 4
+// IR-NEXT:    [[CMP6:%.*]] = icmp sle i32 [[TMP7]], 0
+// IR-NEXT:    br i1 [[CMP6]], label %[[COND_TRUE7:.*]], label %[[COND_FALSE8:.*]]
+// IR:       [[COND_TRUE7]]:
+// IR-NEXT:    br label %[[COND_END9:.*]]
+// IR:       [[COND_FALSE8]]:
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TS_ADDR]], align 4
+// IR-NEXT:    br label %[[COND_END9]]
+// IR:       [[COND_END9]]:
+// IR-NEXT:    [[COND10:%.*]] = phi i32 [ 1, %[[COND_TRUE7]] ], [ [[TMP8]], %[[COND_FALSE8]] ]
+// IR-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP6]], [[COND10]]
+// IR-NEXT:    br label %[[COND_END12]]
+// IR:       [[COND_END12]]:
+// IR-NEXT:    [[COND13:%.*]] = phi i32 [ 12, %[[COND_TRUE4]] ], [ [[ADD11]], %[[COND_END9]] ]
+// IR-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP2]], [[COND13]]
+// IR-NEXT:    br i1 [[CMP14]], label %[[FOR_BODY15:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY15]]:
+// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP9]], 1
+// IR-NEXT:    [[ADD16:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD16]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void @body(i32 noundef [[TMP10]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP7:![0-9]+]]
 // IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC8:.*]]
-// IR:       [[FOR_INC8]]:
-// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TS_ADDR]], align 4
-// IR-NEXT:    [[CMP9:%.*]] = icmp sle i32 [[TMP11]], 0
-// IR-NEXT:    br i1 [[CMP9]], label %[[COND_TRUE10:.*]], label %[[COND_FALSE11:.*]]
-// IR:       [[COND_TRUE10]]:
-// IR-NEXT:    br label %[[COND_END12:.*]]
-// IR:       [[COND_FALSE11]]:
+// IR-NEXT:    br label %[[FOR_INC17:.*]]
+// IR:       [[FOR_INC17]]:
 // IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TS_ADDR]], align 4
-// IR-NEXT:    br label %[[COND_END12]]
-// IR:       [[COND_END12]]:
-// IR-NEXT:    [[COND13:%.*]] = phi i32 [ 1, %[[COND_TRUE10]] ], [ [[TMP12]], %[[COND_FALSE11]] ]
-// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD14:%.*]] = add nsw i32 [[TMP13]], [[COND13]]
-// IR-NEXT:    store i32 [[ADD14]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[CMP18:%.*]] = icmp sle i32 [[TMP12]], 0
+// IR-NEXT:    br i1 [[CMP18]], label %[[COND_TRUE19:.*]], label %[[COND_FALSE20:.*]]
+// IR:       [[COND_TRUE19]]:
+// IR-NEXT:    br label %[[COND_END21:.*]]
+// IR:       [[COND_FALSE20]]:
+// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TS_ADDR]], align 4
+// IR-NEXT:    br label %[[COND_END21]]
+// IR:       [[COND_END21]]:
+// IR-NEXT:    [[COND22:%.*]] = phi i32 [ 1, %[[COND_TRUE19]] ], [ [[TMP13]], %[[COND_FALSE20]] ]
+// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP14]], [[COND22]]
+// IR-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
-// IR:       [[FOR_END15]]:
+// IR:       [[FOR_END24]]:
 // IR-NEXT:    ret void
 //
 void var_tile(int ts) {
@@ -212,13 +241,11 @@ void var_tile(int ts) {
 // IR-SAME: ) #[[ATTR0]] {
 // IR-NEXT:  [[ENTRY:.*:]]
 // IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[J:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // IR-NEXT:    store i32 0, ptr [[I]], align 4
 // IR-NEXT:    store i32 0, ptr [[J]], align 4
 // IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
@@ -226,91 +253,95 @@ void var_tile(int ts) {
 // IR:       [[FOR_COND]]:
 // IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 3
-// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END27:.*]]
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END31:.*]]
 // IR:       [[FOR_BODY]]:
 // IR-NEXT:    store i32 0, ptr [[DOTFLOOR_1_IV_J]], align 4
 // IR-NEXT:    br label %[[FOR_COND1:.*]]
 // IR:       [[FOR_COND1]]:
 // IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 3
-// IR-NEXT:    br i1 [[CMP2]], label %[[FOR_BODY3:.*]], label %[[FOR_END24:.*]]
+// IR-NEXT:    br i1 [[CMP2]], label %[[FOR_BODY3:.*]], label %[[FOR_END28:.*]]
 // IR:       [[FOR_BODY3]]:
-// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP2]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND4:.*]]
 // IR:       [[FOR_COND4]]:
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP2]], 2
-// IR-NEXT:    br i1 [[CMP5]], label %[[FOR_BODY6:.*]], label %[[FOR_END21:.*]]
-// IR:       [[FOR_BODY6]]:
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[TMP4]]
-// IR-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
-// IR-NEXT:    [[ADD7:%.*]] = add nsw i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// IR-NEXT:    br label %[[FOR_COND8:.*]]
-// IR:       [[FOR_COND8]]:
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// IR-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP6]], 2
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP7]], [[TMP8]]
-// IR-NEXT:    [[CMP11:%.*]] = icmp slt i32 [[ADD10]], 3
-// IR-NEXT:    br i1 [[CMP11]], label %[[LAND_RHS:.*]], label %[[LAND_END:.*]]
-// IR:       [[LAND_RHS]]:
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], 2
+// IR-NEXT:    [[CMP5:%.*]] = icmp slt i32 3, [[ADD]]
+// IR-NEXT:    br i1 [[CMP5]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP5]], 2
+// IR-NEXT:    br label %[[COND_END]]
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ 3, %[[COND_TRUE]] ], [ [[ADD6]], %[[COND_FALSE]] ]
+// IR-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP3]], [[COND]]
+// IR-NEXT:    br i1 [[CMP7]], label %[[FOR_BODY8:.*]], label %[[FOR_END25:.*]]
+// IR:       [[FOR_BODY8]]:
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 1
+// IR-NEXT:    [[ADD9:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// IR-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_1_IV_J]], align 4
+// IR-NEXT:    br label %[[FOR_COND10:.*]]
+// IR:       [[FOR_COND10]]:
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
 // IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// IR-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
-// IR-NEXT:    [[CMP13:%.*]] = icmp slt i32 [[ADD12]], 3
-// IR-NEXT:    br label %[[LAND_END]]
-// IR:       [[LAND_END]]:
-// IR-NEXT:    [[TMP11:%.*]] = phi i1 [ false, %[[FOR_COND8]] ], [ [[CMP13]], %[[LAND_RHS]] ]
-// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP9]], [[TMP11]]
-// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label %[[FOR_BODY14:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY14]]:
-// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// IR-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP12]], [[TMP13]]
-// IR-NEXT:    store i32 [[ADD15]], ptr [[DOTTILE_1_IV_J]], align 4
-// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// IR-NEXT:    [[MUL16:%.*]] = mul nsw i32 [[TMP14]], 1
-// IR-NEXT:    [[ADD17:%.*]] = add nsw i32 0, [[MUL16]]
-// IR-NEXT:    store i32 [[ADD17]], ptr [[J]], align 4
-// IR-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    [[TMP16:%.*]] = load i32, ptr [[J]], align 4
-// IR-NEXT:    [[ADD18:%.*]] = add nsw i32 [[TMP15]], [[TMP16]]
-// IR-NEXT:    call void @body(i32 noundef [[ADD18]])
+// IR-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP9]], 2
+// IR-NEXT:    [[CMP12:%.*]] = icmp slt i32 3, [[ADD11]]
+// IR-NEXT:    br i1 [[CMP12]], label %[[COND_TRUE13:.*]], label %[[COND_FALSE14:.*]]
+// IR:       [[COND_TRUE13]]:
+// IR-NEXT:    br label %[[COND_END16:.*]]
+// IR:       [[COND_FALSE14]]:
+// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// IR-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP10]], 2
+// IR-NEXT:    br label %[[COND_END16]]
+// IR:       [[COND_END16]]:
+// IR-NEXT:    [[COND17:%.*]] = phi i32 [ 3, %[[COND_TRUE13]] ], [ [[ADD15]], %[[COND_FALSE14]] ]
+// IR-NEXT:    [[CMP18:%.*]] = icmp slt i32 [[TMP8]], [[COND17]]
+// IR-NEXT:    br i1 [[CMP18]], label %[[FOR_BODY19:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY19]]:
+// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// IR-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[TMP11]], 1
+// IR-NEXT:    [[ADD21:%.*]] = add nsw i32 0, [[MUL20]]
+// IR-NEXT:    store i32 [[ADD21]], ptr [[J]], align 4
+// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
+// IR-NEXT:    [[ADD22:%.*]] = add nsw i32 [[TMP12]], [[TMP13]]
+// IR-NEXT:    call void @body(i32 noundef [[ADD22]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP17]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// IR-NEXT:    br label %[[FOR_COND8]], !llvm.loop [[LOOP9:![0-9]+]]
+// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
+// IR-NEXT:    br label %[[FOR_COND10]], !llvm.loop [[LOOP9:![0-9]+]]
 // IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC19:.*]]
-// IR:       [[FOR_INC19]]:
-// IR-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[INC20:%.*]] = add nsw i32 [[TMP18]], 1
-// IR-NEXT:    store i32 [[INC20]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_INC23:.*]]
+// IR:       [[FOR_INC23]]:
+// IR-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[INC24:%.*]] = add nsw i32 [[TMP15]], 1
+// IR-NEXT:    store i32 [[INC24]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND4]], !llvm.loop [[LOOP10:![0-9]+]]
-// IR:       [[FOR_END21]]:
-// IR-NEXT:    br label %[[FOR_INC22:.*]]
-// IR:       [[FOR_INC22]]:
-// IR-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// IR-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP19]], 2
-// IR-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// IR:       [[FOR_END25]]:
+// IR-NEXT:    br label %[[FOR_INC26:.*]]
+// IR:       [[FOR_INC26]]:
+// IR-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// IR-NEXT:    [[ADD27:%.*]] = add nsw i32 [[TMP16]], 2
+// IR-NEXT:    store i32 [[ADD27]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP11:![0-9]+]]
-// IR:       [[FOR_END24]]:
-// IR-NEXT:    br label %[[FOR_INC25:.*]]
-// IR:       [[FOR_INC25]]:
-// IR-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD26:%.*]] = add nsw i32 [[TMP20]], 2
-// IR-NEXT:    store i32 [[ADD26]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR:       [[FOR_END28]]:
+// IR-NEXT:    br label %[[FOR_INC29:.*]]
+// IR:       [[FOR_INC29]]:
+// IR-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD30:%.*]] = add nsw i32 [[TMP17]], 2
+// IR-NEXT:    store i32 [[ADD30]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP12:![0-9]+]]
-// IR:       [[FOR_END27]]:
+// IR:       [[FOR_END31]]:
 // IR-NEXT:    ret void
 //
 void two_dim(void) {
@@ -320,15 +351,15 @@ void two_dim(void) {
       body(i + j);
 }
 
-// Nested loop in tile body: outer nest still uses .tile.cnt for tiled dim.
+// Nested loop in the tile body: the tiled `i` dim is the floor/tile pair in
+// min-bounded form; the inner `j` loop stays a plain nested loop.
 // IR-LABEL: define dso_local void @_Z14nested_inner_ji(
 // IR-SAME: i32 noundef [[N:%.*]]) #[[ATTR0]] {
 // IR-NEXT:  [[ENTRY:.*:]]
 // IR-NEXT:    [[N_ADDR:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[J:%.*]] = alloca i32, align 4
 // IR-NEXT:    store i32 [[N]], ptr [[N_ADDR]], align 4
 // IR-NEXT:    store i32 0, ptr [[I]], align 4
@@ -339,56 +370,60 @@ void two_dim(void) {
 // IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 6
 // IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END16:.*]]
 // IR:       [[FOR_BODY]]:
-// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1:.*]]
 // IR:       [[FOR_COND1]]:
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 4
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
-// IR-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[ADD]], 6
-// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP2]], [[CMP3]]
-// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label %[[FOR_BODY4:.*]], label %[[FOR_END13:.*]]
-// IR:       [[FOR_BODY4]]:
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 6, [[ADD]]
+// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
 // IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD5:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
-// IR-NEXT:    store i32 [[ADD5]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 1
+// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 4
+// IR-NEXT:    br label %[[COND_END]]
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ 6, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
+// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
+// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END13:.*]]
+// IR:       [[FOR_BODY5]]:
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
 // IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
 // IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
 // IR-NEXT:    store i32 0, ptr [[J]], align 4
 // IR-NEXT:    br label %[[FOR_COND7:.*]]
 // IR:       [[FOR_COND7]]:
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[J]], align 4
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[N_ADDR]], align 4
-// IR-NEXT:    [[CMP8:%.*]] = icmp slt i32 [[TMP7]], [[TMP8]]
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[J]], align 4
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[N_ADDR]], align 4
+// IR-NEXT:    [[CMP8:%.*]] = icmp slt i32 [[TMP6]], [[TMP7]]
 // IR-NEXT:    br i1 [[CMP8]], label %[[FOR_BODY9:.*]], label %[[FOR_END:.*]]
 // IR:       [[FOR_BODY9]]:
-// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[J]], align 4
-// IR-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[J]], align 4
+// IR-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP8]], [[TMP9]]
 // IR-NEXT:    call void @body(i32 noundef [[ADD10]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[J]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
+// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[J]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
 // IR-NEXT:    store i32 [[INC]], ptr [[J]], align 4
 // IR-NEXT:    br label %[[FOR_COND7]], !llvm.loop [[LOOP13:![0-9]+]]
 // IR:       [[FOR_END]]:
 // IR-NEXT:    br label %[[FOR_INC11:.*]]
 // IR:       [[FOR_INC11]]:
-// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[INC12:%.*]] = add nsw i32 [[TMP12]], 1
-// IR-NEXT:    store i32 [[INC12]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[INC12:%.*]] = add nsw i32 [[TMP11]], 1
+// IR-NEXT:    store i32 [[INC12]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP14:![0-9]+]]
 // IR:       [[FOR_END13]]:
 // IR-NEXT:    br label %[[FOR_INC14:.*]]
 // IR:       [[FOR_INC14]]:
-// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP13]], 4
+// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP12]], 4
 // IR-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP15:![0-9]+]]
 // IR:       [[FOR_END16]]:
@@ -401,54 +436,63 @@ void nested_inner_j(int n) {
       body(i + j);
 }
 
-// Predicate elision: when trip count is evenly divisible by constant tile size,
-// no if-guard is needed (no icmp against trip count in tile body).
+// Exact division (trip count 12, tile 4): a standalone tile still emits the
+// min() cond-phi upper bound (it is always present, regardless of divisibility)
+// and never a body-guard predicate.
 // IR-LABEL: define dso_local void @_Z16elided_predicatev(
 // IR-SAME: ) #[[ATTR0]] {
 // IR-NEXT:  [[ENTRY:.*:]]
 // IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    store i32 0, ptr [[I]], align 4
 // IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND:.*]]
 // IR:       [[FOR_COND]]:
 // IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 12
-// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END7:.*]]
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END9:.*]]
 // IR:       [[FOR_BODY]]:
-// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1:.*]]
 // IR:       [[FOR_COND1]]:
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 4
-// IR-NEXT:    br i1 [[CMP2]], label %[[FOR_BODY3:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY3]]:
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
-// IR-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP4]], 1
-// IR-NEXT:    [[ADD4:%.*]] = add nsw i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD4]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void @body(i32 noundef [[TMP5]])
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 12, [[ADD]]
+// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 4
+// IR-NEXT:    br label %[[COND_END]]
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ 12, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
+// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
+// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY5]]:
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
+// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void @body(i32 noundef [[TMP6]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP6]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP7]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP16:![0-9]+]]
 // IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC5:.*]]
-// IR:       [[FOR_INC5]]:
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP7]], 4
-// IR-NEXT:    store i32 [[ADD6]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_INC7:.*]]
+// IR:       [[FOR_INC7]]:
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP8]], 4
+// IR-NEXT:    store i32 [[ADD8]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP17:![0-9]+]]
-// IR:       [[FOR_END7]]:
+// IR:       [[FOR_END9]]:
 // IR-NEXT:    ret void
 //
 void elided_predicate(void) {
diff --git a/clang/test/OpenMP/tile_vectorize_codegen.cpp b/clang/test/OpenMP/tile_vectorize_codegen.cpp
new file mode 100644
index 0000000000000..46fa51b8da57c
--- /dev/null
+++ b/clang/test/OpenMP/tile_vectorize_codegen.cpp
@@ -0,0 +1,14 @@
+// RUN: %clang_cc1 -verify -triple x86_64-unknown-linux-gnu -fclang-abi-compat=latest -std=c++17 -fopenmp -O2 -vectorize-loops -vectorize-slp -emit-llvm %s -o - | FileCheck %s
+// REQUIRES: x86-registered-target
+// expected-no-diagnostics
+
+// Standalone tile stays min-bounded (no body predicate) and vectorizes.
+
+// CHECK-LABEL: define {{.*}} @vec_tile(
+// CHECK: fadd <4 x float>
+// CHECK-NOT: omp.body.next
+extern "C" void vec_tile(float *a, float *b, int n) {
+#pragma omp tile sizes(64)
+  for (int i = 0; i < n; ++i)
+    a[i] = a[i] + b[i];
+}
diff --git a/clang/test/OpenMP/unroll_codegen_tile_for.cpp b/clang/test/OpenMP/unroll_codegen_tile_for.cpp
index cd28ff73a11ec..2e80731c4465d 100644
--- a/clang/test/OpenMP/unroll_codegen_tile_for.cpp
+++ b/clang/test/OpenMP/unroll_codegen_tile_for.cpp
@@ -30,7 +30,6 @@ extern "C" void body(...) {}
 // IR-NEXT:    [[DOTUNROLLED_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTCAPTURE_EXPR_5:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV__UNROLLED_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTCAPTURE_EXPR_13:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_0_IV__UNROLLED_IV_I:%.*]] = alloca i32, align 4
@@ -39,7 +38,7 @@ extern "C" void body(...) {}
 // IR-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_0_IV__UNROLLED_IV_I17:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_CNT_0_IV__UNROLLED_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV__UNROLLED_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTUNROLL_INNER_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
 // IR-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
@@ -118,53 +117,59 @@ extern "C" void body(...) {}
 // IR-NEXT:    [[MUL:%.*]] = mul i32 [[TMP22]], 4
 // IR-NEXT:    [[ADD21:%.*]] = add i32 0, [[MUL]]
 // IR-NEXT:    store i32 [[ADD21]], ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I17]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I17]], align 4
+// IR-NEXT:    store i32 [[TMP23]], ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND:.*]]
 // IR:       [[FOR_COND]]:
-// IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    [[CMP22:%.*]] = icmp ult i32 [[TMP23]], 4
-// IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I17]], align 4
-// IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    [[ADD23:%.*]] = add i32 [[TMP24]], [[TMP25]]
-// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    [[ADD24:%.*]] = add i32 [[TMP26]], 1
-// IR-NEXT:    [[CMP25:%.*]] = icmp ult i32 [[ADD23]], [[ADD24]]
-// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP22]], [[CMP25]]
-// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label %[[FOR_BODY:.*]], label %[[FOR_END39:.*]]
+// IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    [[ADD22:%.*]] = add i32 [[TMP25]], 1
+// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I17]], align 4
+// IR-NEXT:    [[ADD23:%.*]] = add i32 [[TMP26]], 4
+// IR-NEXT:    [[CMP24:%.*]] = icmp ult i32 [[ADD22]], [[ADD23]]
+// IR-NEXT:    br i1 [[CMP24]], label %[[COND_TRUE25:.*]], label %[[COND_FALSE27:.*]]
+// IR:       [[COND_TRUE25]]:
+// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    [[ADD26:%.*]] = add i32 [[TMP27]], 1
+// IR-NEXT:    br label %[[COND_END29:.*]]
+// IR:       [[COND_FALSE27]]:
+// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I17]], align 4
+// IR-NEXT:    [[ADD28:%.*]] = add i32 [[TMP28]], 4
+// IR-NEXT:    br label %[[COND_END29]]
+// IR:       [[COND_END29]]:
+// IR-NEXT:    [[COND30:%.*]] = phi i32 [ [[ADD26]], %[[COND_TRUE25]] ], [ [[ADD28]], %[[COND_FALSE27]] ]
+// IR-NEXT:    [[CMP31:%.*]] = icmp ult i32 [[TMP24]], [[COND30]]
+// IR-NEXT:    br i1 [[CMP31]], label %[[FOR_BODY:.*]], label %[[FOR_END44:.*]]
 // IR:       [[FOR_BODY]]:
-// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I17]], align 4
-// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    [[ADD26:%.*]] = add i32 [[TMP27]], [[TMP28]]
-// IR-NEXT:    store i32 [[ADD26]], ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
 // IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    [[MUL27:%.*]] = mul i32 [[TMP29]], 2
-// IR-NEXT:    [[ADD28:%.*]] = add i32 0, [[MUL27]]
-// IR-NEXT:    store i32 [[ADD28]], ptr [[DOTUNROLLED_IV_I]], align 4
+// IR-NEXT:    [[MUL32:%.*]] = mul i32 [[TMP29]], 2
+// IR-NEXT:    [[ADD33:%.*]] = add i32 0, [[MUL32]]
+// IR-NEXT:    store i32 [[ADD33]], ptr [[DOTUNROLLED_IV_I]], align 4
 // IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTUNROLLED_IV_I]], align 4
 // IR-NEXT:    store i32 [[TMP30]], ptr [[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND29:.*]]
-// IR:       [[FOR_COND29]]:
+// IR-NEXT:    br label %[[FOR_COND34:.*]]
+// IR:       [[FOR_COND34]]:
 // IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
 // IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTUNROLLED_IV_I]], align 4
-// IR-NEXT:    [[ADD30:%.*]] = add i32 [[TMP32]], 2
-// IR-NEXT:    [[CMP31:%.*]] = icmp ult i32 [[TMP31]], [[ADD30]]
-// IR-NEXT:    br i1 [[CMP31]], label %[[LAND_RHS:.*]], label %[[LAND_END:.*]]
+// IR-NEXT:    [[ADD35:%.*]] = add i32 [[TMP32]], 2
+// IR-NEXT:    [[CMP36:%.*]] = icmp ult i32 [[TMP31]], [[ADD35]]
+// IR-NEXT:    br i1 [[CMP36]], label %[[LAND_RHS:.*]], label %[[LAND_END:.*]]
 // IR:       [[LAND_RHS]]:
 // IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
 // IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[ADD32:%.*]] = add i32 [[TMP34]], 1
-// IR-NEXT:    [[CMP33:%.*]] = icmp ult i32 [[TMP33]], [[ADD32]]
+// IR-NEXT:    [[ADD37:%.*]] = add i32 [[TMP34]], 1
+// IR-NEXT:    [[CMP38:%.*]] = icmp ult i32 [[TMP33]], [[ADD37]]
 // IR-NEXT:    br label %[[LAND_END]]
 // IR:       [[LAND_END]]:
-// IR-NEXT:    [[TMP35:%.*]] = phi i1 [ false, %[[FOR_COND29]] ], [ [[CMP33]], %[[LAND_RHS]] ]
-// IR-NEXT:    br i1 [[TMP35]], label %[[FOR_BODY34:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY34]]:
+// IR-NEXT:    [[TMP35:%.*]] = phi i1 [ false, %[[FOR_COND34]] ], [ [[CMP38]], %[[LAND_RHS]] ]
+// IR-NEXT:    br i1 [[TMP35]], label %[[FOR_BODY39:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY39]]:
 // IR-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
 // IR-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
 // IR-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[MUL35:%.*]] = mul i32 [[TMP37]], [[TMP38]]
-// IR-NEXT:    [[ADD36:%.*]] = add i32 [[TMP36]], [[MUL35]]
-// IR-NEXT:    store i32 [[ADD36]], ptr [[I]], align 4
+// IR-NEXT:    [[MUL40:%.*]] = mul i32 [[TMP37]], [[TMP38]]
+// IR-NEXT:    [[ADD41:%.*]] = add i32 [[TMP36]], [[MUL40]]
+// IR-NEXT:    store i32 [[ADD41]], ptr [[I]], align 4
 // IR-NEXT:    [[TMP39:%.*]] = load i32, ptr [[START_ADDR]], align 4
 // IR-NEXT:    [[TMP40:%.*]] = load i32, ptr [[END_ADDR]], align 4
 // IR-NEXT:    [[TMP41:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
@@ -175,22 +180,22 @@ extern "C" void body(...) {}
 // IR-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
 // IR-NEXT:    [[INC:%.*]] = add i32 [[TMP43]], 1
 // IR-NEXT:    store i32 [[INC]], ptr [[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND29]], !llvm.loop [[LOOP2:![0-9]+]]
+// IR-NEXT:    br label %[[FOR_COND34]], !llvm.loop [[LOOP2:![0-9]+]]
 // IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC37:.*]]
-// IR:       [[FOR_INC37]]:
-// IR-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    [[INC38:%.*]] = add i32 [[TMP44]], 1
-// IR-NEXT:    store i32 [[INC38]], ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_INC42:.*]]
+// IR:       [[FOR_INC42]]:
+// IR-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    [[INC43:%.*]] = add i32 [[TMP44]], 1
+// IR-NEXT:    store i32 [[INC43]], ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP5:![0-9]+]]
-// IR:       [[FOR_END39]]:
+// IR:       [[FOR_END44]]:
 // IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.*]]
 // IR:       [[OMP_BODY_CONTINUE]]:
 // IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.*]]
 // IR:       [[OMP_INNER_FOR_INC]]:
 // IR-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[ADD40:%.*]] = add i32 [[TMP45]], 1
-// IR-NEXT:    store i32 [[ADD40]], ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[ADD45:%.*]] = add i32 [[TMP45]], 1
+// IR-NEXT:    store i32 [[ADD45]], ptr [[DOTOMP_IV]], align 4
 // IR-NEXT:    br label %[[OMP_INNER_FOR_COND]]
 // IR:       [[OMP_INNER_FOR_END]]:
 // IR-NEXT:    br label %[[OMP_LOOP_EXIT:.*]]
diff --git a/openmp/runtime/test/transform/tile/foreach.cpp b/openmp/runtime/test/transform/tile/foreach.cpp
index 608a1c896fcb4..4fb3595760974 100644
--- a/openmp/runtime/test/transform/tile/foreach.cpp
+++ b/openmp/runtime/test/transform/tile/foreach.cpp
@@ -206,10 +206,6 @@ int main() {
 // CHECK-NEXT: v=2 w=1
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
-// CHECK-NEXT: [A] iterator advance: 0 += 3
-// CHECK-NEXT: [A] iterator move assign
-// CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 2
@@ -219,10 +215,6 @@ int main() {
 // CHECK-NEXT: v=2 w=2
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
-// CHECK-NEXT: [A] iterator advance: 0 += 3
-// CHECK-NEXT: [A] iterator move assign
-// CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
diff --git a/openmp/runtime/test/transform/tile/intfor.c b/openmp/runtime/test/transform/tile/intfor.c
index c5ceddd83da70..05ef5e85a0dad 100644
--- a/openmp/runtime/test/transform/tile/intfor.c
+++ b/openmp/runtime/test/transform/tile/intfor.c
@@ -30,49 +30,83 @@ int main() {
 // CHECK:      do
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: i=7 j=7
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: i=7 j=10
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: i=7 j=13
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: i=10 j=7
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: i=10 j=10
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: i=10 j=13
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: i=13 j=7
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: i=13 j=10
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: i=13 j=13
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: i=7 j=16
@@ -83,6 +117,8 @@ int main() {
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: i=10 j=16
@@ -93,6 +129,8 @@ int main() {
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: i=13 j=16
@@ -103,6 +141,8 @@ int main() {
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
@@ -111,21 +151,21 @@ int main() {
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: i=16 j=7
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: i=16 j=10
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=16 j=13
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: i=16 j=13
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
@@ -148,12 +188,4 @@ int main() {
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/tile/iterfor.cpp b/openmp/runtime/test/transform/tile/iterfor.cpp
index 4b9bf6f41ada6..05b5677bf9c4f 100644
--- a/openmp/runtime/test/transform/tile/iterfor.cpp
+++ b/openmp/runtime/test/transform/tile/iterfor.cpp
@@ -215,9 +215,6 @@ int main() {
 // CHECK-NEXT: i=2 j=1
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
-// CHECK-NEXT: [A] iterator advance: 0 += 3
-// CHECK-NEXT: [A] iterator move assign
-// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [B] iterator advance: 0 += 2
@@ -227,9 +224,6 @@ int main() {
 // CHECK-NEXT: i=2 j=2
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
-// CHECK-NEXT: [A] iterator advance: 0 += 3
-// CHECK-NEXT: [A] iterator move assign
-// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
diff --git a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-foreach.cpp b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-foreach.cpp
index a007a369453d4..b1f4d98a52ddc 100644
--- a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-foreach.cpp
+++ b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-foreach.cpp
@@ -206,10 +206,6 @@ int main() {
 // CHECK-NEXT: i=0 v=2 w=1
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
-// CHECK-NEXT: [A] iterator advance: 0 += 3
-// CHECK-NEXT: [A] iterator move assign
-// CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 2
@@ -219,10 +215,6 @@ int main() {
 // CHECK-NEXT: i=0 v=2 w=2
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
-// CHECK-NEXT: [A] iterator advance: 0 += 3
-// CHECK-NEXT: [A] iterator move assign
-// CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 0
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 0
@@ -283,10 +275,6 @@ int main() {
 // CHECK-NEXT: i=1 v=2 w=1
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
-// CHECK-NEXT: [A] iterator advance: 0 += 3
-// CHECK-NEXT: [A] iterator move assign
-// CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 2
@@ -296,10 +284,6 @@ int main() {
 // CHECK-NEXT: i=1 v=2 w=2
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
-// CHECK-NEXT: [A] iterator advance: 0 += 3
-// CHECK-NEXT: [A] iterator move assign
-// CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 0
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 0
@@ -360,10 +344,6 @@ int main() {
 // CHECK-NEXT: i=2 v=2 w=1
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
-// CHECK-NEXT: [A] iterator advance: 0 += 3
-// CHECK-NEXT: [A] iterator move assign
-// CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 2
@@ -373,10 +353,6 @@ int main() {
 // CHECK-NEXT: i=2 v=2 w=2
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
-// CHECK-NEXT: [A] iterator advance: 0 += 3
-// CHECK-NEXT: [A] iterator move assign
-// CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
diff --git a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-stacked-tile-deep.cpp b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-stacked-tile-deep.cpp
new file mode 100644
index 0000000000000..1e835db20fb81
--- /dev/null
+++ b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-stacked-tile-deep.cpp
@@ -0,0 +1,32 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+
+// XFAIL: *
+// collapse(3) through stacked tiles is not supported.
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdio>
+
+int main() {
+  printf("do\n");
+#pragma omp parallel for collapse(3) num_threads(1)
+#pragma omp tile sizes(2)
+#pragma omp tile sizes(4)
+  for (int i = 0; i < 6; ++i)
+    printf("i=%d\n", i);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: i=0
+// CHECK-NEXT: i=1
+// CHECK-NEXT: i=2
+// CHECK-NEXT: i=3
+// CHECK-NEXT: i=4
+// CHECK-NEXT: i=5
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-stacked-tile.cpp b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-stacked-tile.cpp
new file mode 100644
index 0000000000000..feb780e8d3aec
--- /dev/null
+++ b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-stacked-tile.cpp
@@ -0,0 +1,31 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+
+// collapse(2) on stacked tiles: only outer floor+tile consumed (works).
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdio>
+
+int main() {
+  printf("do\n");
+#pragma omp parallel for collapse(2) num_threads(1)
+#pragma omp tile sizes(2)
+#pragma omp tile sizes(4)
+  for (int i = 0; i < 6; ++i)
+    printf("i=%d\n", i);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: i=0
+// CHECK-NEXT: i=1
+// CHECK-NEXT: i=2
+// CHECK-NEXT: i=3
+// CHECK-NEXT: i=4
+// CHECK-NEXT: i=5
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile-2d.cpp b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile-2d.cpp
new file mode 100644
index 0000000000000..64c31fd9470a5
--- /dev/null
+++ b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile-2d.cpp
@@ -0,0 +1,56 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+// RUN: %libomp-cxx-compile -O2 && %libomp-run | FileCheck %s --match-full-lines
+
+// 2-D tile consumed by collapse(4) (full) and collapse(3) (partial).
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdio>
+
+int main() {
+  printf("full4\n");
+#pragma omp parallel for collapse(4) num_threads(1)
+#pragma omp tile sizes(2, 2)
+  for (int i = 0; i < 3; ++i)
+    for (int j = 0; j < 3; ++j)
+      printf("i=%d j=%d\n", i, j);
+
+  printf("part3\n");
+#pragma omp parallel for collapse(3) num_threads(1)
+#pragma omp tile sizes(2, 2)
+  for (int i = 0; i < 3; ++i)
+    for (int j = 0; j < 4; ++j)
+      printf("i=%d j=%d\n", i, j);
+
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      full4
+// CHECK-NEXT: i=0 j=0
+// CHECK-NEXT: i=0 j=1
+// CHECK-NEXT: i=1 j=0
+// CHECK-NEXT: i=1 j=1
+// CHECK-NEXT: i=0 j=2
+// CHECK-NEXT: i=1 j=2
+// CHECK-NEXT: i=2 j=0
+// CHECK-NEXT: i=2 j=1
+// CHECK-NEXT: i=2 j=2
+// CHECK-NEXT: part3
+// CHECK-NEXT: i=0 j=0
+// CHECK-NEXT: i=0 j=1
+// CHECK-NEXT: i=1 j=0
+// CHECK-NEXT: i=1 j=1
+// CHECK-NEXT: i=0 j=2
+// CHECK-NEXT: i=0 j=3
+// CHECK-NEXT: i=1 j=2
+// CHECK-NEXT: i=1 j=3
+// CHECK-NEXT: i=2 j=0
+// CHECK-NEXT: i=2 j=1
+// CHECK-NEXT: i=2 j=2
+// CHECK-NEXT: i=2 j=3
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile-compose.cpp b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile-compose.cpp
new file mode 100644
index 0000000000000..15fe9e72564fc
--- /dev/null
+++ b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile-compose.cpp
@@ -0,0 +1,51 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+// RUN: %libomp-cxx-compile -O2 && %libomp-run | FileCheck %s --match-full-lines
+
+// collapse consuming a tile composed with reverse.
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdio>
+
+int main() {
+  printf("rev2\n");
+#pragma omp parallel for collapse(2) num_threads(1)
+#pragma omp tile sizes(2)
+#pragma omp reverse
+  for (int i = 0; i < 5; ++i)
+    printf("i=%d\n", i);
+
+  printf("rev3\n");
+#pragma omp parallel for collapse(3) num_threads(1)
+#pragma omp tile sizes(2)
+#pragma omp reverse
+  for (int i = 0; i < 5; ++i)
+    for (int j = 0; j < 2; ++j)
+      printf("i=%d j=%d\n", i, j);
+
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      rev2
+// CHECK-NEXT: i=4
+// CHECK-NEXT: i=3
+// CHECK-NEXT: i=2
+// CHECK-NEXT: i=1
+// CHECK-NEXT: i=0
+// CHECK-NEXT: rev3
+// CHECK-NEXT: i=4 j=0
+// CHECK-NEXT: i=4 j=1
+// CHECK-NEXT: i=3 j=0
+// CHECK-NEXT: i=3 j=1
+// CHECK-NEXT: i=2 j=0
+// CHECK-NEXT: i=2 j=1
+// CHECK-NEXT: i=1 j=0
+// CHECK-NEXT: i=1 j=1
+// CHECK-NEXT: i=0 j=0
+// CHECK-NEXT: i=0 j=1
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile-iterator.cpp b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile-iterator.cpp
new file mode 100644
index 0000000000000..94f3399aef9e9
--- /dev/null
+++ b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile-iterator.cpp
@@ -0,0 +1,74 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+// RUN: %libomp-cxx-compile -O2 && %libomp-run | FileCheck %s --match-full-lines
+
+// collapse consuming a tiled range-based-for loop.
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdio>
+
+struct Range {
+  int lo, hi;
+  struct It {
+    int v;
+    int operator*() const { return v; }
+    It &operator++() {
+      ++v;
+      return *this;
+    }
+    It operator++(int) {
+      It t = *this;
+      ++v;
+      return t;
+    }
+    bool operator==(const It &o) const { return v == o.v; }
+    bool operator!=(const It &o) const { return v != o.v; }
+    int operator-(const It &o) const { return v - o.v; }
+    It operator+(int n) const { return It{v + n}; }
+  };
+  It begin() const { return It{lo}; }
+  It end() const { return It{hi}; }
+};
+
+int main() {
+  Range r{0, 5};
+
+  printf("iter2\n");
+#pragma omp parallel for collapse(2) num_threads(1)
+#pragma omp tile sizes(2)
+  for (int v : r)
+    printf("v=%d\n", v);
+
+  printf("iter3\n");
+#pragma omp parallel for collapse(3) num_threads(1)
+#pragma omp tile sizes(2)
+  for (int v : r)
+    for (int j = 0; j < 2; ++j)
+      printf("v=%d j=%d\n", v, j);
+
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      iter2
+// CHECK-NEXT: v=0
+// CHECK-NEXT: v=1
+// CHECK-NEXT: v=2
+// CHECK-NEXT: v=3
+// CHECK-NEXT: v=4
+// CHECK-NEXT: iter3
+// CHECK-NEXT: v=0 j=0
+// CHECK-NEXT: v=0 j=1
+// CHECK-NEXT: v=1 j=0
+// CHECK-NEXT: v=1 j=1
+// CHECK-NEXT: v=2 j=0
+// CHECK-NEXT: v=2 j=1
+// CHECK-NEXT: v=3 j=0
+// CHECK-NEXT: v=3 j=1
+// CHECK-NEXT: v=4 j=0
+// CHECK-NEXT: v=4 j=1
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile.cpp b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile.cpp
index c7f7e143f7983..2cdd3ea33703f 100644
--- a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile.cpp
+++ b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile.cpp
@@ -1,9 +1,7 @@
 // RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+// RUN: %libomp-cxx-compile -O2 && %libomp-run | FileCheck %s --match-full-lines
 
-// After tiling `i`, the logical loops are `floor_i`, `tile_i`, and `j`, so
-// `collapse(3)` must reach through the tile guard to include `j`. CodeGen must
-// then keep that guard around the collapsed body once, instead of emitting `j`
-// again as a normal nested loop.
+// collapse(3) reaches through a tiled loop to include the inner `j` loop.
 
 #ifndef HEADER
 #define HEADER
@@ -25,8 +23,6 @@ int main() {
 #endif /* HEADER */
 
 // CHECK:      do
-
-// Full tile (.floor.i=0): i=0..3
 // CHECK-NEXT: i=0 j=0
 // CHECK-NEXT: i=0 j=1
 // CHECK-NEXT: i=1 j=0
@@ -35,8 +31,6 @@ int main() {
 // CHECK-NEXT: i=2 j=1
 // CHECK-NEXT: i=3 j=0
 // CHECK-NEXT: i=3 j=1
-
-// Partial tile (.floor.i=4): i=4,5 in bounds; i=6,7 skipped by the predicate
 // CHECK-NEXT: i=4 j=0
 // CHECK-NEXT: i=4 j=1
 // CHECK-NEXT: i=5 j=0

>From 8444507cc1cf84cc91ad87a6a920a40d7e9a8257 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Tue, 11 Aug 2026 15:08:11 -0400
Subject: [PATCH 17/18] revised

---
 clang/include/clang/AST/StmtOpenMP.h          |  36 ++++-
 clang/include/clang/Basic/Attr.td             |   4 +-
 .../clang/Basic/DiagnosticSemaKinds.td        |   3 +
 clang/lib/AST/StmtOpenMP.cpp                  |  47 +++---
 clang/lib/CodeGen/CGStmtOpenMP.cpp            |  17 +--
 clang/lib/Sema/SemaOpenMP.cpp                 | 143 ++++++++++++------
 6 files changed, 163 insertions(+), 87 deletions(-)

diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index dbc76e7df8ecd..33aada14bb642 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -25,6 +25,8 @@
 
 namespace clang {
 
+class OMPInvariantPredicateBoundAttr;
+
 //===----------------------------------------------------------------------===//
 // AST classes for directives.
 //===----------------------------------------------------------------------===//
@@ -892,6 +894,31 @@ class OMPLoopBasedDirective : public OMPExecutableDirective {
                                   TryImperfectlyNestedLoops);
   }
 
+  /// Returns the intra-tile reinterpretation hint attached to \p S, or nullptr
+  /// if \p S does not carry one. See OMPInvariantPredicateBoundAttr.
+  static const OMPInvariantPredicateBoundAttr *getIntraTileHint(const Stmt *S);
+
+  /// If \p S is an intra-tile reinterpretation wrapper, returns the loop it
+  /// annotates; otherwise returns \p S unchanged.
+  static Stmt *ignoreIntraTileHint(Stmt *S);
+  static const Stmt *ignoreIntraTileHint(const Stmt *S) {
+    return ignoreIntraTileHint(const_cast<Stmt *>(S));
+  }
+
+  /// Calls the specified callback function for all the loops in \p CurStmt,
+  /// from the outermost to the innermost. The callback is always passed a
+  /// ForStmt or CXXForRangeStmt; when that loop carries the intra-tile
+  /// reinterpretation hint, it is delivered as a separate argument instead of
+  /// wrapping the loop, so callers that only expect a loop cannot trip over it.
+  static bool
+  doForAllLoops(Stmt *CurStmt, bool TryImperfectlyNestedLoops,
+                unsigned NumLoops,
+                llvm::function_ref<bool(unsigned, Stmt *,
+                                        const OMPInvariantPredicateBoundAttr *)>
+                    Callback,
+                llvm::function_ref<void(OMPLoopTransformationDirective *)>
+                    OnTransformationCallback);
+
   /// Calls the specified callback function for all the loops in \p CurStmt,
   /// from the outermost to the innermost.
   static bool
@@ -899,7 +926,14 @@ class OMPLoopBasedDirective : public OMPExecutableDirective {
                 unsigned NumLoops,
                 llvm::function_ref<bool(unsigned, Stmt *)> Callback,
                 llvm::function_ref<void(OMPLoopTransformationDirective *)>
-                    OnTransformationCallback);
+                    OnTransformationCallback) {
+    auto &&NewCallback = [Callback](unsigned Cnt, Stmt *Loop,
+                                    const OMPInvariantPredicateBoundAttr *) {
+      return Callback(Cnt, Loop);
+    };
+    return doForAllLoops(CurStmt, TryImperfectlyNestedLoops, NumLoops,
+                         NewCallback, OnTransformationCallback);
+  }
   static bool
   doForAllLoops(const Stmt *CurStmt, bool TryImperfectlyNestedLoops,
                 unsigned NumLoops,
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 7d90c40c5ecf9..443a4874fd3e0 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -4881,8 +4881,8 @@ def OMPInvariantPredicateBound : StmtAttr {
   let Spellings = [];
   let SemaHandler = 0;
   let Subjects = SubjectList<[Stmt], ErrorDiag>;
-  let Args = [ExprArgument<"RectCond">, ExprArgument<"Predicate">,
-              ExprArgument<"TileSize">];
+  let Args = [ExprArgument<"RectCond">, ExprArgument<"TileSize">,
+              ExprArgument<"Predicate", 1>];
   let Documentation = [InternalOnly];
 }
 
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 01527e87c903f..b27e4e9e03e04 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12303,6 +12303,9 @@ def err_omp_loop_bad_collapse_var : Error<
   "cannot use variable %1 in collapsed imperfectly-nested loop %select{init|condition|increment}0 statement">;
 def err_omp_loop_var_reused_in_collapsed_loop : Error<
   "loop iteration variable %0 cannot be reused in a nested loop of a collapsed loop nest">;
+def err_omp_collapse_stacked_tile : Error<
+  "cannot collapse the intra-tile loop of a '#pragma omp tile' that is nested "
+  "inside another loop-transforming directive">;
 def err_omp_simd_region_cannot_use_stmt : Error<
   "'%0' statement cannot be used in OpenMP simd region">;
 def warn_omp_loop_64_bit_var : Warning<
diff --git a/clang/lib/AST/StmtOpenMP.cpp b/clang/lib/AST/StmtOpenMP.cpp
index 26000198518a8..827f8143cf744 100644
--- a/clang/lib/AST/StmtOpenMP.cpp
+++ b/clang/lib/AST/StmtOpenMP.cpp
@@ -18,22 +18,19 @@
 using namespace clang;
 using namespace llvm::omp;
 
-/// Returns true if the attributed statement carries the internal `omp tile`
-/// intra-tile reinterpretation hint (see OMPInvariantPredicateBoundAttr).
-static bool hasOMPInvariantPredicateBound(const AttributedStmt *AS) {
-  for (const Attr *A : AS->getAttrs())
-    if (isa<OMPInvariantPredicateBoundAttr>(A))
-      return true;
-  return false;
-}
-
-/// Peeks through an intra-tile hint wrapper to return the underlying loop;
-/// otherwise returns \p S unchanged. Used by the loop walker once the hint has
-/// already been delivered to the analysis callback and we need to look through
-/// the wrapper to reach the loop body.
-static Stmt *ignoreOMPIntraTileHint(Stmt *S) {
+/// Returns the intra-tile hint attribute for the given statement.
+const OMPInvariantPredicateBoundAttr *
+OMPLoopBasedDirective::getIntraTileHint(const Stmt *S) {
+  if (const auto *AS = dyn_cast_or_null<AttributedStmt>(S))
+    return getSpecificAttr<OMPInvariantPredicateBoundAttr>(AS->getAttrs());
+  return nullptr;
+}
+
+/// Peek through an intra-tile hint wrapper to return the underlying loop.
+/// The hint is delivered as the third argument to the loop callback.
+Stmt *OMPLoopBasedDirective::ignoreIntraTileHint(Stmt *S) {
   if (auto *AS = dyn_cast_or_null<AttributedStmt>(S))
-    if (hasOMPInvariantPredicateBound(AS))
+    if (hasSpecificAttr<OMPInvariantPredicateBoundAttr>(AS->getAttrs()))
       return AS->getSubStmt();
   return S;
 }
@@ -45,7 +42,7 @@ static Stmt *ignoreOMPIntraTileHint(Stmt *S) {
 static Stmt *ignoreContainersKeepingIntraTileHint(Stmt *S) {
   while (true) {
     if (auto *AS = dyn_cast_or_null<AttributedStmt>(S)) {
-      if (hasOMPInvariantPredicateBound(AS))
+      if (hasSpecificAttr<OMPInvariantPredicateBoundAttr>(AS->getAttrs()))
         break;
       S = AS->getSubStmt();
       continue;
@@ -144,7 +141,7 @@ OMPLoopBasedDirective::tryToFindNextInnerLoop(Stmt *CurStmt,
           Stmt *Inner = S;
           if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(Inner))
             Inner = CanonLoop->getLoopStmt();
-          Inner = ignoreOMPIntraTileHint(Inner);
+          Inner = OMPLoopBasedDirective::ignoreIntraTileHint(Inner);
           if (isa<ForStmt>(Inner) || isa<CXXForRangeStmt>(Inner) ||
               (isa<OMPLoopBasedDirective>(Inner) &&
                !isa<OMPLoopDirective>(Inner))) {
@@ -176,7 +173,9 @@ OMPLoopBasedDirective::tryToFindNextInnerLoop(Stmt *CurStmt,
 
 bool OMPLoopBasedDirective::doForAllLoops(
     Stmt *CurStmt, bool TryImperfectlyNestedLoops, unsigned NumLoops,
-    llvm::function_ref<bool(unsigned, Stmt *)> Callback,
+    llvm::function_ref<bool(unsigned, Stmt *,
+                            const OMPInvariantPredicateBoundAttr *)>
+        Callback,
     llvm::function_ref<void(OMPLoopTransformationDirective *)>
         OnTransformationCallback) {
   CurStmt = ignoreContainersKeepingIntraTileHint(CurStmt);
@@ -209,12 +208,12 @@ bool OMPLoopBasedDirective::doForAllLoops(
     }
     if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(CurStmt))
       CurStmt = CanonLoop->getLoopStmt();
-    // Deliver the (possibly hint-wrapped) loop node to the callback so it can
-    // read the intra-tile reinterpretation hint, then look through the wrapper
-    // to reach the loop body.
-    if (Callback(Cnt, CurStmt))
+    // The walker keeps the intra-tile hint wrapper around so that it can be
+    // handed to the analysis, but the callback only ever sees the loop itself.
+    const OMPInvariantPredicateBoundAttr *Hint = getIntraTileHint(CurStmt);
+    Stmt *LoopStmt = ignoreIntraTileHint(CurStmt);
+    if (Callback(Cnt, LoopStmt, Hint))
       return false;
-    Stmt *LoopStmt = ignoreOMPIntraTileHint(CurStmt);
     // Move on to the next nested for loop, or to the loop body.
     // OpenMP [2.8.1, simd construct, Restrictions]
     // All loops associated with the construct must be perfectly nested; that
@@ -239,8 +238,6 @@ void OMPLoopBasedDirective::doForAllLoopsBodies(
   bool Res = OMPLoopBasedDirective::doForAllLoops(
       CurStmt, TryImperfectlyNestedLoops, NumLoops,
       [Callback](unsigned Cnt, Stmt *Loop) {
-        // Look through an intra-tile hint wrapper to reach the loop itself.
-        Loop = ignoreOMPIntraTileHint(Loop);
         Stmt *Body = nullptr;
         if (auto *For = dyn_cast<ForStmt>(Loop)) {
           Body = For->getBody();
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 509d4f3d35815..3e87db54d9fd1 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -2231,17 +2231,6 @@ class OMPTransformDirectiveScopeRAII {
 };
 } // namespace
 
-/// Helper function to look through the droppable intra-tile hint wrapper
-/// (OMPInvariantPredicateBoundAttr) to the loop statement it annotates.
-static const Stmt *lookThroughOMPIntraTileHint(const Stmt *S) {
-  if (const auto *AS = dyn_cast_or_null<AttributedStmt>(S))
-    if (llvm::any_of(AS->getAttrs(), [](const Attr *A) {
-          return isa<OMPInvariantPredicateBoundAttr>(A);
-        }))
-      return AS->getSubStmt();
-  return S;
-}
-
 static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop,
                      int MaxLevel, int Level = 0) {
   assert(Level < MaxLevel && "Too deep lookup during loop body codegen.");
@@ -2258,10 +2247,10 @@ static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop,
     return;
   }
 
-  // The loop walker may hand back the intra-tile hint wrapper as `NextLoop`;
-  // match against the loop it annotates. The tile overshoot guard is emitted
+  // `tryToFindNextInnerLoop` keeps the intra-tile hint wrapper around, so match
+  // against the loop it annotates. The tile overshoot guard is emitted
   // separately via EmitOMPLoopBody's finals-conditions handling.
-  if (SimplifiedS == lookThroughOMPIntraTileHint(NextLoop)) {
+  if (SimplifiedS == OMPLoopBasedDirective::ignoreIntraTileHint(NextLoop)) {
     if (auto *Dir = dyn_cast<OMPLoopTransformationDirective>(SimplifiedS))
       SimplifiedS = Dir->getTransformedStmt();
     if (const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(SimplifiedS))
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 701ba2d9f127d..e1097e75c66d4 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -17,6 +17,7 @@
 #include "TreeTransform.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/ASTMutationListener.h"
+#include "clang/AST/Attr.h"
 #include "clang/AST/CXXInheritance.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclCXX.h"
@@ -8021,6 +8022,8 @@ struct LoopIterationSpace final {
   /// True if this is a reinterpreted intra-tile loop whose lower bound is a
   /// live floor dependence.
   bool ReinterpretTileLB = false;
+  /// Location of the reinterpreted intra-tile loop, for diagnostics.
+  SourceLocation ReinterpretTileLoc;
 };
 
 /// Scan an AST subtree, checking that no decls in the CollapsedLoopVarDecls
@@ -9578,24 +9581,24 @@ static bool checkOpenMPIterationSpace(
     llvm::MutableArrayRef<LoopIterationSpace> ResultIterSpaces,
     llvm::MapVector<const Expr *, DeclRefExpr *> &Captures,
     const llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopVarDecls,
-    llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopInductionVars) {
+    llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopInductionVars,
+    const OMPInvariantPredicateBoundAttr *IntraTileHint) {
   bool SupportsNonRectangular = !isOpenMPLoopTransformationDirective(DKind);
   // See the tile reinterpretation design note in ActOnOpenMPTileDirective.
   // If the loop carries the hint, analyze its rectangular form instead.
+  //
+  // Only directives that emit the per-iteration body guard (i.e. those going
+  // through EmitOMPLoopBody's finals-conditions handling) may reinterpret the
+  // loop; a loop transformation such as an enclosing 'tile' has nowhere to put
+  // the overshoot guard, so for those the stored min-bounded form is analyzed
+  // exactly as it is on a build without this hint.
   Expr *TileRectCond = nullptr;
   Expr *TileBodyPredicate = nullptr;
   Expr *TileTripCount = nullptr;
-  if (auto *AS = dyn_cast_or_null<AttributedStmt>(S)) {
-    for (const Attr *A : AS->getAttrs()) {
-      if (const auto *H = dyn_cast<OMPInvariantPredicateBoundAttr>(A)) {
-        TileRectCond = H->getRectCond();
-        TileBodyPredicate = H->getPredicate();
-        TileTripCount = H->getTileSize();
-        break;
-      }
-    }
-    if (TileRectCond)
-      S = AS->getSubStmt();
+  if (IntraTileHint && !isOpenMPLoopTransformationDirective(DKind)) {
+    TileRectCond = IntraTileHint->getRectCond();
+    TileBodyPredicate = IntraTileHint->getPredicate();
+    TileTripCount = IntraTileHint->getTileSize();
   }
   // OpenMP [2.9.1, Canonical Loop Form]
   //   for (init-expr; test-expr; incr-expr) structured-block
@@ -9693,11 +9696,13 @@ static bool checkOpenMPIterationSpace(
     return HasErrors;
 
   // Build the loop's iteration space representation.
-  ResultIterSpaces[CurrentNestedLoopCount].PreCond =
-      ISC.buildPreCond(DSA.getCurScope(), EffectiveCond, Captures);
-  // Reinterpreted intra-tile: precondition is trivially true (.floor.iv
-  // unreadable here); overshoot handled by Predicate.
-  if (TileRectCond)
+  //
+  // A reinterpreted intra-tile loop takes all three of these from the hint
+  // instead of deriving them: the precondition is trivially true (the floor
+  // loop's own precondition already covers an empty iteration space), the trip
+  // count is the constant tile size, and any overshoot on the remainder tile
+  // is handled by the body guard.
+  if (TileRectCond) {
     ResultIterSpaces[CurrentNestedLoopCount].PreCond =
         SemaRef
             .PerformImplicitConversion(
@@ -9705,17 +9710,19 @@ static bool checkOpenMPIterationSpace(
                 SemaRef.Context.BoolTy, AssignmentAction::Casting,
                 /*AllowExplicit=*/true)
             .get();
-  ResultIterSpaces[CurrentNestedLoopCount].NumIterations =
-      ISC.buildNumIterations(DSA.getCurScope(), ResultIterSpaces,
-                             (isOpenMPWorksharingDirective(DKind) ||
-                              isOpenMPGenericLoopDirective(DKind) ||
-                              isOpenMPTaskLoopDirective(DKind) ||
-                              isOpenMPDistributeDirective(DKind) ||
-                              isOpenMPLoopTransformationDirective(DKind)),
-                             Captures);
-  // Reinterpreted intra-tile: trip count is the constant TileSize.
-  if (TileTripCount)
     ResultIterSpaces[CurrentNestedLoopCount].NumIterations = TileTripCount;
+  } else {
+    ResultIterSpaces[CurrentNestedLoopCount].PreCond =
+        ISC.buildPreCond(DSA.getCurScope(), EffectiveCond, Captures);
+    ResultIterSpaces[CurrentNestedLoopCount].NumIterations =
+        ISC.buildNumIterations(DSA.getCurScope(), ResultIterSpaces,
+                               (isOpenMPWorksharingDirective(DKind) ||
+                                isOpenMPGenericLoopDirective(DKind) ||
+                                isOpenMPTaskLoopDirective(DKind) ||
+                                isOpenMPDistributeDirective(DKind) ||
+                                isOpenMPLoopTransformationDirective(DKind)),
+                               Captures);
+  }
   ResultIterSpaces[CurrentNestedLoopCount].CounterVar =
       ISC.buildCounterVar(Captures, DSA);
   ResultIterSpaces[CurrentNestedLoopCount].PrivateCounterVar =
@@ -9733,15 +9740,19 @@ static bool checkOpenMPIterationSpace(
   std::tie(ResultIterSpaces[CurrentNestedLoopCount].MinValue,
            ResultIterSpaces[CurrentNestedLoopCount].MaxValue) =
       ISC.buildMinMaxValues(DSA.getCurScope(), Captures);
-  ResultIterSpaces[CurrentNestedLoopCount].FinalCondition =
-      ISC.buildFinalCondition(DSA.getCurScope());
-  // Reinterpreted intra-tile: body guard is the overshoot Predicate.
-  if (TileBodyPredicate)
+  if (TileRectCond) {
+    // The body guard is the overshoot predicate, which is absent when the tile
+    // can never be partial. Lower bound is the live .floor.iv; wire as a floor
+    // dependence for collapse
+    // (see design note in ActOnOpenMPTileDirective).
     ResultIterSpaces[CurrentNestedLoopCount].FinalCondition = TileBodyPredicate;
-  // Lower bound is the live .floor.iv; wire as a floor dependence for collapse
-  // (see design note in ActOnOpenMPTileDirective).
-  if (TileRectCond)
     ResultIterSpaces[CurrentNestedLoopCount].ReinterpretTileLB = true;
+    ResultIterSpaces[CurrentNestedLoopCount].ReinterpretTileLoc =
+        ISC.getInitSrcRange().getBegin();
+  } else {
+    ResultIterSpaces[CurrentNestedLoopCount].FinalCondition =
+        ISC.buildFinalCondition(DSA.getCurScope());
+  }
   ResultIterSpaces[CurrentNestedLoopCount].IsNonRectangularLB =
       ISC.doesInitDependOnLC();
   ResultIterSpaces[CurrentNestedLoopCount].IsNonRectangularUB =
@@ -10139,12 +10150,15 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
           [DKind, &SemaRef, &DSA, NumLoops, NestedLoopCount,
            CollapseLoopCountExpr, OrderedLoopCountExpr, &VarsWithImplicitDSA,
            &IterSpaces, &Captures, &CollapsedLoopVarDecls,
-           &CollapsedLoopInductionVars](unsigned Cnt, Stmt *CurStmt) {
+           &CollapsedLoopInductionVars](
+              unsigned Cnt, Stmt *CurStmt,
+              const OMPInvariantPredicateBoundAttr *IntraTileHint) {
             if (checkOpenMPIterationSpace(
                     DKind, CurStmt, SemaRef, DSA, Cnt, NestedLoopCount,
                     NumLoops, CollapseLoopCountExpr, OrderedLoopCountExpr,
                     VarsWithImplicitDSA, IterSpaces, Captures,
-                    CollapsedLoopVarDecls, CollapsedLoopInductionVars))
+                    CollapsedLoopVarDecls, CollapsedLoopInductionVars,
+                    IntraTileHint))
               return true;
             if (Cnt > 0 && Cnt >= NestedLoopCount &&
                 IterSpaces[Cnt].CounterVar) {
@@ -10669,6 +10683,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
       // Wire the reinterpreted lower bound to its floor dependence (see design
       // note in ActOnOpenMPTileDirective).
       if (IS.ReinterpretTileLB && IS.CounterInit) {
+        bool FoundFloor = false;
         if (const auto *LBRef =
                 dyn_cast<DeclRefExpr>(IS.CounterInit->IgnoreParenImpCasts())) {
           const Decl *FloorDecl = LBRef->getDecl()->getCanonicalDecl();
@@ -10677,10 +10692,22 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
             if (CV && CV->getDecl()->getCanonicalDecl() == FloorDecl) {
               IS.IsNonRectangularLB = true;
               IS.LoopDependentIdx = K + 1;
+              FoundFloor = true;
               break;
             }
           }
         }
+        if (!FoundFloor) {
+          // The floor this intra-tile loop starts from is not one of the
+          // collapsed counters: it is assigned in the body of an enclosing
+          // transformed loop, so the collapsed nest would read a stale value
+          // instead of recomputing it per iteration. Diagnose to avoid silently
+          // producing the wrong iteration space.
+          SemaRef.Diag(IS.ReinterpretTileLoc,
+                       diag::err_omp_collapse_stacked_tile);
+          HasErrors = true;
+          break;
+        }
       }
 
       // Compute prod
@@ -15242,9 +15269,11 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
   // below carrying an equivalent rectangular reinterpretation:
   //   RectCond  : .tile.iv < .floor.iv + T   -- rectangular bound, analyzed by
   //               checkOpenMPIterationSpace in place of the stored condition
-  //   Predicate : .tile.iv < N               -- remainder-tile overshoot,
-  //               applied as a body guard instead of shortening the trip count
   //   TileSize  : T                          -- constant per-tile trip count
+  //   Predicate : .tile.iv < N               -- remainder-tile overshoot,
+  //               applied as a body guard instead of shortening the trip count;
+  //               omitted when N is known to be a multiple of T, as there is
+  //               then no partial tile to guard against
   // Because the reinterpreted lower bound is still `.floor.iv`, checkOpenMPLoop
   // wires it as a live dependence on the matching floor counter, so a
   // collapsed `.tile.iv` re-reads the floor's current value instead of a
@@ -15314,8 +15343,8 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     if (!IncrStmt.isUsable())
       return StmtError();
 
-    // Build OMPInvariantPredicateBoundAttr hint (RectCond, Predicate,
-    // TileSize).
+    // Build OMPInvariantPredicateBoundAttr hint (RectCond, TileSize,
+    // Predicate).
     Expr *RectDimTileSize = MakeDimTileSize(I);
     if (!RectDimTileSize)
       return StmtError();
@@ -15328,18 +15357,41 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     ExprResult RectCond =
         SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
                            MakeTileIVRef(), RectEndOfTile.get());
-    ExprResult Predicate =
-        SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
-                           MakeTileIVRef(), NumIterations);
     Expr *TileSizeExpr = MakeDimTileSize(I);
     if (!TileSizeExpr)
       return StmtError();
     ExprResult TileSize = SemaRef.PerformImplicitConversion(
         TileSizeExpr, IVTy, AssignmentAction::Converting,
         /*AllowExplicit=*/true);
-    if (!RectCond.isUsable() || !Predicate.isUsable() || !TileSize.isUsable())
+    if (!RectCond.isUsable() || !TileSize.isUsable())
       return StmtError();
 
+    // The overshoot guard is only needed if the last tile can be partial. When
+    // both the trip count and the tile size are known at compile time and the
+    // former is a multiple of the latter, every tile is full, so leave the
+    // predicate out. A dependent or run-time value makes
+    // the evaluation fail, which keeps the guard.
+    bool NoPartialTile = false;
+    Expr::EvalResult TileSizeVal, NumIterationsVal;
+    if (TileSizeExpr->EvaluateAsInt(TileSizeVal, Context) &&
+        NumIterations->EvaluateAsInt(NumIterationsVal, Context)) {
+      llvm::APSInt TS = TileSizeVal.Val.getInt();
+      llvm::APSInt N = NumIterationsVal.Val.getInt();
+      unsigned Width = std::max(TS.getBitWidth(), N.getBitWidth());
+      TS = TS.extend(Width);
+      N = N.extend(Width);
+      NoPartialTile =
+          TS.isStrictlyPositive() && N.isStrictlyPositive() && N.urem(TS) == 0;
+    }
+
+    ExprResult Predicate;
+    if (!NoPartialTile) {
+      Predicate = SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(),
+                                     BO_LT, MakeTileIVRef(), NumIterations);
+      if (!Predicate.isUsable())
+        return StmtError();
+    }
+
     // Statements to set the original iteration variable's value from the
     // logical iteration number.
     // Generated for loop is:
@@ -15370,7 +15422,8 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
 
     // Attach the droppable reinterpretation attribute to the intra-tile loop.
     auto *Hint = OMPInvariantPredicateBoundAttr::CreateImplicit(
-        Context, RectCond.get(), Predicate.get(), TileSize.get(),
+        Context, RectCond.get(), TileSize.get(),
+        Predicate.isUsable() ? Predicate.get() : nullptr,
         Inner->getSourceRange());
     Inner =
         AttributedStmt::Create(Context, Inner->getBeginLoc(), {Hint}, Inner);

>From 0cff8d5501e2430d82a04594da6190f7fddc20e7 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Tue, 11 Aug 2026 16:22:13 -0400
Subject: [PATCH 18/18] tests-updated

---
 .../OpenMP/tile_codegen_for_dependent.cpp     | 310 ++++++-------
 clang/test/OpenMP/tile_codegen_tile_for.cpp   | 421 +++++++++---------
 clang/test/OpenMP/tile_collapse_messages.cpp  |  63 +++
 .../tile_collapse_reinterpret_codegen.cpp     | 116 +++--
 clang/test/OpenMP/tile_messages.cpp           |  30 ++
 clang/test/OpenMP/tile_rect_codegen.cpp       |  51 ---
 clang/test/OpenMP/tile_rect_codegen_ir.cpp    | 375 +---------------
 clang/test/OpenMP/tile_vectorize_codegen.cpp  |  18 +-
 clang/test/OpenMP/unroll_codegen_tile_for.cpp | 408 +++++++++--------
 clang/unittests/AST/AttrTest.cpp              |  66 +++
 openmp/runtime/test/transform/tile/intfor.c   |   2 +-
 ...llel-wsloop-collapse-stacked-tile-deep.cpp |  32 --
 12 files changed, 875 insertions(+), 1017 deletions(-)
 create mode 100644 clang/test/OpenMP/tile_collapse_messages.cpp
 delete mode 100644 clang/test/OpenMP/tile_rect_codegen.cpp
 delete mode 100644 openmp/runtime/test/transform/tile/parallel-wsloop-collapse-stacked-tile-deep.cpp

diff --git a/clang/test/OpenMP/tile_codegen_for_dependent.cpp b/clang/test/OpenMP/tile_codegen_for_dependent.cpp
index 7401ab1a2ded3..820d33d15287b 100644
--- a/clang/test/OpenMP/tile_codegen_for_dependent.cpp
+++ b/clang/test/OpenMP/tile_codegen_for_dependent.cpp
@@ -1,4 +1,3 @@
-// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
 // Check code generation
 // RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fopenmp -emit-llvm %s -o - | FileCheck %s --check-prefix=IR
 
@@ -18,155 +17,172 @@
 extern "C" void body(...) {}
 
 
-// IR-LABEL: define dso_local void @func(
-// IR-SAME: i32 noundef [[START:%.*]], i32 noundef [[END:%.*]], i32 noundef [[STEP:%.*]]) #[[ATTR0:[0-9]+]] {
-// IR-NEXT:  [[ENTRY:.*:]]
-// IR-NEXT:    [[START_ADDR:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[TMP:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_5:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV_I11:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
-// IR-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
-// IR-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
-// IR-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// IR-NEXT:    store i32 [[TMP1]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// IR-NEXT:    store i32 [[TMP2]], ptr [[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// IR-NEXT:    store i32 [[TMP3]], ptr [[DOTCAPTURE_EXPR_1]], align 4
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// IR-NEXT:    store i32 [[TMP4]], ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    [[SUB:%.*]] = sub i32 [[TMP5]], [[TMP6]]
-// IR-NEXT:    [[SUB3:%.*]] = sub i32 [[SUB]], 1
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add i32 [[SUB3]], [[TMP7]]
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP8]]
-// IR-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
-// IR-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[ADD6:%.*]] = add i32 [[TMP9]], 1
-// IR-NEXT:    store i32 [[ADD6]], ptr [[DOTCAPTURE_EXPR_5]], align 4
-// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// IR-NEXT:    [[SUB8:%.*]] = sub i32 [[TMP10]], -3
-// IR-NEXT:    [[DIV9:%.*]] = udiv i32 [[SUB8]], 4
-// IR-NEXT:    [[SUB10:%.*]] = sub i32 [[DIV9]], 1
-// IR-NEXT:    store i32 [[SUB10]], ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// IR-NEXT:    [[CMP:%.*]] = icmp ult i32 0, [[TMP11]]
-// IR-NEXT:    br i1 [[CMP]], label %[[OMP_PRECOND_THEN:.*]], label %[[OMP_PRECOND_END:.*]]
-// IR:       [[OMP_PRECOND_THEN]]:
-// IR-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
-// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    store i32 [[TMP12]], ptr [[DOTOMP_UB]], align 4
-// IR-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
-// IR-NEXT:    call void @__kmpc_for_static_init_4u(ptr @[[GLOB1:[0-9]+]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
-// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    [[CMP12:%.*]] = icmp ugt i32 [[TMP13]], [[TMP14]]
-// IR-NEXT:    br i1 [[CMP12]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
-// IR:       [[COND_TRUE]]:
-// IR-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    br label %[[COND_END:.*]]
-// IR:       [[COND_FALSE]]:
-// IR-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// IR-LABEL: define {{.*}}@func(
+// IR-NEXT:  [[ENTRY:.*]]:
+// IR-NEXT:    %[[START_ADDR:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[END_ADDR:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[STEP_ADDR:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_IV:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[TMP:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[I:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_1:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTNEW_STEP:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_2:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_5:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_7:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTFLOOR_0_IV_I:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_LB:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_UB:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_STRIDE:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_IS_LAST:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTFLOOR_0_IV_I11:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTTILE_0_IV_I:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[TMP0:.+]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:.+]])
+// IR-NEXT:    store i32 %[[START:.+]], ptr %[[START_ADDR]], align 4
+// IR-NEXT:    store i32 %[[END:.+]], ptr %[[END_ADDR]], align 4
+// IR-NEXT:    store i32 %[[STEP:.+]], ptr %[[STEP_ADDR]], align 4
+// IR-NEXT:    %[[TMP1:.+]] = load i32, ptr %[[START_ADDR]], align 4
+// IR-NEXT:    store i32 %[[TMP1]], ptr %[[I]], align 4
+// IR-NEXT:    %[[TMP2:.+]] = load i32, ptr %[[START_ADDR]], align 4
+// IR-NEXT:    store i32 %[[TMP2]], ptr %[[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    %[[TMP3:.+]] = load i32, ptr %[[END_ADDR]], align 4
+// IR-NEXT:    store i32 %[[TMP3]], ptr %[[DOTCAPTURE_EXPR_1]], align 4
+// IR-NEXT:    %[[TMP4:.+]] = load i32, ptr %[[STEP_ADDR]], align 4
+// IR-NEXT:    store i32 %[[TMP4]], ptr %[[DOTNEW_STEP]], align 4
+// IR-NEXT:    %[[TMP5:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_1]], align 4
+// IR-NEXT:    %[[TMP6:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    %[[SUB:.+]] = sub i32 %[[TMP5]], %[[TMP6]]
+// IR-NEXT:    %[[SUB3:.+]] = sub i32 %[[SUB]], 1
+// IR-NEXT:    %[[TMP7:.+]] = load i32, ptr %[[DOTNEW_STEP]], align 4
+// IR-NEXT:    %[[ADD:.+]] = add i32 %[[SUB3]], %[[TMP7]]
+// IR-NEXT:    %[[TMP8:.+]] = load i32, ptr %[[DOTNEW_STEP]], align 4
+// IR-NEXT:    %[[DIV:.+]] = udiv i32 %[[ADD]], %[[TMP8]]
+// IR-NEXT:    %[[SUB4:.+]] = sub i32 %[[DIV]], 1
+// IR-NEXT:    store i32 %[[SUB4]], ptr %[[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    %[[TMP9:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    %[[ADD6:.+]] = add i32 %[[TMP9]], 1
+// IR-NEXT:    store i32 %[[ADD6]], ptr %[[DOTCAPTURE_EXPR_5]], align 4
+// IR-NEXT:    %[[TMP10:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_5]], align 4
+// IR-NEXT:    %[[SUB8:.+]] = sub i32 %[[TMP10]], -3
+// IR-NEXT:    %[[DIV9:.+]] = udiv i32 %[[SUB8]], 4
+// IR-NEXT:    %[[SUB10:.+]] = sub i32 %[[DIV9]], 1
+// IR-NEXT:    store i32 %[[SUB10]], ptr %[[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    store i32 0, ptr %[[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    %[[TMP11:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_5]], align 4
+// IR-NEXT:    %[[CMP:.+]] = icmp ult i32 0, %[[TMP11]]
+// IR-NEXT:    br i1 %[[CMP]], label %[[OMP_PRECOND_THEN:.+]], label %[[OMP_PRECOND_END:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_PRECOND_THEN]]:
+// IR-NEXT:    store i32 0, ptr %[[DOTOMP_LB]], align 4
+// IR-NEXT:    %[[TMP12:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    store i32 %[[TMP12]], ptr %[[DOTOMP_UB]], align 4
+// IR-NEXT:    store i32 1, ptr %[[DOTOMP_STRIDE]], align 4
+// IR-NEXT:    store i32 0, ptr %[[DOTOMP_IS_LAST]], align 4
+// IR-NEXT:    call void @__kmpc_for_static_init_4u(ptr @[[GLOB1:.+]], i32 %[[TMP0]], i32 34, ptr %[[DOTOMP_IS_LAST]], ptr %[[DOTOMP_LB]], ptr %[[DOTOMP_UB]], ptr %[[DOTOMP_STRIDE]], i32 1, i32 1)
+// IR-NEXT:    %[[TMP13:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
+// IR-NEXT:    %[[TMP14:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    %[[CMP12:.+]] = icmp ugt i32 %[[TMP13]], %[[TMP14]]
+// IR-NEXT:    br i1 %[[CMP12]], label %[[COND_TRUE:.+]], label %[[COND_FALSE:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_TRUE]]:
+// IR-NEXT:    %[[TMP15:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    br label %[[COND_END:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_FALSE]]:
+// IR-NEXT:    %[[TMP16:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
 // IR-NEXT:    br label %[[COND_END]]
-// IR:       [[COND_END]]:
-// IR-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP15]], %[[COND_TRUE]] ], [ [[TMP16]], %[[COND_FALSE]] ]
-// IR-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
-// IR-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
-// IR-NEXT:    store i32 [[TMP17]], ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    br label %[[OMP_INNER_FOR_COND:.*]]
-// IR:       [[OMP_INNER_FOR_COND]]:
-// IR-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// IR-NEXT:    [[ADD13:%.*]] = add i32 [[TMP19]], 1
-// IR-NEXT:    [[CMP14:%.*]] = icmp ult i32 [[TMP18]], [[ADD13]]
-// IR-NEXT:    br i1 [[CMP14]], label %[[OMP_INNER_FOR_BODY:.*]], label %[[OMP_INNER_FOR_END:.*]]
-// IR:       [[OMP_INNER_FOR_BODY]]:
-// IR-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul i32 [[TMP20]], 4
-// IR-NEXT:    [[ADD15:%.*]] = add i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I11]], align 4
-// IR-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I11]], align 4
-// IR-NEXT:    store i32 [[TMP21]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND:.*]]
-// IR:       [[FOR_COND]]:
-// IR-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[ADD16:%.*]] = add i32 [[TMP23]], 1
-// IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I11]], align 4
-// IR-NEXT:    [[ADD17:%.*]] = add i32 [[TMP24]], 4
-// IR-NEXT:    [[CMP18:%.*]] = icmp ult i32 [[ADD16]], [[ADD17]]
-// IR-NEXT:    br i1 [[CMP18]], label %[[COND_TRUE19:.*]], label %[[COND_FALSE21:.*]]
-// IR:       [[COND_TRUE19]]:
-// IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[ADD20:%.*]] = add i32 [[TMP25]], 1
-// IR-NEXT:    br label %[[COND_END23:.*]]
-// IR:       [[COND_FALSE21]]:
-// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I11]], align 4
-// IR-NEXT:    [[ADD22:%.*]] = add i32 [[TMP26]], 4
+// IR-EMPTY:
+// IR-NEXT:  [[COND_END]]:
+// IR-NEXT:    %[[COND:.+]] = phi i32 [ %[[TMP15]], %[[COND_TRUE]] ], [ %[[TMP16]], %[[COND_FALSE]] ]
+// IR-NEXT:    store i32 %[[COND]], ptr %[[DOTOMP_UB]], align 4
+// IR-NEXT:    %[[TMP17:.+]] = load i32, ptr %[[DOTOMP_LB]], align 4
+// IR-NEXT:    store i32 %[[TMP17]], ptr %[[DOTOMP_IV]], align 4
+// IR-NEXT:    br label %[[OMP_INNER_FOR_COND:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_INNER_FOR_COND]]:
+// IR-NEXT:    %[[TMP18:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
+// IR-NEXT:    %[[TMP19:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
+// IR-NEXT:    %[[ADD13:.+]] = add i32 %[[TMP19]], 1
+// IR-NEXT:    %[[CMP14:.+]] = icmp ult i32 %[[TMP18]], %[[ADD13]]
+// IR-NEXT:    br i1 %[[CMP14]], label %[[OMP_INNER_FOR_BODY:.+]], label %[[OMP_INNER_FOR_END:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_INNER_FOR_BODY]]:
+// IR-NEXT:    %[[TMP20:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
+// IR-NEXT:    %[[MUL:.+]] = mul i32 %[[TMP20]], 4
+// IR-NEXT:    %[[ADD15:.+]] = add i32 0, %[[MUL]]
+// IR-NEXT:    store i32 %[[ADD15]], ptr %[[DOTFLOOR_0_IV_I11]], align 4
+// IR-NEXT:    %[[TMP21:.+]] = load i32, ptr %[[DOTFLOOR_0_IV_I11]], align 4
+// IR-NEXT:    store i32 %[[TMP21]], ptr %[[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_COND]]:
+// IR-NEXT:    %[[TMP22:.+]] = load i32, ptr %[[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    %[[TMP23:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    %[[ADD16:.+]] = add i32 %[[TMP23]], 1
+// IR-NEXT:    %[[TMP24:.+]] = load i32, ptr %[[DOTFLOOR_0_IV_I11]], align 4
+// IR-NEXT:    %[[ADD17:.+]] = add i32 %[[TMP24]], 4
+// IR-NEXT:    %[[CMP18:.+]] = icmp ult i32 %[[ADD16]], %[[ADD17]]
+// IR-NEXT:    br i1 %[[CMP18]], label %[[COND_TRUE19:.+]], label %[[COND_FALSE21:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_TRUE19]]:
+// IR-NEXT:    %[[TMP25:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    %[[ADD20:.+]] = add i32 %[[TMP25]], 1
+// IR-NEXT:    br label %[[COND_END23:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_FALSE21]]:
+// IR-NEXT:    %[[TMP26:.+]] = load i32, ptr %[[DOTFLOOR_0_IV_I11]], align 4
+// IR-NEXT:    %[[ADD22:.+]] = add i32 %[[TMP26]], 4
 // IR-NEXT:    br label %[[COND_END23]]
-// IR:       [[COND_END23]]:
-// IR-NEXT:    [[COND24:%.*]] = phi i32 [ [[ADD20]], %[[COND_TRUE19]] ], [ [[ADD22]], %[[COND_FALSE21]] ]
-// IR-NEXT:    [[CMP25:%.*]] = icmp ult i32 [[TMP22]], [[COND24]]
-// IR-NEXT:    br i1 [[CMP25]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY]]:
-// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[MUL26:%.*]] = mul i32 [[TMP28]], [[TMP29]]
-// IR-NEXT:    [[ADD27:%.*]] = add i32 [[TMP27]], [[MUL26]]
-// IR-NEXT:    store i32 [[ADD27]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void (...) @body(i32 noundef [[TMP30]], i32 noundef [[TMP31]], i32 noundef [[TMP32]], i32 noundef [[TMP33]])
-// IR-NEXT:    br label %[[FOR_INC:.*]]
-// IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add i32 [[TMP34]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP2:![0-9]+]]
-// IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.*]]
-// IR:       [[OMP_BODY_CONTINUE]]:
-// IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.*]]
-// IR:       [[OMP_INNER_FOR_INC]]:
-// IR-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[ADD28:%.*]] = add i32 [[TMP35]], 1
-// IR-NEXT:    store i32 [[ADD28]], ptr [[DOTOMP_IV]], align 4
+// IR-EMPTY:
+// IR-NEXT:  [[COND_END23]]:
+// IR-NEXT:    %[[COND24:.+]] = phi i32 [ %[[ADD20]], %[[COND_TRUE19]] ], [ %[[ADD22]], %[[COND_FALSE21]] ]
+// IR-NEXT:    %[[CMP25:.+]] = icmp ult i32 %[[TMP22]], %[[COND24]]
+// IR-NEXT:    br i1 %[[CMP25]], label %[[FOR_BODY:.+]], label %[[FOR_END:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_BODY]]:
+// IR-NEXT:    %[[TMP27:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    %[[TMP28:.+]] = load i32, ptr %[[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    %[[TMP29:.+]] = load i32, ptr %[[DOTNEW_STEP]], align 4
+// IR-NEXT:    %[[MUL26:.+]] = mul i32 %[[TMP28]], %[[TMP29]]
+// IR-NEXT:    %[[ADD27:.+]] = add i32 %[[TMP27]], %[[MUL26]]
+// IR-NEXT:    store i32 %[[ADD27]], ptr %[[I]], align 4
+// IR-NEXT:    %[[TMP30:.+]] = load i32, ptr %[[START_ADDR]], align 4
+// IR-NEXT:    %[[TMP31:.+]] = load i32, ptr %[[END_ADDR]], align 4
+// IR-NEXT:    %[[TMP32:.+]] = load i32, ptr %[[STEP_ADDR]], align 4
+// IR-NEXT:    %[[TMP33:.+]] = load i32, ptr %[[I]], align 4
+// IR-NEXT:    call void (...) @body(i32 noundef %[[TMP30]], i32 noundef %[[TMP31]], i32 noundef %[[TMP32]], i32 noundef %[[TMP33]])
+// IR-NEXT:    br label %[[FOR_INC:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_INC]]:
+// IR-NEXT:    %[[TMP34:.+]] = load i32, ptr %[[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    %[[INC:.+]] = add i32 %[[TMP34]], 1
+// IR-NEXT:    store i32 %[[INC]], ptr %[[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop ![[LOOP3:[0-9]+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_END]]:
+// IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_BODY_CONTINUE]]:
+// IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_INNER_FOR_INC]]:
+// IR-NEXT:    %[[TMP35:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
+// IR-NEXT:    %[[ADD28:.+]] = add i32 %[[TMP35]], 1
+// IR-NEXT:    store i32 %[[ADD28]], ptr %[[DOTOMP_IV]], align 4
 // IR-NEXT:    br label %[[OMP_INNER_FOR_COND]]
-// IR:       [[OMP_INNER_FOR_END]]:
-// IR-NEXT:    br label %[[OMP_LOOP_EXIT:.*]]
-// IR:       [[OMP_LOOP_EXIT]]:
-// IR-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP0]])
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_INNER_FOR_END]]:
+// IR-NEXT:    br label %[[OMP_LOOP_EXIT:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_LOOP_EXIT]]:
+// IR-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 %[[TMP0]])
 // IR-NEXT:    br label %[[OMP_PRECOND_END]]
-// IR:       [[OMP_PRECOND_END]]:
-// IR-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3:[0-9]+]], i32 [[TMP0]])
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_PRECOND_END]]:
+// IR-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3:.+]], i32 %[[TMP0]])
 // IR-NEXT:    ret void
-//
+// IR-NEXT:  }
 extern "C" void func(int start, int end, int step) {
 #pragma omp for
 #pragma omp tile sizes(4)
@@ -175,7 +191,3 @@ extern "C" void func(int start, int end, int step) {
 }
 
 #endif /* HEADER */
-//.
-// IR: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]]}
-// IR: [[META3]] = !{!"llvm.loop.mustprogress"}
-//.
diff --git a/clang/test/OpenMP/tile_codegen_tile_for.cpp b/clang/test/OpenMP/tile_codegen_tile_for.cpp
index 3d0092042cdfc..8a844c78c54a6 100644
--- a/clang/test/OpenMP/tile_codegen_tile_for.cpp
+++ b/clang/test/OpenMP/tile_codegen_tile_for.cpp
@@ -1,4 +1,3 @@
-// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
 // Check code generation
 // RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fopenmp -emit-llvm %s -o - | FileCheck %s --check-prefix=IR
 
@@ -17,205 +16,229 @@
 extern "C" void body(...) {}
 
 
-
-
-// IR-LABEL: define dso_local void @func(
-// IR-SAME: i32 noundef [[START:%.*]], i32 noundef [[END:%.*]], i32 noundef [[STEP:%.*]]) #[[ATTR0:[0-9]+]] {
-// IR-NEXT:  [[ENTRY:.*:]]
-// IR-NEXT:    [[START_ADDR:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[TMP:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_5:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_13:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV__FLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV__FLOOR_0_IV_I17:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV__FLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
-// IR-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
-// IR-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
-// IR-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// IR-NEXT:    store i32 [[TMP1]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// IR-NEXT:    store i32 [[TMP2]], ptr [[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// IR-NEXT:    store i32 [[TMP3]], ptr [[DOTCAPTURE_EXPR_1]], align 4
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// IR-NEXT:    store i32 [[TMP4]], ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    [[SUB:%.*]] = sub i32 [[TMP5]], [[TMP6]]
-// IR-NEXT:    [[SUB3:%.*]] = sub i32 [[SUB]], 1
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add i32 [[SUB3]], [[TMP7]]
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP8]]
-// IR-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
-// IR-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[ADD6:%.*]] = add i32 [[TMP9]], 1
-// IR-NEXT:    store i32 [[ADD6]], ptr [[DOTCAPTURE_EXPR_5]], align 4
-// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// IR-NEXT:    [[SUB8:%.*]] = sub i32 [[TMP10]], -3
-// IR-NEXT:    [[DIV9:%.*]] = udiv i32 [[SUB8]], 4
-// IR-NEXT:    [[SUB10:%.*]] = sub i32 [[DIV9]], 1
-// IR-NEXT:    store i32 [[SUB10]], ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    [[ADD12:%.*]] = add i32 [[TMP11]], 1
-// IR-NEXT:    store i32 [[ADD12]], ptr [[DOTCAPTURE_EXPR_11]], align 4
-// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// IR-NEXT:    [[SUB14:%.*]] = sub i32 [[TMP12]], -2
-// IR-NEXT:    [[DIV15:%.*]] = udiv i32 [[SUB14]], 3
-// IR-NEXT:    [[SUB16:%.*]] = sub i32 [[DIV15]], 1
-// IR-NEXT:    store i32 [[SUB16]], ptr [[DOTCAPTURE_EXPR_13]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// IR-NEXT:    [[CMP:%.*]] = icmp ult i32 0, [[TMP13]]
-// IR-NEXT:    br i1 [[CMP]], label %[[OMP_PRECOND_THEN:.*]], label %[[OMP_PRECOND_END:.*]]
-// IR:       [[OMP_PRECOND_THEN]]:
-// IR-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
-// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_13]], align 4
-// IR-NEXT:    store i32 [[TMP14]], ptr [[DOTOMP_UB]], align 4
-// IR-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
-// IR-NEXT:    call void @__kmpc_for_static_init_4u(ptr @[[GLOB1:[0-9]+]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
-// IR-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// IR-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_13]], align 4
-// IR-NEXT:    [[CMP18:%.*]] = icmp ugt i32 [[TMP15]], [[TMP16]]
-// IR-NEXT:    br i1 [[CMP18]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
-// IR:       [[COND_TRUE]]:
-// IR-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_13]], align 4
-// IR-NEXT:    br label %[[COND_END:.*]]
-// IR:       [[COND_FALSE]]:
-// IR-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// IR-LABEL: define {{.*}}@func(
+// IR-NEXT:  [[ENTRY:.*]]:
+// IR-NEXT:    %[[START_ADDR:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[END_ADDR:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[STEP_ADDR:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_IV:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[TMP:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[I:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_1:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTNEW_STEP:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_2:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTFLOOR_0_IV_I:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_5:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_7:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_11:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_13:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTFLOOR_0_IV__FLOOR_0_IV_I:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_LB:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_UB:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_STRIDE:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_IS_LAST:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTFLOOR_0_IV__FLOOR_0_IV_I17:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTTILE_0_IV__FLOOR_0_IV_I:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTTILE_0_IV_I:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[TMP0:.+]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:.+]])
+// IR-NEXT:    store i32 %[[START:.+]], ptr %[[START_ADDR]], align 4
+// IR-NEXT:    store i32 %[[END:.+]], ptr %[[END_ADDR]], align 4
+// IR-NEXT:    store i32 %[[STEP:.+]], ptr %[[STEP_ADDR]], align 4
+// IR-NEXT:    %[[TMP1:.+]] = load i32, ptr %[[START_ADDR]], align 4
+// IR-NEXT:    store i32 %[[TMP1]], ptr %[[I]], align 4
+// IR-NEXT:    %[[TMP2:.+]] = load i32, ptr %[[START_ADDR]], align 4
+// IR-NEXT:    store i32 %[[TMP2]], ptr %[[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    %[[TMP3:.+]] = load i32, ptr %[[END_ADDR]], align 4
+// IR-NEXT:    store i32 %[[TMP3]], ptr %[[DOTCAPTURE_EXPR_1]], align 4
+// IR-NEXT:    %[[TMP4:.+]] = load i32, ptr %[[STEP_ADDR]], align 4
+// IR-NEXT:    store i32 %[[TMP4]], ptr %[[DOTNEW_STEP]], align 4
+// IR-NEXT:    %[[TMP5:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_1]], align 4
+// IR-NEXT:    %[[TMP6:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    %[[SUB:.+]] = sub i32 %[[TMP5]], %[[TMP6]]
+// IR-NEXT:    %[[SUB3:.+]] = sub i32 %[[SUB]], 1
+// IR-NEXT:    %[[TMP7:.+]] = load i32, ptr %[[DOTNEW_STEP]], align 4
+// IR-NEXT:    %[[ADD:.+]] = add i32 %[[SUB3]], %[[TMP7]]
+// IR-NEXT:    %[[TMP8:.+]] = load i32, ptr %[[DOTNEW_STEP]], align 4
+// IR-NEXT:    %[[DIV:.+]] = udiv i32 %[[ADD]], %[[TMP8]]
+// IR-NEXT:    %[[SUB4:.+]] = sub i32 %[[DIV]], 1
+// IR-NEXT:    store i32 %[[SUB4]], ptr %[[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    store i32 0, ptr %[[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    %[[TMP9:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    %[[ADD6:.+]] = add i32 %[[TMP9]], 1
+// IR-NEXT:    store i32 %[[ADD6]], ptr %[[DOTCAPTURE_EXPR_5]], align 4
+// IR-NEXT:    %[[TMP10:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_5]], align 4
+// IR-NEXT:    %[[SUB8:.+]] = sub i32 %[[TMP10]], -3
+// IR-NEXT:    %[[DIV9:.+]] = udiv i32 %[[SUB8]], 4
+// IR-NEXT:    %[[SUB10:.+]] = sub i32 %[[DIV9]], 1
+// IR-NEXT:    store i32 %[[SUB10]], ptr %[[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    %[[TMP11:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    %[[ADD12:.+]] = add i32 %[[TMP11]], 1
+// IR-NEXT:    store i32 %[[ADD12]], ptr %[[DOTCAPTURE_EXPR_11]], align 4
+// IR-NEXT:    %[[TMP12:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_11]], align 4
+// IR-NEXT:    %[[SUB14:.+]] = sub i32 %[[TMP12]], -2
+// IR-NEXT:    %[[DIV15:.+]] = udiv i32 %[[SUB14]], 3
+// IR-NEXT:    %[[SUB16:.+]] = sub i32 %[[DIV15]], 1
+// IR-NEXT:    store i32 %[[SUB16]], ptr %[[DOTCAPTURE_EXPR_13]], align 4
+// IR-NEXT:    store i32 0, ptr %[[DOTFLOOR_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    %[[TMP13:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_11]], align 4
+// IR-NEXT:    %[[CMP:.+]] = icmp ult i32 0, %[[TMP13]]
+// IR-NEXT:    br i1 %[[CMP]], label %[[OMP_PRECOND_THEN:.+]], label %[[OMP_PRECOND_END:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_PRECOND_THEN]]:
+// IR-NEXT:    store i32 0, ptr %[[DOTOMP_LB]], align 4
+// IR-NEXT:    %[[TMP14:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_13]], align 4
+// IR-NEXT:    store i32 %[[TMP14]], ptr %[[DOTOMP_UB]], align 4
+// IR-NEXT:    store i32 1, ptr %[[DOTOMP_STRIDE]], align 4
+// IR-NEXT:    store i32 0, ptr %[[DOTOMP_IS_LAST]], align 4
+// IR-NEXT:    call void @__kmpc_for_static_init_4u(ptr @[[GLOB1:.+]], i32 %[[TMP0]], i32 34, ptr %[[DOTOMP_IS_LAST]], ptr %[[DOTOMP_LB]], ptr %[[DOTOMP_UB]], ptr %[[DOTOMP_STRIDE]], i32 1, i32 1)
+// IR-NEXT:    %[[TMP15:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
+// IR-NEXT:    %[[TMP16:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_13]], align 4
+// IR-NEXT:    %[[CMP18:.+]] = icmp ugt i32 %[[TMP15]], %[[TMP16]]
+// IR-NEXT:    br i1 %[[CMP18]], label %[[COND_TRUE:.+]], label %[[COND_FALSE:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_TRUE]]:
+// IR-NEXT:    %[[TMP17:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_13]], align 4
+// IR-NEXT:    br label %[[COND_END:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_FALSE]]:
+// IR-NEXT:    %[[TMP18:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
 // IR-NEXT:    br label %[[COND_END]]
-// IR:       [[COND_END]]:
-// IR-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP17]], %[[COND_TRUE]] ], [ [[TMP18]], %[[COND_FALSE]] ]
-// IR-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
-// IR-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
-// IR-NEXT:    store i32 [[TMP19]], ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    br label %[[OMP_INNER_FOR_COND:.*]]
-// IR:       [[OMP_INNER_FOR_COND]]:
-// IR-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// IR-NEXT:    [[ADD19:%.*]] = add i32 [[TMP21]], 1
-// IR-NEXT:    [[CMP20:%.*]] = icmp ult i32 [[TMP20]], [[ADD19]]
-// IR-NEXT:    br i1 [[CMP20]], label %[[OMP_INNER_FOR_BODY:.*]], label %[[OMP_INNER_FOR_END:.*]]
-// IR:       [[OMP_INNER_FOR_BODY]]:
-// IR-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul i32 [[TMP22]], 3
-// IR-NEXT:    [[ADD21:%.*]] = add i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD21]], ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
-// IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
-// IR-NEXT:    store i32 [[TMP23]], ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND:.*]]
-// IR:       [[FOR_COND]]:
-// IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    [[ADD22:%.*]] = add i32 [[TMP25]], 1
-// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
-// IR-NEXT:    [[ADD23:%.*]] = add i32 [[TMP26]], 3
-// IR-NEXT:    [[CMP24:%.*]] = icmp ult i32 [[ADD22]], [[ADD23]]
-// IR-NEXT:    br i1 [[CMP24]], label %[[COND_TRUE25:.*]], label %[[COND_FALSE27:.*]]
-// IR:       [[COND_TRUE25]]:
-// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    [[ADD26:%.*]] = add i32 [[TMP27]], 1
-// IR-NEXT:    br label %[[COND_END29:.*]]
-// IR:       [[COND_FALSE27]]:
-// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
-// IR-NEXT:    [[ADD28:%.*]] = add i32 [[TMP28]], 3
+// IR-EMPTY:
+// IR-NEXT:  [[COND_END]]:
+// IR-NEXT:    %[[COND:.+]] = phi i32 [ %[[TMP17]], %[[COND_TRUE]] ], [ %[[TMP18]], %[[COND_FALSE]] ]
+// IR-NEXT:    store i32 %[[COND]], ptr %[[DOTOMP_UB]], align 4
+// IR-NEXT:    %[[TMP19:.+]] = load i32, ptr %[[DOTOMP_LB]], align 4
+// IR-NEXT:    store i32 %[[TMP19]], ptr %[[DOTOMP_IV]], align 4
+// IR-NEXT:    br label %[[OMP_INNER_FOR_COND:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_INNER_FOR_COND]]:
+// IR-NEXT:    %[[TMP20:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
+// IR-NEXT:    %[[TMP21:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
+// IR-NEXT:    %[[ADD19:.+]] = add i32 %[[TMP21]], 1
+// IR-NEXT:    %[[CMP20:.+]] = icmp ult i32 %[[TMP20]], %[[ADD19]]
+// IR-NEXT:    br i1 %[[CMP20]], label %[[OMP_INNER_FOR_BODY:.+]], label %[[OMP_INNER_FOR_END:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_INNER_FOR_BODY]]:
+// IR-NEXT:    %[[TMP22:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
+// IR-NEXT:    %[[MUL:.+]] = mul i32 %[[TMP22]], 3
+// IR-NEXT:    %[[ADD21:.+]] = add i32 0, %[[MUL]]
+// IR-NEXT:    store i32 %[[ADD21]], ptr %[[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
+// IR-NEXT:    %[[TMP23:.+]] = load i32, ptr %[[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
+// IR-NEXT:    store i32 %[[TMP23]], ptr %[[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_COND]]:
+// IR-NEXT:    %[[TMP24:.+]] = load i32, ptr %[[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    %[[TMP25:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    %[[ADD22:.+]] = add i32 %[[TMP25]], 1
+// IR-NEXT:    %[[TMP26:.+]] = load i32, ptr %[[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
+// IR-NEXT:    %[[ADD23:.+]] = add i32 %[[TMP26]], 3
+// IR-NEXT:    %[[CMP24:.+]] = icmp ult i32 %[[ADD22]], %[[ADD23]]
+// IR-NEXT:    br i1 %[[CMP24]], label %[[COND_TRUE25:.+]], label %[[COND_FALSE27:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_TRUE25]]:
+// IR-NEXT:    %[[TMP27:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    %[[ADD26:.+]] = add i32 %[[TMP27]], 1
+// IR-NEXT:    br label %[[COND_END29:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_FALSE27]]:
+// IR-NEXT:    %[[TMP28:.+]] = load i32, ptr %[[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
+// IR-NEXT:    %[[ADD28:.+]] = add i32 %[[TMP28]], 3
 // IR-NEXT:    br label %[[COND_END29]]
-// IR:       [[COND_END29]]:
-// IR-NEXT:    [[COND30:%.*]] = phi i32 [ [[ADD26]], %[[COND_TRUE25]] ], [ [[ADD28]], %[[COND_FALSE27]] ]
-// IR-NEXT:    [[CMP31:%.*]] = icmp ult i32 [[TMP24]], [[COND30]]
-// IR-NEXT:    br i1 [[CMP31]], label %[[FOR_BODY:.*]], label %[[FOR_END50:.*]]
-// IR:       [[FOR_BODY]]:
-// IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[MUL32:%.*]] = mul i32 [[TMP29]], 4
-// IR-NEXT:    [[ADD33:%.*]] = add i32 0, [[MUL32]]
-// IR-NEXT:    store i32 [[ADD33]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    store i32 [[TMP30]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND34:.*]]
-// IR:       [[FOR_COND34]]:
-// IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[ADD35:%.*]] = add i32 [[TMP32]], 1
-// IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD36:%.*]] = add i32 [[TMP33]], 4
-// IR-NEXT:    [[CMP37:%.*]] = icmp ult i32 [[ADD35]], [[ADD36]]
-// IR-NEXT:    br i1 [[CMP37]], label %[[COND_TRUE38:.*]], label %[[COND_FALSE40:.*]]
-// IR:       [[COND_TRUE38]]:
-// IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[ADD39:%.*]] = add i32 [[TMP34]], 1
-// IR-NEXT:    br label %[[COND_END42:.*]]
-// IR:       [[COND_FALSE40]]:
-// IR-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD41:%.*]] = add i32 [[TMP35]], 4
+// IR-EMPTY:
+// IR-NEXT:  [[COND_END29]]:
+// IR-NEXT:    %[[COND30:.+]] = phi i32 [ %[[ADD26]], %[[COND_TRUE25]] ], [ %[[ADD28]], %[[COND_FALSE27]] ]
+// IR-NEXT:    %[[CMP31:.+]] = icmp ult i32 %[[TMP24]], %[[COND30]]
+// IR-NEXT:    br i1 %[[CMP31]], label %[[FOR_BODY:.+]], label %[[FOR_END50:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_BODY]]:
+// IR-NEXT:    %[[TMP29:.+]] = load i32, ptr %[[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    %[[MUL32:.+]] = mul i32 %[[TMP29]], 4
+// IR-NEXT:    %[[ADD33:.+]] = add i32 0, %[[MUL32]]
+// IR-NEXT:    store i32 %[[ADD33]], ptr %[[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    %[[TMP30:.+]] = load i32, ptr %[[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 %[[TMP30]], ptr %[[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND34:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_COND34]]:
+// IR-NEXT:    %[[TMP31:.+]] = load i32, ptr %[[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    %[[TMP32:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    %[[ADD35:.+]] = add i32 %[[TMP32]], 1
+// IR-NEXT:    %[[TMP33:.+]] = load i32, ptr %[[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    %[[ADD36:.+]] = add i32 %[[TMP33]], 4
+// IR-NEXT:    %[[CMP37:.+]] = icmp ult i32 %[[ADD35]], %[[ADD36]]
+// IR-NEXT:    br i1 %[[CMP37]], label %[[COND_TRUE38:.+]], label %[[COND_FALSE40:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_TRUE38]]:
+// IR-NEXT:    %[[TMP34:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    %[[ADD39:.+]] = add i32 %[[TMP34]], 1
+// IR-NEXT:    br label %[[COND_END42:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_FALSE40]]:
+// IR-NEXT:    %[[TMP35:.+]] = load i32, ptr %[[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    %[[ADD41:.+]] = add i32 %[[TMP35]], 4
 // IR-NEXT:    br label %[[COND_END42]]
-// IR:       [[COND_END42]]:
-// IR-NEXT:    [[COND43:%.*]] = phi i32 [ [[ADD39]], %[[COND_TRUE38]] ], [ [[ADD41]], %[[COND_FALSE40]] ]
-// IR-NEXT:    [[CMP44:%.*]] = icmp ult i32 [[TMP31]], [[COND43]]
-// IR-NEXT:    br i1 [[CMP44]], label %[[FOR_BODY45:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY45]]:
-// IR-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[MUL46:%.*]] = mul i32 [[TMP37]], [[TMP38]]
-// IR-NEXT:    [[ADD47:%.*]] = add i32 [[TMP36]], [[MUL46]]
-// IR-NEXT:    store i32 [[ADD47]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP39:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// IR-NEXT:    [[TMP40:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// IR-NEXT:    [[TMP41:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// IR-NEXT:    [[TMP42:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void (...) @body(i32 noundef [[TMP39]], i32 noundef [[TMP40]], i32 noundef [[TMP41]], i32 noundef [[TMP42]])
-// IR-NEXT:    br label %[[FOR_INC:.*]]
-// IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add i32 [[TMP43]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND34]], !llvm.loop [[LOOP2:![0-9]+]]
-// IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC48:.*]]
-// IR:       [[FOR_INC48]]:
-// IR-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[INC49:%.*]] = add i32 [[TMP44]], 1
-// IR-NEXT:    store i32 [[INC49]], ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP4:![0-9]+]]
-// IR:       [[FOR_END50]]:
-// IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.*]]
-// IR:       [[OMP_BODY_CONTINUE]]:
-// IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.*]]
-// IR:       [[OMP_INNER_FOR_INC]]:
-// IR-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[ADD51:%.*]] = add i32 [[TMP45]], 1
-// IR-NEXT:    store i32 [[ADD51]], ptr [[DOTOMP_IV]], align 4
+// IR-EMPTY:
+// IR-NEXT:  [[COND_END42]]:
+// IR-NEXT:    %[[COND43:.+]] = phi i32 [ %[[ADD39]], %[[COND_TRUE38]] ], [ %[[ADD41]], %[[COND_FALSE40]] ]
+// IR-NEXT:    %[[CMP44:.+]] = icmp ult i32 %[[TMP31]], %[[COND43]]
+// IR-NEXT:    br i1 %[[CMP44]], label %[[FOR_BODY45:.+]], label %[[FOR_END:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_BODY45]]:
+// IR-NEXT:    %[[TMP36:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    %[[TMP37:.+]] = load i32, ptr %[[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    %[[TMP38:.+]] = load i32, ptr %[[DOTNEW_STEP]], align 4
+// IR-NEXT:    %[[MUL46:.+]] = mul i32 %[[TMP37]], %[[TMP38]]
+// IR-NEXT:    %[[ADD47:.+]] = add i32 %[[TMP36]], %[[MUL46]]
+// IR-NEXT:    store i32 %[[ADD47]], ptr %[[I]], align 4
+// IR-NEXT:    %[[TMP39:.+]] = load i32, ptr %[[START_ADDR]], align 4
+// IR-NEXT:    %[[TMP40:.+]] = load i32, ptr %[[END_ADDR]], align 4
+// IR-NEXT:    %[[TMP41:.+]] = load i32, ptr %[[STEP_ADDR]], align 4
+// IR-NEXT:    %[[TMP42:.+]] = load i32, ptr %[[I]], align 4
+// IR-NEXT:    call void (...) @body(i32 noundef %[[TMP39]], i32 noundef %[[TMP40]], i32 noundef %[[TMP41]], i32 noundef %[[TMP42]])
+// IR-NEXT:    br label %[[FOR_INC:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_INC]]:
+// IR-NEXT:    %[[TMP43:.+]] = load i32, ptr %[[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    %[[INC:.+]] = add i32 %[[TMP43]], 1
+// IR-NEXT:    store i32 %[[INC]], ptr %[[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND34]], !llvm.loop ![[LOOP2:[0-9]+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_END]]:
+// IR-NEXT:    br label %[[FOR_INC48:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_INC48]]:
+// IR-NEXT:    %[[TMP44:.+]] = load i32, ptr %[[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    %[[INC49:.+]] = add i32 %[[TMP44]], 1
+// IR-NEXT:    store i32 %[[INC49]], ptr %[[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop ![[LOOP4:[0-9]+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_END50]]:
+// IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_BODY_CONTINUE]]:
+// IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_INNER_FOR_INC]]:
+// IR-NEXT:    %[[TMP45:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
+// IR-NEXT:    %[[ADD51:.+]] = add i32 %[[TMP45]], 1
+// IR-NEXT:    store i32 %[[ADD51]], ptr %[[DOTOMP_IV]], align 4
 // IR-NEXT:    br label %[[OMP_INNER_FOR_COND]]
-// IR:       [[OMP_INNER_FOR_END]]:
-// IR-NEXT:    br label %[[OMP_LOOP_EXIT:.*]]
-// IR:       [[OMP_LOOP_EXIT]]:
-// IR-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP0]])
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_INNER_FOR_END]]:
+// IR-NEXT:    br label %[[OMP_LOOP_EXIT:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_LOOP_EXIT]]:
+// IR-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 %[[TMP0]])
 // IR-NEXT:    br label %[[OMP_PRECOND_END]]
-// IR:       [[OMP_PRECOND_END]]:
-// IR-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3:[0-9]+]], i32 [[TMP0]])
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_PRECOND_END]]:
+// IR-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3:.+]], i32 %[[TMP0]])
 // IR-NEXT:    ret void
-//
+// IR-NEXT:  }
+
+
 extern "C" void func(int start, int end, int step) {
 #pragma omp for
 #pragma omp tile sizes(3)
@@ -226,8 +249,8 @@ extern "C" void func(int start, int end, int step) {
 
 #endif /* HEADER */
 
-//.
-// IR: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]]}
-// IR: [[META3]] = !{!"llvm.loop.mustprogress"}
-// IR: [[LOOP4]] = distinct !{[[LOOP4]], [[META3]]}
-//.
+// IR: ![[META0:[0-9]+]] = !{i32 7, !"openmp", i32 51}
+// IR: ![[META1:[0-9]+]] =
+// IR: ![[LOOP2]] = distinct !{![[LOOP2]], ![[LOOPPROP3:[0-9]+]]}
+// IR: ![[LOOPPROP3]] = !{!"llvm.loop.mustprogress"}
+// IR: ![[LOOP4]] = distinct !{![[LOOP4]], ![[LOOPPROP3]]}
diff --git a/clang/test/OpenMP/tile_collapse_messages.cpp b/clang/test/OpenMP/tile_collapse_messages.cpp
new file mode 100644
index 0000000000000..6c5cb24b30289
--- /dev/null
+++ b/clang/test/OpenMP/tile_collapse_messages.cpp
@@ -0,0 +1,63 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++17 -fopenmp -fsyntax-only -Wuninitialized -verify %s
+
+// A collapsed intra-tile loop keeps a live dependence on its floor counter, so
+// that floor must itself be one of the collapsed loops. Inside another loop
+// transformation it is assigned in an enclosing loop's body and the nest would
+// read a stale value, which is diagnosed rather than silently miscomputed.
+
+extern void body(int);
+
+void collapse_through_single_tile() {
+#pragma omp for collapse(2)
+#pragma omp tile sizes(4)
+  for (int i = 0; i < 6; ++i)
+    body(i);
+}
+
+void collapse_through_tile_2d() {
+#pragma omp for collapse(4)
+#pragma omp tile sizes(2, 3)
+  for (int i = 0; i < 6; ++i)
+    for (int j = 0; j < 7; ++j)
+      body(i + j);
+}
+
+void collapse_outer_of_stacked_tiles() {
+  // Only the outer tile's floor and intra-tile loops are collapsed, and that
+  // floor is a collapsed counter, so this is fine.
+#pragma omp for collapse(2)
+#pragma omp tile sizes(2)
+#pragma omp tile sizes(4)
+  for (int i = 0; i < 6; ++i)
+    body(i);
+}
+
+void collapse_into_stacked_tiles() {
+  // The third loop is the inner tile's intra-tile loop, whose floor is computed
+  // inside the outer intra-tile loop's body.
+#pragma omp for collapse(3)
+#pragma omp tile sizes(2)
+#pragma omp tile sizes(4)
+  // expected-error at +1 {{cannot collapse the intra-tile loop of a '#pragma omp tile' that is nested inside another loop-transforming directive}}
+  for (int i = 0; i < 6; ++i)
+    body(i);
+}
+
+void collapse_simd_over_tile() {
+  // Rejected by the pre-existing restriction on nesting inside a 'simd' region.
+#pragma omp simd collapse(2)
+  // expected-error at +1 {{OpenMP constructs may not be nested inside a simd region except for ordered simd, simd, scan, or atomic directive}}
+#pragma omp tile sizes(4)
+  for (int i = 0; i < 6; ++i)
+    body(i);
+}
+
+void collapse_outer_of_stacked_tiles_2d() {
+  // The outer tile consumes both loops of the inner tile, so both of its
+  // intra-tile loops find their floor among the collapsed counters.
+#pragma omp for collapse(4)
+#pragma omp tile sizes(2, 3)
+#pragma omp tile sizes(4)
+  for (int i = 0; i < 6; ++i)
+    body(i);
+}
diff --git a/clang/test/OpenMP/tile_collapse_reinterpret_codegen.cpp b/clang/test/OpenMP/tile_collapse_reinterpret_codegen.cpp
index f88db73778f2b..477c86b4ead44 100644
--- a/clang/test/OpenMP/tile_collapse_reinterpret_codegen.cpp
+++ b/clang/test/OpenMP/tile_collapse_reinterpret_codegen.cpp
@@ -1,17 +1,23 @@
 // RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fclang-abi-compat=latest -std=c++20 -fopenmp -ast-print %s | FileCheck %s --check-prefix=PRINT
 // RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fclang-abi-compat=latest -std=c++20 -fopenmp -emit-llvm %s -o - | FileCheck %s --check-prefix=IR
+
+// Check same results after serialization round-trip, which exercises reading
+// and writing OMPInvariantPredicateBoundAttr including its optional predicate.
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fclang-abi-compat=latest -std=c++20 -fopenmp -emit-pch -o %t %s
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fclang-abi-compat=latest -std=c++20 -fopenmp -include-pch %t -ast-print %s | FileCheck %s --check-prefix=PRINT
+
 // expected-no-diagnostics
 
-// Codegen tests for how a loop-associated directive (e.g. 'omp for' with a
-// 'collapse' clause) consumes a '#pragma omp tile' intra-tile loop.
-//
-// These tests pin the key IR invariants of the below design:
-// The intra-tile loop is stored in its min-bounded form
-//   for (.tile.iv = .floor.iv; .tile.iv < min(.floor.iv + tilesize, N); ++.tile.iv)
-// and carries an internal OMPInvariantPredicateBoundAttr hint. When a directive
-// consumes the loop, it is reinterpreted as a rectangular loop with a constant
-// per-tile trip count (the tile size), and the overshoot of a partial last tile
-// is handled by a predicate emitted as a body guard.
+// How a loop-associated directive consumes a '#pragma omp tile': the stored
+// min-bounded intra-tile loop is reinterpreted as a rectangular one with a
+// constant trip count (the tile size), and a partial last tile is handled by a
+// body guard. Worksharing, 'distribute' and 'ordered(n)' consumers each read
+// the iteration space differently, so all three are covered. The hint is
+// internal and never dumped, so it is observed through the guard it produces
+// ('omp.body.next') and through the PCH round-trip above.
+
+#ifndef HEADER
+#define HEADER
 
 extern "C" void body(...) {}
 
@@ -21,7 +27,6 @@ extern "C" void body(...) {}
 // IR-LABEL: define internal void @collapse2_tile1d.omp_outlined
 // IR:       %.floor_0.iv{{[0-9a-z.]*}} = alloca i32
 // IR:       %.tile_0.iv{{[0-9a-z.]*}} = alloca i32
-// IR-NOT:   .tile.cnt
 // IR:       omp.inner.for.body:
 // IR:       store i32 %{{.*}}, ptr %.tile_0.iv
 // IR:       %[[PRED:.*]] = icmp slt i32 %{{.*}}, %{{.*}}
@@ -37,18 +42,18 @@ extern "C" void collapse2_tile1d(int n) {
     body(i);
 }
 
+// A non-tiled inner loop joins the collapsed nest as a third counter, and the
+// guard still applies to the tile counter alone.
 // PRINT-LABEL: void collapse3_tile1d_nested(
 // PRINT:       #pragma omp parallel for collapse(3)
 // PRINT-NEXT:  #pragma omp tile sizes(5)
 // IR-LABEL: define internal void @collapse3_tile1d_nested.omp_outlined
-// IR-NOT:   .tile.cnt
 // IR:       omp.inner.for.body:
 // IR:       store i32 %{{.*}}, ptr %.tile_0.iv
-// IR:       %[[PRED:.*]] = icmp slt i32 %{{.*}}, %{{.*}}
+// IR:       store i32 %{{.*}}, ptr %j
+// IR:       %[[TILEIV:.*]] = load i32, ptr %.tile_0.iv
+// IR:       %[[PRED:.*]] = icmp slt i32 %[[TILEIV]], %{{.*}}
 // IR-NEXT:  br i1 %[[PRED]], label %omp.body.next, label %omp.body.continue
-// IR:       omp.body.next:
-// IR:       call void (...) @body
-// IR-NEXT:  br label %omp.body.continue
 extern "C" void collapse3_tile1d_nested(int n) {
 #pragma omp parallel for collapse(3)
 #pragma omp tile sizes(5)
@@ -57,12 +62,13 @@ extern "C" void collapse3_tile1d_nested(int n) {
       body(i, j);
 }
 
+// An enclosing 'tile' is another loop transformation rather than a consumer, so
+// the inner intra-tile loop stays min-bounded and gets no guard.
 // PRINT-LABEL: void tile2d_inside_tile1d(
 // PRINT:       #pragma omp tile sizes(3)
 // PRINT-NEXT:  #pragma omp tile sizes(2)
 // IR-LABEL: define {{.*}} @tile2d_inside_tile1d(
 // IR:       %.tile_0.iv{{[0-9a-z.]*}} = alloca i32
-// IR-NOT:   .tile.cnt
 // IR:       cond.true:
 // IR:       cond.false:
 // IR:       cond.end:
@@ -79,7 +85,6 @@ extern "C" void tile2d_inside_tile1d(int n) {
 // PRINT:       #pragma omp parallel for collapse(3)
 // PRINT-NEXT:  #pragma omp tile sizes(2, 3)
 // IR-LABEL: define internal void @collapse3_tile2d.omp_outlined
-// IR-NOT:   .tile.cnt
 // IR:       omp.inner.for.body:
 // IR:       store i32 %{{.*}}, ptr %.tile_0.iv
 // IR:       br i1 %{{.*}}, label %omp.body.next, label %omp.body.continue
@@ -93,16 +98,77 @@ extern "C" void collapse3_tile2d(int n) {
       body(i, j);
 }
 
-// PRINT-LABEL: void tile1d_constant_div_no_predicate(
-// PRINT:       #pragma omp tile sizes(5)
-// IR-LABEL: define {{.*}} @tile1d_constant_div_no_predicate(
-// IR:       %.floor_0.iv{{[0-9a-z.]*}} = alloca i32
-// IR:       %.tile_0.iv{{[0-9a-z.]*}} = alloca i32
-// IR-NOT:   .tile.cnt
+// Trip count 10 is a multiple of the tile size 5, so no tile is partial, the
+// hint carries no predicate, and the body is emitted with no guard.
+// PRINT-LABEL: void collapse2_tile1d_no_partial_tile(
+// PRINT:       #pragma omp parallel for collapse(2)
+// PRINT-NEXT:  #pragma omp tile sizes(5)
+// IR-LABEL: define internal void @collapse2_tile1d_no_partial_tile.omp_outlined
+// IR:       omp.inner.for.body:
 // IR-NOT:   omp.body.next
 // IR:       call void (...) @body
-extern "C" void tile1d_constant_div_no_predicate() {
+// IR:       omp.body.continue:
+extern "C" void collapse2_tile1d_no_partial_tile() {
+#pragma omp parallel for collapse(2)
 #pragma omp tile sizes(5)
   for (int i = 0; i < 10; ++i)
     body(i);
 }
+
+// Trip count 11 is not a multiple of the tile size 5, so the predicate is kept
+// even though both values are compile-time constants.
+// PRINT-LABEL: void collapse2_tile1d_constant_partial_tile(
+// PRINT:       #pragma omp parallel for collapse(2)
+// PRINT-NEXT:  #pragma omp tile sizes(5)
+// IR-LABEL: define internal void @collapse2_tile1d_constant_partial_tile.omp_outlined
+// IR:       omp.inner.for.body:
+// IR:       br i1 %{{.*}}, label %omp.body.next, label %omp.body.continue
+// IR:       omp.body.next:
+// IR:       call void (...) @body
+extern "C" void collapse2_tile1d_constant_partial_tile() {
+#pragma omp parallel for collapse(2)
+#pragma omp tile sizes(5)
+  for (int i = 0; i < 11; ++i)
+    body(i);
+}
+
+// 'distribute' computes bounds through its own LB/UB/EUB/DistCond fields, so
+// check that the reinterpretation and its guard survive that path too.
+// PRINT-LABEL: void dist_collapse2_tile1d(
+// PRINT:       #pragma omp teams distribute collapse(2)
+// PRINT-NEXT:  #pragma omp tile sizes(5)
+// IR-LABEL: define internal void @dist_collapse2_tile1d.omp_outlined
+// IR:       omp.inner.for.body:
+// IR:       store i32 %{{.*}}, ptr %.tile_0.iv
+// IR:       br i1 %{{.*}}, label %omp.body.next, label %omp.body.continue
+// IR:       omp.body.next:
+// IR:       call void (...) @body
+extern "C" void dist_collapse2_tile1d(int n) {
+#pragma omp teams distribute collapse(2)
+#pragma omp tile sizes(5)
+  for (int i = 0; i < n; ++i)
+    body(i);
+}
+
+// 'ordered(n)' feeds each collapsed loop's trip count to __kmpc_doacross_init,
+// so the intra-tile dimension must report the constant tile size, matching the
+// space 'collapse' linearizes.
+// PRINT-LABEL: void ordered_collapse2_tile1d(
+// PRINT:       #pragma omp for collapse(2) ordered(2)
+// PRINT-NEXT:  #pragma omp tile sizes(5)
+// IR-LABEL: define {{.*}} @ordered_collapse2_tile1d(
+// IR:       %[[DIM1:.*]] = getelementptr inbounds [2 x %struct.kmp_dim], ptr %dims, i64 0, i64 1
+// IR:       %[[UP1:.*]] = getelementptr inbounds nuw %struct.kmp_dim, ptr %[[DIM1]], i32 0, i32 1
+// IR-NEXT:  store i64 5, ptr %[[UP1]]
+// IR:       call void @__kmpc_doacross_init(ptr @{{[0-9]+}}, i32 %{{.*}}, i32 2, ptr %{{.*}})
+// IR:       omp.inner.for.body:
+// IR:       br i1 %{{.*}}, label %omp.body.next, label %omp.body.continue
+// IR:       call void @__kmpc_doacross_fini(
+extern "C" void ordered_collapse2_tile1d(int n) {
+#pragma omp for collapse(2) ordered(2)
+#pragma omp tile sizes(5)
+  for (int i = 0; i < n; ++i)
+    body(i);
+}
+
+#endif /* HEADER */
diff --git a/clang/test/OpenMP/tile_messages.cpp b/clang/test/OpenMP/tile_messages.cpp
index e1f5155c924e5..6ac905d937159 100644
--- a/clang/test/OpenMP/tile_messages.cpp
+++ b/clang/test/OpenMP/tile_messages.cpp
@@ -170,3 +170,33 @@ void f(void) {
     ;
 }
 } // namespace GH139073
+
+// A tile's intra-tile loop is wrapped in an internal reinterpretation hint. A
+// loop transformation consuming that loop must see through the wrapper, so
+// check that these still parse and analyze without diagnostics.
+namespace consumed_by_transformation {
+// Codegen: tile_collapse_reinterpret_codegen.cpp and tile_codegen_tile_for.cpp;
+// run time: libomp transform/tile/parallel-wsloop-collapse-stacked-tile.cpp.
+void tile_of_tile() {
+#pragma omp tile sizes(3, 5)
+#pragma omp tile sizes(2)
+  for (int i = 0; i < 100; ++i)
+    ;
+}
+
+// Same, for a range-based for loop.
+void tile_of_tile_range_for(int (&A)[100]) {
+#pragma omp tile sizes(3, 5)
+#pragma omp tile sizes(2)
+  for (int &v : A)
+    (void)v;
+}
+
+// 'unroll' is a loop transformation too, so it consumes the tile the same way.
+void unroll_of_tile() {
+#pragma omp unroll partial(4)
+#pragma omp tile sizes(2)
+  for (int i = 0; i < 100; ++i)
+    ;
+}
+} // namespace consumed_by_transformation
diff --git a/clang/test/OpenMP/tile_rect_codegen.cpp b/clang/test/OpenMP/tile_rect_codegen.cpp
deleted file mode 100644
index ab63de9343fd2..0000000000000
--- a/clang/test/OpenMP/tile_rect_codegen.cpp
+++ /dev/null
@@ -1,51 +0,0 @@
-// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fclang-abi-compat=latest -std=c++20 -fopenmp -ast-print %s | FileCheck %s
-// expected-no-diagnostics
-
-// -ast-print round-trip tests: a `#pragma omp tile` directive and its associated
-// loop must print back as written. This checks parsing/printing only; for the
-// generated IR (min-bounded lowering) see tile_rect_codegen_ir.cpp.
-
-extern "C" void body(...) {}
-
-// CHECK-LABEL: void rect_tile_1d(
-void rect_tile_1d() {
-  // Partial last tile (trip count 4, tile 5); -ast-print prints the original loop.
-  // CHECK: #pragma omp tile sizes(5)
-  // CHECK-NEXT: for (int i = 7; i < 17; i += 3)
-  #pragma omp tile sizes(5)
-  for (int i = 7; i < 17; i += 3)
-    body(i);
-}
-
-// CHECK-LABEL: void rect_tile_2d(
-void rect_tile_2d() {
-  // CHECK: #pragma omp tile sizes(5, 5)
-  // CHECK-NEXT: for (int i = 7; i < 17; i += 3)
-  // CHECK-NEXT:   for (int j = 7; j < 17; j += 3)
-  #pragma omp tile sizes(5, 5)
-  for (int i = 7; i < 17; i += 3)
-    for (int j = 7; j < 17; j += 3)
-      body(i, j);
-}
-
-// CHECK-LABEL: void rect_tile_exact_div(
-void rect_tile_exact_div() {
-  // Exact-dividing trip count (10, tile 5); -ast-print prints the original loop.
-  // CHECK: #pragma omp tile sizes(5)
-  // CHECK-NEXT: for (int i = 0; i < 10; i += 1)
-  #pragma omp tile sizes(5)
-  for (int i = 0; i < 10; i += 1)
-    body(i);
-}
-
-// CHECK-LABEL: void rect_tile_nested_body_loop(
-void rect_tile_nested_body_loop(int n) {
-  // The inner `j` loop lives in the tiled construct's body; -ast-print shows the
-  // directive and the outer `i` loop printed back.
-  // CHECK: #pragma omp tile sizes(4)
-  // CHECK-NEXT: for (int i = 0; i < 6; i += 1)
-  #pragma omp tile sizes(4)
-  for (int i = 0; i < 6; i += 1)
-    for (int j = 0; j < n; ++j)
-      body(i, j);
-}
diff --git a/clang/test/OpenMP/tile_rect_codegen_ir.cpp b/clang/test/OpenMP/tile_rect_codegen_ir.cpp
index 027cd8385044c..33b3415d36b6d 100644
--- a/clang/test/OpenMP/tile_rect_codegen_ir.cpp
+++ b/clang/test/OpenMP/tile_rect_codegen_ir.cpp
@@ -1,11 +1,8 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
-// Regression tests for standalone `#pragma omp tile` lowering (no consuming
-// directive). The intra-tile loop is emitted in its min-bounded form
-//   for (.tile.iv = .floor.iv; .tile.iv < min(.floor.iv + tilesize, N); ++.tile.iv)
-// so the upper bound is materialized as a `min()` (cond.true/cond.false/phi over
-// the two `.tile_*.iv`/`.floor_*.iv` counters). There is no `.tile.cnt`
-// rectangular counter and no body-guard predicate here; those only appear when a
-// directive (e.g. `collapse`) consumes the tile -- see
+// A standalone `#pragma omp tile` keeps the intra-tile loop min-bounded: the
+// upper bound `min(.floor.iv + tilesize, N)` is materialized as a
+// cond.true/cond.false/phi and there is no body guard. The guarded rectangular
+// form appears only once a directive consumes the tile, see
 // tile_collapse_reinterpret_codegen.cpp.
 //
 // RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fopenmp -emit-llvm -o - %s | FileCheck %s --check-prefix=IR
@@ -13,8 +10,8 @@
 
 extern "C" void body(int);
 
-// Trip count 6, tile 4 → the last tile is partial; the min() upper bound stops
-// the tile loop at the trip count (no separate body guard).
+// Trip count 6, tile 4: the last tile is partial, and the min() bound alone
+// stops it at the trip count.
 // IR-LABEL: define dso_local void @_Z18remainder_6_tile_4v(
 // IR-SAME: ) #[[ATTR0:[0-9]+]] {
 // IR-NEXT:  [[ENTRY:.*:]]
@@ -72,287 +69,13 @@ extern "C" void body(int);
 // IR-NEXT:    ret void
 //
 void remainder_6_tile_4(void) {
-  // Rectangular tile counter and constant tile-size bound.
 #pragma omp tile sizes(4)
   for (int i = 0; i < 6; ++i)
     body(i);
 }
 
-// Simple stride-1 loop; tile size matches full tiles only (10, tile 5).
-// IR-LABEL: define dso_local void @_Z15full_tiles_10_5v(
-// IR-SAME: ) #[[ATTR0]] {
-// IR-NEXT:  [[ENTRY:.*:]]
-// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    store i32 0, ptr [[I]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND:.*]]
-// IR:       [[FOR_COND]]:
-// IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 10
-// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END9:.*]]
-// IR:       [[FOR_BODY]]:
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND1:.*]]
-// IR:       [[FOR_COND1]]:
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 5
-// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 10, [[ADD]]
-// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
-// IR:       [[COND_TRUE]]:
-// IR-NEXT:    br label %[[COND_END:.*]]
-// IR:       [[COND_FALSE]]:
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 5
-// IR-NEXT:    br label %[[COND_END]]
-// IR:       [[COND_END]]:
-// IR-NEXT:    [[COND:%.*]] = phi i32 [ 10, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
-// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
-// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY5]]:
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
-// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void @body(i32 noundef [[TMP6]])
-// IR-NEXT:    br label %[[FOR_INC:.*]]
-// IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP7]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP5:![0-9]+]]
-// IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC7:.*]]
-// IR:       [[FOR_INC7]]:
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP8]], 5
-// IR-NEXT:    store i32 [[ADD8]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
-// IR:       [[FOR_END9]]:
-// IR-NEXT:    ret void
-//
-void full_tiles_10_5(void) {
-#pragma omp tile sizes(5)
-  for (int i = 0; i < 10; ++i)
-    body(i);
-}
-
-// Variable tile size: clamp (TS<=0 ? 1 : TS) is inlined; bound compares to %cond.
-// IR-LABEL: define dso_local void @_Z8var_tilei(
-// IR-SAME: i32 noundef [[TS:%.*]]) #[[ATTR0]] {
-// IR-NEXT:  [[ENTRY:.*:]]
-// IR-NEXT:    [[TS_ADDR:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    store i32 [[TS]], ptr [[TS_ADDR]], align 4
-// IR-NEXT:    store i32 0, ptr [[I]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND:.*]]
-// IR:       [[FOR_COND]]:
-// IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 12
-// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END24:.*]]
-// IR:       [[FOR_BODY]]:
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND1:.*]]
-// IR:       [[FOR_COND1]]:
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TS_ADDR]], align 4
-// IR-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], 0
-// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
-// IR:       [[COND_TRUE]]:
-// IR-NEXT:    br label %[[COND_END:.*]]
-// IR:       [[COND_FALSE]]:
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TS_ADDR]], align 4
-// IR-NEXT:    br label %[[COND_END]]
-// IR:       [[COND_END]]:
-// IR-NEXT:    [[COND:%.*]] = phi i32 [ 1, %[[COND_TRUE]] ], [ [[TMP5]], %[[COND_FALSE]] ]
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[COND]]
-// IR-NEXT:    [[CMP3:%.*]] = icmp slt i32 12, [[ADD]]
-// IR-NEXT:    br i1 [[CMP3]], label %[[COND_TRUE4:.*]], label %[[COND_FALSE5:.*]]
-// IR:       [[COND_TRUE4]]:
-// IR-NEXT:    br label %[[COND_END12:.*]]
-// IR:       [[COND_FALSE5]]:
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TS_ADDR]], align 4
-// IR-NEXT:    [[CMP6:%.*]] = icmp sle i32 [[TMP7]], 0
-// IR-NEXT:    br i1 [[CMP6]], label %[[COND_TRUE7:.*]], label %[[COND_FALSE8:.*]]
-// IR:       [[COND_TRUE7]]:
-// IR-NEXT:    br label %[[COND_END9:.*]]
-// IR:       [[COND_FALSE8]]:
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TS_ADDR]], align 4
-// IR-NEXT:    br label %[[COND_END9]]
-// IR:       [[COND_END9]]:
-// IR-NEXT:    [[COND10:%.*]] = phi i32 [ 1, %[[COND_TRUE7]] ], [ [[TMP8]], %[[COND_FALSE8]] ]
-// IR-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP6]], [[COND10]]
-// IR-NEXT:    br label %[[COND_END12]]
-// IR:       [[COND_END12]]:
-// IR-NEXT:    [[COND13:%.*]] = phi i32 [ 12, %[[COND_TRUE4]] ], [ [[ADD11]], %[[COND_END9]] ]
-// IR-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP2]], [[COND13]]
-// IR-NEXT:    br i1 [[CMP14]], label %[[FOR_BODY15:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY15]]:
-// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP9]], 1
-// IR-NEXT:    [[ADD16:%.*]] = add nsw i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD16]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void @body(i32 noundef [[TMP10]])
-// IR-NEXT:    br label %[[FOR_INC:.*]]
-// IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP7:![0-9]+]]
-// IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC17:.*]]
-// IR:       [[FOR_INC17]]:
-// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TS_ADDR]], align 4
-// IR-NEXT:    [[CMP18:%.*]] = icmp sle i32 [[TMP12]], 0
-// IR-NEXT:    br i1 [[CMP18]], label %[[COND_TRUE19:.*]], label %[[COND_FALSE20:.*]]
-// IR:       [[COND_TRUE19]]:
-// IR-NEXT:    br label %[[COND_END21:.*]]
-// IR:       [[COND_FALSE20]]:
-// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TS_ADDR]], align 4
-// IR-NEXT:    br label %[[COND_END21]]
-// IR:       [[COND_END21]]:
-// IR-NEXT:    [[COND22:%.*]] = phi i32 [ 1, %[[COND_TRUE19]] ], [ [[TMP13]], %[[COND_FALSE20]] ]
-// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP14]], [[COND22]]
-// IR-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
-// IR:       [[FOR_END24]]:
-// IR-NEXT:    ret void
-//
-void var_tile(int ts) {
-#pragma omp tile sizes(ts)
-  for (int i = 0; i < 12; ++i)
-    body(i);
-}
-
-// Two tiled dimensions → two tile counters.
-// IR-LABEL: define dso_local void @_Z7two_dimv(
-// IR-SAME: ) #[[ATTR0]] {
-// IR-NEXT:  [[ENTRY:.*:]]
-// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[J:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
-// IR-NEXT:    store i32 0, ptr [[I]], align 4
-// IR-NEXT:    store i32 0, ptr [[J]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND:.*]]
-// IR:       [[FOR_COND]]:
-// IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 3
-// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END31:.*]]
-// IR:       [[FOR_BODY]]:
-// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_1_IV_J]], align 4
-// IR-NEXT:    br label %[[FOR_COND1:.*]]
-// IR:       [[FOR_COND1]]:
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 3
-// IR-NEXT:    br i1 [[CMP2]], label %[[FOR_BODY3:.*]], label %[[FOR_END28:.*]]
-// IR:       [[FOR_BODY3]]:
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    store i32 [[TMP2]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND4:.*]]
-// IR:       [[FOR_COND4]]:
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], 2
-// IR-NEXT:    [[CMP5:%.*]] = icmp slt i32 3, [[ADD]]
-// IR-NEXT:    br i1 [[CMP5]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
-// IR:       [[COND_TRUE]]:
-// IR-NEXT:    br label %[[COND_END:.*]]
-// IR:       [[COND_FALSE]]:
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP5]], 2
-// IR-NEXT:    br label %[[COND_END]]
-// IR:       [[COND_END]]:
-// IR-NEXT:    [[COND:%.*]] = phi i32 [ 3, %[[COND_TRUE]] ], [ [[ADD6]], %[[COND_FALSE]] ]
-// IR-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP3]], [[COND]]
-// IR-NEXT:    br i1 [[CMP7]], label %[[FOR_BODY8:.*]], label %[[FOR_END25:.*]]
-// IR:       [[FOR_BODY8]]:
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 1
-// IR-NEXT:    [[ADD9:%.*]] = add nsw i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// IR-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_1_IV_J]], align 4
-// IR-NEXT:    br label %[[FOR_COND10:.*]]
-// IR:       [[FOR_COND10]]:
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// IR-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP9]], 2
-// IR-NEXT:    [[CMP12:%.*]] = icmp slt i32 3, [[ADD11]]
-// IR-NEXT:    br i1 [[CMP12]], label %[[COND_TRUE13:.*]], label %[[COND_FALSE14:.*]]
-// IR:       [[COND_TRUE13]]:
-// IR-NEXT:    br label %[[COND_END16:.*]]
-// IR:       [[COND_FALSE14]]:
-// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// IR-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP10]], 2
-// IR-NEXT:    br label %[[COND_END16]]
-// IR:       [[COND_END16]]:
-// IR-NEXT:    [[COND17:%.*]] = phi i32 [ 3, %[[COND_TRUE13]] ], [ [[ADD15]], %[[COND_FALSE14]] ]
-// IR-NEXT:    [[CMP18:%.*]] = icmp slt i32 [[TMP8]], [[COND17]]
-// IR-NEXT:    br i1 [[CMP18]], label %[[FOR_BODY19:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY19]]:
-// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// IR-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[TMP11]], 1
-// IR-NEXT:    [[ADD21:%.*]] = add nsw i32 0, [[MUL20]]
-// IR-NEXT:    store i32 [[ADD21]], ptr [[J]], align 4
-// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
-// IR-NEXT:    [[ADD22:%.*]] = add nsw i32 [[TMP12]], [[TMP13]]
-// IR-NEXT:    call void @body(i32 noundef [[ADD22]])
-// IR-NEXT:    br label %[[FOR_INC:.*]]
-// IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
-// IR-NEXT:    br label %[[FOR_COND10]], !llvm.loop [[LOOP9:![0-9]+]]
-// IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC23:.*]]
-// IR:       [[FOR_INC23]]:
-// IR-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[INC24:%.*]] = add nsw i32 [[TMP15]], 1
-// IR-NEXT:    store i32 [[INC24]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND4]], !llvm.loop [[LOOP10:![0-9]+]]
-// IR:       [[FOR_END25]]:
-// IR-NEXT:    br label %[[FOR_INC26:.*]]
-// IR:       [[FOR_INC26]]:
-// IR-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// IR-NEXT:    [[ADD27:%.*]] = add nsw i32 [[TMP16]], 2
-// IR-NEXT:    store i32 [[ADD27]], ptr [[DOTFLOOR_1_IV_J]], align 4
-// IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP11:![0-9]+]]
-// IR:       [[FOR_END28]]:
-// IR-NEXT:    br label %[[FOR_INC29:.*]]
-// IR:       [[FOR_INC29]]:
-// IR-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD30:%.*]] = add nsw i32 [[TMP17]], 2
-// IR-NEXT:    store i32 [[ADD30]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP12:![0-9]+]]
-// IR:       [[FOR_END31]]:
-// IR-NEXT:    ret void
-//
-void two_dim(void) {
-#pragma omp tile sizes(2, 2)
-  for (int i = 0; i < 3; ++i)
-    for (int j = 0; j < 3; ++j)
-      body(i + j);
-}
-
-// Nested loop in the tile body: the tiled `i` dim is the floor/tile pair in
-// min-bounded form; the inner `j` loop stays a plain nested loop.
+// The inner `j` loop is not one of the tiled loops, so it is emitted untouched
+// inside the intra-tile loop.
 // IR-LABEL: define dso_local void @_Z14nested_inner_ji(
 // IR-SAME: i32 noundef [[N:%.*]]) #[[ATTR0]] {
 // IR-NEXT:  [[ENTRY:.*:]]
@@ -411,21 +134,21 @@ void two_dim(void) {
 // IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[J]], align 4
 // IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
 // IR-NEXT:    store i32 [[INC]], ptr [[J]], align 4
-// IR-NEXT:    br label %[[FOR_COND7]], !llvm.loop [[LOOP13:![0-9]+]]
+// IR-NEXT:    br label %[[FOR_COND7]], !llvm.loop [[LOOP5:![0-9]+]]
 // IR:       [[FOR_END]]:
 // IR-NEXT:    br label %[[FOR_INC11:.*]]
 // IR:       [[FOR_INC11]]:
 // IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    [[INC12:%.*]] = add nsw i32 [[TMP11]], 1
 // IR-NEXT:    store i32 [[INC12]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP14:![0-9]+]]
+// IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP6:![0-9]+]]
 // IR:       [[FOR_END13]]:
 // IR-NEXT:    br label %[[FOR_INC14:.*]]
 // IR:       [[FOR_INC14]]:
 // IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP12]], 4
 // IR-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP15:![0-9]+]]
+// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP7:![0-9]+]]
 // IR:       [[FOR_END16]]:
 // IR-NEXT:    ret void
 //
@@ -435,72 +158,6 @@ void nested_inner_j(int n) {
     for (int j = 0; j < n; ++j)
       body(i + j);
 }
-
-// Exact division (trip count 12, tile 4): a standalone tile still emits the
-// min() cond-phi upper bound (it is always present, regardless of divisibility)
-// and never a body-guard predicate.
-// IR-LABEL: define dso_local void @_Z16elided_predicatev(
-// IR-SAME: ) #[[ATTR0]] {
-// IR-NEXT:  [[ENTRY:.*:]]
-// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    store i32 0, ptr [[I]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND:.*]]
-// IR:       [[FOR_COND]]:
-// IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 12
-// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END9:.*]]
-// IR:       [[FOR_BODY]]:
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND1:.*]]
-// IR:       [[FOR_COND1]]:
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 4
-// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 12, [[ADD]]
-// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
-// IR:       [[COND_TRUE]]:
-// IR-NEXT:    br label %[[COND_END:.*]]
-// IR:       [[COND_FALSE]]:
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 4
-// IR-NEXT:    br label %[[COND_END]]
-// IR:       [[COND_END]]:
-// IR-NEXT:    [[COND:%.*]] = phi i32 [ 12, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
-// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
-// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY5]]:
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
-// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void @body(i32 noundef [[TMP6]])
-// IR-NEXT:    br label %[[FOR_INC:.*]]
-// IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP7]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP16:![0-9]+]]
-// IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC7:.*]]
-// IR:       [[FOR_INC7]]:
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP8]], 4
-// IR-NEXT:    store i32 [[ADD8]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP17:![0-9]+]]
-// IR:       [[FOR_END9]]:
-// IR-NEXT:    ret void
-//
-void elided_predicate(void) {
-  // Trip count 12, tile size 4 → 12 % 4 == 0, no predicate.
-#pragma omp tile sizes(4)
-  for (int i = 0; i < 12; ++i)
-    body(i);
-}
 //.
 // IR: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]]}
 // IR: [[META3]] = !{!"llvm.loop.mustprogress"}
@@ -508,14 +165,4 @@ void elided_predicate(void) {
 // IR: [[LOOP5]] = distinct !{[[LOOP5]], [[META3]]}
 // IR: [[LOOP6]] = distinct !{[[LOOP6]], [[META3]]}
 // IR: [[LOOP7]] = distinct !{[[LOOP7]], [[META3]]}
-// IR: [[LOOP8]] = distinct !{[[LOOP8]], [[META3]]}
-// IR: [[LOOP9]] = distinct !{[[LOOP9]], [[META3]]}
-// IR: [[LOOP10]] = distinct !{[[LOOP10]], [[META3]]}
-// IR: [[LOOP11]] = distinct !{[[LOOP11]], [[META3]]}
-// IR: [[LOOP12]] = distinct !{[[LOOP12]], [[META3]]}
-// IR: [[LOOP13]] = distinct !{[[LOOP13]], [[META3]]}
-// IR: [[LOOP14]] = distinct !{[[LOOP14]], [[META3]]}
-// IR: [[LOOP15]] = distinct !{[[LOOP15]], [[META3]]}
-// IR: [[LOOP16]] = distinct !{[[LOOP16]], [[META3]]}
-// IR: [[LOOP17]] = distinct !{[[LOOP17]], [[META3]]}
 //.
diff --git a/clang/test/OpenMP/tile_vectorize_codegen.cpp b/clang/test/OpenMP/tile_vectorize_codegen.cpp
index 46fa51b8da57c..f2f110c0372c4 100644
--- a/clang/test/OpenMP/tile_vectorize_codegen.cpp
+++ b/clang/test/OpenMP/tile_vectorize_codegen.cpp
@@ -2,13 +2,27 @@
 // REQUIRES: x86-registered-target
 // expected-no-diagnostics
 
-// Standalone tile stays min-bounded (no body predicate) and vectorizes.
+// Why the intra-tile loop is stored min-bounded: that form carries its trip
+// count in the loop bound and vectorizes, while the rectangular form a
+// 'collapse' consumer needs adds a body guard that blocks the vectorizer. Both
+// sides are checked here so the tradeoff is visible in one place.
 
 // CHECK-LABEL: define {{.*}} @vec_tile(
 // CHECK: fadd <4 x float>
-// CHECK-NOT: omp.body.next
 extern "C" void vec_tile(float *a, float *b, int n) {
 #pragma omp tile sizes(64)
   for (int i = 0; i < n; ++i)
     a[i] = a[i] + b[i];
 }
+
+// CHECK-LABEL: define internal void @vec_tile_collapsed.omp_outlined(
+// CHECK-NOT: fadd <4 x float>
+// CHECK: br i1 %{{.*}}, label %omp.body.next, label %omp.inner.for.inc
+// CHECK-NOT: fadd <4 x float>
+// CHECK: ret void
+extern "C" void vec_tile_collapsed(float *a, float *b, int n) {
+#pragma omp parallel for collapse(2)
+#pragma omp tile sizes(64)
+  for (int i = 0; i < n; ++i)
+    a[i] = a[i] + b[i];
+}
diff --git a/clang/test/OpenMP/unroll_codegen_tile_for.cpp b/clang/test/OpenMP/unroll_codegen_tile_for.cpp
index 2e80731c4465d..ccfc9f38f5a24 100644
--- a/clang/test/OpenMP/unroll_codegen_tile_for.cpp
+++ b/clang/test/OpenMP/unroll_codegen_tile_for.cpp
@@ -1,4 +1,3 @@
-// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
 // Check code generation
 // RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fopenmp -emit-llvm %s -o - | FileCheck %s --check-prefix=IR
 
@@ -14,198 +13,221 @@
 extern "C" void body(...) {}
 
 
-// IR-LABEL: define dso_local void @func(
-// IR-SAME: i32 noundef [[START:%.*]], i32 noundef [[END:%.*]], i32 noundef [[STEP:%.*]]) #[[ATTR0:[0-9]+]] {
-// IR-NEXT:  [[ENTRY:.*:]]
-// IR-NEXT:    [[START_ADDR:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[TMP:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTUNROLLED_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_5:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_13:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV__UNROLLED_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV__UNROLLED_IV_I17:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV__UNROLLED_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTUNROLL_INNER_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
-// IR-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
-// IR-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
-// IR-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// IR-NEXT:    store i32 [[TMP1]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// IR-NEXT:    store i32 [[TMP2]], ptr [[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// IR-NEXT:    store i32 [[TMP3]], ptr [[DOTCAPTURE_EXPR_1]], align 4
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// IR-NEXT:    store i32 [[TMP4]], ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    [[SUB:%.*]] = sub i32 [[TMP5]], [[TMP6]]
-// IR-NEXT:    [[SUB3:%.*]] = sub i32 [[SUB]], 1
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add i32 [[SUB3]], [[TMP7]]
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP8]]
-// IR-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
-// IR-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTUNROLLED_IV_I]], align 4
-// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[ADD6:%.*]] = add i32 [[TMP9]], 1
-// IR-NEXT:    store i32 [[ADD6]], ptr [[DOTCAPTURE_EXPR_5]], align 4
-// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// IR-NEXT:    [[SUB8:%.*]] = sub i32 [[TMP10]], -1
-// IR-NEXT:    [[DIV9:%.*]] = udiv i32 [[SUB8]], 2
-// IR-NEXT:    [[SUB10:%.*]] = sub i32 [[DIV9]], 1
-// IR-NEXT:    store i32 [[SUB10]], ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    [[ADD12:%.*]] = add i32 [[TMP11]], 1
-// IR-NEXT:    store i32 [[ADD12]], ptr [[DOTCAPTURE_EXPR_11]], align 4
-// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// IR-NEXT:    [[SUB14:%.*]] = sub i32 [[TMP12]], -3
-// IR-NEXT:    [[DIV15:%.*]] = udiv i32 [[SUB14]], 4
-// IR-NEXT:    [[SUB16:%.*]] = sub i32 [[DIV15]], 1
-// IR-NEXT:    store i32 [[SUB16]], ptr [[DOTCAPTURE_EXPR_13]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// IR-NEXT:    [[CMP:%.*]] = icmp ult i32 0, [[TMP13]]
-// IR-NEXT:    br i1 [[CMP]], label %[[OMP_PRECOND_THEN:.*]], label %[[OMP_PRECOND_END:.*]]
-// IR:       [[OMP_PRECOND_THEN]]:
-// IR-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
-// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_13]], align 4
-// IR-NEXT:    store i32 [[TMP14]], ptr [[DOTOMP_UB]], align 4
-// IR-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
-// IR-NEXT:    call void @__kmpc_for_static_init_4u(ptr @[[GLOB1:[0-9]+]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
-// IR-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// IR-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_13]], align 4
-// IR-NEXT:    [[CMP18:%.*]] = icmp ugt i32 [[TMP15]], [[TMP16]]
-// IR-NEXT:    br i1 [[CMP18]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
-// IR:       [[COND_TRUE]]:
-// IR-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_13]], align 4
-// IR-NEXT:    br label %[[COND_END:.*]]
-// IR:       [[COND_FALSE]]:
-// IR-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// IR-LABEL: @func(
+// IR-NEXT:  [[ENTRY:.*]]:
+// IR-NEXT:    %[[START_ADDR:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[END_ADDR:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[STEP_ADDR:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_IV:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[TMP:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[I:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_1:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_2:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_3:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTUNROLLED_IV_I:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_6:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_8:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_12:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_14:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTFLOOR_0_IV__UNROLLED_IV_I:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_LB:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_UB:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_STRIDE:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_IS_LAST:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTFLOOR_0_IV__UNROLLED_IV_I18:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTTILE_0_IV__UNROLLED_IV_I:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTUNROLL_INNER_IV_I:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[TMP0:.+]] = call i32 @__kmpc_global_thread_num(ptr @2)
+// IR-NEXT:    store i32 %[[START:.+]], ptr %[[START_ADDR]], align 4
+// IR-NEXT:    store i32 %[[END:.+]], ptr %[[END_ADDR]], align 4
+// IR-NEXT:    store i32 %[[STEP:.+]], ptr %[[STEP_ADDR]], align 4
+// IR-NEXT:    %[[TMP1:.+]] = load i32, ptr %[[START_ADDR]], align 4
+// IR-NEXT:    store i32 %[[TMP1]], ptr %[[I]], align 4
+// IR-NEXT:    %[[TMP2:.+]] = load i32, ptr %[[START_ADDR]], align 4
+// IR-NEXT:    store i32 %[[TMP2]], ptr %[[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    %[[TMP3:.+]] = load i32, ptr %[[END_ADDR]], align 4
+// IR-NEXT:    store i32 %[[TMP3]], ptr %[[DOTCAPTURE_EXPR_1]], align 4
+// IR-NEXT:    %[[TMP4:.+]] = load i32, ptr %[[STEP_ADDR]], align 4
+// IR-NEXT:    store i32 %[[TMP4]], ptr %[[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    %[[TMP5:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_1]], align 4
+// IR-NEXT:    %[[TMP6:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    %[[SUB:.+]] = sub i32 %[[TMP5]], %[[TMP6]]
+// IR-NEXT:    %[[SUB4:.+]] = sub i32 %[[SUB]], 1
+// IR-NEXT:    %[[TMP7:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    %[[ADD:.+]] = add i32 %[[SUB4]], %[[TMP7]]
+// IR-NEXT:    %[[TMP8:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    %[[DIV:.+]] = udiv i32 %[[ADD]], %[[TMP8]]
+// IR-NEXT:    %[[SUB5:.+]] = sub i32 %[[DIV]], 1
+// IR-NEXT:    store i32 %[[SUB5]], ptr %[[DOTCAPTURE_EXPR_3]], align 4
+// IR-NEXT:    store i32 0, ptr %[[DOTUNROLLED_IV_I]], align 4
+// IR-NEXT:    %[[TMP9:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_3]], align 4
+// IR-NEXT:    %[[ADD7:.+]] = add i32 %[[TMP9]], 1
+// IR-NEXT:    store i32 %[[ADD7]], ptr %[[DOTCAPTURE_EXPR_6]], align 4
+// IR-NEXT:    %[[TMP10:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_6]], align 4
+// IR-NEXT:    %[[SUB9:.+]] = sub i32 %[[TMP10]], -1
+// IR-NEXT:    %[[DIV10:.+]] = udiv i32 %[[SUB9]], 2
+// IR-NEXT:    %[[SUB11:.+]] = sub i32 %[[DIV10]], 1
+// IR-NEXT:    store i32 %[[SUB11]], ptr %[[DOTCAPTURE_EXPR_8]], align 4
+// IR-NEXT:    %[[TMP11:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_8]], align 4
+// IR-NEXT:    %[[ADD13:.+]] = add i32 %[[TMP11]], 1
+// IR-NEXT:    store i32 %[[ADD13]], ptr %[[DOTCAPTURE_EXPR_12]], align 4
+// IR-NEXT:    %[[TMP12:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_12]], align 4
+// IR-NEXT:    %[[SUB15:.+]] = sub i32 %[[TMP12]], -3
+// IR-NEXT:    %[[DIV16:.+]] = udiv i32 %[[SUB15]], 4
+// IR-NEXT:    %[[SUB17:.+]] = sub i32 %[[DIV16]], 1
+// IR-NEXT:    store i32 %[[SUB17]], ptr %[[DOTCAPTURE_EXPR_14]], align 4
+// IR-NEXT:    store i32 0, ptr %[[DOTFLOOR_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    %[[TMP13:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_12]], align 4
+// IR-NEXT:    %[[CMP:.+]] = icmp ult i32 0, %[[TMP13]]
+// IR-NEXT:    br i1 %[[CMP]], label %[[OMP_PRECOND_THEN:.+]], label %[[OMP_PRECOND_END:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_PRECOND_THEN]]:
+// IR-NEXT:    store i32 0, ptr %[[DOTOMP_LB]], align 4
+// IR-NEXT:    %[[TMP14:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_14]], align 4
+// IR-NEXT:    store i32 %[[TMP14]], ptr %[[DOTOMP_UB]], align 4
+// IR-NEXT:    store i32 1, ptr %[[DOTOMP_STRIDE]], align 4
+// IR-NEXT:    store i32 0, ptr %[[DOTOMP_IS_LAST]], align 4
+// IR-NEXT:    call void @__kmpc_for_static_init_4u(ptr @1, i32 %[[TMP0]], i32 34, ptr %[[DOTOMP_IS_LAST]], ptr %[[DOTOMP_LB]], ptr %[[DOTOMP_UB]], ptr %[[DOTOMP_STRIDE]], i32 1, i32 1)
+// IR-NEXT:    %[[TMP15:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
+// IR-NEXT:    %[[TMP16:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_14]], align 4
+// IR-NEXT:    %[[CMP19:.+]] = icmp ugt i32 %[[TMP15]], %[[TMP16]]
+// IR-NEXT:    br i1 %[[CMP19]], label %[[COND_TRUE:.+]], label %[[COND_FALSE:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_TRUE]]:
+// IR-NEXT:    %[[TMP17:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_14]], align 4
+// IR-NEXT:    br label %[[COND_END:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_FALSE]]:
+// IR-NEXT:    %[[TMP18:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
 // IR-NEXT:    br label %[[COND_END]]
-// IR:       [[COND_END]]:
-// IR-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP17]], %[[COND_TRUE]] ], [ [[TMP18]], %[[COND_FALSE]] ]
-// IR-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
-// IR-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
-// IR-NEXT:    store i32 [[TMP19]], ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    br label %[[OMP_INNER_FOR_COND:.*]]
-// IR:       [[OMP_INNER_FOR_COND]]:
-// IR-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// IR-NEXT:    [[ADD19:%.*]] = add i32 [[TMP21]], 1
-// IR-NEXT:    [[CMP20:%.*]] = icmp ult i32 [[TMP20]], [[ADD19]]
-// IR-NEXT:    br i1 [[CMP20]], label %[[OMP_INNER_FOR_BODY:.*]], label %[[OMP_INNER_FOR_END:.*]]
-// IR:       [[OMP_INNER_FOR_BODY]]:
-// IR-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul i32 [[TMP22]], 4
-// IR-NEXT:    [[ADD21:%.*]] = add i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD21]], ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I17]], align 4
-// IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I17]], align 4
-// IR-NEXT:    store i32 [[TMP23]], ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND:.*]]
-// IR:       [[FOR_COND]]:
-// IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    [[ADD22:%.*]] = add i32 [[TMP25]], 1
-// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I17]], align 4
-// IR-NEXT:    [[ADD23:%.*]] = add i32 [[TMP26]], 4
-// IR-NEXT:    [[CMP24:%.*]] = icmp ult i32 [[ADD22]], [[ADD23]]
-// IR-NEXT:    br i1 [[CMP24]], label %[[COND_TRUE25:.*]], label %[[COND_FALSE27:.*]]
-// IR:       [[COND_TRUE25]]:
-// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    [[ADD26:%.*]] = add i32 [[TMP27]], 1
-// IR-NEXT:    br label %[[COND_END29:.*]]
-// IR:       [[COND_FALSE27]]:
-// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I17]], align 4
-// IR-NEXT:    [[ADD28:%.*]] = add i32 [[TMP28]], 4
-// IR-NEXT:    br label %[[COND_END29]]
-// IR:       [[COND_END29]]:
-// IR-NEXT:    [[COND30:%.*]] = phi i32 [ [[ADD26]], %[[COND_TRUE25]] ], [ [[ADD28]], %[[COND_FALSE27]] ]
-// IR-NEXT:    [[CMP31:%.*]] = icmp ult i32 [[TMP24]], [[COND30]]
-// IR-NEXT:    br i1 [[CMP31]], label %[[FOR_BODY:.*]], label %[[FOR_END44:.*]]
-// IR:       [[FOR_BODY]]:
-// IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    [[MUL32:%.*]] = mul i32 [[TMP29]], 2
-// IR-NEXT:    [[ADD33:%.*]] = add i32 0, [[MUL32]]
-// IR-NEXT:    store i32 [[ADD33]], ptr [[DOTUNROLLED_IV_I]], align 4
-// IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTUNROLLED_IV_I]], align 4
-// IR-NEXT:    store i32 [[TMP30]], ptr [[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND34:.*]]
-// IR:       [[FOR_COND34]]:
-// IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTUNROLLED_IV_I]], align 4
-// IR-NEXT:    [[ADD35:%.*]] = add i32 [[TMP32]], 2
-// IR-NEXT:    [[CMP36:%.*]] = icmp ult i32 [[TMP31]], [[ADD35]]
-// IR-NEXT:    br i1 [[CMP36]], label %[[LAND_RHS:.*]], label %[[LAND_END:.*]]
-// IR:       [[LAND_RHS]]:
-// IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[ADD37:%.*]] = add i32 [[TMP34]], 1
-// IR-NEXT:    [[CMP38:%.*]] = icmp ult i32 [[TMP33]], [[ADD37]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_END]]:
+// IR-NEXT:    %[[COND:.+]] = phi i32 [ %[[TMP17]], %[[COND_TRUE]] ], [ %[[TMP18]], %[[COND_FALSE]] ]
+// IR-NEXT:    store i32 %[[COND]], ptr %[[DOTOMP_UB]], align 4
+// IR-NEXT:    %[[TMP19:.+]] = load i32, ptr %[[DOTOMP_LB]], align 4
+// IR-NEXT:    store i32 %[[TMP19]], ptr %[[DOTOMP_IV]], align 4
+// IR-NEXT:    br label %[[OMP_INNER_FOR_COND:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_INNER_FOR_COND]]:
+// IR-NEXT:    %[[TMP20:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
+// IR-NEXT:    %[[TMP21:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
+// IR-NEXT:    %[[ADD20:.+]] = add i32 %[[TMP21]], 1
+// IR-NEXT:    %[[CMP21:.+]] = icmp ult i32 %[[TMP20]], %[[ADD20]]
+// IR-NEXT:    br i1 %[[CMP21]], label %[[OMP_INNER_FOR_BODY:.+]], label %[[OMP_INNER_FOR_END:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_INNER_FOR_BODY]]:
+// IR-NEXT:    %[[TMP22:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
+// IR-NEXT:    %[[MUL:.+]] = mul i32 %[[TMP22]], 4
+// IR-NEXT:    %[[ADD22:.+]] = add i32 0, %[[MUL]]
+// IR-NEXT:    store i32 %[[ADD22]], ptr %[[DOTFLOOR_0_IV__UNROLLED_IV_I18]], align 4
+// IR-NEXT:    %[[TMP23:.+]] = load i32, ptr %[[DOTFLOOR_0_IV__UNROLLED_IV_I18]], align 4
+// IR-NEXT:    store i32 %[[TMP23]], ptr %[[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_COND]]:
+// IR-NEXT:    %[[TMP24:.+]] = load i32, ptr %[[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    %[[TMP25:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_8]], align 4
+// IR-NEXT:    %[[ADD23:.+]] = add i32 %[[TMP25]], 1
+// IR-NEXT:    %[[TMP26:.+]] = load i32, ptr %[[DOTFLOOR_0_IV__UNROLLED_IV_I18]], align 4
+// IR-NEXT:    %[[ADD24:.+]] = add i32 %[[TMP26]], 4
+// IR-NEXT:    %[[CMP25:.+]] = icmp ult i32 %[[ADD23]], %[[ADD24]]
+// IR-NEXT:    br i1 %[[CMP25]], label %[[COND_TRUE26:.+]], label %[[COND_FALSE28:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_TRUE26]]:
+// IR-NEXT:    %[[TMP27:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_8]], align 4
+// IR-NEXT:    %[[ADD27:.+]] = add i32 %[[TMP27]], 1
+// IR-NEXT:    br label %[[COND_END30:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_FALSE28]]:
+// IR-NEXT:    %[[TMP28:.+]] = load i32, ptr %[[DOTFLOOR_0_IV__UNROLLED_IV_I18]], align 4
+// IR-NEXT:    %[[ADD29:.+]] = add i32 %[[TMP28]], 4
+// IR-NEXT:    br label %[[COND_END30]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_END30]]:
+// IR-NEXT:    %[[COND31:.+]] = phi i32 [ %[[ADD27]], %[[COND_TRUE26]] ], [ %[[ADD29]], %[[COND_FALSE28]] ]
+// IR-NEXT:    %[[CMP32:.+]] = icmp ult i32 %[[TMP24]], %[[COND31]]
+// IR-NEXT:    br i1 %[[CMP32]], label %[[FOR_BODY:.+]], label %[[FOR_END45:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_BODY]]:
+// IR-NEXT:    %[[TMP29:.+]] = load i32, ptr %[[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    %[[MUL33:.+]] = mul i32 %[[TMP29]], 2
+// IR-NEXT:    %[[ADD34:.+]] = add i32 0, %[[MUL33]]
+// IR-NEXT:    store i32 %[[ADD34]], ptr %[[DOTUNROLLED_IV_I]], align 4
+// IR-NEXT:    %[[TMP30:.+]] = load i32, ptr %[[DOTUNROLLED_IV_I]], align 4
+// IR-NEXT:    store i32 %[[TMP30]], ptr %[[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND35:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_COND35]]:
+// IR-NEXT:    %[[TMP31:.+]] = load i32, ptr %[[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    %[[TMP32:.+]] = load i32, ptr %[[DOTUNROLLED_IV_I]], align 4
+// IR-NEXT:    %[[ADD36:.+]] = add i32 %[[TMP32]], 2
+// IR-NEXT:    %[[CMP37:.+]] = icmp ult i32 %[[TMP31]], %[[ADD36]]
+// IR-NEXT:    br i1 %[[CMP37]], label %[[LAND_RHS:.+]], label %[[LAND_END:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[LAND_RHS]]:
+// IR-NEXT:    %[[TMP33:.+]] = load i32, ptr %[[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    %[[TMP34:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_3]], align 4
+// IR-NEXT:    %[[ADD38:.+]] = add i32 %[[TMP34]], 1
+// IR-NEXT:    %[[CMP39:.+]] = icmp ult i32 %[[TMP33]], %[[ADD38]]
 // IR-NEXT:    br label %[[LAND_END]]
-// IR:       [[LAND_END]]:
-// IR-NEXT:    [[TMP35:%.*]] = phi i1 [ false, %[[FOR_COND34]] ], [ [[CMP38]], %[[LAND_RHS]] ]
-// IR-NEXT:    br i1 [[TMP35]], label %[[FOR_BODY39:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY39]]:
-// IR-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[MUL40:%.*]] = mul i32 [[TMP37]], [[TMP38]]
-// IR-NEXT:    [[ADD41:%.*]] = add i32 [[TMP36]], [[MUL40]]
-// IR-NEXT:    store i32 [[ADD41]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP39:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// IR-NEXT:    [[TMP40:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// IR-NEXT:    [[TMP41:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// IR-NEXT:    [[TMP42:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void (...) @body(i32 noundef [[TMP39]], i32 noundef [[TMP40]], i32 noundef [[TMP41]], i32 noundef [[TMP42]])
-// IR-NEXT:    br label %[[FOR_INC:.*]]
-// IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add i32 [[TMP43]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND34]], !llvm.loop [[LOOP2:![0-9]+]]
-// IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC42:.*]]
-// IR:       [[FOR_INC42]]:
-// IR-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    [[INC43:%.*]] = add i32 [[TMP44]], 1
-// IR-NEXT:    store i32 [[INC43]], ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP5:![0-9]+]]
-// IR:       [[FOR_END44]]:
-// IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.*]]
-// IR:       [[OMP_BODY_CONTINUE]]:
-// IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.*]]
-// IR:       [[OMP_INNER_FOR_INC]]:
-// IR-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[ADD45:%.*]] = add i32 [[TMP45]], 1
-// IR-NEXT:    store i32 [[ADD45]], ptr [[DOTOMP_IV]], align 4
+// IR-EMPTY:
+// IR-NEXT:  [[LAND_END]]:
+// IR-NEXT:    %[[TMP35:.+]] = phi i1 [ false, %[[FOR_COND35]] ], [ %[[CMP39]], %[[LAND_RHS]] ]
+// IR-NEXT:    br i1 %[[TMP35]], label %[[FOR_BODY40:.+]], label %[[FOR_END:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_BODY40]]:
+// IR-NEXT:    %[[TMP36:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    %[[TMP37:.+]] = load i32, ptr %[[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    %[[TMP38:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    %[[MUL41:.+]] = mul i32 %[[TMP37]], %[[TMP38]]
+// IR-NEXT:    %[[ADD42:.+]] = add i32 %[[TMP36]], %[[MUL41]]
+// IR-NEXT:    store i32 %[[ADD42]], ptr %[[I]], align 4
+// IR-NEXT:    %[[TMP39:.+]] = load i32, ptr %[[START_ADDR]], align 4
+// IR-NEXT:    %[[TMP40:.+]] = load i32, ptr %[[END_ADDR]], align 4
+// IR-NEXT:    %[[TMP41:.+]] = load i32, ptr %[[STEP_ADDR]], align 4
+// IR-NEXT:    %[[TMP42:.+]] = load i32, ptr %[[I]], align 4
+// IR-NEXT:    call void (...) @body(i32 noundef %[[TMP39]], i32 noundef %[[TMP40]], i32 noundef %[[TMP41]], i32 noundef %[[TMP42]])
+// IR-NEXT:    br label %[[FOR_INC:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_INC]]:
+// IR-NEXT:    %[[TMP43:.+]] = load i32, ptr %[[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    %[[INC:.+]] = add i32 %[[TMP43]], 1
+// IR-NEXT:    store i32 %[[INC]], ptr %[[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND35]], !llvm.loop ![[LOOP2:[0-9]+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_END]]:
+// IR-NEXT:    br label %[[FOR_INC43:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_INC43]]:
+// IR-NEXT:    %[[TMP44:.+]] = load i32, ptr %[[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    %[[INC44:.+]] = add i32 %[[TMP44]], 1
+// IR-NEXT:    store i32 %[[INC44]], ptr %[[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop ![[LOOP5:[0-9]+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_END45]]:
+// IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_BODY_CONTINUE]]:
+// IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_INNER_FOR_INC]]:
+// IR-NEXT:    %[[TMP45:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
+// IR-NEXT:    %[[ADD46:.+]] = add i32 %[[TMP45]], 1
+// IR-NEXT:    store i32 %[[ADD46]], ptr %[[DOTOMP_IV]], align 4
 // IR-NEXT:    br label %[[OMP_INNER_FOR_COND]]
-// IR:       [[OMP_INNER_FOR_END]]:
-// IR-NEXT:    br label %[[OMP_LOOP_EXIT:.*]]
-// IR:       [[OMP_LOOP_EXIT]]:
-// IR-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP0]])
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_INNER_FOR_END]]:
+// IR-NEXT:    br label %[[OMP_LOOP_EXIT:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_LOOP_EXIT]]:
+// IR-NEXT:    call void @__kmpc_for_static_fini(ptr @1, i32 %[[TMP0]])
 // IR-NEXT:    br label %[[OMP_PRECOND_END]]
-// IR:       [[OMP_PRECOND_END]]:
-// IR-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3:[0-9]+]], i32 [[TMP0]])
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_PRECOND_END]]:
+// IR-NEXT:    call void @__kmpc_barrier(ptr @3, i32 %[[TMP0]])
 // IR-NEXT:    ret void
-//
+// IR-NEXT:  }
 extern "C" void func(int start, int end, int step) {
   #pragma omp for
   #pragma omp tile sizes(4)
@@ -217,9 +239,7 @@ extern "C" void func(int start, int end, int step) {
 #endif /* HEADER */
 
 
-//.
-// IR: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]], [[META4:![0-9]+]]}
-// IR: [[META3]] = !{!"llvm.loop.mustprogress"}
-// IR: [[META4]] = !{!"llvm.loop.unroll.count", i32 2}
-// IR: [[LOOP5]] = distinct !{[[LOOP5]], [[META3]]}
-//.
+// IR: ![[LOOP2]] = distinct !{![[LOOP2]], ![[LOOPPROP3:[0-9]+]], ![[LOOPPROP4:[0-9]+]]}
+// IR: ![[LOOPPROP3]] = !{!"llvm.loop.mustprogress"}
+// IR: ![[LOOPPROP4]] = !{!"llvm.loop.unroll.count", i32 2}
+// IR: ![[LOOP5]] = distinct !{![[LOOP5]], ![[LOOPPROP3]]}
diff --git a/clang/unittests/AST/AttrTest.cpp b/clang/unittests/AST/AttrTest.cpp
index cbdc3a1cc59a9..56096054dd267 100644
--- a/clang/unittests/AST/AttrTest.cpp
+++ b/clang/unittests/AST/AttrTest.cpp
@@ -8,6 +8,7 @@
 
 #include "clang/AST/Attr.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/StmtOpenMP.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
 #include "clang/ASTMatchers/ASTMatchers.h"
 #include "clang/Basic/AttrKinds.h"
@@ -272,4 +273,69 @@ TEST(Attr, UnknownScopedAttributeSpellingIndex) {
   EXPECT_EQ(CI.getAttributeSpellingListIndex(), 0u);
 }
 
+template <typename NodeT> const NodeT *findFirst(const Stmt *S) {
+  if (!S)
+    return nullptr;
+  if (const auto *N = dyn_cast<NodeT>(S))
+    return N;
+  for (const Stmt *C : S->children())
+    if (const NodeT *N = findFirst<NodeT>(C))
+      return N;
+  return nullptr;
+}
+
+const OMPInvariantPredicateBoundAttr *findIntraTileHint(const Stmt *S) {
+  if (const auto *AS = findFirst<AttributedStmt>(S))
+    for (const Attr *A : AS->getAttrs())
+      if (const auto *Hint = dyn_cast<OMPInvariantPredicateBoundAttr>(A))
+        return Hint;
+  return nullptr;
+}
+
+// The intra-tile loop generated by '#pragma omp tile' carries an internal
+// OMPInvariantPredicateBoundAttr; that marker is what lets a consuming
+// directive such as 'collapse' reinterpret the min-bounded loop as a
+// rectangular one with a constant trip count.
+//
+// Assert on the attribute directly here,
+// including the fact that the overshoot predicate is dropped when the trip
+// count is a compile-time multiple of the tile size.
+TEST(Attr, OMPInvariantPredicateBoundOnIntraTileLoop) {
+  auto AST = buildASTFromCodeWithArgs(
+      R"cpp(
+        void body(int);
+        void partial_last_tile() {
+          #pragma omp tile sizes(5)
+          for (int i = 0; i < 11; ++i)
+            body(i);
+        }
+        void full_tiles_only() {
+          #pragma omp tile sizes(5)
+          for (int i = 0; i < 10; ++i)
+            body(i);
+        }
+      )cpp",
+      {"-fopenmp"});
+  ASSERT_TRUE(AST);
+
+  auto GetHint = [&AST](const char *FuncName) {
+    const FunctionDecl *Func = getFunctionNode(AST.get(), FuncName);
+    const auto *Tile = findFirst<OMPTileDirective>(Func->getBody());
+    EXPECT_TRUE(Tile);
+    return Tile ? findIntraTileHint(Tile->getTransformedStmt()) : nullptr;
+  };
+
+  const OMPInvariantPredicateBoundAttr *Partial = GetHint("partial_last_tile");
+  ASSERT_TRUE(Partial);
+  EXPECT_TRUE(Partial->getRectCond());
+  EXPECT_TRUE(Partial->getTileSize());
+  EXPECT_TRUE(Partial->getPredicate());
+
+  const OMPInvariantPredicateBoundAttr *Full = GetHint("full_tiles_only");
+  ASSERT_TRUE(Full);
+  EXPECT_TRUE(Full->getRectCond());
+  EXPECT_TRUE(Full->getTileSize());
+  EXPECT_FALSE(Full->getPredicate());
+}
+
 } // namespace
diff --git a/openmp/runtime/test/transform/tile/intfor.c b/openmp/runtime/test/transform/tile/intfor.c
index 05ef5e85a0dad..4a930eab6730a 100644
--- a/openmp/runtime/test/transform/tile/intfor.c
+++ b/openmp/runtime/test/transform/tile/intfor.c
@@ -188,4 +188,4 @@ int main() {
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: done
+// CHECK-NEXT: done
\ No newline at end of file
diff --git a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-stacked-tile-deep.cpp b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-stacked-tile-deep.cpp
deleted file mode 100644
index 1e835db20fb81..0000000000000
--- a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-stacked-tile-deep.cpp
+++ /dev/null
@@ -1,32 +0,0 @@
-// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
-
-// XFAIL: *
-// collapse(3) through stacked tiles is not supported.
-
-#ifndef HEADER
-#define HEADER
-
-#include <cstdlib>
-#include <cstdio>
-
-int main() {
-  printf("do\n");
-#pragma omp parallel for collapse(3) num_threads(1)
-#pragma omp tile sizes(2)
-#pragma omp tile sizes(4)
-  for (int i = 0; i < 6; ++i)
-    printf("i=%d\n", i);
-  printf("done\n");
-  return EXIT_SUCCESS;
-}
-
-#endif /* HEADER */
-
-// CHECK:      do
-// CHECK-NEXT: i=0
-// CHECK-NEXT: i=1
-// CHECK-NEXT: i=2
-// CHECK-NEXT: i=3
-// CHECK-NEXT: i=4
-// CHECK-NEXT: i=5
-// CHECK-NEXT: done


From openmp-commits at lists.llvm.org  Wed Aug 12 03:28:48 2026
From: openmp-commits at lists.llvm.org (Michael Kruse via Openmp-commits)
Date: Wed, 12 Aug 2026 03:28:48 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [Clang][OpenMP] Add
 `#pragma omp flatten` loop directive + `depth` clause (PR #206977)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206977@github.com>
Message-ID: <6a7c4ae0.e78280df.145547.f6c9@mx.google.com>


https://github.com/Meinersbur commented:

I think we need more tests:

Diagnostics output
 * Diagnostics when applied to a non-perfectly nested loop
 * depth argument diagnostics: `0`, `-1`, insufficient number of loops
 * depth argument is not a constant

Codegen/execution
 * When applied to a non-rectangular loop
 * Applied to the generated loops of other constructs: fuse, reverse, unroll, etc
 * Other loop-associated constructs applied to it, e.g. flatten+for, flatten+unroll, etc.
 * Applied onto itself: flatten + flatten ≈ flatten depth(3)
 * depth argument is a template parameter
 * Degenerate loop: `for (int i = 0; i < -1; ++i)`

Optional: A warning diagnostic if `depth` is omitted (so the default of 2 applies), but there are 3 or more loops that could be flattened

https://github.com/llvm/llvm-project/pull/206977

From openmp-commits at lists.llvm.org  Wed Aug 12 03:28:48 2026
From: openmp-commits at lists.llvm.org (Michael Kruse via Openmp-commits)
Date: Wed, 12 Aug 2026 03:28:48 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [Clang][OpenMP] Add
 `#pragma omp flatten` loop directive + `depth` clause (PR #206977)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206977@github.com>
Message-ID: <6a7c4ae0.28a0c303.2b4f83.f6c6@mx.google.com>


================
@@ -16382,6 +16395,277 @@ StmtResult SemaOpenMP::ActOnOpenMPInterchangeDirective(
                                          buildPreInits(Context, PreInits));
 }
 
+/// Counts the perfectly nested canonical loop depth at \p AStmt. Returns
+/// std::nullopt if a loop-generating transformation prevents a static count.
+static std::optional<unsigned> getCanonicalLoopNestDepth(Stmt *AStmt) {
----------------
Meinersbur wrote:

Please try to use existing functions. If you cannot adapt `doForAllLoops`, at least use `OMPLoopBasedDirective::tryToFindNextInnerLoop`.

https://github.com/llvm/llvm-project/pull/206977

From openmp-commits at lists.llvm.org  Wed Aug 12 03:28:48 2026
From: openmp-commits at lists.llvm.org (Michael Kruse via Openmp-commits)
Date: Wed, 12 Aug 2026 03:28:48 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [Clang][OpenMP] Add
 `#pragma omp flatten` loop directive + `depth` clause (PR #206977)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206977@github.com>
Message-ID: <6a7c4ae0.8f0cbdd2.259ad8.808a@mx.google.com>


================
@@ -16382,6 +16395,277 @@ StmtResult SemaOpenMP::ActOnOpenMPInterchangeDirective(
                                          buildPreInits(Context, PreInits));
 }
 
+/// Counts the perfectly nested canonical loop depth at \p AStmt. Returns
+/// std::nullopt if a loop-generating transformation prevents a static count.
+static std::optional<unsigned> getCanonicalLoopNestDepth(Stmt *AStmt) {
+  unsigned Depth = 0;
+  Stmt *CurStmt = AStmt ? AStmt->IgnoreContainers() : nullptr;
+  while (CurStmt) {
+    if (isa<OMPLoopTransformationDirective>(CurStmt))
+      return std::nullopt;
+    if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(CurStmt))
+      CurStmt = CanonLoop->getLoopStmt();
+    Stmt *Body = nullptr;
+    if (auto *For = dyn_cast<ForStmt>(CurStmt))
+      Body = For->getBody();
+    else if (auto *RangeFor = dyn_cast<CXXForRangeStmt>(CurStmt))
+      Body = RangeFor->getBody();
+    else
+      break;
+    ++Depth;
+    CurStmt = Body ? Body->IgnoreContainers() : nullptr;
+  }
+  return Depth;
+}
+
+StmtResult
+SemaOpenMP::ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
+                                        Stmt *AStmt, SourceLocation StartLoc,
+                                        SourceLocation EndLoc) {
+  ASTContext &Context = getASTContext();
+  DeclContext *CurContext = SemaRef.CurContext;
+  Scope *CurScope = SemaRef.getCurScope();
+
+  // Empty statement should only be possible if there already was an error.
+  if (!AStmt)
+    return StmtError();
+
+  // flatten without 'depth' clause combines two loops; 'depth(k)' selects k.
+  unsigned NumLoops = 2;
+  bool DepthIsValueDependent = false;
+  const auto *DepthClause =
+      OMPExecutableDirective::getSingleClause<OMPDepthClause>(Clauses);
+  if (DepthClause) {
+    Expr *DepthExpr = DepthClause->getDepth();
+    if (DepthExpr && DepthExpr->isValueDependent()) {
+      DepthIsValueDependent = true;
+    } else if (DepthExpr) {
+      Expr::EvalResult EvalResult;
+      if (DepthExpr->EvaluateAsInt(EvalResult, Context))
+        NumLoops = EvalResult.Val.getInt().getZExtValue();
+    }
+  }
+
+  // Report "expected k for loops, but found only n" when depth exceeds the
+  // perfect nest (same form as 'collapse'); skip if depth or nest is unknown.
+  if (DepthClause && !DepthIsValueDependent) {
+    if (std::optional<unsigned> NestDepth = getCanonicalLoopNestDepth(AStmt);
+        NestDepth && NumLoops > *NestDepth) {
+      Diag(AStmt->getBeginLoc(), diag::err_omp_not_for)
+          << /*expected N for loops form=*/1
+          << getOpenMPDirectiveName(OMPD_flatten) << NumLoops
+          << (*NestDepth > 0) << *NestDepth;
+      return StmtError();
+    }
+  }
+
+  // Defer when 'depth' is value-dependent (concrete k unknown until
+  // instantiation).
+  if (DepthIsValueDependent)
+    return OMPFlattenDirective::Create(Context, StartLoc, EndLoc, Clauses,
+                                       NumLoops, AStmt, nullptr, nullptr);
+
+  // Verify and diagnose loop nest.
+  SmallVector<OMPLoopBasedDirective::HelperExprs, 4> LoopHelpers(NumLoops);
+  Stmt *Body = nullptr;
+  SmallVector<SmallVector<Stmt *>, 4> OriginalInits;
+  if (!checkTransformableLoopNest(OMPD_flatten, AStmt, NumLoops, LoopHelpers,
+                                  Body, OriginalInits))
+    return StmtError();
+
+  // Delay flattening to when template is completely instantiated.
+  if (CurContext->isDependentContext())
+    return OMPFlattenDirective::Create(Context, StartLoc, EndLoc, Clauses,
+                                       NumLoops, AStmt, nullptr, nullptr);
+
+  assert(LoopHelpers.size() == NumLoops &&
+         "Expecting loop iteration space dimensionality to match number of "
+         "affected loops");
+  assert(OriginalInits.size() == NumLoops &&
+         "Expecting loop iteration space dimensionality to match number of "
+         "affected loops");
+
+  // Find the affected loops.
+  SmallVector<Stmt *> LoopStmts(NumLoops, nullptr);
+  collectLoopStmts(AStmt, LoopStmts);
+
+  // Collect pre-init statements in outer-to-inner order.
+  SmallVector<Stmt *> PreInits;
+  for (auto I : llvm::seq<int>(NumLoops)) {
+    OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
+    assert(LoopHelper.Counters.size() == 1 &&
+           "Single-dimensional loop iteration space expected");
+    addLoopPreInits(Context, LoopHelper, LoopStmts[I], OriginalInits[I],
+                    PreInits);
+  }
+
+  CaptureVars CopyTransformer(SemaRef);
+  auto MakeNumIterations = [&CopyTransformer,
+                            &LoopHelpers](unsigned I) -> Expr * {
+    return AssertSuccess(
+        CopyTransformer.TransformExpr(LoopHelpers[I].NumIterations));
+  };
+
+  OMPLoopBasedDirective::HelperExprs &OutermostHelper = LoopHelpers[0];
+  auto *OutermostCntVar = cast<DeclRefExpr>(OutermostHelper.Counters.front());
+  SourceLocation OrigVarLoc = OutermostCntVar->getExprLoc();
+  SourceLocation OrigVarLocBegin = OutermostCntVar->getBeginLoc();
+  SourceLocation OrigVarLocEnd = OutermostCntVar->getEndLoc();
+  SourceLocation CondLoc = OutermostHelper.Cond->getExprLoc();
+
+  // Product of trip counts; mirror 'collapse' IV-width selection to avoid
+  // overflow when several counts are multiplied.
+  auto BuildTripCount = [&](unsigned Bits) -> ExprResult {
+    ExprResult Product;
+    for (unsigned I = 0; I < NumLoops; ++I) {
+      ExprResult N = widenIterationCount(Bits, MakeNumIterations(I), SemaRef);
+      if (!N.isUsable())
+        return ExprError();
+      if (I == 0)
+        Product = N;
+      else
+        Product = SemaRef.BuildBinOp(CurScope, CondLoc, BO_Mul, Product.get(),
+                                     N.get());
+      if (!Product.isUsable())
+        return ExprError();
+    }
+    return Product;
+  };
+
+  bool AllCountsLessThan32Bits = true;
+  for (unsigned I = 0; I < NumLoops; ++I)
+    AllCountsLessThan32Bits &=
+        Context.getTypeSize(LoopHelpers[I].NumIterations->getType()) < 32;
+
+  ExprResult TripCount32 = BuildTripCount(/*Bits=*/32);
+  ExprResult TripCount64 = BuildTripCount(/*Bits=*/64);
----------------
Meinersbur wrote:

Don't create both expressions if you are going to use just one of them.

https://github.com/llvm/llvm-project/pull/206977

From openmp-commits at lists.llvm.org  Wed Aug 12 03:28:48 2026
From: openmp-commits at lists.llvm.org (Michael Kruse via Openmp-commits)
Date: Wed, 12 Aug 2026 03:28:48 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [Clang][OpenMP] Add
 `#pragma omp flatten` loop directive + `depth` clause (PR #206977)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206977@github.com>
Message-ID: <6a7c4ae0.94610230.382472.666b@mx.google.com>


================
@@ -5959,6 +5959,79 @@ class OMPInterchangeDirective final
   }
 };
 
+/// Represents the '#pragma omp flatten' loop transformation directive.
+///
+/// \code{c}
+///   #pragma omp flatten
+///   for (int i = 0; i < m; ++i)
+///     for (int j = 0; j < n; ++j)
+///       ..
+/// \endcode
+class OMPFlattenDirective final
+    : public OMPCanonicalLoopNestTransformationDirective {
+  friend class ASTStmtReader;
+  friend class OMPExecutableDirective;
+
+  /// Offsets of child members.
+  enum {
+    PreInitsOffset = 0,
+    TransformedStmtOffset,
+  };
+
+  explicit OMPFlattenDirective(SourceLocation StartLoc, SourceLocation EndLoc,
+                               unsigned NumLoops)
+      : OMPCanonicalLoopNestTransformationDirective(
+            OMPFlattenDirectiveClass, llvm::omp::OMPD_flatten, StartLoc, EndLoc,
+            NumLoops) {}
+
+  void setPreInits(Stmt *PreInits) {
+    Data->getChildren()[PreInitsOffset] = PreInits;
+  }
+
+  void setTransformedStmt(Stmt *S) {
+    Data->getChildren()[TransformedStmtOffset] = S;
+  }
+
+public:
+  /// Create a new AST node representation for '#pragma omp flatten'.
+  ///
+  /// \param C         Context of the AST.
+  /// \param StartLoc  Location of the introducer (e.g. the 'omp' token).
+  /// \param EndLoc    Location of the directive's end (e.g. the tok::eod).
+  /// \param Clauses   The directive's clauses.
+  /// \param NumLoops  Number of affected loops (the flatten depth, currently
+  ///                  always 2).
----------------
Meinersbur wrote:

There are tests in this PR where it is 3.

https://github.com/llvm/llvm-project/pull/206977

From openmp-commits at lists.llvm.org  Wed Aug 12 03:28:48 2026
From: openmp-commits at lists.llvm.org (Michael Kruse via Openmp-commits)
Date: Wed, 12 Aug 2026 03:28:48 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [Clang][OpenMP] Add
 `#pragma omp flatten` loop directive + `depth` clause (PR #206977)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206977@github.com>
Message-ID: <6a7c4ae0.a70a48cb.48415.412c@mx.google.com>


================
@@ -161,6 +161,7 @@ def OMPC_Depobj : Clause<[Spelling<"depobj">]> {
   let isImplicit = true;
 }
 def OMPC_Depth : Clause<[Spelling<"depth">]> {
+  let clangClass = "OMPDepthClause";
----------------
Meinersbur wrote:

This file claims `depth` is also supported on `fuse`. Can you make it an error diagnostic to be not yet supported when applied to fuse?

https://github.com/llvm/llvm-project/pull/206977

From openmp-commits at lists.llvm.org  Wed Aug 12 03:28:48 2026
From: openmp-commits at lists.llvm.org (Michael Kruse via Openmp-commits)
Date: Wed, 12 Aug 2026 03:28:48 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [Clang][OpenMP] Add
 `#pragma omp flatten` loop directive + `depth` clause (PR #206977)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206977@github.com>
Message-ID: <6a7c4ae0.dbf3bad1.2ce02c.fc30@mx.google.com>


================

----------------
Meinersbur wrote:

Could you add tests where what `flatten` does changes the output?
For instance:
```
#pragma omp parallel for schedule(static,1) num_threads(2)
#pragma omp flatten 
  for (int i = 0; i < 2; i += 1)
    for (int j = 0; j < 2; j += 1)
       printf("i=%d j=%d tid=%d\n", i, j, omp_get_thread_num());
```

Output with flatten should be:
```
i=0 j=0 tid=0
i=0 j=1 tid=1
i=1 j=0 tid=0
i=1 j=1 tid=0
```

without flatten:
```
i=0 j=0 tid=0
i=0 j=1 tid=0
i=1 j=0 tid=1
i=1 j=1 tid=1
```

Generally, add test that check the composability with other directives, that's what its intended use it


https://github.com/llvm/llvm-project/pull/206977

From openmp-commits at lists.llvm.org  Wed Aug 12 03:28:48 2026
From: openmp-commits at lists.llvm.org (Michael Kruse via Openmp-commits)
Date: Wed, 12 Aug 2026 03:28:48 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [Clang][OpenMP] Add
 `#pragma omp flatten` loop directive + `depth` clause (PR #206977)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206977@github.com>
Message-ID: <6a7c4ae0.a70a48cb.48415.4133@mx.google.com>


https://github.com/Meinersbur edited https://github.com/llvm/llvm-project/pull/206977

From openmp-commits at lists.llvm.org  Wed Aug 12 03:28:49 2026
From: openmp-commits at lists.llvm.org (Michael Kruse via Openmp-commits)
Date: Wed, 12 Aug 2026 03:28:49 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [Clang][OpenMP] Add
 `#pragma omp flatten` loop directive + `depth` clause (PR #206977)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206977@github.com>
Message-ID: <6a7c4ae1.05c456d6.2bad36.1d3a@mx.google.com>


================
@@ -16382,6 +16395,277 @@ StmtResult SemaOpenMP::ActOnOpenMPInterchangeDirective(
                                          buildPreInits(Context, PreInits));
 }
 
+/// Counts the perfectly nested canonical loop depth at \p AStmt. Returns
+/// std::nullopt if a loop-generating transformation prevents a static count.
+static std::optional<unsigned> getCanonicalLoopNestDepth(Stmt *AStmt) {
+  unsigned Depth = 0;
+  Stmt *CurStmt = AStmt ? AStmt->IgnoreContainers() : nullptr;
+  while (CurStmt) {
+    if (isa<OMPLoopTransformationDirective>(CurStmt))
+      return std::nullopt;
+    if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(CurStmt))
+      CurStmt = CanonLoop->getLoopStmt();
+    Stmt *Body = nullptr;
+    if (auto *For = dyn_cast<ForStmt>(CurStmt))
+      Body = For->getBody();
+    else if (auto *RangeFor = dyn_cast<CXXForRangeStmt>(CurStmt))
+      Body = RangeFor->getBody();
+    else
+      break;
+    ++Depth;
+    CurStmt = Body ? Body->IgnoreContainers() : nullptr;
+  }
+  return Depth;
+}
+
+StmtResult
+SemaOpenMP::ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
+                                        Stmt *AStmt, SourceLocation StartLoc,
+                                        SourceLocation EndLoc) {
+  ASTContext &Context = getASTContext();
+  DeclContext *CurContext = SemaRef.CurContext;
+  Scope *CurScope = SemaRef.getCurScope();
+
+  // Empty statement should only be possible if there already was an error.
+  if (!AStmt)
+    return StmtError();
+
+  // flatten without 'depth' clause combines two loops; 'depth(k)' selects k.
+  unsigned NumLoops = 2;
+  bool DepthIsValueDependent = false;
+  const auto *DepthClause =
+      OMPExecutableDirective::getSingleClause<OMPDepthClause>(Clauses);
+  if (DepthClause) {
+    Expr *DepthExpr = DepthClause->getDepth();
+    if (DepthExpr && DepthExpr->isValueDependent()) {
+      DepthIsValueDependent = true;
+    } else if (DepthExpr) {
+      Expr::EvalResult EvalResult;
+      if (DepthExpr->EvaluateAsInt(EvalResult, Context))
+        NumLoops = EvalResult.Val.getInt().getZExtValue();
+    }
+  }
+
+  // Report "expected k for loops, but found only n" when depth exceeds the
+  // perfect nest (same form as 'collapse'); skip if depth or nest is unknown.
+  if (DepthClause && !DepthIsValueDependent) {
+    if (std::optional<unsigned> NestDepth = getCanonicalLoopNestDepth(AStmt);
----------------
Meinersbur wrote:

Can you use `OMPLoopBasedDirective::doForAllLoop` instead? It returns false if there are insufficient loops.

https://github.com/llvm/llvm-project/pull/206977

From openmp-commits at lists.llvm.org  Wed Aug 12 03:28:49 2026
From: openmp-commits at lists.llvm.org (Michael Kruse via Openmp-commits)
Date: Wed, 12 Aug 2026 03:28:49 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [Clang][OpenMP] Add
 `#pragma omp flatten` loop directive + `depth` clause (PR #206977)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206977@github.com>
Message-ID: <6a7c4ae1.2bb73dab.357588.1e5d@mx.google.com>


================

----------------
Meinersbur wrote:

Other tests use this to override the OpenMP version:
```
// RUN: %libomp-cxx-compile -fopenmp-version=61  && %libomp-run
```

https://github.com/llvm/llvm-project/pull/206977

From openmp-commits at lists.llvm.org  Wed Aug 12 03:30:11 2026
From: openmp-commits at lists.llvm.org (Michael Kruse via Openmp-commits)
Date: Wed, 12 Aug 2026 03:30:11 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [Clang][OpenMP] Add
 `#pragma omp flatten` loop directive + `depth` clause (PR #206977)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206977@github.com>
Message-ID: <6a7c4b33.39bfcb41.96a3f.17aa@mx.google.com>


https://github.com/Meinersbur edited https://github.com/llvm/llvm-project/pull/206977

From openmp-commits at lists.llvm.org  Wed Aug 12 04:37:57 2026
From: openmp-commits at lists.llvm.org (Alex Duran via Openmp-commits)
Date: Wed, 12 Aug 2026 04:37:57 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [Offload] Add logger in Plugin API
 functions for OpenMP (PR #215582)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/215582@github.com>
Message-ID: <6a7c5b15.b8c88a87.38394d.14c5@mx.google.com>


https://github.com/adurang edited https://github.com/llvm/llvm-project/pull/215582

From openmp-commits at lists.llvm.org  Wed Aug 12 04:37:57 2026
From: openmp-commits at lists.llvm.org (Alex Duran via Openmp-commits)
Date: Wed, 12 Aug 2026 04:37:57 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [Offload] Add logger in Plugin API
 functions for OpenMP (PR #215582)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/215582@github.com>
Message-ID: <6a7c5b15.e12b82d4.561a0.0e43@mx.google.com>


https://github.com/adurang commented:

could we have an enable/disable routine (which would also allow for selective profiling) in the CommonInterface and just let OpenMP call that when OMP_INFOTYPE_API_TRACE is enabled?

https://github.com/llvm/llvm-project/pull/215582

From openmp-commits at lists.llvm.org  Wed Aug 12 04:37:57 2026
From: openmp-commits at lists.llvm.org (Alex Duran via Openmp-commits)
Date: Wed, 12 Aug 2026 04:37:57 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [Offload] Add logger in Plugin API
 functions for OpenMP (PR #215582)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/215582@github.com>
Message-ID: <6a7c5b15.f6bcba31.11169f.8b9c@mx.google.com>


================
@@ -0,0 +1,162 @@
+//===-- Shared/APITrace.h - Tracing of offload API calls --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Prints one line per offload API call when the OMP_INFOTYPE_API_TRACE bit of
+// LIBOMPTARGET_INFO is set:
+//
+//   ---> init_device(.DeviceId = 0)-> OFFLOAD_SUCCESS (1053 us)
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OMPTARGET_SHARED_API_TRACE_H
+#define OMPTARGET_SHARED_API_TRACE_H
+
+#include "APITypes.h"
+#include "Debug.h"
+#include "omptarget.h"
+
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <chrono>
+#include <type_traits>
+#include <utility>
+
+namespace llvm::offload::trace {
+
+/// The level is read on every call so that __tgt_set_info_flag can turn tracing
+/// on and off around a region of interest.
+inline bool isTraceEnabled() { return getInfoLevel() & OMP_INFOTYPE_API_TRACE; }
----------------
adurang wrote:

Do we really want to enable tracing at this level by default in all builds? Seems a bit of heavy hammer to me.

https://github.com/llvm/llvm-project/pull/215582

From openmp-commits at lists.llvm.org  Wed Aug 12 04:49:18 2026
From: openmp-commits at lists.llvm.org (Alex Duran via Openmp-commits)
Date: Wed, 12 Aug 2026 04:49:18 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [Offload] Add logger in Plugin API
 functions for OpenMP (PR #215582)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/215582@github.com>
Message-ID: <6a7c5dbe.4a46f23c.26208d.ccad@mx.google.com>


https://github.com/adurang edited https://github.com/llvm/llvm-project/pull/215582

From openmp-commits at lists.llvm.org  Wed Aug 12 06:13:08 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Wed, 12 Aug 2026 06:13:08 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP] Support up to 32 arguments in
 the generic microtask dispatcher (PR #211071)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/211071@github.com>
Message-ID: <6a7c7164.af4834b4.3cab8e.6ff5@mx.google.com>


arsnyder16 wrote:

@sbc100 Anything else here? Or who to ping to give it a look? 

https://github.com/llvm/llvm-project/pull/211071

From openmp-commits at lists.llvm.org  Wed Aug 12 06:19:27 2026
From: openmp-commits at lists.llvm.org (Michael Kruse via Openmp-commits)
Date: Wed, 12 Aug 2026 06:19:27 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [mlir] [openmp] [Clang][OpenMP]
 Canonicalize Intra-tiles in Loop Tiling (PR #191114)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/191114@github.com>
Message-ID: <6a7c72df.0d5ebfa7.3a9d6a.689c@mx.google.com>


================
@@ -9575,8 +9581,25 @@ static bool checkOpenMPIterationSpace(
     llvm::MutableArrayRef<LoopIterationSpace> ResultIterSpaces,
     llvm::MapVector<const Expr *, DeclRefExpr *> &Captures,
     const llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopVarDecls,
-    llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopInductionVars) {
+    llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopInductionVars,
+    const OMPInvariantPredicateBoundAttr *IntraTileHint) {
   bool SupportsNonRectangular = !isOpenMPLoopTransformationDirective(DKind);
+  // See the tile reinterpretation design note in ActOnOpenMPTileDirective.
----------------
Meinersbur wrote:

```suggestion

  // See the tile reinterpretation design note in ActOnOpenMPTileDirective.
```
[nit] Usually we put an empty line before comment lines

https://github.com/llvm/llvm-project/pull/191114

From openmp-commits at lists.llvm.org  Wed Aug 12 06:19:28 2026
From: openmp-commits at lists.llvm.org (Michael Kruse via Openmp-commits)
Date: Wed, 12 Aug 2026 06:19:28 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [mlir] [openmp] [Clang][OpenMP]
 Canonicalize Intra-tiles in Loop Tiling (PR #191114)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/191114@github.com>
Message-ID: <6a7c72e0.dbb26164.2c51dd.4e54@mx.google.com>


================
@@ -9575,8 +9581,25 @@ static bool checkOpenMPIterationSpace(
     llvm::MutableArrayRef<LoopIterationSpace> ResultIterSpaces,
     llvm::MapVector<const Expr *, DeclRefExpr *> &Captures,
     const llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopVarDecls,
-    llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopInductionVars) {
+    llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopInductionVars,
+    const OMPInvariantPredicateBoundAttr *IntraTileHint) {
----------------
Meinersbur wrote:

Would it be possible to not need to propagate `IntraTileHint` through multiple functions, but instead make `checkOpenMPIterationSpace` find it itself? For instance `Stmt *S` can be that AttributedStmt. If so, extract the `OMPInvariantPredicateBoundAttr` and continue with the Stmt that S had attributed.

https://github.com/llvm/llvm-project/pull/191114

From openmp-commits at lists.llvm.org  Wed Aug 12 06:19:28 2026
From: openmp-commits at lists.llvm.org (Michael Kruse via Openmp-commits)
Date: Wed, 12 Aug 2026 06:19:28 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [mlir] [openmp] [Clang][OpenMP]
 Canonicalize Intra-tiles in Loop Tiling (PR #191114)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/191114@github.com>
Message-ID: <6a7c72e0.a1accbb4.2cf7.2a6c@mx.google.com>


================
@@ -12303,6 +12303,9 @@ def err_omp_loop_bad_collapse_var : Error<
   "cannot use variable %1 in collapsed imperfectly-nested loop %select{init|condition|increment}0 statement">;
 def err_omp_loop_var_reused_in_collapsed_loop : Error<
   "loop iteration variable %0 cannot be reused in a nested loop of a collapsed loop nest">;
+def err_omp_collapse_stacked_tile : Error<
+  "cannot collapse the intra-tile loop of a '#pragma omp tile' that is nested "
+  "inside another loop-transforming directive">;
----------------
Meinersbur wrote:

Could you adjust the error message to say that it is a current limitation of the implementation, not something that OpenMP does not allow?

https://github.com/llvm/llvm-project/pull/191114

From openmp-commits at lists.llvm.org  Wed Aug 12 06:19:28 2026
From: openmp-commits at lists.llvm.org (Michael Kruse via Openmp-commits)
Date: Wed, 12 Aug 2026 06:19:28 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [mlir] [openmp] [Clang][OpenMP]
 Canonicalize Intra-tiles in Loop Tiling (PR #191114)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/191114@github.com>
Message-ID: <6a7c72e0.c4d3d80a.5c373.2ec9@mx.google.com>


================
@@ -15257,6 +15367,13 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
         ForStmt(Context, InitStmt.get(), CondExpr.get(), nullptr,
                 IncrStmt.get(), Inner, LoopHelper.Init->getBeginLoc(),
                 LoopHelper.Init->getBeginLoc(), LoopHelper.Inc->getEndLoc());
+
+    // Attach the droppable reinterpretation attribute to the intra-tile loop.
+    auto *Hint = OMPInvariantPredicateBoundAttr::CreateImplicit(
----------------
Meinersbur wrote:

The OMPInvariantPredicateBoundAttr doesn't need to be added either in this case.

https://github.com/llvm/llvm-project/pull/191114

From openmp-commits at lists.llvm.org  Wed Aug 12 06:28:23 2026
From: openmp-commits at lists.llvm.org (Robert Imschweiler via Openmp-commits)
Date: Wed, 12 Aug 2026 06:28:23 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [offload] Use HSA SVM for AMDGPU
	shared memory (PR #215801)
Message-ID: <llvm.org/llvm/llvm-project/pull/215801@github.com>


https://github.com/ro-i created https://github.com/llvm/llvm-project/pull/215801

Claude assisted with this patch.

>From aa8d5ec4fed93d50c65bfe81035d2ea2ebbf7ec5 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Wed, 12 Aug 2026 07:33:34 -0500
Subject: [PATCH] [offload] Use HSA SVM for AMDGPU shared memory

---
 .../amdgpu/dynamic_hsa/hsa.cpp                |   1 +
 .../plugins-nextgen/amdgpu/dynamic_hsa/hsa.h  |   1 +
 .../amdgpu/dynamic_hsa/hsa_ext_amd.h          |  15 +
 offload/plugins-nextgen/amdgpu/src/rtl.cpp    | 288 +++++++++++++++++-
 .../api/amdgpu_managed_memory_accessible.c    |  31 ++
 .../test/api/omp_device_managed_memory_ops.c  |  68 +++++
 openmp/docs/design/Runtimes.rst               |  13 +
 7 files changed, 406 insertions(+), 11 deletions(-)
 create mode 100644 offload/test/api/amdgpu_managed_memory_accessible.c
 create mode 100644 offload/test/api/omp_device_managed_memory_ops.c

diff --git a/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa.cpp b/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa.cpp
index 5c7ec186b0ceb..eb206a37634b9 100644
--- a/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa.cpp
+++ b/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa.cpp
@@ -66,6 +66,7 @@ DLWRAP(hsa_amd_agents_allow_access, 4)
 DLWRAP(hsa_amd_memory_lock, 5)
 DLWRAP(hsa_amd_memory_unlock, 1)
 DLWRAP(hsa_amd_memory_fill, 3)
+DLWRAP(hsa_amd_svm_attributes_set, 4)
 DLWRAP(hsa_amd_register_system_event_handler, 2)
 DLWRAP(hsa_amd_signal_create, 5)
 DLWRAP(hsa_amd_signal_async_handler, 5)
diff --git a/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa.h b/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa.h
index f66326a7f240e..1ed9a6275c3dc 100644
--- a/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa.h
+++ b/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa.h
@@ -123,6 +123,7 @@ typedef enum {
   HSA_SYSTEM_INFO_VERSION_MINOR = 1,
   HSA_SYSTEM_INFO_TIMESTAMP = 2,
   HSA_SYSTEM_INFO_TIMESTAMP_FREQUENCY = 3,
+  HSA_AMD_SYSTEM_INFO_SVM_SUPPORTED = 0x201,
 } hsa_system_info_t;
 
 typedef enum {
diff --git a/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa_ext_amd.h b/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa_ext_amd.h
index d26f9248e27ef..a78e6d7c5cb4a 100644
--- a/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa_ext_amd.h
+++ b/offload/plugins-nextgen/amdgpu/dynamic_hsa/hsa_ext_amd.h
@@ -125,6 +125,21 @@ hsa_status_t hsa_amd_memory_unlock(void* host_ptr);
 
 hsa_status_t hsa_amd_memory_fill(void *ptr, uint32_t value, size_t count);
 
+typedef enum hsa_amd_svm_attribute_s {
+  HSA_AMD_SVM_ATTRIB_GLOBAL_FLAG = 0,
+  HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE = 0x200,
+} hsa_amd_svm_attribute_t;
+
+typedef struct hsa_amd_svm_attribute_pair_s {
+  uint64_t attribute;
+  uint64_t value;
+} hsa_amd_svm_attribute_pair_t;
+
+hsa_status_t
+hsa_amd_svm_attributes_set(void *ptr, size_t size,
+                           hsa_amd_svm_attribute_pair_t *attribute_list,
+                           size_t attribute_count);
+
 typedef enum hsa_amd_event_type_s {
   HSA_AMD_GPU_MEMORY_FAULT_EVENT = 0,
 } hsa_amd_event_type_t;
diff --git a/offload/plugins-nextgen/amdgpu/src/rtl.cpp b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
index 0f0de67b20c7c..b363bbd472013 100644
--- a/offload/plugins-nextgen/amdgpu/src/rtl.cpp
+++ b/offload/plugins-nextgen/amdgpu/src/rtl.cpp
@@ -10,12 +10,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include <algorithm>
 #include <atomic>
 #include <cassert>
 #include <cstddef>
 #include <cstdint>
+#include <cstring>
 #include <deque>
 #include <functional>
+#include <map>
 #include <mutex>
 #include <string>
 #include <system_error>
@@ -46,7 +49,9 @@
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FileOutputBuffer.h"
 #include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Memory.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Process.h"
 #include "llvm/Support/Program.h"
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/raw_ostream.h"
@@ -450,6 +455,157 @@ struct AMDGPUMemoryPoolTy {
   size_t PoolAllocationAlignment;
 };
 
+/// Class that implements shared (managed) allocations on top of the HSA shared
+/// virtual memory (SVM) interface.
+///
+/// SVM allocations are backed by ordinary system memory that is made
+/// accessible to all the kernel agents. With XNACK enabled, the driver
+/// migrates the pages between the host and the devices on demand; otherwise
+/// they remain resident in system memory.
+struct AMDGPUSVMManagerTy {
+  /// Determine whether the SVM interface can be used for shared allocations.
+  void init() {
+    if (!OMPX_UseSVM)
+      return;
+
+    bool SVMSupported = false;
+    if (hsa_system_get_info(HSA_AMD_SYSTEM_INFO_SVM_SUPPORTED, &SVMSupported) !=
+        HSA_STATUS_SUCCESS)
+      return;
+
+    Supported = SVMSupported;
+    PageSize = llvm::sys::Process::getPageSizeEstimate();
+  }
+
+  /// Release the allocations that are still live.
+  Error deinit() {
+    std::lock_guard<std::mutex> Lock(Mutex);
+
+    Error Err = Plugin::success();
+    for (auto &Allocation : Allocations)
+      if (std::error_code EC =
+              llvm::sys::Memory::releaseMappedMemory(Allocation.second))
+        Err = joinErrors(std::move(Err),
+                         Plugin::error(ErrorCode::UNKNOWN,
+                                       "error releasing SVM memory: %s",
+                                       EC.message().c_str()));
+
+    Allocations.clear();
+    return Err;
+  }
+
+  /// Whether new SVM allocations can be created.
+  bool isSupported() const { return Supported; }
+
+  /// Allocate system memory and make it accessible to all the \p Agents.
+  /// Returns a null pointer if the request cannot be served, either because
+  /// the requested alignment is too large or because the driver rejected the
+  /// allocation.
+  Expected<void *> allocate(size_t Size, size_t Alignment,
+                            ArrayRef<hsa_agent_t> Agents) {
+    // A concurrent allocation may have disabled the SVM interface.
+    if (!Supported)
+      return nullptr;
+
+    // Mapped memory is only page aligned.
+    if (Alignment > PageSize)
+      return nullptr;
+
+    std::error_code EC;
+    llvm::sys::MemoryBlock Block = llvm::sys::Memory::allocateMappedMemory(
+        Size, /*NearBlock=*/nullptr,
+        llvm::sys::Memory::MF_READ | llvm::sys::Memory::MF_WRITE, EC);
+    if (EC)
+      return Plugin::error(ErrorCode::OUT_OF_RESOURCES,
+                           "error allocating SVM memory: %s",
+                           EC.message().c_str());
+
+    // Give all the kernel agents access to the allocation. Accesses may incur
+    // a page fault and the migration of the memory to the accessing agent.
+    llvm::SmallVector<hsa_amd_svm_attribute_pair_t> Attrs;
+    for (hsa_agent_t Agent : Agents)
+      Attrs.push_back({HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE, Agent.handle});
+
+    hsa_status_t Status = hsa_amd_svm_attributes_set(
+        Block.base(), Block.allocatedSize(), Attrs.data(), Attrs.size());
+    if (auto Err =
+            Plugin::check(Status, "error in hsa_amd_svm_attributes_set: %s")) {
+      // The driver rejected the allocation, e.g., because one of the agents
+      // does not support SVM. Stop creating SVM allocations, reporting the
+      // rejection once.
+      if (Supported.exchange(false))
+        REPORT() << "Serving shared allocations from host memory: "
+                 << toString(std::move(Err));
+      else
+        consumeError(std::move(Err));
+
+      // Ignore any error from undoing the allocation.
+      consumeError(llvm::errorCodeToError(
+          llvm::sys::Memory::releaseMappedMemory(Block)));
+      return nullptr;
+    }
+
+    std::lock_guard<std::mutex> Lock(Mutex);
+    Allocations[reinterpret_cast<uintptr_t>(Block.base())] = Block;
+    return Block.base();
+  }
+
+  /// Release the SVM allocation starting at \p Ptr. Returns whether \p Ptr is
+  /// an SVM allocation.
+  Expected<bool> deallocate(void *Ptr) {
+    llvm::sys::MemoryBlock Block;
+    {
+      std::lock_guard<std::mutex> Lock(Mutex);
+      auto It = Allocations.find(reinterpret_cast<uintptr_t>(Ptr));
+      if (It == Allocations.end())
+        return false;
+      Block = It->second;
+      Allocations.erase(It);
+    }
+
+    if (std::error_code EC = llvm::sys::Memory::releaseMappedMemory(Block))
+      return Plugin::error(ErrorCode::UNKNOWN, "error releasing SVM memory: %s",
+                           EC.message().c_str());
+    return true;
+  }
+
+  /// Whether the \p Size bytes starting at \p Ptr are within an SVM
+  /// allocation. Allocations stay live after the SVM interface is disabled, so
+  /// this must not check Supported.
+  bool contains(const void *Ptr, size_t Size = 1) const {
+    const uintptr_t Addr = reinterpret_cast<uintptr_t>(Ptr);
+    std::lock_guard<std::mutex> Lock(Mutex);
+
+    // Find the allocation with the largest base address not above Addr.
+    auto It = Allocations.upper_bound(Addr);
+    if (It == Allocations.begin())
+      return false;
+    --It;
+
+    const size_t Offset = Addr - It->first;
+    const size_t AllocatedSize = It->second.allocatedSize();
+    return Offset < AllocatedSize && Size <= AllocatedSize - Offset;
+  }
+
+private:
+  /// Whether the user allows serving shared allocations through SVM.
+  BoolEnvar OMPX_UseSVM =
+      BoolEnvar("LIBOMPTARGET_AMDGPU_SHARED_ALLOC_SVM", true);
+
+  /// Whether new SVM allocations can be created. Cleared if the driver rejects
+  /// one.
+  std::atomic<bool> Supported = false;
+
+  /// The granularity of the mapped memory allocations.
+  size_t PageSize = 0;
+
+  /// The live SVM allocations, indexed by their base address.
+  std::map<uintptr_t, llvm::sys::MemoryBlock> Allocations;
+
+  /// Mutex for safe access to the allocations.
+  mutable std::mutex Mutex;
+};
+
 /// Class that implements a memory manager that gets memory from a specific
 /// memory pool.
 struct AMDGPUMemoryManagerTy : public DeviceAllocatorTy {
@@ -2701,6 +2857,9 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     return AMDImage;
   }
 
+  /// Get the plugin's manager of the shared (managed) SVM allocations.
+  AMDGPUSVMManagerTy &getSVMManager();
+
   /// Allocate memory on the device or related to the device.
   Expected<void *> allocate(size_t Size, void *, TargetAllocTy Kind,
                             size_t Alignment) override;
@@ -2710,6 +2869,15 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     if (TgtPtr == nullptr)
       return Plugin::success();
 
+    // Shared allocations are SVM allocations, or memory pool allocations when
+    // SVM cannot serve them. The kind is not checked here; the generic layer
+    // already diagnoses deallocations with a mismatching kind.
+    auto WasSVMAllocOrErr = getSVMManager().deallocate(TgtPtr);
+    if (!WasSVMAllocOrErr)
+      return WasSVMAllocOrErr.takeError();
+    if (*WasSVMAllocOrErr)
+      return Plugin::success();
+
     AMDGPUMemoryPoolTy *MemoryPool = nullptr;
     switch (Kind) {
     case TARGET_ALLOC_DEFAULT:
@@ -3023,9 +3191,63 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     return Plugin::success();
   }
 
+  /// Fill \p TgtPtr with the given pattern on the host. Requires \p TgtPtr to
+  /// be host accessible.
+  Error dataFillHostImpl(void *TgtPtr, const void *PatternPtr,
+                         int64_t PatternSize, int64_t Size,
+                         AsyncInfoWrapperTy &AsyncInfoWrapper) {
+    struct HostFillArgsTy {
+      void *Dst;
+      llvm::SmallVector<char, 8> Pattern;
+      int64_t NumTimes;
+    };
+    auto Args = std::make_unique<HostFillArgsTy>(HostFillArgsTy{
+        TgtPtr,
+        {reinterpret_cast<const char *>(PatternPtr),
+         reinterpret_cast<const char *>(PatternPtr) + PatternSize},
+        Size / PatternSize});
+
+    auto Fill = [](void *Data) {
+      std::unique_ptr<HostFillArgsTy> Args(static_cast<HostFillArgsTy *>(Data));
+      auto *Dst = reinterpret_cast<char *>(Args->Dst);
+      if (Args->Pattern.size() == 1) {
+        std::memset(Dst, Args->Pattern.front(), Args->NumTimes);
+      } else {
+        for (int64_t I = 0; I < Args->NumTimes; ++I)
+          Dst = std::copy(Args->Pattern.begin(), Args->Pattern.end(), Dst);
+      }
+    };
+
+    // Defer the fill until all the previously enqueued operations completed.
+    auto HasPendingWorkOrErr = hasPendingWorkImpl(AsyncInfoWrapper);
+    if (!HasPendingWorkOrErr)
+      return HasPendingWorkOrErr.takeError();
+
+    if (*HasPendingWorkOrErr) {
+      AMDGPUStreamTy *Stream = nullptr;
+      if (auto Err = getStream(AsyncInfoWrapper, Stream))
+        return Err;
+
+      if (auto Err = Stream->pushHostCallback(Fill, Args.get()))
+        return Err;
+
+      Args.release();
+      return Plugin::success();
+    }
+
+    Fill(Args.release());
+    return Plugin::success();
+  }
+
   Error dataFillImpl(void *TgtPtr, const void *PatternPtr, int64_t PatternSize,
                      int64_t Size,
                      AsyncInfoWrapperTy &AsyncInfoWrapper) override {
+    // hsa_amd_memory_fill only operates on memory allocated by the HSA
+    // runtime. SVM allocations are host accessible, so fill them on the host.
+    if (getSVMManager().contains(TgtPtr, Size))
+      return dataFillHostImpl(TgtPtr, PatternPtr, PatternSize, Size,
+                              AsyncInfoWrapper);
+
     // Fast case, where we can use the 4 byte hsa_amd_memory_fill
     if (Size % 4 == 0 &&
         (PatternSize == 4 || PatternSize == 2 || PatternSize == 1)) {
@@ -3043,7 +3265,11 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
         llvm_unreachable("Invalid pattern size");
       }
 
-      if (hasPendingWorkImpl(AsyncInfoWrapper)) {
+      auto HasPendingWorkOrErr = hasPendingWorkImpl(AsyncInfoWrapper);
+      if (!HasPendingWorkOrErr)
+        return HasPendingWorkOrErr.takeError();
+
+      if (*HasPendingWorkOrErr) {
         AMDGPUStreamTy *Stream = nullptr;
         if (auto Err = getStream(AsyncInfoWrapper, Stream))
           return Err;
@@ -3053,14 +3279,14 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
           uint32_t Pattern;
           int64_t Size;
         };
-        auto *Args = new MemFillArgsTy{TgtPtr, Pattern, Size / 4};
+        auto Args = std::make_unique<MemFillArgsTy>(
+            MemFillArgsTy{TgtPtr, Pattern, Size / 4});
         auto Fill = [](void *Data) {
-          MemFillArgsTy *Args = reinterpret_cast<MemFillArgsTy *>(Data);
-          assert(Args && "Invalid arguments");
+          std::unique_ptr<MemFillArgsTy> Args(
+              static_cast<MemFillArgsTy *>(Data));
 
           auto Status =
               hsa_amd_memory_fill(Args->Dst, Args->Pattern, Args->Size);
-          delete Args;
           auto Err =
               Plugin::check(Status, "error in hsa_amd_memory_fill: %s\n");
           if (Err) {
@@ -3071,7 +3297,11 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
 
         // hsa_amd_memory_fill doesn't signal completion using a signal, so use
         // the existing host callback logic to handle that instead
-        return Stream->pushHostCallback(Fill, Args);
+        if (auto Err = Stream->pushHostCallback(Fill, Args.get()))
+          return Err;
+
+        Args.release();
+        return Plugin::success();
       }
       // If there is no pending work, do the fill synchronously
       auto Status = hsa_amd_memory_fill(TgtPtr, Pattern, Size / 4);
@@ -3488,6 +3718,11 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
   }
 
   Expected<bool> isAccessiblePtrImpl(const void *Ptr, size_t Size) override {
+    // SVM allocations are accessible by all the kernel agents but are unknown
+    // to the HSA runtime.
+    if (getSVMManager().contains(Ptr, Size))
+      return true;
+
     hsa_amd_pointer_info_t Info;
     Info.size = sizeof(hsa_amd_pointer_info_t);
 
@@ -4056,6 +4291,8 @@ struct AMDGPUPluginTy final : public GenericPluginTy {
     if (auto Err = HostDevice->init())
       return std::move(Err);
 
+    SVMManager.init();
+
     return NumDevices;
   }
 
@@ -4070,9 +4307,12 @@ struct AMDGPUPluginTy final : public GenericPluginTy {
       if (auto Err = HostDevice->deinit())
         return Err;
 
+    Error Err = SVMManager.deinit();
+
     // Finalize the HSA runtime.
     hsa_status_t Status = hsa_shut_down();
-    return Plugin::check(Status, "error in hsa_shut_down: %s");
+    return joinErrors(std::move(Err),
+                      Plugin::check(Status, "error in hsa_shut_down: %s"));
   }
 
   /// Creates an AMDGPU device.
@@ -4138,6 +4378,11 @@ struct AMDGPUPluginTy final : public GenericPluginTy {
     return KernelAgents;
   }
 
+  /// Get the manager of the shared (managed) SVM allocations. A single manager
+  /// serves all the devices because SVM allocations are accessible by all the
+  /// agents.
+  AMDGPUSVMManagerTy &getSVMManager() { return SVMManager; }
+
   /// Create an HSA signal for the RPC doorbell and return the fields needed
   /// for the GPU to fire interrupts that wake the server thread.
   Error initRPCDoorbell(uint64_t *&Value, uint64_t *&Mailbox,
@@ -4278,6 +4523,9 @@ struct AMDGPUPluginTy final : public GenericPluginTy {
 
   /// The device representing all HSA host agents.
   AMDHostDeviceTy *HostDevice;
+
+  /// The manager of the shared (managed) SVM allocations.
+  AMDGPUSVMManagerTy SVMManager;
 };
 
 Error AMDGPUKernelTy::launchImpl(GenericDeviceTy &GenericDevice,
@@ -4486,12 +4734,31 @@ Expected<void *> AMDGPUMemoryManagerTy::allocate(size_t Size, void *HstPtr,
   return Ptr;
 }
 
+AMDGPUSVMManagerTy &AMDGPUDeviceTy::getSVMManager() {
+  return static_cast<AMDGPUPluginTy &>(Plugin).getSVMManager();
+}
+
 Expected<void *> AMDGPUDeviceTy::allocate(size_t Size, void *,
                                           TargetAllocTy Kind,
                                           size_t Alignment) {
   if (Size == 0)
     return nullptr;
 
+  auto &AMDGPUPlugin = static_cast<AMDGPUPluginTy &>(Plugin);
+
+  // Shared (managed) allocations use the SVM interface, which lets the driver
+  // migrate the memory between the host and the devices. Requests that SVM
+  // cannot serve fall back to the fine-grained host memory pool below.
+  auto &SVMManager = AMDGPUPlugin.getSVMManager();
+  if (Kind == TARGET_ALLOC_SHARED && SVMManager.isSupported()) {
+    auto AllocOrErr =
+        SVMManager.allocate(Size, Alignment, AMDGPUPlugin.getKernelAgents());
+    if (!AllocOrErr)
+      return AllocOrErr.takeError();
+    if (*AllocOrErr)
+      return *AllocOrErr;
+  }
+
   // Find the correct memory pool.
   AMDGPUMemoryPoolTy *MemoryPool = nullptr;
   switch (Kind) {
@@ -4521,10 +4788,9 @@ Expected<void *> AMDGPUDeviceTy::allocate(size_t Size, void *,
     // necessary for host or shared allocations Also enabled for device memory
     // to allow device to device memcpy
     llvm::SmallVector<hsa_agent_t> Agents;
-    llvm::copy_if(static_cast<AMDGPUPluginTy &>(Plugin).getKernelAgents(),
-                  std::back_inserter(Agents), [&](hsa_agent_t Agent) {
-                    return MemoryPool->canAccess(Agent);
-                  });
+    llvm::copy_if(
+        AMDGPUPlugin.getKernelAgents(), std::back_inserter(Agents),
+        [&](hsa_agent_t Agent) { return MemoryPool->canAccess(Agent); });
 
     // Enable all valid kernel agents to access the buffer.
     if (auto Err = MemoryPool->enableAccess(Alloc, Size, Agents))
diff --git a/offload/test/api/amdgpu_managed_memory_accessible.c b/offload/test/api/amdgpu_managed_memory_accessible.c
new file mode 100644
index 0000000000000..9cf01aed56d83
--- /dev/null
+++ b/offload/test/api/amdgpu_managed_memory_accessible.c
@@ -0,0 +1,31 @@
+// RUN: %libomptarget-compile-run-and-check-amdgcn-amd-amdhsa
+
+// REQUIRES: amdgcn-amd-amdhsa
+
+// Device managed memory is accessible by the device it was allocated for.
+
+#include <omp.h>
+#include <stdio.h>
+
+void *llvm_omp_target_alloc_shared(size_t, int);
+void llvm_omp_target_free_shared(void *, int);
+
+int main() {
+  const size_t Size = 1024;
+  const int Device = omp_get_default_device();
+
+  char *Shared = llvm_omp_target_alloc_shared(Size, Device);
+  if (!Shared) {
+    printf("FAIL: allocation\n");
+    return 1;
+  }
+
+  int Accessible =
+      omp_target_is_accessible(Shared, Size, Device) &&
+      omp_target_is_accessible(&Shared[Size / 2], Size / 2, Device);
+
+  llvm_omp_target_free_shared(Shared, Device);
+
+  // CHECK: PASS
+  printf(Accessible ? "PASS\n" : "FAIL\n");
+}
diff --git a/offload/test/api/omp_device_managed_memory_ops.c b/offload/test/api/omp_device_managed_memory_ops.c
new file mode 100644
index 0000000000000..b60ef512d3a5b
--- /dev/null
+++ b/offload/test/api/omp_device_managed_memory_ops.c
@@ -0,0 +1,68 @@
+// RUN: %libomptarget-compile-run-and-check-generic
+
+// Check the data movement API on device managed memory.
+
+#include <omp.h>
+#include <stdio.h>
+
+void *llvm_omp_target_alloc_shared(size_t, int);
+void llvm_omp_target_free_shared(void *, int);
+
+int main() {
+  const int N = 128;
+  const int Device = omp_get_default_device();
+  const int Host = omp_get_initial_device();
+
+  int *Shared = llvm_omp_target_alloc_shared(N * sizeof(int), Device);
+  int Buffer[N];
+  int Failures = 0;
+
+  if (!Shared) {
+    printf("FAIL: allocation\n");
+    return 1;
+  }
+
+  // The host can access the allocation directly.
+  for (int I = 0; I < N; ++I)
+    Shared[I] = I;
+
+  // The device can access the allocation directly.
+#pragma omp target teams distribute parallel for device(Device)                \
+    is_device_ptr(Shared)
+  for (int I = 0; I < N; ++I)
+    Shared[I] += 1;
+
+  for (int I = 0; I < N; ++I)
+    Failures += (Shared[I] != I + 1);
+
+  // Filling the whole allocation.
+  if (omp_target_memset(Shared, 0, N * sizeof(int), Device) != Shared)
+    ++Failures;
+  for (int I = 0; I < N; ++I)
+    Failures += (Shared[I] != 0);
+
+  // Filling a subrange of the allocation.
+  if (omp_target_memset(&Shared[1], 0xFF, sizeof(int), Device) != &Shared[1])
+    ++Failures;
+  Failures += (Shared[0] != 0) + (Shared[1] != -1) + (Shared[2] != 0);
+
+  // Copying into and out of the allocation.
+  for (int I = 0; I < N; ++I)
+    Buffer[I] = 2 * I;
+  Failures += (omp_target_memcpy(Shared, Buffer, N * sizeof(int), 0, 0, Device,
+                                 Host) != 0);
+  for (int I = 0; I < N; ++I)
+    Buffer[I] = 0;
+  Failures += (omp_target_memcpy(Buffer, Shared, N * sizeof(int), 0, 0, Host,
+                                 Device) != 0);
+  for (int I = 0; I < N; ++I)
+    Failures += (Buffer[I] != 2 * I);
+
+  llvm_omp_target_free_shared(Shared, Device);
+
+  // CHECK: PASS
+  if (!Failures)
+    printf("PASS\n");
+  else
+    printf("FAIL: %d\n", Failures);
+}
diff --git a/openmp/docs/design/Runtimes.rst b/openmp/docs/design/Runtimes.rst
index 14e300a0f531f..f6fca089559d0 100644
--- a/openmp/docs/design/Runtimes.rst
+++ b/openmp/docs/design/Runtimes.rst
@@ -1398,6 +1398,7 @@ There are several environment variables to change the behavior of the plugins:
 * ``LIBOMPTARGET_AMDGPU_MAX_ASYNC_COPY_BYTES``
 * ``LIBOMPTARGET_AMDGPU_NUM_INITIAL_HSA_SIGNALS``
 * ``LIBOMPTARGET_AMDGPU_STREAM_BUSYWAIT``
+* ``LIBOMPTARGET_AMDGPU_SHARED_ALLOC_SVM``
 
 The environment variables ``LIBOMPTARGET_STACK_SIZE`` and
 ``LIBOMPTARGET_HEAP_SIZE`` are described in
@@ -1498,6 +1499,18 @@ HSA wait state within the AMDGPU plugin. For the duration of this value
 the HSA runtime may busy wait. This can reduce overall latency.
 The default value is ``2000000``.
 
+LIBOMPTARGET_AMDGPU_SHARED_ALLOC_SVM
+""""""""""""""""""""""""""""""""""""
+
+This environment variable controls whether shared (managed) allocations, e.g.,
+the ones performed by ``omp_target_alloc_shared``, are served through the HSA
+shared virtual memory (SVM) interface in the AMDGPU plugin. SVM allocations are
+backed by ordinary system memory that the driver can migrate between the host
+and the devices on demand, which greatly improves the device access bandwidth on
+systems with XNACK enabled. If SVM is disabled, or if it is not supported by the
+driver, shared allocations are served from a fine-grained host memory pool and
+therefore remain resident on the host. The default value is ``true``.
+
 .. _remote_offloading_plugin:
 
 Remote Offloading Plugin:


From openmp-commits at lists.llvm.org  Wed Aug 12 06:29:23 2026
From: openmp-commits at lists.llvm.org (Robert Imschweiler via Openmp-commits)
Date: Wed, 12 Aug 2026 06:29:23 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [offload] Pool host and shared
	allocations (PR #214754)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/214754@github.com>
Message-ID: <6a7c7533.b71ce897.28ae90.db0e@mx.google.com>


================
@@ -4529,6 +4535,12 @@ Expected<void *> AMDGPUDeviceTy::allocate(size_t Size, void *,
     // Enable all valid kernel agents to access the buffer.
     if (auto Err = MemoryPool->enableAccess(Alloc, Size, Agents))
       return std::move(Err);
+
+    // Register host / shared memory as pinned memory, so that transfers reading
+    // from it can take a device-accessible path.
+    if (Kind == TARGET_ALLOC_HOST || Kind == TARGET_ALLOC_SHARED)
+      if (auto Err = PinnedAllocs.registerHostBuffer(Alloc, Alloc, Size))
+        return std::move(Err);
----------------
ro-i wrote:

> Honestly, amd sharing this for host and shared is just wrong. That might be my fault because I think I just aliased them because we didn't have anything similar readily available at the time. But now that we have SVM we probably should actually use that.

this is something I'd like to handle separately, if possible, so that it doesn't block this stack. I already made a draft PR: https://github.com/llvm/llvm-project/pull/215801

https://github.com/llvm/llvm-project/pull/214754

From openmp-commits at lists.llvm.org  Wed Aug 12 06:30:44 2026
From: openmp-commits at lists.llvm.org (Robert Imschweiler via Openmp-commits)
Date: Wed, 12 Aug 2026 06:30:44 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [offload] Use HSA SVM for AMDGPU
	shared memory (PR #215801)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/215801@github.com>
Message-ID: <6a7c7584.fe909fa6.190e27.32a6@mx.google.com>


ro-i wrote:

(this PR is actually still a draft :) )

https://github.com/llvm/llvm-project/pull/215801

From openmp-commits at lists.llvm.org  Wed Aug 12 07:03:17 2026
From: openmp-commits at lists.llvm.org (=?UTF-8?Q?Jan_Andr=C3=A9_Reuter?= via Openmp-commits)
Date: Wed, 12 Aug 2026 07:03:17 -0700 (PDT)
Subject: [Openmp-commits] [clang] [openmp] [Clang][OpenMP] Use loop as ident
 flag for sections (PR #210038)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/210038@github.com>
Message-ID: <6a7c7d25.28d1163c.3495ec.2f9d@mx.google.com>


https://github.com/Thyre updated https://github.com/llvm/llvm-project/pull/210038

>From 223da05d52980521d0053828a2235349746b80d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Andr=C3=A9=20Reuter?= <jan at zyten.de>
Date: Wed, 15 Jul 2026 16:04:44 +0200
Subject: [PATCH] [Clang][OpenMP] Use loop as ident flag for sections
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

For the OpenMP Tools Interface, a number of callbacks should get dispatched
for the `sections` construct, and the `section` directive.
For each _sections-begin_ event, a work callback with `ompt_scope_begin` as
the endpoint and `ompt_work_sections` as the work_type dispatched. For each
_section-begin_, a dispatch callback with the kind set to
`ompt_dispatch_section` is dispatched. And finally, on _sections-end_,
another work callback with `ompt_scope_end` is emitted.

In Clang and Flang, the `sections` construct and `section` directive
is rewritten during code generation. A user code such as:

```
{
  #pragma omp section
  {}

  #pragma omp section
  {}
}
```

is transformed to:

```
[...]

omp.inner.for.cond:
  %6 = load i32, ptr %.omp.sections.iv., align 4
  %7 = load i32, ptr %.omp.sections.ub., align 4
  %cmp = icmp sle i32 %6, %7
  br i1 %cmp, label %omp.inner.for.body, label %omp.inner.for.end

omp.inner.for.body:
  %8 = load i32, ptr %.omp.sections.iv., align 4
  switch i32 %8, label %.omp.sections.exit [
    i32 0, label %.omp.sections.case
    i32 1, label %.omp.sections.case1
  ]

.omp.sections.case:
  br label %.omp.sections.exit

.omp.sections.case1:
  br label %.omp.sections.exit

.omp.sections.exit:
  br label %omp.inner.for.inc

[...]
```

Clang and Flang both use a static loop, splitting work across threads in the
team, with a switch-case to select the correct `section` directive.
On the call to `__kmpc_for_static_init`, Clang passes `OMP_IDENT_WORK_SECTIONS`
as part of the ident flags. As such, the OpenMP runtime identifies the
transformed construct as a `sections` construct with one or more `section`
directives.

In #207746, it was found that this handling can lead to an incorrect event
sequence emitted to tools. Since `__kmpc_for_static_init` is only called _once_
at the beginning of the loop for each thread in the team, and each thread
dispatches one work and dispatch callback to an attached tool, regardless of
the actual number of section directives handles, a tool might therefore
incorrectly assume that the number of `section` directives matches the number
of threads.

To improve the dispatched events to an attached tool, use
`OMP_IDENT_WORK_LOOP` for the `sections` construct and `section` directives
as well. With this, the behavior of Clang is consistent with the behavior of
Flang. A tool is able to identify the number of loop iterations done per
thread, hence is able to filter threads not executing any `section` directive.

Closes #207746

Signed-off-by: Jan André Reuter <jan at zyten.de>
---
 clang/lib/CodeGen/CGOpenMPRuntime.cpp              | 10 +++-------
 clang/test/OpenMP/sections_codegen.cpp             |  2 +-
 openmp/runtime/test/ompt/worksharing/sections.c    |  8 ++++----
 .../test/ompt/worksharing/sections_dispatch.c      | 14 ++++++--------
 4 files changed, 14 insertions(+), 20 deletions(-)

diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index eb2f92cdbf972..f68fbb8b50824 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -2752,10 +2752,8 @@ void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF,
                                Values.Ordered);
   assert((isOpenMPWorksharingDirective(DKind) || (DKind == OMPD_loop)) &&
          "Expected loop-based or sections-based directive.");
-  llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc,
-                                             isOpenMPLoopDirective(DKind)
-                                                 ? OMP_IDENT_WORK_LOOP
-                                                 : OMP_IDENT_WORK_SECTIONS);
+  llvm::Value *UpdatedLocation =
+      emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_LOOP);
   llvm::Value *ThreadId = getThreadID(CGF, Loc);
   llvm::FunctionCallee StaticInitFunction =
       OMPBuilder.createForStaticInitFunction(Values.IVSize, Values.IVSigned,
@@ -2799,9 +2797,7 @@ void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF,
                          isOpenMPDistributeDirective(DKind) ||
                                  (DKind == OMPD_target_teams_loop)
                              ? OMP_IDENT_WORK_DISTRIBUTE
-                         : isOpenMPLoopDirective(DKind)
-                             ? OMP_IDENT_WORK_LOOP
-                             : OMP_IDENT_WORK_SECTIONS),
+                             : OMP_IDENT_WORK_LOOP),
       getThreadID(CGF, Loc)};
   auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc);
   if (isOpenMPDistributeDirective(DKind) &&
diff --git a/clang/test/OpenMP/sections_codegen.cpp b/clang/test/OpenMP/sections_codegen.cpp
index 5a5e32751c677..957833e60daaa 100644
--- a/clang/test/OpenMP/sections_codegen.cpp
+++ b/clang/test/OpenMP/sections_codegen.cpp
@@ -10,7 +10,7 @@
 #ifndef HEADER
 #define HEADER
 // CHECK-DAG: [[IMPLICIT_BARRIER_SECTIONS_LOC:@.+]] = private unnamed_addr constant %{{.+}} { i32 0, i32 194, i32 0, i32 {{[0-9]+}}, ptr
-// CHECK-DAG: [[SECTIONS_LOC:@.+]] = private unnamed_addr constant %{{.+}} { i32 0, i32 1026, i32 0, i32 {{[0-9]+}}, ptr
+// CHECK-DAG: [[SECTIONS_LOC:@.+]] = private unnamed_addr constant %{{.+}} { i32 0, i32 514, i32 0, i32 {{[0-9]+}}, ptr
 // CHECK-LABEL: foo
 void foo() { extern void mayThrow(); mayThrow(); };
 // CHECK-LABEL: bar
diff --git a/openmp/runtime/test/ompt/worksharing/sections.c b/openmp/runtime/test/ompt/worksharing/sections.c
index 5cf927def6905..b319a0f1a9a3f 100644
--- a/openmp/runtime/test/ompt/worksharing/sections.c
+++ b/openmp/runtime/test/ompt/worksharing/sections.c
@@ -28,11 +28,11 @@ int main() {
 
   // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_sections_begin: parallel_id=[[PARALLEL_ID:[0-f]+]], task_id=[[TASK_ID:[0-f]+]], codeptr_ra=[[SECT_BEGIN:(0x)?[0-f]+]], count=2
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_sections_end: parallel_id=[[PARALLEL_ID]], task_id={{[0-f]+}}, codeptr_ra=[[SECT_END:(0x)?[0-f]+]]
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_loop_static_begin: parallel_id=[[PARALLEL_ID:[0-f]+]], task_id=[[TASK_ID:[0-f]+]], codeptr_ra=[[SECT_BEGIN:(0x)?[0-f]+]], count=2
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_loop_static_end: parallel_id=[[PARALLEL_ID]], task_id={{[0-f]+}}, codeptr_ra=[[SECT_END:(0x)?[0-f]+]]
 
-  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_sections_begin: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID:[0-f]+]], codeptr_ra=[[SECT_BEGIN]], count=2
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_sections_end: parallel_id=[[PARALLEL_ID]], task_id={{[0-f]+}}, codeptr_ra=[[SECT_END]]
+  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_loop_static_begin: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID:[0-f]+]], codeptr_ra=[[SECT_BEGIN]], count=2
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_loop_static_end: parallel_id=[[PARALLEL_ID]], task_id={{[0-f]+}}, codeptr_ra=[[SECT_END]]
   // clang-format on
 
   return 0;
diff --git a/openmp/runtime/test/ompt/worksharing/sections_dispatch.c b/openmp/runtime/test/ompt/worksharing/sections_dispatch.c
index ec3bb5bd7552d..fc573ee5ae971 100644
--- a/openmp/runtime/test/ompt/worksharing/sections_dispatch.c
+++ b/openmp/runtime/test/ompt/worksharing/sections_dispatch.c
@@ -29,24 +29,22 @@ int main() {
 
   // CHECK: 0: NULL_POINTER=[[NULL:.*$]]
 
-  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_sections_begin:
+  // CHECK: {{^}}[[MASTER_ID:[0-9]+]]: ompt_event_loop_static_begin:
   // CHECK-SAME: parallel_id=[[PARALLEL_ID:[0-f]+]],
   // CHECK-SAME: task_id=[[TASK_ID:[0-f]+]],
   // CHECK-SAME: codeptr_ra=[[SECT_BEGIN:(0x)?[0-f]+]], count=2
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_section_begin:
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_ws_loop_chunk_begin:
   // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID]]
-  // CHECK-SAME: codeptr_ra=[[SECT_BEGIN]]
-  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_sections_end:
+  // CHECK: {{^}}[[MASTER_ID]]: ompt_event_loop_static_end:
   // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id={{[0-f]+}},
   // CHECK-SAME: codeptr_ra=[[SECT_END:(0x)?[0-f]+]]
 
-  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_sections_begin:
+  // CHECK: {{^}}[[THREAD_ID:[0-9]+]]: ompt_event_loop_static_begin:
   // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID:[0-f]+]],
   // CHECK-SAME: codeptr_ra=[[SECT_BEGIN]], count=2
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_section_begin:
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_ws_loop_chunk_begin:
   // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id=[[TASK_ID]]
-  // CHECK-SAME: codeptr_ra=[[SECT_BEGIN]]
-  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_sections_end:
+  // CHECK: {{^}}[[THREAD_ID]]: ompt_event_loop_static_end:
   // CHECK-SAME: parallel_id=[[PARALLEL_ID]], task_id={{[0-f]+}},
   // CHECK-SAME: codeptr_ra=[[SECT_END]]
   // clang-format on


From openmp-commits at lists.llvm.org  Wed Aug 12 07:09:04 2026
From: openmp-commits at lists.llvm.org (Joseph Huber via Openmp-commits)
Date: Wed, 12 Aug 2026 07:09:04 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [offload] Use HSA SVM for AMDGPU
	shared memory (PR #215801)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/215801@github.com>
Message-ID: <6a7c7e80.d61da0cb.1660a1.2426@mx.google.com>


================
@@ -450,6 +455,157 @@ struct AMDGPUMemoryPoolTy {
   size_t PoolAllocationAlignment;
 };
 
+/// Class that implements shared (managed) allocations on top of the HSA shared
+/// virtual memory (SVM) interface.
+///
+/// SVM allocations are backed by ordinary system memory that is made
+/// accessible to all the kernel agents. With XNACK enabled, the driver
+/// migrates the pages between the host and the devices on demand; otherwise
+/// they remain resident in system memory.
+struct AMDGPUSVMManagerTy {
+  /// Determine whether the SVM interface can be used for shared allocations.
+  void init() {
+    if (!OMPX_UseSVM)
+      return;
+
+    bool SVMSupported = false;
+    if (hsa_system_get_info(HSA_AMD_SYSTEM_INFO_SVM_SUPPORTED, &SVMSupported) !=
+        HSA_STATUS_SUCCESS)
+      return;
+
+    Supported = SVMSupported;
+    PageSize = llvm::sys::Process::getPageSizeEstimate();
+  }
+
+  /// Release the allocations that are still live.
+  Error deinit() {
+    std::lock_guard<std::mutex> Lock(Mutex);
+
+    Error Err = Plugin::success();
+    for (auto &Allocation : Allocations)
+      if (std::error_code EC =
+              llvm::sys::Memory::releaseMappedMemory(Allocation.second))
+        Err = joinErrors(std::move(Err),
+                         Plugin::error(ErrorCode::UNKNOWN,
+                                       "error releasing SVM memory: %s",
+                                       EC.message().c_str()));
+
+    Allocations.clear();
+    return Err;
+  }
+
+  /// Whether new SVM allocations can be created.
+  bool isSupported() const { return Supported; }
+
+  /// Allocate system memory and make it accessible to all the \p Agents.
+  /// Returns a null pointer if the request cannot be served, either because
+  /// the requested alignment is too large or because the driver rejected the
+  /// allocation.
+  Expected<void *> allocate(size_t Size, size_t Alignment,
+                            ArrayRef<hsa_agent_t> Agents) {
+    // A concurrent allocation may have disabled the SVM interface.
+    if (!Supported)
+      return nullptr;
+
+    // Mapped memory is only page aligned.
+    if (Alignment > PageSize)
+      return nullptr;
+
+    std::error_code EC;
+    llvm::sys::MemoryBlock Block = llvm::sys::Memory::allocateMappedMemory(
+        Size, /*NearBlock=*/nullptr,
+        llvm::sys::Memory::MF_READ | llvm::sys::Memory::MF_WRITE, EC);
+    if (EC)
+      return Plugin::error(ErrorCode::OUT_OF_RESOURCES,
+                           "error allocating SVM memory: %s",
+                           EC.message().c_str());
+
+    // Give all the kernel agents access to the allocation. Accesses may incur
+    // a page fault and the migration of the memory to the accessing agent.
+    llvm::SmallVector<hsa_amd_svm_attribute_pair_t> Attrs;
+    for (hsa_agent_t Agent : Agents)
+      Attrs.push_back({HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE, Agent.handle});
+
+    hsa_status_t Status = hsa_amd_svm_attributes_set(
+        Block.base(), Block.allocatedSize(), Attrs.data(), Attrs.size());
+    if (auto Err =
+            Plugin::check(Status, "error in hsa_amd_svm_attributes_set: %s")) {
----------------
jhuber6 wrote:

This should just be an error and handled from the user's side. If you requested SVM and the agent could not fulfill it that is an error.

https://github.com/llvm/llvm-project/pull/215801

From openmp-commits at lists.llvm.org  Wed Aug 12 07:09:06 2026
From: openmp-commits at lists.llvm.org (Joseph Huber via Openmp-commits)
Date: Wed, 12 Aug 2026 07:09:06 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [offload] Use HSA SVM for AMDGPU
	shared memory (PR #215801)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/215801@github.com>
Message-ID: <6a7c7e82.433f7c1d.1ca091.3030@mx.google.com>


================
@@ -450,6 +455,157 @@ struct AMDGPUMemoryPoolTy {
   size_t PoolAllocationAlignment;
 };
 
+/// Class that implements shared (managed) allocations on top of the HSA shared
+/// virtual memory (SVM) interface.
+///
+/// SVM allocations are backed by ordinary system memory that is made
+/// accessible to all the kernel agents. With XNACK enabled, the driver
+/// migrates the pages between the host and the devices on demand; otherwise
+/// they remain resident in system memory.
+struct AMDGPUSVMManagerTy {
+  /// Determine whether the SVM interface can be used for shared allocations.
+  void init() {
+    if (!OMPX_UseSVM)
+      return;
+
+    bool SVMSupported = false;
+    if (hsa_system_get_info(HSA_AMD_SYSTEM_INFO_SVM_SUPPORTED, &SVMSupported) !=
+        HSA_STATUS_SUCCESS)
+      return;
+
+    Supported = SVMSupported;
+    PageSize = llvm::sys::Process::getPageSizeEstimate();
+  }
+
+  /// Release the allocations that are still live.
+  Error deinit() {
+    std::lock_guard<std::mutex> Lock(Mutex);
+
+    Error Err = Plugin::success();
+    for (auto &Allocation : Allocations)
+      if (std::error_code EC =
+              llvm::sys::Memory::releaseMappedMemory(Allocation.second))
+        Err = joinErrors(std::move(Err),
+                         Plugin::error(ErrorCode::UNKNOWN,
+                                       "error releasing SVM memory: %s",
+                                       EC.message().c_str()));
+
+    Allocations.clear();
+    return Err;
+  }
+
+  /// Whether new SVM allocations can be created.
+  bool isSupported() const { return Supported; }
+
+  /// Allocate system memory and make it accessible to all the \p Agents.
+  /// Returns a null pointer if the request cannot be served, either because
+  /// the requested alignment is too large or because the driver rejected the
+  /// allocation.
+  Expected<void *> allocate(size_t Size, size_t Alignment,
+                            ArrayRef<hsa_agent_t> Agents) {
+    // A concurrent allocation may have disabled the SVM interface.
+    if (!Supported)
+      return nullptr;
+
+    // Mapped memory is only page aligned.
+    if (Alignment > PageSize)
+      return nullptr;
+
+    std::error_code EC;
+    llvm::sys::MemoryBlock Block = llvm::sys::Memory::allocateMappedMemory(
+        Size, /*NearBlock=*/nullptr,
+        llvm::sys::Memory::MF_READ | llvm::sys::Memory::MF_WRITE, EC);
+    if (EC)
+      return Plugin::error(ErrorCode::OUT_OF_RESOURCES,
+                           "error allocating SVM memory: %s",
+                           EC.message().c_str());
+
+    // Give all the kernel agents access to the allocation. Accesses may incur
+    // a page fault and the migration of the memory to the accessing agent.
+    llvm::SmallVector<hsa_amd_svm_attribute_pair_t> Attrs;
+    for (hsa_agent_t Agent : Agents)
+      Attrs.push_back({HSA_AMD_SVM_ATTRIB_AGENT_ACCESSIBLE, Agent.handle});
+
+    hsa_status_t Status = hsa_amd_svm_attributes_set(
+        Block.base(), Block.allocatedSize(), Attrs.data(), Attrs.size());
+    if (auto Err =
+            Plugin::check(Status, "error in hsa_amd_svm_attributes_set: %s")) {
+      // The driver rejected the allocation, e.g., because one of the agents
+      // does not support SVM. Stop creating SVM allocations, reporting the
+      // rejection once.
+      if (Supported.exchange(false))
+        REPORT() << "Serving shared allocations from host memory: "
+                 << toString(std::move(Err));
+      else
+        consumeError(std::move(Err));
+
+      // Ignore any error from undoing the allocation.
+      consumeError(llvm::errorCodeToError(
+          llvm::sys::Memory::releaseMappedMemory(Block)));
+      return nullptr;
+    }
+
+    std::lock_guard<std::mutex> Lock(Mutex);
+    Allocations[reinterpret_cast<uintptr_t>(Block.base())] = Block;
+    return Block.base();
+  }
+
+  /// Release the SVM allocation starting at \p Ptr. Returns whether \p Ptr is
+  /// an SVM allocation.
+  Expected<bool> deallocate(void *Ptr) {
+    llvm::sys::MemoryBlock Block;
+    {
+      std::lock_guard<std::mutex> Lock(Mutex);
+      auto It = Allocations.find(reinterpret_cast<uintptr_t>(Ptr));
+      if (It == Allocations.end())
+        return false;
+      Block = It->second;
+      Allocations.erase(It);
+    }
+
+    if (std::error_code EC = llvm::sys::Memory::releaseMappedMemory(Block))
+      return Plugin::error(ErrorCode::UNKNOWN, "error releasing SVM memory: %s",
+                           EC.message().c_str());
+    return true;
+  }
+
+  /// Whether the \p Size bytes starting at \p Ptr are within an SVM
+  /// allocation. Allocations stay live after the SVM interface is disabled, so
+  /// this must not check Supported.
+  bool contains(const void *Ptr, size_t Size = 1) const {
----------------
jhuber6 wrote:

Do we need a full region check? The user shouldn't corrupt the pointers returned by the allocator. I'd only expect to need this if we were caching pages. i.e. several allocations share a 4KiB page and before freeing it all others need to be freed.

https://github.com/llvm/llvm-project/pull/215801

From openmp-commits at lists.llvm.org  Wed Aug 12 07:09:06 2026
From: openmp-commits at lists.llvm.org (Joseph Huber via Openmp-commits)
Date: Wed, 12 Aug 2026 07:09:06 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [offload] Use HSA SVM for AMDGPU
	shared memory (PR #215801)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/215801@github.com>
Message-ID: <6a7c7e82.8bb56669.2ef474.5535@mx.google.com>


================
@@ -450,6 +455,157 @@ struct AMDGPUMemoryPoolTy {
   size_t PoolAllocationAlignment;
 };
 
+/// Class that implements shared (managed) allocations on top of the HSA shared
+/// virtual memory (SVM) interface.
+///
+/// SVM allocations are backed by ordinary system memory that is made
+/// accessible to all the kernel agents. With XNACK enabled, the driver
+/// migrates the pages between the host and the devices on demand; otherwise
+/// they remain resident in system memory.
+struct AMDGPUSVMManagerTy {
+  /// Determine whether the SVM interface can be used for shared allocations.
+  void init() {
+    if (!OMPX_UseSVM)
+      return;
+
+    bool SVMSupported = false;
+    if (hsa_system_get_info(HSA_AMD_SYSTEM_INFO_SVM_SUPPORTED, &SVMSupported) !=
+        HSA_STATUS_SUCCESS)
+      return;
+
+    Supported = SVMSupported;
+    PageSize = llvm::sys::Process::getPageSizeEstimate();
+  }
+
+  /// Release the allocations that are still live.
+  Error deinit() {
+    std::lock_guard<std::mutex> Lock(Mutex);
+
+    Error Err = Plugin::success();
+    for (auto &Allocation : Allocations)
+      if (std::error_code EC =
+              llvm::sys::Memory::releaseMappedMemory(Allocation.second))
+        Err = joinErrors(std::move(Err),
+                         Plugin::error(ErrorCode::UNKNOWN,
+                                       "error releasing SVM memory: %s",
+                                       EC.message().c_str()));
+
+    Allocations.clear();
+    return Err;
+  }
+
+  /// Whether new SVM allocations can be created.
+  bool isSupported() const { return Supported; }
+
+  /// Allocate system memory and make it accessible to all the \p Agents.
+  /// Returns a null pointer if the request cannot be served, either because
+  /// the requested alignment is too large or because the driver rejected the
+  /// allocation.
+  Expected<void *> allocate(size_t Size, size_t Alignment,
+                            ArrayRef<hsa_agent_t> Agents) {
+    // A concurrent allocation may have disabled the SVM interface.
+    if (!Supported)
+      return nullptr;
+
+    // Mapped memory is only page aligned.
+    if (Alignment > PageSize)
----------------
jhuber6 wrote:

We could handle this more intentionally but I'm unsure if it's worthwhile. The standard heuristic is to over-allocate then `munmap` the pages before the alignment you needed.

https://github.com/llvm/llvm-project/pull/215801

From openmp-commits at lists.llvm.org  Wed Aug 12 07:09:06 2026
From: openmp-commits at lists.llvm.org (Joseph Huber via Openmp-commits)
Date: Wed, 12 Aug 2026 07:09:06 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [offload] Use HSA SVM for AMDGPU
	shared memory (PR #215801)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/215801@github.com>
Message-ID: <6a7c7e82.ea52d05b.1f7229.a5f2@mx.google.com>


================
@@ -3024,9 +3191,63 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     return Plugin::success();
   }
 
+  /// Fill \p TgtPtr with the given pattern on the host. Requires \p TgtPtr to
+  /// be host accessible.
+  Error dataFillHostImpl(void *TgtPtr, const void *PatternPtr,
----------------
jhuber6 wrote:

Is this an optimization to fill via SVM? Should be a separate PR if so.

https://github.com/llvm/llvm-project/pull/215801

From openmp-commits at lists.llvm.org  Wed Aug 12 07:09:38 2026
From: openmp-commits at lists.llvm.org (Joseph Huber via Openmp-commits)
Date: Wed, 12 Aug 2026 07:09:38 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [offload] Use HSA SVM for AMDGPU
	shared memory (PR #215801)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/215801@github.com>
Message-ID: <6a7c7ea2.a2448446.2671a1.9d25@mx.google.com>


jhuber6 wrote:

Thanks for cleaning up random things like this I never got around to doing

https://github.com/llvm/llvm-project/pull/215801

From openmp-commits at lists.llvm.org  Wed Aug 12 07:16:51 2026
From: openmp-commits at lists.llvm.org (=?UTF-8?Q?Jan_Andr=C3=A9_Reuter?= via Openmp-commits)
Date: Wed, 12 Aug 2026 07:16:51 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP][OMPT] Mark dispatch callback as
 sometimes on registration (PR #212991)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/212991@github.com>
Message-ID: <6a7c8053.4987c784.12dc3.26b5@mx.google.com>


https://github.com/Thyre updated https://github.com/llvm/llvm-project/pull/212991

>From e440f031bad7c58ad3e2c227b09e91a878c692f0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Andr=C3=A9=20Reuter?= <j.reuter at fz-juelich.de>
Date: Thu, 30 Jul 2026 12:31:17 +0200
Subject: [PATCH] [OpenMP][OMPT] Mark dispatch callback as sometimes on
 registration
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Recently, multiple issues in the implementation of the `dispatch`
callback have been found (e.g., #212784, #207746). In these, the code
generation done by Clang and Flang does not provide a sufficient amount of
information for the runtime to dispatch the correct amount of flags, or a
callback for a given event at all.

These missing and incorrect events do not satisfy OpenMP's requirement
for `ompt_set_always` as a registration result, as the runtime does not
trace or invoke the correct callback whenever an associated event occurs
(OpenMP v5.2, p. 457, l.5-6).
As such, change the registration result for the `dispatch` callback to
`ompt_set_sometimes` for now, which is fulfilled. Here, an
implementation can dispatch callbacks or trace an implementation-defined
subset of associated event occurrences.

Based on the changed registration result, a tool can then decide if they are
still interested in recording the information provided by the runtime
implementation.

Signed-off-by: Jan André Reuter <j.reuter at fz-juelich.de>
---
 openmp/runtime/src/ompt-event-specific.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/openmp/runtime/src/ompt-event-specific.h b/openmp/runtime/src/ompt-event-specific.h
index 7736ba8531635..539fe1548b666 100644
--- a/openmp/runtime/src/ompt-event-specific.h
+++ b/openmp/runtime/src/ompt-event-specific.h
@@ -103,7 +103,7 @@
 
 #define ompt_callback_reduction_implemented ompt_event_MAY_ALWAYS_OPTIONAL
 
-#define ompt_callback_dispatch_implemented ompt_event_MAY_ALWAYS_OPTIONAL
+#define ompt_callback_dispatch_implemented ompt_event_MAY_CONVENIENT
 
 #define ompt_callback_error_implemented ompt_event_MAY_ALWAYS_OPTIONAL
 


From openmp-commits at lists.llvm.org  Wed Aug 12 09:00:42 2026
From: openmp-commits at lists.llvm.org (Robert Imschweiler via Openmp-commits)
Date: Wed, 12 Aug 2026 09:00:42 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [offload] Use HSA SVM for AMDGPU
	shared memory (PR #215801)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/215801@github.com>
Message-ID: <6a7c98aa.834f8ed8.12ecb3.173b@mx.google.com>


================
@@ -3024,9 +3191,63 @@ struct AMDGPUDeviceTy : public GenericDeviceTy, AMDGenericDeviceTy {
     return Plugin::success();
   }
 
+  /// Fill \p TgtPtr with the given pattern on the host. Requires \p TgtPtr to
+  /// be host accessible.
+  Error dataFillHostImpl(void *TgtPtr, const void *PatternPtr,
----------------
ro-i wrote:

hsa_amd_memory_fill fails for SVM memory with HSA_STATUS_ERROR_INVALID_ALLOCATION. So, there needs to be some other path. That's the correctness part that requires some action.
And, afaict, the existing alternative is creating the memory pattern on the host and then copying it in a loop to the device until the whole size is filled. That would be madness and that's the performance part that should be covered directly here if we want to benefit from the SVM performance-wise, I think. (?)
(Other optimizations, such as prefetch, shall be reserved for potential future PRs, ofc.)

https://github.com/llvm/llvm-project/pull/215801

From openmp-commits at lists.llvm.org  Wed Aug 12 09:04:12 2026
From: openmp-commits at lists.llvm.org (Robert Imschweiler via Openmp-commits)
Date: Wed, 12 Aug 2026 09:04:12 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [libomp] OpenMP 6.0: Add device trait
	parser (PR #176164)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/176164@github.com>
Message-ID: <6a7c997c.0f490953.35172.cfdf@mx.google.com>


https://github.com/ro-i updated https://github.com/llvm/llvm-project/pull/176164

>From 9bf1f7b4191f9f25fb0af1e6aabba16792998208 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Thu, 15 Jan 2026 04:54:17 -0600
Subject: [PATCH 1/5] [libomp] OpenMP 6.0: Add device trait parser

OpenMP 6.0 introduced a device trait specification language for the
environment variables OMP_AVAILABLE_DEVICES (4.3.7) and
OMP_DEFAULT_DEVICE (4.3.8).

This commit defines a grammar for that language and implements a parser
for a large part of this grammar.
---
 openmp/runtime/src/CMakeLists.txt             |    1 +
 openmp/runtime/src/i18n/en_US.txt             |    5 +
 openmp/runtime/src/kmp_traits.cpp             |  306 +++++
 openmp/runtime/src/kmp_traits.h               |  436 +++++++
 openmp/runtime/unittests/CMakeLists.txt       |    1 +
 .../runtime/unittests/Traits/CMakeLists.txt   |    6 +
 openmp/runtime/unittests/Traits/MockOMP.cpp   |   24 +
 .../unittests/Traits/TestOMPTraitParser.cpp   | 1038 +++++++++++++++
 .../unittests/Traits/TestOMPTraits.cpp        | 1132 +++++++++++++++++
 9 files changed, 2949 insertions(+)
 create mode 100644 openmp/runtime/src/kmp_traits.cpp
 create mode 100644 openmp/runtime/src/kmp_traits.h
 create mode 100644 openmp/runtime/unittests/Traits/CMakeLists.txt
 create mode 100644 openmp/runtime/unittests/Traits/MockOMP.cpp
 create mode 100644 openmp/runtime/unittests/Traits/TestOMPTraitParser.cpp
 create mode 100644 openmp/runtime/unittests/Traits/TestOMPTraits.cpp

diff --git a/openmp/runtime/src/CMakeLists.txt b/openmp/runtime/src/CMakeLists.txt
index f8b9260af8c17..a2659262762ea 100644
--- a/openmp/runtime/src/CMakeLists.txt
+++ b/openmp/runtime/src/CMakeLists.txt
@@ -98,6 +98,7 @@ else()
     kmp_str.cpp
     kmp_tasking.cpp
     kmp_threadprivate.cpp
+    kmp_traits.cpp
     kmp_utility.cpp
     kmp_barrier.cpp
     kmp_wait_release.cpp
diff --git a/openmp/runtime/src/i18n/en_US.txt b/openmp/runtime/src/i18n/en_US.txt
index b3699df6db65d..bc74ffaf23465 100644
--- a/openmp/runtime/src/i18n/en_US.txt
+++ b/openmp/runtime/src/i18n/en_US.txt
@@ -362,6 +362,11 @@ TopologyGeneric              "%1$s: %2$s (%3$d total cores)"
 AffGranularityBad            "%1$s: granularity setting: %2$s does not exist in topology.  Using granularity=%3$s instead."
 TopologyHybrid               "%1$s: hybrid core type detected: %2$d %3$s cores."
 TopologyHybridCoreEff        "%1$s:   %2$d with core efficiency %3$d."
+TraitParserInvalidDeviceKind "trait parser while parsing %1$s: invalid device kind (allowed: host/nohost/cpu/gpu/fpga)"
+TraitParserInvalidUID        "trait parser while parsing %1$s: invalid uid (%2$s)"
+TraitParserMaxRecursion      "trait parser while parsing %1$s: max recursion depth (%2$d) exceeded"
+TraitParserFailed            "trait parser while parsing %1$s: failed to parse trait specification (%2$s)"
+TraitParserValueTooLarge     "trait parser while parsing %1$s: value %2$d above limit (%3$d)"
 
 # --- OpenMP errors detected at runtime ---
 #
diff --git a/openmp/runtime/src/kmp_traits.cpp b/openmp/runtime/src/kmp_traits.cpp
new file mode 100644
index 0000000000000..9d44ca2affe2b
--- /dev/null
+++ b/openmp/runtime/src/kmp_traits.cpp
@@ -0,0 +1,306 @@
+/*
+ * kmp_traits.cpp -- Handle OpenMP context traits
+ *
+ * OpenMP 6.0 specifies the following trait sets:
+ * - construct
+ * - device
+ * - target device
+ * - implementation
+ * - extension
+ * - dynamic
+ * Currently, the implementation in this file supports traits from the (target)
+ * device and implementation trait sets that are relevant for implementing the
+ * OMP_DEFAULT_DEVICE and OMP_AVAILABLE_DEVICES environment variables.
+ */
+
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "kmp_traits.h"
+#include "kmp_i18n.h"
+
+using namespace kmp_traits;
+
+// OpenMP trait grammar (in EBNF), currently used for parsing the
+// OMP_DEFAULT_DEVICE/OMP_AVAILABLE_DEVICES environment variables
+//
+// Notes about the grammar:
+// - Device traits are going to be translated into device numbers (aka integers)
+// later in the runtime. The parser handles device numbers as device traits that
+// have already been translated.
+// - "*" is also not a trait, strictly speaking. But it's also supported by the
+// parser and converted into a "match any" wildcard trait.
+// - OpenMP 6.0 explicitly excludes "&&" and "||" from appearing in the same
+// grouping level.
+// - This grammar currently only supports plain integers for array subsripts /
+// sections, no expressions.
+// - TODO:
+//   - Add support for more traits
+//
+// TODOs regarding the implementation (not the grammar):
+// - Implement array subscript/section parsing
+// - Implement grammar TODOs after they have been incorporated into the grammar
+//
+// list = [clause {',' clause}]
+// clause =
+//       device_number
+//     | "*" [index_expr]
+//     | trait_expr_group
+//     | trait_expr index_expr
+// device_number = ["-"] integer0
+// trait_expr_group =
+//       trait_expr
+//     | trait_expr {"&&" trait_expr}
+//     | trait_expr {"||" trait_expr}
+// trait_expr =
+//       trait_expr_single
+//     | trait_expr_group_paren
+// trait_expr_single = ["!"] trait
+// trait_expr_group_paren = ["!"] "(" trait_expr_group ")"
+// trait =
+//       "uid" "(" uid_value ")"
+// uid_value = (letter | digit0 | symbol) {letter | digit0 | symbol}
+//
+// index_expr = "[" integer0 "]" | "[" array_section "]"
+// array_section =
+//       lower_bound ":" length ":" stride
+//     | lower_bound ":" length ":"
+//     | lower_bound ":" length
+//     | lower_bound "::" stride
+//     | lower_bound "::"
+//     | lower_bound ":"
+//     | ":" length ":" stride
+//     | ":" length ":"
+//     | ":" length
+//     | "::" stride
+//     | "::"
+//     | ":"
+// lower_bound = integer0
+// length = integer0
+// stride = integer
+//
+// integer0 = 0 | integer
+// integer = digit {digit0}
+//
+// letter =
+//       "A" | "B" | "C" | "D" | "E" | "F" | "G" | "H" | "I" | "J" | "K" | "L"
+//     | "M" | "N" | "O" | "P" | "Q" | "R" | "S" | "T" | "U" | "V" | "W" | "X"
+//     | "Y" | "Z" | "a" | "b" | "c" | "d" | "e" | "f" | "g" | "h" | "i" | "j"
+//     | "k" | "l" | "m" | "n" | "o" | "p" | "q" | "r" | "s" | "t" | "u" | "v"
+//     | "w" | "x" | "y" | "z"
+// digit0 = "0" | digit
+// digit = "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9"
+// symbol = "-" | "_"
+
+namespace parser {
+
+#define MAX_RECURSION_DEPTH 64
+
+using namespace kmp_traits;
+
+static kmp_str_ref consume_uid_value(kmp_str_ref &scan, const char *dbg_name) {
+  scan.skip_space();
+  kmp_str_ref uid = scan.take_while([](char c) {
+    return isalnum(static_cast<unsigned char>(c)) || c == '-' || c == '_';
+  });
+  scan.drop_front(uid.length());
+  if (uid.empty() || !scan.consume_front(")"))
+    KMP_FATAL(TraitParserInvalidUID, dbg_name, uid.copy());
+  return uid;
+}
+
+static bool consume_trait(kmp_trait_expr_single &expr, kmp_str_ref &scan,
+                          const char *dbg_name) {
+  scan.skip_space();
+  if (!scan.consume_front("uid("))
+    return false;
+  kmp_str_ref uid = consume_uid_value(scan, dbg_name);
+  expr.set_trait(new kmp_uid_trait(uid));
+  return true;
+}
+
+static bool consume_trait_expr_single(kmp_trait_expr_single &expr,
+                                      kmp_str_ref &scan, const char *dbg_name) {
+  kmp_str_ref orig_scan = scan;
+
+  scan.skip_space();
+  if (scan.consume_front("!"))
+    expr.set_negated();
+  if (consume_trait(expr, scan, dbg_name))
+    return true;
+  scan = orig_scan;
+  return false;
+}
+
+// forward declaration
+static bool consume_trait_expr_group(kmp_trait_expr_group &group,
+                                     kmp_str_ref &scan, int max_recursion,
+                                     const char *dbg_name);
+
+static bool consume_trait_expr_group_paren(kmp_trait_expr_group &group,
+                                           kmp_str_ref &scan, int max_recursion,
+                                           const char *dbg_name) {
+  if (max_recursion-- <= 0)
+    KMP_FATAL(TraitParserMaxRecursion, dbg_name, MAX_RECURSION_DEPTH);
+  kmp_str_ref orig_scan = scan;
+
+  scan.skip_space();
+  if (scan.consume_front("!"))
+    group.set_negated();
+
+  scan.skip_space();
+  if (!scan.consume_front("(") ||
+      !consume_trait_expr_group(group, scan, max_recursion, dbg_name)) {
+    scan = orig_scan;
+    return false;
+  }
+
+  scan.skip_space();
+  if (!scan.consume_front(")")) {
+    scan = orig_scan;
+    return false;
+  }
+  return true;
+}
+
+static bool consume_trait_expr(kmp_trait_expr *&expr, kmp_str_ref &scan,
+                               int max_recursion, const char *dbg_name) {
+  if (max_recursion-- <= 0)
+    KMP_FATAL(TraitParserMaxRecursion, dbg_name, MAX_RECURSION_DEPTH);
+
+  // Parse a single trait expression
+  kmp_trait_expr_single *single_expr = new kmp_trait_expr_single();
+  if (consume_trait_expr_single(*single_expr, scan, dbg_name)) {
+    expr = single_expr;
+    return true;
+  }
+  delete single_expr;
+
+  // Parse a parenthesized group trait expression
+  kmp_trait_expr_group *group_expr = new kmp_trait_expr_group();
+  if (consume_trait_expr_group_paren(*group_expr, scan, max_recursion,
+                                     dbg_name)) {
+    expr = group_expr;
+    return true;
+  }
+  delete group_expr;
+
+  return false;
+}
+
+static bool consume_trait_expr_group(kmp_trait_expr_group &group,
+                                     kmp_str_ref &scan, int max_recursion,
+                                     const char *dbg_name) {
+  if (max_recursion-- <= 0)
+    KMP_FATAL(TraitParserMaxRecursion, dbg_name, MAX_RECURSION_DEPTH);
+
+  kmp_trait_expr *expr = nullptr;
+  if (!consume_trait_expr(expr, scan, max_recursion, dbg_name))
+    return false;
+
+  group.add_expr(expr);
+  const char *op = nullptr;
+
+  scan.skip_space();
+  if (scan.consume_front("||")) {
+    group.set_group_type(kmp_trait_expr_group::OR);
+    op = "||";
+  } else if (scan.consume_front("&&")) {
+    group.set_group_type(kmp_trait_expr_group::AND);
+    op = "&&";
+  } else {
+    return true; // single trait expression, no group
+  }
+
+  // Now that we got an operator, we need at least one more trait expr.
+  do {
+    if (!consume_trait_expr(expr, scan, max_recursion, dbg_name))
+      return false;
+    group.add_expr(expr);
+    scan.skip_space();
+  } while (scan.consume_front(op));
+
+  return true;
+}
+
+static bool consume_clause(kmp_trait_clause &clause, kmp_str_ref &scan,
+                           const char *dbg_name) {
+  kmp_str_ref orig_scan = scan;
+  scan.skip_space();
+
+  // Parse wildcard "trait"
+  if (scan.consume_front("*")) {
+    clause.set_expr(new kmp_wildcard_trait());
+    return true;
+  }
+
+  // Parse a literal device number
+  int value;
+  if (scan.consume_integer(value)) {
+    clause.set_expr(new kmp_literal_trait(value));
+    return true;
+  }
+
+  // Parse a trait expression group
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+  if (consume_trait_expr_group(*group, scan, MAX_RECURSION_DEPTH, dbg_name)) {
+    clause.set_expr(group);
+    return true;
+  }
+  delete group;
+
+  scan = orig_scan;
+  return false;
+}
+
+static bool consume_list(kmp_trait_context &context, kmp_str_ref &scan,
+                         const char *dbg_name) {
+  kmp_str_ref orig_scan = scan;
+  scan.skip_space();
+
+  while (!scan.empty()) {
+    kmp_trait_clause *clause = new kmp_trait_clause();
+    if (!consume_clause(*clause, scan, dbg_name)) {
+      delete clause;
+      scan = orig_scan;
+      return false;
+    }
+    context.add_clause(clause);
+    orig_scan = scan;
+
+    scan.skip_space();
+    if (!scan.consume_front(",") && !scan.empty()) {
+      scan = orig_scan;
+      return false;
+    }
+  }
+
+  return true;
+}
+
+} // namespace parser
+
+kmp_trait_context *kmp_trait_context::parse_from_spec(kmp_str_ref spec,
+                                                      const char *dbg_name) {
+  kmp_trait_context *context = new kmp_trait_context();
+  if (!parser::consume_list(*context, spec, dbg_name))
+    KMP_FATAL(TraitParserFailed, dbg_name, spec.copy());
+  return context;
+}
+
+int kmp_trait_context::parse_single_device(kmp_str_ref spec,
+                                           int device_num_limit,
+                                           const char *dbg_name) {
+  int device_num;
+  spec.skip_space();
+  if (!spec.consume_integer(device_num))
+    KMP_FATAL(TraitParserFailed, dbg_name, spec.copy());
+  if (device_num > device_num_limit)
+    KMP_FATAL(TraitParserValueTooLarge, dbg_name, device_num, device_num_limit);
+  return device_num;
+}
diff --git a/openmp/runtime/src/kmp_traits.h b/openmp/runtime/src/kmp_traits.h
new file mode 100644
index 0000000000000..3f93fa74f0b74
--- /dev/null
+++ b/openmp/runtime/src/kmp_traits.h
@@ -0,0 +1,436 @@
+//===----------- Traits.h - OpenMP context traits -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementation of OpenMP context traits.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OPENMP_TRAITS_H
+#define OPENMP_TRAITS_H
+
+#include "kmp.h"
+#include "kmp_adt.h"
+
+namespace kmp_traits {
+
+extern "C" int omp_get_num_devices();
+extern "C" const char *omp_get_uid_from_device(int device_num);
+
+class kmp_trait {
+protected:
+  enum trait_type { WILDCARD_T, LITERAL_T, UID_T };
+  trait_type _type;
+
+  kmp_trait(trait_type type) : _type(type) {}
+
+public:
+  virtual ~kmp_trait() = default;
+
+  kmp_trait(const kmp_trait &) = delete;
+  kmp_trait(kmp_trait &&) = delete;
+  kmp_trait &operator=(const kmp_trait &) = delete;
+  kmp_trait &operator=(kmp_trait &&) = delete;
+
+  virtual bool match(int device) const = 0;
+
+  // Use KMP_INTERNAL_MALLOC/KMP_INTERNAL_FREE for memory management.
+  void *operator new(size_t size) { return KMP_INTERNAL_MALLOC(size); }
+  void operator delete(void *ptr) { KMP_INTERNAL_FREE(ptr); }
+
+  virtual bool operator==(const kmp_trait &other) const {
+    return _type == other._type;
+  }
+};
+
+/// Represents a wildcard trait that matches any device.
+class kmp_wildcard_trait final : public kmp_trait {
+public:
+  kmp_wildcard_trait() : kmp_trait(WILDCARD_T) {}
+
+  bool match([[maybe_unused]] int device) const override { return true; }
+
+  bool operator==(const kmp_trait &other) const override {
+    return kmp_trait::operator==(other);
+  }
+};
+
+/// Represents a specific device number.
+class kmp_literal_trait final : public kmp_trait {
+  int device_num;
+
+public:
+  kmp_literal_trait(int device_num)
+      : kmp_trait(LITERAL_T), device_num(device_num) {
+    assert(device_num >= 0 && "Device number must be non-negative");
+  }
+
+  bool match(int device) const override { return device_num == device; }
+
+  bool operator==(const kmp_trait &other) const override {
+    return kmp_trait::operator==(other) &&
+           device_num ==
+               static_cast<const kmp_literal_trait &>(other).device_num;
+  }
+};
+
+/// Represents a specific UID.
+/// UID is deliberately not resolved at construction time since libomptarget
+/// might not be initialized yet. This is why we delay calls to
+/// omp_get_uid_from_device / omp_get_device_from_uid until the trait is
+/// evaluated.
+class kmp_uid_trait final : public kmp_trait {
+  char *uid;
+  // Can be used by unit tests to mock omp_get_uid_from_device.
+  const char *(*get_uid_from_device)(int device) = omp_get_uid_from_device;
+
+public:
+  kmp_uid_trait(kmp_str_ref uid) : kmp_trait(UID_T), uid(uid.copy()) {}
+
+  ~kmp_uid_trait() override {
+    if (uid)
+      KMP_INTERNAL_FREE(uid);
+  }
+
+  bool match(int device) const override {
+    const char *device_uid = get_uid_from_device(device);
+    if (!device_uid || !uid)
+      return false;
+    return strcmp(device_uid, uid) == 0;
+  }
+
+  // For testing purposes only: set the function that returns the UID from a
+  // device.
+  void set_uid_from_device(const char *(*uid_from_device)(int)) {
+    get_uid_from_device = uid_from_device;
+  }
+
+  bool operator==(const kmp_trait &other) const override {
+    if (!kmp_trait::operator==(other))
+      return false;
+    const char *other_uid = static_cast<const kmp_uid_trait &>(other).uid;
+    return uid && other_uid ? strcmp(uid, other_uid) == 0 : uid == other_uid;
+  }
+};
+
+/// Abstract class representing either a single trait expression or a collection
+/// of trait expressions that are ANDed or ORed together.
+class kmp_trait_expr {
+protected:
+  enum expr_type { SINGLE_T, GROUP_T };
+  expr_type _type;
+  // Determines if the expression is negated (true) or not (false).
+  bool negated = false;
+  // Can be used by unit tests to mock omp_get_num_devices.
+  int (*get_num_devices)() = omp_get_num_devices;
+
+  kmp_trait_expr(expr_type type) : _type(type) {}
+  kmp_trait_expr(expr_type type, bool negated)
+      : _type(type), negated(negated) {}
+
+  virtual bool match_impl(int device, int num_devices) const = 0;
+
+public:
+  virtual ~kmp_trait_expr() = default;
+
+  kmp_trait_expr(const kmp_trait_expr &) = delete;
+  kmp_trait_expr(kmp_trait_expr &&) = delete;
+  kmp_trait_expr &operator=(const kmp_trait_expr &) = delete;
+  kmp_trait_expr &operator=(kmp_trait_expr &&) = delete;
+
+  bool is_negated() const { return negated; }
+
+  // Check if the device matches the expression.
+  bool match(int device, int num_devices = -1) const {
+    if (num_devices == -1)
+      num_devices = get_num_devices();
+    if (device < 0 || device >= num_devices)
+      return false;
+    return match_impl(device, num_devices);
+  }
+
+  void set_negated(bool neg = true) { negated = neg; }
+
+  // For testing purposes only: set the function that returns the number of
+  // devices.
+  void set_num_devices(int (*num_devices)()) { get_num_devices = num_devices; }
+
+  // Use KMP_INTERNAL_MALLOC/KMP_INTERNAL_FREE for memory management.
+  void *operator new(size_t size) { return KMP_INTERNAL_MALLOC(size); }
+  void operator delete(void *ptr) { KMP_INTERNAL_FREE(ptr); }
+
+  virtual bool operator==(const kmp_trait_expr &other) const {
+    return _type == other._type && negated == other.negated;
+  }
+};
+
+/// Represents a single (possibly negated) trait.
+class kmp_trait_expr_single final : public kmp_trait_expr {
+  kmp_trait *trait = nullptr;
+
+protected:
+  bool match_impl(int device, [[maybe_unused]] int num_devices) const override {
+    assert(trait);
+    bool result = trait->match(device);
+    return negated ? !result : result;
+  }
+
+public:
+  kmp_trait_expr_single() : kmp_trait_expr(SINGLE_T) {}
+  kmp_trait_expr_single(bool negated) : kmp_trait_expr(SINGLE_T, negated) {}
+  kmp_trait_expr_single(kmp_trait *trait)
+      : kmp_trait_expr(SINGLE_T), trait(trait) {
+    assert(trait && "kmp_trait_expr_single requires a non-null trait");
+  }
+  ~kmp_trait_expr_single() override { delete trait; }
+
+  void set_trait(kmp_trait *new_trait) {
+    assert(new_trait);
+    if (trait)
+      delete trait;
+    trait = new_trait;
+  }
+
+  bool operator==(const kmp_trait_expr &other) const override {
+    if (!kmp_trait_expr::operator==(other))
+      return false;
+    const kmp_trait_expr_single &other_single =
+        static_cast<const kmp_trait_expr_single &>(other);
+    return trait && other_single.trait ? *trait == *other_single.trait
+                                       : trait == other_single.trait;
+  }
+};
+
+/// Represents a (possibly negated) collection of traits that are either ANDed
+/// or ORed together.
+class kmp_trait_expr_group final : public kmp_trait_expr {
+public:
+  enum group_type { AND, OR };
+
+private:
+  kmp_vector<kmp_trait_expr *> exprs;
+  // Determines if all traits have to match (true) or any of them (false).
+  group_type type = OR;
+
+protected:
+  bool match_impl(int device, int num_devices) const override {
+    size_t matched = 0;
+    for (const kmp_trait_expr *expr : exprs) {
+      if (expr->match(device, num_devices))
+        matched++;
+    }
+    // Note: AND evaluates to true for an empty group.
+    bool result = type == AND ? matched == exprs.size() : matched > 0;
+    return negated ? !result : result;
+  }
+
+public:
+  kmp_trait_expr_group() : kmp_trait_expr(GROUP_T) {}
+  kmp_trait_expr_group(bool negated) : kmp_trait_expr(GROUP_T, negated) {}
+  ~kmp_trait_expr_group() override {
+    for (kmp_trait_expr *expr : exprs)
+      delete expr;
+  }
+
+  void add_expr(kmp_trait *trait) {
+    assert(trait);
+    add_expr(new kmp_trait_expr_single(trait));
+  }
+  void add_expr(kmp_trait_expr *expr) {
+    assert(expr);
+    exprs.push_back(expr);
+    // Propagate get_num_devices to the expression.
+    expr->set_num_devices(get_num_devices);
+  }
+
+  group_type get_group_type() const { return type; }
+
+  void set_group_type(group_type new_type) { type = new_type; }
+
+  void set_num_devices(int (*num_devices)()) {
+    kmp_trait_expr::set_num_devices(num_devices);
+    for (kmp_trait_expr *expr : exprs)
+      expr->set_num_devices(num_devices);
+  }
+
+  bool operator==(const kmp_trait_expr &other) const override {
+    if (!kmp_trait_expr::operator==(other))
+      return false;
+    const kmp_trait_expr_group &other_group =
+        static_cast<const kmp_trait_expr_group &>(other);
+    return exprs.is_set_equal(
+        other_group.exprs, [](kmp_trait_expr *const &a,
+                              kmp_trait_expr *const &b) { return *a == *b; });
+  }
+};
+
+class kmp_trait_clause final {
+  kmp_trait_expr *expr = nullptr;
+
+public:
+  kmp_trait_clause() = default;
+  ~kmp_trait_clause() { delete expr; }
+
+  kmp_trait_clause(const kmp_trait_clause &) = delete;
+  kmp_trait_clause(kmp_trait_clause &&) = delete;
+  kmp_trait_clause &operator=(const kmp_trait_clause &) = delete;
+  kmp_trait_clause &operator=(kmp_trait_clause &&) = delete;
+
+  kmp_trait_expr *get_expr() { return expr; }
+
+  bool match(int device, int num_devices = -1) const {
+    assert(expr);
+    return expr->match(device, num_devices);
+  }
+
+  void set_expr(kmp_trait *trait) {
+    assert(trait);
+    if (expr)
+      delete expr;
+    expr = new kmp_trait_expr_single(trait);
+  }
+  void set_expr(kmp_trait_expr *new_expr) {
+    assert(new_expr);
+    if (expr)
+      delete expr;
+    expr = new_expr;
+  }
+
+  // Use KMP_INTERNAL_MALLOC/KMP_INTERNAL_FREE for memory management.
+  void *operator new(size_t size) { return KMP_INTERNAL_MALLOC(size); }
+  void operator delete(void *ptr) { KMP_INTERNAL_FREE(ptr); }
+
+  bool operator==(const kmp_trait_clause &other) const {
+    return expr && other.expr ? *expr == *other.expr : expr == other.expr;
+  }
+};
+
+} // namespace kmp_traits
+
+class kmp_trait_context final {
+  using kmp_trait_clause = kmp_traits::kmp_trait_clause;
+  using kmp_trait_expr = kmp_traits::kmp_trait_expr;
+
+  kmp_vector<kmp_trait_clause *> clauses;
+  // List of devices that have been evaluated.
+  kmp_vector<int> devices;
+  bool evaluated = false;
+  // Can be used by unit tests to mock omp_get_num_devices.
+  int (*get_num_devices)() = kmp_traits::omp_get_num_devices;
+
+  void _evaluate() {
+    devices.clear();
+    for (int d = 0; d < get_num_devices(); ++d) {
+      if (_match(d))
+        devices.push_back(d);
+    }
+    evaluated = true;
+  }
+
+  bool _match(int device) const {
+    if (device < 0 || device >= get_num_devices())
+      return false;
+    for (kmp_trait_clause *clause : clauses) {
+      if (clause->match(device))
+        return true;
+    }
+    return false;
+  }
+
+public:
+  kmp_trait_context() = default;
+  ~kmp_trait_context() {
+    for (kmp_trait_clause *clause : clauses)
+      delete clause;
+  }
+
+  kmp_trait_context(const kmp_trait_context &) = delete;
+  kmp_trait_context(kmp_trait_context &&) = delete;
+  kmp_trait_context &operator=(const kmp_trait_context &) = delete;
+  kmp_trait_context &operator=(kmp_trait_context &&) = delete;
+
+  // Parse a trait specification from a string.
+  // If dbg_name is provided, it will be used in error messages to identify the
+  // source of the trait specification.
+  static kmp_trait_context *parse_from_spec(kmp_str_ref spec,
+                                            const char *dbg_name = nullptr);
+
+  // Parse only a single device number from the spec.
+  // This is useful for backward compatibility with legacy code.
+  // If dbg_name is provided, it will be used in error messages to identify the
+  // source of the device number.
+  static int parse_single_device(kmp_str_ref spec, int device_num_limit,
+                                 const char *dbg_name = nullptr);
+
+  void add_clause(kmp_trait_clause *clause) {
+    assert(clause);
+    clauses.push_back(clause);
+    // Propagate get_num_devices to the clause.
+    if (kmp_trait_expr *expr = clause->get_expr())
+      expr->set_num_devices(get_num_devices);
+  }
+
+  // Returns the list of devices that match the trait specification represented
+  // by the context. The list contains devices numbers forming a set and sorted
+  // in ascending order.
+  // Note to future developers: if we want to add an option to force
+  // re-evaluation, we need to consider that the devices vector and thus the
+  // context iterators are invalidated.
+  const kmp_vector<int> &evaluate() {
+    trigger_evaluation();
+    return devices;
+  }
+
+  const kmp_vector<int> &evaluate() const {
+    assert(evaluated && "kmp_trait_context not evaluated");
+    return devices;
+  }
+
+  // Check if the device matches the trait specification represented by the
+  // context.
+  bool match(int device) { return evaluate().contains(device); }
+
+  bool match(int device) const {
+    assert(evaluated && "kmp_trait_context not evaluated");
+    return devices.contains(device);
+  }
+
+  // For testing purposes only: set the function that returns the number of
+  // devices.
+  void set_num_devices(int (*num_devices)()) {
+    get_num_devices = num_devices;
+    for (kmp_trait_clause *clause : clauses) {
+      if (kmp_trait_expr *expr = clause->get_expr())
+        expr->set_num_devices(num_devices);
+    }
+  }
+
+  // Triggers lazy evaluation if not already evaluated.
+  void trigger_evaluation() {
+    if (!evaluated)
+      _evaluate();
+  }
+
+  // Use KMP_INTERNAL_MALLOC/KMP_INTERNAL_FREE for memory management.
+  void *operator new(size_t size) { return KMP_INTERNAL_MALLOC(size); }
+  void operator delete(void *ptr) { KMP_INTERNAL_FREE(ptr); }
+
+  bool operator==(const kmp_trait_context &other) const {
+    auto clause_comp = [](kmp_trait_clause *const &a,
+                          kmp_trait_clause *const &b) { return *a == *b; };
+    return clauses.is_set_equal(other.clauses, clause_comp);
+  }
+
+  // Iterator support (returns the iterators of the devices vector; triggers
+  // lazy evaluation if not already evaluated and if the context is not const).
+  const int *begin() { return evaluate().begin(); }
+  const int *end() { return evaluate().end(); }
+  const int *begin() const { return evaluate().begin(); }
+  const int *end() const { return evaluate().end(); }
+};
+
+#endif // OPENMP_TRAITS_H
diff --git a/openmp/runtime/unittests/CMakeLists.txt b/openmp/runtime/unittests/CMakeLists.txt
index 8335c94cfb6d9..96884397f2b30 100644
--- a/openmp/runtime/unittests/CMakeLists.txt
+++ b/openmp/runtime/unittests/CMakeLists.txt
@@ -69,3 +69,4 @@ add_openmp_testsuite(
 
 add_subdirectory(ADT)
 add_subdirectory(String)
+add_subdirectory(Traits)
diff --git a/openmp/runtime/unittests/Traits/CMakeLists.txt b/openmp/runtime/unittests/Traits/CMakeLists.txt
new file mode 100644
index 0000000000000..d53db903c9f3a
--- /dev/null
+++ b/openmp/runtime/unittests/Traits/CMakeLists.txt
@@ -0,0 +1,6 @@
+add_openmp_unittest(TraitsTests
+  MockOMP.cpp
+  TestOMPTraits.cpp
+  TestOMPTraitParser.cpp
+)
+
diff --git a/openmp/runtime/unittests/Traits/MockOMP.cpp b/openmp/runtime/unittests/Traits/MockOMP.cpp
new file mode 100644
index 0000000000000..2c8623798b7d7
--- /dev/null
+++ b/openmp/runtime/unittests/Traits/MockOMP.cpp
@@ -0,0 +1,24 @@
+//===- MockOMP.cpp - Mock OMP functions for testing ----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Mock implementations for OMP functions (libomptarget not available in tests).
+//
+//===----------------------------------------------------------------------===//
+
+// Mock device count for testing
+static int MockNumDevices = 4;
+static const char *MockDeviceUIDs[] = {"device-0", "device-1", "device-2",
+                                       "device-3"};
+
+extern "C" int omp_get_num_devices() { return MockNumDevices; }
+
+extern "C" const char *omp_get_uid_from_device(int DeviceNum) {
+  if (DeviceNum >= 0 && DeviceNum < MockNumDevices)
+    return MockDeviceUIDs[DeviceNum];
+  return "";
+}
diff --git a/openmp/runtime/unittests/Traits/TestOMPTraitParser.cpp b/openmp/runtime/unittests/Traits/TestOMPTraitParser.cpp
new file mode 100644
index 0000000000000..7d78ca0d202cf
--- /dev/null
+++ b/openmp/runtime/unittests/Traits/TestOMPTraitParser.cpp
@@ -0,0 +1,1038 @@
+//===- TestOMPTraitParser.cpp - Tests for OMP Trait Parser ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "kmp_traits.h"
+#include "gtest/gtest.h"
+
+namespace {
+
+//===----------------------------------------------------------------------===//
+// Helper to parse and auto-cleanup
+//===----------------------------------------------------------------------===//
+
+class ParserTest : public ::testing::Test {
+protected:
+  kmp_trait_context *context = nullptr;
+
+  void parse(const char *spec, const char *dbg_name = nullptr) {
+    context = kmp_trait_context::parse_from_spec(kmp_str_ref(spec), dbg_name);
+  }
+
+  void TearDown() override {
+    if (context) {
+      delete context;
+      context = nullptr;
+    }
+  }
+};
+
+template <bool expected_result>
+static void check_result_single(kmp_trait_context *context,
+                                const kmp_vector<int> &result,
+                                int expected_device_num) {
+  EXPECT_EQ(context->match(expected_device_num), expected_result);
+  EXPECT_EQ(result.contains(expected_device_num), expected_result);
+}
+
+template <bool expected_result, int... device_nums>
+static void check_result(kmp_trait_context *context,
+                         const kmp_vector<int> &result) {
+  (check_result_single<expected_result>(context, result, device_nums), ...);
+}
+
+template <bool expected_result, int... device_nums>
+static void check_result(kmp_trait_context *context) {
+  const kmp_vector<int> &result = context->evaluate();
+  check_result<expected_result, device_nums...>(context, result);
+}
+
+//===----------------------------------------------------------------------===//
+// Literal Device Numbers
+//===----------------------------------------------------------------------===//
+
+TEST_F(ParserTest, SingleLiteral) {
+  parse("5");
+
+  ASSERT_NE(context, nullptr);
+  // Device 5 is out of range (mock has 4 devices: 0-3), so match returns false
+
+  EXPECT_EQ(context->evaluate().size(), 0u);
+  check_result<false, 5, 0, 4, 6>(context);
+}
+
+TEST_F(ParserTest, ZeroLiteral) {
+  parse("0");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 0>(context, result);
+  check_result<false, 1>(context, result);
+}
+
+TEST_F(ParserTest, MultipleLiterals) {
+  parse("1,2,3");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 3u);
+  check_result<true, 1, 2, 3>(context, result);
+  check_result<false, 0, 4>(context, result);
+}
+
+TEST_F(ParserTest, LiteralsWithSpaces) {
+  parse("1, 2, 3");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 3u);
+  check_result<true, 1, 2, 3>(context, result);
+  check_result<false, 0, 4>(context, result);
+}
+
+TEST_F(ParserTest, LiteralsWithLeadingSpaces) {
+  parse("  1,  2,  3");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 3u);
+  check_result<true, 1, 2, 3>(context, result);
+  check_result<false, 0, 4>(context, result);
+}
+
+TEST_F(ParserTest, LargeLiteral) {
+  parse("12345");
+
+  ASSERT_NE(context, nullptr);
+  // Device 12345 is out of range, so match returns false
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 0u);
+  check_result<false, 12345, 0>(context, result);
+}
+
+//===----------------------------------------------------------------------===//
+// Wildcard
+//===----------------------------------------------------------------------===//
+
+TEST_F(ParserTest, Wildcard) {
+  parse("*");
+
+  ASSERT_NE(context, nullptr);
+  // Wildcard matches all 4 mock devices
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+  check_result<false, 100>(context, result);
+}
+
+TEST_F(ParserTest, WildcardWithLiterals) {
+  parse("1, *, 3");
+
+  ASSERT_NE(context, nullptr);
+  // Wildcard makes all in-range devices match
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+  check_result<false, 100>(context, result);
+}
+
+//===----------------------------------------------------------------------===//
+// UID Traits
+//===----------------------------------------------------------------------===//
+
+TEST_F(ParserTest, UIDTrait) {
+  parse("uid(device-0)");
+
+  ASSERT_NE(context, nullptr);
+  // Uses mock: device-0 is at index 0
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 0>(context, result);
+  check_result<false, 1>(context, result);
+}
+
+TEST_F(ParserTest, UIDTraitWithUnderscore) {
+  parse("uid(my_device_123)");
+
+  ASSERT_NE(context, nullptr);
+  // This UID doesn't match any mock device
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 0u);
+  check_result<false, 0, 1>(context, result);
+}
+
+TEST_F(ParserTest, UIDTraitWithDash) {
+  parse("uid(device-2)");
+
+  ASSERT_NE(context, nullptr);
+  // Uses mock: device-2 is at index 2
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 2>(context, result);
+  check_result<false, 0, 1, 3>(context, result);
+}
+
+TEST_F(ParserTest, MultipleUIDTraits) {
+  parse("uid(device-1), uid(device-3)");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 2u);
+  check_result<true, 1, 3>(context, result);
+  check_result<false, 0, 2>(context, result);
+}
+
+TEST_F(ParserTest, MixedLiteralsAndUIDs) {
+  parse("0, uid(device-2), 1, uid(device-3)");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+}
+
+//===----------------------------------------------------------------------===//
+// Negation
+//===----------------------------------------------------------------------===//
+
+TEST_F(ParserTest, NegatedUID) {
+  parse("!uid(device-0)");
+
+  ASSERT_NE(context, nullptr);
+  // Negated: everything except device-0 matches
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 3u);
+  check_result<false, 0>(context, result);
+  check_result<true, 1, 2, 3>(context, result);
+}
+
+//===----------------------------------------------------------------------===//
+// Grouping with Parentheses
+//===----------------------------------------------------------------------===//
+
+TEST_F(ParserTest, SimpleGroup) {
+  parse("(uid(device-1))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<false, 0>(context, result);
+  check_result<true, 1>(context, result);
+}
+
+TEST_F(ParserTest, GroupWithOR) {
+  parse("(uid(device-0) || uid(device-2))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 2u);
+  check_result<true, 0, 2>(context, result);
+  check_result<false, 1, 3>(context, result);
+}
+
+TEST_F(ParserTest, GroupWithAND) {
+  parse("(uid(device-0) && uid(device-0))");
+
+  ASSERT_NE(context, nullptr);
+  // Both refer to same device, so AND passes for device 0
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 0>(context, result);
+  check_result<false, 1>(context, result);
+}
+
+TEST_F(ParserTest, NegatedGroup) {
+  parse("!(uid(device-0) || uid(device-1))");
+
+  ASSERT_NE(context, nullptr);
+  // Negated: matches devices NOT in {0, 1}
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 2u);
+  check_result<false, 0, 1>(context, result);
+  check_result<true, 2, 3>(context, result);
+}
+
+//===----------------------------------------------------------------------===//
+// Complex Expressions
+//===----------------------------------------------------------------------===//
+
+TEST_F(ParserTest, ComplexMixed) {
+  parse("0, 1, uid(device-2), *");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+  check_result<false, 100>(context, result);
+}
+
+TEST_F(ParserTest, MultipleORGroups) {
+  parse("(uid(device-0) || uid(device-1)), (uid(device-2) || uid(device-3))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+}
+
+//===----------------------------------------------------------------------===//
+// Complex Boolean Operators
+//===----------------------------------------------------------------------===//
+
+TEST_F(ParserTest, ThreeWayOR) {
+  // Three UIDs combined with OR
+  parse("(uid(device-0) || uid(device-1) || uid(device-2))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 3u);
+  check_result<true, 0, 1, 2>(context, result);
+  check_result<false, 3>(context, result);
+}
+
+TEST_F(ParserTest, FourWayOR) {
+  // All four mock devices via OR
+  parse("(uid(device-0) || uid(device-1) || uid(device-2) || uid(device-3))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, ThreeWayAND) {
+  // Three identical UIDs with AND - all must match same device
+  parse("(uid(device-1) && uid(device-1) && uid(device-1))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 1>(context, result);
+  check_result<false, 0, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, ANDWithDifferentUIDs) {
+  // AND with different UIDs - can never match (device can't have two UIDs)
+  parse("(uid(device-0) && uid(device-1))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 0u);
+  check_result<false, 0, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, NegatedThreeWayOR) {
+  // Negate a group of three UIDs - matches devices NOT in {0, 1, 2}
+  parse("!(uid(device-0) || uid(device-1) || uid(device-2))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<false, 0, 1, 2>(context, result);
+  check_result<true, 3>(context, result);
+}
+
+TEST_F(ParserTest, NegatedAND) {
+  // Negate an AND group - since AND never matches, negation matches all
+  parse("!(uid(device-0) && uid(device-1))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, NegatedANDWithSameUID) {
+  // Negate an AND that matches device-0 - matches everything except 0
+  parse("!(uid(device-0) && uid(device-0))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 3u);
+  check_result<false, 0>(context, result);
+  check_result<true, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, NestedParensWithOR) {
+  // Nested parentheses around OR
+  parse("((uid(device-0) || uid(device-1)))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 2u);
+  check_result<true, 0, 1>(context, result);
+  check_result<false, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, NestedParensWithAND) {
+  // Nested parentheses around AND
+  parse("((uid(device-2) && uid(device-2)))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 2>(context, result);
+  check_result<false, 0, 1, 3>(context, result);
+}
+
+TEST_F(ParserTest, DoubleNegation) {
+  // Double negation: !!uid(device-0) should match device-0
+  parse("!(!uid(device-0))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 0>(context, result);
+  check_result<false, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, NegatedNestedOR) {
+  // Negate nested OR group
+  parse("!((uid(device-0) || uid(device-1)))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 2u);
+  check_result<false, 0, 1>(context, result);
+  check_result<true, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, MultipleNegatedExprs) {
+  // Multiple negated clauses - OR semantics between clauses
+  parse("!uid(device-0), !uid(device-1)");
+
+  ASSERT_NE(context, nullptr);
+  // First clause matches 1,2,3; Second clause matches 0,2,3
+  // OR between clauses: union = all devices
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, MixedNegatedAndNonNegated) {
+  // Mix negated and non-negated clauses
+  parse("uid(device-0), !uid(device-0)");
+
+  ASSERT_NE(context, nullptr);
+  // First matches 0, second matches 1,2,3 -> union = all
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, ComplexORGroupsInSeparateExprs) {
+  // Complex OR groups as separate clauses
+  parse("(uid(device-0) || uid(device-1)), (uid(device-2) || uid(device-3))");
+
+  ASSERT_NE(context, nullptr);
+  // First matches 0,1; Second matches 2,3 -> union = all
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, NegatedORGroupWithLiteral) {
+  // Negated OR group combined with literal in separate clauses
+  parse("!(uid(device-0) || uid(device-1)), 0");
+
+  ASSERT_NE(context, nullptr);
+  // First matches 2,3; Second matches 0 -> union = 0,2,3
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 3u);
+  check_result<true, 0, 2, 3>(context, result);
+  check_result<false, 1>(context, result);
+}
+
+TEST_F(ParserTest, DeeplyNestedWithOperators) {
+  // Deeply nested with operators
+  parse("(((uid(device-0) || uid(device-1))))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 2u);
+  check_result<true, 0, 1>(context, result);
+  check_result<false, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, ORWithSpacesAroundOperators) {
+  // OR with lots of whitespace
+  parse("(  uid(device-0)   ||   uid(device-2)   ||   uid(device-3)  )");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 3u);
+  check_result<true, 0, 2, 3>(context, result);
+  check_result<false, 1>(context, result);
+}
+
+TEST_F(ParserTest, ANDWithSpacesAroundOperators) {
+  // AND with lots of whitespace
+  parse("(  uid(device-1)   &&   uid(device-1)  )");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 1>(context, result);
+  check_result<false, 0, 2, 3>(context, result);
+}
+
+//===----------------------------------------------------------------------===//
+// Mixed && and || (in separate clauses/groups)
+//===----------------------------------------------------------------------===//
+
+TEST_F(ParserTest, ORExprAndANDExpr) {
+  // OR group in first clause, AND group in second clause
+  parse("(uid(device-0) || uid(device-1)), (uid(device-2) && uid(device-2))");
+
+  ASSERT_NE(context, nullptr);
+  // First clause matches 0,1; Second clause matches 2 -> union = 0,1,2
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 3u);
+  check_result<true, 0, 1, 2>(context, result);
+  check_result<false, 3>(context, result);
+}
+
+TEST_F(ParserTest, ANDExprAndORExpr) {
+  // AND group first, OR group second
+  parse("(uid(device-0) && uid(device-0)), (uid(device-2) || uid(device-3))");
+
+  ASSERT_NE(context, nullptr);
+  // First clause matches 0; Second clause matches 2,3 -> union = 0,2,3
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 3u);
+  check_result<true, 0, 2, 3>(context, result);
+  check_result<false, 1>(context, result);
+}
+
+TEST_F(ParserTest, MultipleANDAndORExprs) {
+  // Multiple clauses alternating between AND and OR
+  parse("(uid(device-0) && uid(device-0)), (uid(device-1) || uid(device-2)), "
+        "(uid(device-3) && uid(device-3))");
+
+  ASSERT_NE(context, nullptr);
+  // Expr 1 matches 0; Expr 2 matches 1,2; Expr 3 matches 3 -> all
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, NegatedORWithAND) {
+  // Negated OR clause combined with AND clause
+  parse("!(uid(device-0) || uid(device-1)), (uid(device-0) && uid(device-0))");
+
+  ASSERT_NE(context, nullptr);
+  // First clause matches 2,3; Second clause matches 0 -> union = 0,2,3
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 3u);
+  check_result<true, 0, 2, 3>(context, result);
+  check_result<false, 1>(context, result);
+}
+
+TEST_F(ParserTest, NegatedANDWithOR) {
+  // Negated AND clause combined with OR clause
+  parse("!(uid(device-0) && uid(device-0)), (uid(device-0) || uid(device-1))");
+
+  ASSERT_NE(context, nullptr);
+  // First clause matches 1,2,3; Second clause matches 0,1 -> all
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, ComplexMixedOperators) {
+  // Complex mix: OR, AND, negated OR, literal
+  parse("(uid(device-0) || uid(device-1)), (uid(device-2) && uid(device-2)), "
+        "!(uid(device-0) || uid(device-1) || uid(device-2)), 0");
+
+  ASSERT_NE(context, nullptr);
+  // Expr 1: 0,1; Expr 2: 2; Expr 3: NOT(0,1,2) = 3; Expr 4: 0
+  // Union = all
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, ANDNeverMatchesWithOR) {
+  // AND that never matches combined with OR that does
+  parse("(uid(device-0) && uid(device-1)), (uid(device-2) || uid(device-3))");
+
+  ASSERT_NE(context, nullptr);
+  // First clause: never matches (different UIDs); Second: 2,3
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 2u);
+  check_result<false, 0, 1>(context, result);
+  check_result<true, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, ORNeverMatchesWithAND) {
+  // OR with non-existent UIDs combined with AND that matches
+  parse("(uid(nonexistent-a) || uid(nonexistent-b)), (uid(device-0) && "
+        "uid(device-0))");
+
+  ASSERT_NE(context, nullptr);
+  // First clause: no match; Second: 0
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 0>(context, result);
+  check_result<false, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, ThreeWayORAndThreeWayAND) {
+  // Three-way OR and three-way AND in separate clauses
+  parse("(uid(device-0) || uid(device-1) || uid(device-2)), (uid(device-3) && "
+        "uid(device-3) && uid(device-3))");
+
+  ASSERT_NE(context, nullptr);
+  // First: 0,1,2; Second: 3 -> all
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, NegatedMixedExprs) {
+  // Both clauses negated with different operators
+  parse("!(uid(device-0) || uid(device-1)), !(uid(device-2) && uid(device-2))");
+
+  ASSERT_NE(context, nullptr);
+  // First: NOT(0,1) = 2,3; Second: NOT(2) = 0,1,3
+  // Union = all
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, LiteralsWithMixedOperatorExprs) {
+  // Literals combined with both OR and AND clauses
+  parse("0, (uid(device-1) || uid(device-2)), 3, (uid(device-0) && "
+        "uid(device-0))");
+
+  ASSERT_NE(context, nullptr);
+  // Literals: 0,3; OR clause: 1,2; AND clause: 0
+  // Union = all
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+}
+
+//===----------------------------------------------------------------------===//
+// Nested Mixed Operators (|| and && at different nesting levels)
+//===----------------------------------------------------------------------===//
+
+TEST_F(ParserTest, ORContainingANDGroup) {
+  // Outer OR with inner AND group: (A || (B && C))
+  // For (B && C) to match, both B and C must match same device
+  parse("(uid(device-0) || (uid(device-1) && uid(device-1)))");
+
+  ASSERT_NE(context, nullptr);
+  // device-0 matches via first operand
+  // device-1 matches via (device-1 && device-1)
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 2u);
+  check_result<true, 0, 1>(context, result);
+  check_result<false, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, ANDContainingORGroup) {
+  // Outer AND with inner OR group: (A && (B || C))
+  // Both the trait A and the group (B || C) must match
+  // Since A is uid(device-0), only device-0 can satisfy A
+  // (B || C) must also match device-0 for AND to succeed
+  parse("(uid(device-0) && (uid(device-0) || uid(device-1)))");
+
+  ASSERT_NE(context, nullptr);
+  // device-0: uid(device-0) matches AND (uid(device-0) || uid(device-1))
+  // matches -> true device-1: uid(device-0) doesn't match -> false
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 0>(context, result);
+  check_result<false, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, ORWithTwoANDGroups) {
+  // ((A && B) || (C && D)) - OR of two AND groups
+  parse(
+      "((uid(device-0) && uid(device-0)) || (uid(device-2) && uid(device-2)))");
+
+  ASSERT_NE(context, nullptr);
+  // First AND matches device-0; Second AND matches device-2
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 2u);
+  check_result<true, 0, 2>(context, result);
+  check_result<false, 1, 3>(context, result);
+}
+
+TEST_F(ParserTest, ANDWithTwoORGroups) {
+  // ((A || B) && (C || D)) - AND of two OR groups
+  // For a device to match: must match (A || B) AND must match (C || D)
+  parse(
+      "((uid(device-0) || uid(device-1)) && (uid(device-0) || uid(device-2)))");
+
+  ASSERT_NE(context, nullptr);
+  // device-0: matches (0||1) AND matches (0||2) -> true
+  // device-1: matches (0||1) but NOT (0||2) -> false
+  // device-2: NOT (0||1) -> false
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 0>(context, result);
+  check_result<false, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, ORWithNestedANDContainingOR) {
+  // (A || (B && (C || D))) - three levels of nesting
+  parse(
+      "(uid(device-3) || (uid(device-0) && (uid(device-0) || uid(device-1))))");
+
+  ASSERT_NE(context, nullptr);
+  // device-0: inner (0||1) matches, uid(device-0) matches -> AND matches; OR
+  // satisfied device-1: inner (0||1) matches, but uid(device-0) doesn't -> AND
+  // fails; outer uid(device-3) fails device-3: outer uid(device-3) matches
+  // directly
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 2u);
+  check_result<true, 0, 3>(context, result);
+  check_result<false, 1, 2>(context, result);
+}
+
+TEST_F(ParserTest, ANDWithNestedORContainingAND) {
+  // (A && (B || (C && D))) - three levels of nesting
+  parse(
+      "(uid(device-0) && (uid(device-0) || (uid(device-1) && uid(device-1))))");
+
+  ASSERT_NE(context, nullptr);
+  // device-0: uid(device-0) matches; inner (uid(device-0) || ...) matches ->
+  // AND satisfied device-1: uid(device-0) doesn't match -> AND fails
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 0>(context, result);
+  check_result<false, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, NegatedORContainingAND) {
+  // !(A || (B && C)) - negated complex expression
+  parse("!(uid(device-0) || (uid(device-1) && uid(device-1)))");
+
+  ASSERT_NE(context, nullptr);
+  // Without negation: matches 0, 1
+  // With negation: matches 2, 3
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 2u);
+  check_result<false, 0, 1>(context, result);
+  check_result<true, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, NegatedANDContainingOR) {
+  // !(A && (B || C)) - negated complex expression
+  parse("!(uid(device-0) && (uid(device-0) || uid(device-1)))");
+
+  ASSERT_NE(context, nullptr);
+  // Without negation: matches only 0
+  // With negation: matches 1, 2, 3
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 3u);
+  check_result<false, 0>(context, result);
+  check_result<true, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, ComplexNestedWithAllDevices) {
+  // ((A || B) && (C || D)) where union covers all but AND restricts
+  parse(
+      "((uid(device-0) || uid(device-1)) && (uid(device-1) || uid(device-2)))");
+
+  ASSERT_NE(context, nullptr);
+  // device-0: (0||1)=true, (1||2)=false -> AND=false
+  // device-1: (0||1)=true, (1||2)=true -> AND=true
+  // device-2: (0||1)=false -> AND=false
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 1>(context, result);
+  check_result<false, 0, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, TripleNestedMixedOperators) {
+  // (((A || B) && C) || D) - deeply nested with alternating operators
+  parse(
+      "(((uid(device-0) || uid(device-1)) && uid(device-0)) || uid(device-3))");
+
+  ASSERT_NE(context, nullptr);
+  // Inner (0||1): matches 0, 1
+  // Middle ((0||1) && 0): matches only 0
+  // Outer (... || 3): matches 0, 3
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 2u);
+  check_result<true, 0, 3>(context, result);
+  check_result<false, 1, 2>(context, result);
+}
+
+TEST_F(ParserTest, ANDChainWithNestedOR) {
+  // (A && (B || C) && D) - wait, this mixes operators at same level
+  // Actually: ((A && (B || C)) is valid - let's do that
+  // Let's do: (uid(device-0) && (uid(device-0) || uid(device-1)) &&
+  // uid(device-0)) This is three-way AND where middle operand is an OR group
+  parse("(uid(device-0) && (uid(device-0) || uid(device-1)) && uid(device-0))");
+
+  ASSERT_NE(context, nullptr);
+  // All three must match: uid(device-0), (0||1), uid(device-0)
+  // Only device-0 satisfies all
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 0>(context, result);
+  check_result<false, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, ORChainWithNestedAND) {
+  // (A || (B && C) || D) - three-way OR where middle is AND group
+  parse("(uid(device-0) || (uid(device-1) && uid(device-1)) || uid(device-3))");
+
+  ASSERT_NE(context, nullptr);
+  // Any of: device-0, (device-1 && device-1), device-3
+  // Matches: 0, 1, 3
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 3u);
+  check_result<true, 0, 1, 3>(context, result);
+  check_result<false, 2>(context, result);
+}
+
+TEST_F(ParserTest, NestedMixedWithSpaces) {
+  // Nested mixed operators with lots of whitespace
+  parse("(  uid(device-0)  ||  ( uid(device-1)  &&  uid(device-1) )  ||  "
+        "uid(device-2)  )");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 3u);
+  check_result<true, 0, 1, 2>(context, result);
+  check_result<false, 3>(context, result);
+}
+
+//===----------------------------------------------------------------------===//
+// Empty Input
+//===----------------------------------------------------------------------===//
+
+TEST_F(ParserTest, EmptyString) {
+  parse("");
+
+  ASSERT_NE(context, nullptr);
+  // Empty context matches nothing
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 0u);
+  check_result<false, 0, 1>(context, result);
+}
+
+TEST_F(ParserTest, OnlyWhitespace) {
+  parse("   ");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 0u);
+  check_result<false, 0>(context, result);
+}
+
+//===----------------------------------------------------------------------===//
+// Error Cases
+//===----------------------------------------------------------------------===//
+
+TEST_F(ParserTest, OnlyComma) {
+  ASSERT_DEATH(
+      parse(",", "test_only_comma"),
+      "OMP: Error #[0-9]+: trait parser while parsing test_only_comma: "
+      "failed to parse trait specification \\(,\\)");
+}
+
+TEST_F(ParserTest, OnlyCommaNullDbgName) {
+  ASSERT_DEATH(parse(","),
+               "OMP: Error #[0-9]+: trait parser while parsing \\(null\\): "
+               "failed to parse trait specification \\(,\\)");
+}
+
+TEST_F(ParserTest, MixedAndOrSameLevel) {
+  // OpenMP 6.0 explicitly excludes "&&" and "||" from appearing in the same
+  // grouping level.
+  ASSERT_DEATH(
+      parse("uid(a) && uid(b) || uid(c)", "mixed_and_or_same_level"),
+      "OMP: Error #[0-9]+: trait parser while parsing mixed_and_or_same_level: "
+      "failed to parse trait specification "
+      "\\(\\|\\| uid\\(c\\)\\)");
+}
+
+TEST_F(ParserTest, MixedOrAndSameLevel) {
+  ASSERT_DEATH(
+      parse("uid(a) || uid(b) && uid(c)", "mixed_or_and_same_level"),
+      "OMP: Error #[0-9]+: trait parser while parsing mixed_or_and_same_level: "
+      "failed to parse trait specification "
+      "\\(&& uid\\(c\\)\\)");
+}
+
+TEST_F(ParserTest, InvalidUID) {
+  // Empty UID is not allowed
+  ASSERT_DEATH(parse("uid()", "invalid_uid"),
+               "OMP: Error #[0-9]+: trait parser while parsing invalid_uid: "
+               "invalid uid \\(\\)");
+}
+
+TEST_F(ParserTest, UnclosedParenthesis) {
+  ASSERT_DEATH(
+      parse("(uid(a)", "unclosed_parenthesis"),
+      "OMP: Error #[0-9]+: trait parser while parsing unclosed_parenthesis: "
+      "failed to parse trait specification \\(\\(uid\\(a\\)\\)");
+}
+
+TEST_F(ParserTest, UnmatchedClosingParenthesis) {
+  ASSERT_DEATH(parse("uid(a))", "unmatched_closing_parenthesis"),
+               "OMP: Error #[0-9]+: trait parser while parsing "
+               "unmatched_closing_parenthesis: "
+               "failed to parse trait specification \\(\\)\\)");
+}
+
+TEST_F(ParserTest, EmptyParentheses) {
+  ASSERT_DEATH(
+      parse("()", "empty_parentheses"),
+      "OMP: Error #[0-9]+: trait parser while parsing empty_parentheses: "
+      "failed to parse trait specification \\(\\(\\)\\)");
+}
+
+TEST_F(ParserTest, TrailingOperator) {
+  ASSERT_DEATH(
+      parse("uid(a) &&", "trailing_operator"),
+      "OMP: Error #[0-9]+: trait parser while parsing trailing_operator: "
+      "failed to parse trait specification \\(uid\\(a\\) &&\\)");
+}
+
+TEST_F(ParserTest, LeadingOperator) {
+  ASSERT_DEATH(
+      parse("&& uid(a)", "leading_operator"),
+      "OMP: Error #[0-9]+: trait parser while parsing leading_operator: "
+      "failed to parse trait specification \\(&& uid\\(a\\)\\)");
+}
+
+TEST_F(ParserTest, DoubleComma) {
+  ASSERT_DEATH(parse("uid(a),,uid(b)", "double_comma"),
+               "OMP: Error #[0-9]+: trait parser while parsing double_comma: "
+               "failed to parse trait specification \\(,,uid\\(b\\)\\)");
+}
+
+//===----------------------------------------------------------------------===//
+// parse_single_device Tests
+//===----------------------------------------------------------------------===//
+
+TEST(ParseSingleDeviceTest, ValidSingleDigit) {
+  int result = kmp_trait_context::parse_single_device(kmp_str_ref("5"), 10);
+  EXPECT_EQ(result, 5);
+}
+
+TEST(ParseSingleDeviceTest, ValidMultiDigit) {
+  int result = kmp_trait_context::parse_single_device(kmp_str_ref("123"), 200);
+  EXPECT_EQ(result, 123);
+}
+
+TEST(ParseSingleDeviceTest, Zero) {
+  int result = kmp_trait_context::parse_single_device(kmp_str_ref("0"), 10);
+  EXPECT_EQ(result, 0);
+}
+
+TEST(ParseSingleDeviceTest, AtLimit) {
+  int result = kmp_trait_context::parse_single_device(kmp_str_ref("10"), 10);
+  EXPECT_EQ(result, 10);
+}
+
+TEST(ParseSingleDeviceTest, AboveLimit) {
+  ASSERT_DEATH(kmp_trait_context::parse_single_device(kmp_str_ref("11"), 10,
+                                                      "above_limit"),
+               "OMP: Error #[0-9]+: trait parser while parsing above_limit: "
+               "value 11 above limit \\(10\\)");
+}
+
+TEST(ParseSingleDeviceTest, NonInteger) {
+  ASSERT_DEATH(kmp_trait_context::parse_single_device(kmp_str_ref("abc"), 10,
+                                                      "non_integer"),
+               "OMP: Error #[0-9]+: trait parser while parsing non_integer: "
+               "failed to parse trait specification \\(abc\\)");
+}
+
+TEST(ParseSingleDeviceTest, EmptyString) {
+  ASSERT_DEATH(
+      kmp_trait_context::parse_single_device(kmp_str_ref(""), 10, "empty"),
+      "OMP: Error #[0-9]+: trait parser while parsing empty: "
+      "failed to parse trait specification \\(\\)");
+}
+
+TEST(ParseSingleDeviceTest, LeadingSpaces) {
+  // consume_integer skips leading spaces
+  int result = kmp_trait_context::parse_single_device(kmp_str_ref("  7"), 10);
+  EXPECT_EQ(result, 7);
+}
+
+TEST(ParseSingleDeviceTest, LargeNumber) {
+  int result =
+      kmp_trait_context::parse_single_device(kmp_str_ref("999999"), 1000000);
+  EXPECT_EQ(result, 999999);
+}
+
+} // namespace
diff --git a/openmp/runtime/unittests/Traits/TestOMPTraits.cpp b/openmp/runtime/unittests/Traits/TestOMPTraits.cpp
new file mode 100644
index 0000000000000..9af10db803cc3
--- /dev/null
+++ b/openmp/runtime/unittests/Traits/TestOMPTraits.cpp
@@ -0,0 +1,1132 @@
+//===- TestOMPTraits.cpp - Tests for OMP Trait classes -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "kmp_traits.h"
+#include "gtest/gtest.h"
+
+using namespace kmp_traits;
+
+namespace {
+
+//===----------------------------------------------------------------------===//
+// kmp_wildcard_trait Tests
+//===----------------------------------------------------------------------===//
+
+TEST(kmp_wildcard_trait_test, MatchesAnyDevice) {
+  kmp_wildcard_trait *trait = new kmp_wildcard_trait();
+
+  EXPECT_TRUE(trait->match(0));
+  EXPECT_TRUE(trait->match(1));
+  EXPECT_TRUE(trait->match(100));
+  EXPECT_TRUE(trait->match(-1));
+
+  delete trait;
+}
+
+TEST(kmp_wildcard_trait_test, Equality) {
+  kmp_wildcard_trait *t1 = new kmp_wildcard_trait();
+  kmp_wildcard_trait *t2 = new kmp_wildcard_trait();
+
+  EXPECT_TRUE(*t1 == *t2);
+
+  delete t1;
+  delete t2;
+}
+
+//===----------------------------------------------------------------------===//
+// kmp_literal_trait Tests
+//===----------------------------------------------------------------------===//
+
+TEST(kmp_literal_trait_test, MatchesExactDevice) {
+  kmp_literal_trait *trait = new kmp_literal_trait(5);
+
+  EXPECT_TRUE(trait->match(5));
+  EXPECT_FALSE(trait->match(0));
+  EXPECT_FALSE(trait->match(4));
+  EXPECT_FALSE(trait->match(6));
+
+  delete trait;
+}
+
+TEST(kmp_literal_trait_test, MatchesZero) {
+  kmp_literal_trait *trait = new kmp_literal_trait(0);
+
+  EXPECT_TRUE(trait->match(0));
+  EXPECT_FALSE(trait->match(1));
+
+  delete trait;
+}
+
+#ifndef NDEBUG
+TEST(kmp_literal_trait_test, MatchesNegative) {
+  EXPECT_DEATH(new kmp_literal_trait(-1), "Device number must be non-negative");
+}
+#endif
+
+TEST(kmp_literal_trait_test, EqualitySameValue) {
+  kmp_literal_trait *t1 = new kmp_literal_trait(42);
+  kmp_literal_trait *t2 = new kmp_literal_trait(42);
+
+  EXPECT_TRUE(*t1 == *t2);
+
+  delete t1;
+  delete t2;
+}
+
+TEST(kmp_literal_trait_test, EqualityDifferentValue) {
+  kmp_literal_trait *t1 = new kmp_literal_trait(1);
+  kmp_literal_trait *t2 = new kmp_literal_trait(2);
+
+  EXPECT_FALSE(*t1 == *t2);
+
+  delete t1;
+  delete t2;
+}
+
+//===----------------------------------------------------------------------===//
+// kmp_uid_trait Tests
+//===----------------------------------------------------------------------===//
+
+TEST(kmp_uid_trait_test, Construction) {
+  kmp_uid_trait *trait = new kmp_uid_trait(kmp_str_ref("test-uid"));
+
+  // Just verify it can be constructed without crashing
+  delete trait;
+}
+
+TEST(kmp_uid_trait_test, MatchWithMock) {
+  kmp_uid_trait *trait = new kmp_uid_trait(kmp_str_ref("device-0"));
+
+  // Uses the mock omp_get_uid_from_device
+  EXPECT_TRUE(trait->match(0)); // device-0 matches
+  EXPECT_FALSE(trait->match(1)); // device-1 doesn't match
+  EXPECT_FALSE(trait->match(2)); // device-2 doesn't match
+
+  delete trait;
+}
+
+TEST(kmp_uid_trait_test, MatchWithCustomMock) {
+  kmp_uid_trait *trait = new kmp_uid_trait(kmp_str_ref("custom-uid"));
+
+  // Set a custom mock function
+  trait->set_uid_from_device([](int device) -> const char * {
+    return device == 2 ? "custom-uid" : "other";
+  });
+
+  EXPECT_FALSE(trait->match(0));
+  EXPECT_FALSE(trait->match(1));
+  EXPECT_TRUE(trait->match(2)); // custom-uid matches device 2
+  EXPECT_FALSE(trait->match(3));
+
+  delete trait;
+}
+
+TEST(kmp_uid_trait_test, EqualitySameUID) {
+  kmp_uid_trait *t1 = new kmp_uid_trait(kmp_str_ref("my-device"));
+  kmp_uid_trait *t2 = new kmp_uid_trait(kmp_str_ref("my-device"));
+
+  EXPECT_TRUE(*t1 == *t2);
+
+  delete t1;
+  delete t2;
+}
+
+TEST(kmp_uid_trait_test, EqualityDifferentUID) {
+  kmp_uid_trait *t1 = new kmp_uid_trait(kmp_str_ref("device-a"));
+  kmp_uid_trait *t2 = new kmp_uid_trait(kmp_str_ref("device-b"));
+
+  EXPECT_FALSE(*t1 == *t2);
+
+  delete t1;
+  delete t2;
+}
+
+//===----------------------------------------------------------------------===//
+// kmp_trait_expr_single Tests
+//===----------------------------------------------------------------------===//
+
+TEST(kmp_trait_expr_single_test, CreateAndDestroy) {
+  kmp_trait_expr_single *expr = new kmp_trait_expr_single();
+  EXPECT_NE(expr, nullptr);
+  delete expr;
+}
+
+TEST(kmp_trait_expr_single_test, CreateWithTrait) {
+  kmp_trait_expr_single *expr =
+      new kmp_trait_expr_single(new kmp_literal_trait(2));
+
+  // Mock: 4 devices
+  expr->set_num_devices([]() { return 4; });
+
+  EXPECT_TRUE(expr->match(2));
+  EXPECT_FALSE(expr->match(0));
+  EXPECT_FALSE(expr->match(1));
+  EXPECT_FALSE(expr->match(5)); // Out of range
+
+  delete expr;
+}
+
+TEST(kmp_trait_expr_single_test, SetTrait) {
+  kmp_trait_expr_single *expr = new kmp_trait_expr_single();
+  expr->set_trait(new kmp_literal_trait(3));
+
+  // Mock: 4 devices
+  expr->set_num_devices([]() { return 4; });
+
+  EXPECT_TRUE(expr->match(3));
+  EXPECT_FALSE(expr->match(0));
+
+  delete expr;
+}
+
+TEST(kmp_trait_expr_single_test, DefaultNotNegated) {
+  kmp_trait_expr_single *expr = new kmp_trait_expr_single();
+
+  EXPECT_FALSE(expr->is_negated());
+
+  delete expr;
+}
+
+TEST(kmp_trait_expr_single_test, SetNegated) {
+  kmp_trait_expr_single *expr = new kmp_trait_expr_single();
+
+  expr->set_negated(true);
+  EXPECT_TRUE(expr->is_negated());
+
+  expr->set_negated(false);
+  EXPECT_FALSE(expr->is_negated());
+
+  delete expr;
+}
+
+TEST(kmp_trait_expr_single_test, MatchNegated) {
+  kmp_trait_expr_single *expr =
+      new kmp_trait_expr_single(new kmp_literal_trait(2));
+  expr->set_negated(true);
+
+  // Mock: 4 devices
+  expr->set_num_devices([]() { return 4; });
+
+  // Without negation: matches 2
+  // With negation: matches everything in-range except 2
+  EXPECT_FALSE(expr->match(2));
+  EXPECT_TRUE(expr->match(0));
+  EXPECT_TRUE(expr->match(1));
+  EXPECT_TRUE(expr->match(3));
+  // Out of range devices return false regardless of negation
+  EXPECT_FALSE(expr->match(5));
+
+  delete expr;
+}
+
+TEST(kmp_trait_expr_single_test, MatchWildcard) {
+  kmp_trait_expr_single *expr =
+      new kmp_trait_expr_single(new kmp_wildcard_trait());
+
+  // Mock: 4 devices
+  expr->set_num_devices([]() { return 4; });
+
+  // Wildcard matches any in-range device
+  EXPECT_TRUE(expr->match(0));
+  EXPECT_TRUE(expr->match(3));
+  // Out of range devices return false
+  EXPECT_FALSE(expr->match(100));
+
+  delete expr;
+}
+
+TEST(kmp_trait_expr_single_test, Equality) {
+  kmp_trait_expr_single *e1 =
+      new kmp_trait_expr_single(new kmp_literal_trait(1));
+  kmp_trait_expr_single *e2 =
+      new kmp_trait_expr_single(new kmp_literal_trait(1));
+
+  EXPECT_TRUE(*e1 == *e2);
+
+  delete e1;
+  delete e2;
+}
+
+TEST(kmp_trait_expr_single_test, EqualityDifferentTrait) {
+  kmp_trait_expr_single *e1 =
+      new kmp_trait_expr_single(new kmp_literal_trait(1));
+  kmp_trait_expr_single *e2 =
+      new kmp_trait_expr_single(new kmp_literal_trait(2));
+
+  EXPECT_FALSE(*e1 == *e2);
+
+  delete e1;
+  delete e2;
+}
+
+TEST(kmp_trait_expr_single_test, EqualityDifferentNegation) {
+  kmp_trait_expr_single *e1 =
+      new kmp_trait_expr_single(new kmp_literal_trait(1));
+  kmp_trait_expr_single *e2 =
+      new kmp_trait_expr_single(new kmp_literal_trait(1));
+  e2->set_negated(true);
+
+  EXPECT_FALSE(*e1 == *e2);
+
+  delete e1;
+  delete e2;
+}
+
+//===----------------------------------------------------------------------===//
+// kmp_trait_expr_group Tests
+//===----------------------------------------------------------------------===//
+
+TEST(kmp_trait_expr_group_test, CreateAndDestroy) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+  EXPECT_NE(group, nullptr);
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, DefaultTypeIsOR) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+
+  EXPECT_EQ(group->get_group_type(), kmp_trait_expr_group::OR);
+
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, SetTypeAND) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+
+  group->set_group_type(kmp_trait_expr_group::AND);
+  EXPECT_EQ(group->get_group_type(), kmp_trait_expr_group::AND);
+
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, DefaultNotNegated) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+
+  EXPECT_FALSE(group->is_negated());
+
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, SetNegated) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+
+  group->set_negated(true);
+  EXPECT_TRUE(group->is_negated());
+
+  group->set_negated(false);
+  EXPECT_FALSE(group->is_negated());
+
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, AddTraitDirectly) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+
+  group->add_expr(new kmp_wildcard_trait());
+
+  // Mock: 4 devices
+  group->set_num_devices([]() { return 4; });
+
+  // Wildcard matches any in-range device
+  EXPECT_TRUE(group->match(0));
+  EXPECT_TRUE(group->match(3));
+  // Out of range devices return false
+  EXPECT_FALSE(group->match(100));
+
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, AddExpr) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+
+  group->add_expr(new kmp_trait_expr_single(new kmp_literal_trait(2)));
+
+  // Mock: 4 devices
+  group->set_num_devices([]() { return 4; });
+
+  EXPECT_TRUE(group->match(2));
+  EXPECT_FALSE(group->match(0));
+  EXPECT_FALSE(group->match(5)); // Out of range
+
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, MatchORSemantics) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+  group->set_group_type(kmp_trait_expr_group::OR);
+
+  group->add_expr(new kmp_literal_trait(1));
+  group->add_expr(new kmp_literal_trait(2));
+  group->add_expr(new kmp_literal_trait(3));
+
+  // Mock: 5 devices
+  group->set_num_devices([]() { return 5; });
+
+  // OR: matches if ANY trait matches
+  EXPECT_TRUE(group->match(1));
+  EXPECT_TRUE(group->match(2));
+  EXPECT_TRUE(group->match(3));
+  EXPECT_FALSE(group->match(0));
+  EXPECT_FALSE(group->match(4));
+
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, MatchANDSemantics) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+  group->set_group_type(kmp_trait_expr_group::AND);
+
+  // For AND to pass, ALL traits must match the same device
+  // A single literal only matches one device
+  group->add_expr(new kmp_literal_trait(2));
+
+  // Mock: 4 devices
+  group->set_num_devices([]() { return 4; });
+
+  EXPECT_TRUE(group->match(2));
+  EXPECT_FALSE(group->match(0));
+  // Out of range
+  EXPECT_FALSE(group->match(5));
+
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, MatchANDWithWildcard) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+  group->set_group_type(kmp_trait_expr_group::AND);
+
+  group->add_expr(new kmp_wildcard_trait());
+  group->add_expr(new kmp_literal_trait(2));
+
+  // Mock: 4 devices
+  group->set_num_devices([]() { return 4; });
+
+  // Wildcard matches all, literal matches 2
+  // AND: both must match
+  EXPECT_TRUE(group->match(2));
+  EXPECT_FALSE(group->match(0));
+  // Out of range
+  EXPECT_FALSE(group->match(5));
+
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, MatchNegated) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+
+  group->add_expr(new kmp_literal_trait(2));
+  group->set_negated(true);
+
+  // Mock: 4 devices
+  group->set_num_devices([]() { return 4; });
+
+  // Without negation: matches 2
+  // With negation: matches everything in-range except 2
+  EXPECT_FALSE(group->match(2));
+  EXPECT_TRUE(group->match(0));
+  EXPECT_TRUE(group->match(1));
+  EXPECT_TRUE(group->match(3));
+  // Out of range devices return false regardless of negation
+  EXPECT_FALSE(group->match(5));
+
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, MatchEmptyGroupOR) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+  group->set_group_type(kmp_trait_expr_group::OR);
+
+  // Mock: 4 devices
+  group->set_num_devices([]() { return 4; });
+
+  // Empty OR: no traits match, so result is false
+  EXPECT_FALSE(group->match(0));
+  EXPECT_FALSE(group->match(1));
+
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, MatchEmptyGroupAND) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+  group->set_group_type(kmp_trait_expr_group::AND);
+
+  // Mock: 4 devices
+  group->set_num_devices([]() { return 4; });
+
+  // Empty AND: vacuously true (0 out of 0 traits match)
+  EXPECT_TRUE(group->match(0));
+  EXPECT_TRUE(group->match(1));
+
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, Equality) {
+  kmp_trait_expr_group *g1 = new kmp_trait_expr_group();
+  kmp_trait_expr_group *g2 = new kmp_trait_expr_group();
+
+  g1->add_expr(new kmp_literal_trait(1));
+  g2->add_expr(new kmp_literal_trait(1));
+
+  EXPECT_TRUE(*g1 == *g2);
+
+  delete g1;
+  delete g2;
+}
+
+TEST(kmp_trait_expr_group_test, EqualityDifferentNegation) {
+  kmp_trait_expr_group *g1 = new kmp_trait_expr_group();
+  kmp_trait_expr_group *g2 = new kmp_trait_expr_group();
+
+  g1->add_expr(new kmp_literal_trait(1));
+  g2->add_expr(new kmp_literal_trait(1));
+  g2->set_negated(true);
+
+  EXPECT_FALSE(*g1 == *g2);
+
+  delete g1;
+  delete g2;
+}
+
+TEST(kmp_trait_expr_group_test, NestedGroups) {
+  kmp_trait_expr_group *outer = new kmp_trait_expr_group();
+  outer->set_group_type(kmp_trait_expr_group::OR);
+
+  kmp_trait_expr_group *inner = new kmp_trait_expr_group();
+  inner->set_group_type(kmp_trait_expr_group::AND);
+  inner->add_expr(new kmp_literal_trait(1));
+  inner->add_expr(new kmp_wildcard_trait());
+
+  outer->add_expr(inner);
+  outer->add_expr(new kmp_literal_trait(2));
+
+  // Mock: 4 devices
+  outer->set_num_devices([]() { return 4; });
+
+  // Inner matches device 1 (literal 1 AND wildcard)
+  // Outer matches 1 OR 2
+  EXPECT_TRUE(outer->match(1));
+  EXPECT_TRUE(outer->match(2));
+  EXPECT_FALSE(outer->match(0));
+  EXPECT_FALSE(outer->match(3));
+
+  delete outer;
+}
+
+//===----------------------------------------------------------------------===//
+// kmp_trait_clause Tests
+//===----------------------------------------------------------------------===//
+
+TEST(kmp_trait_clause_test, CreateAndDestroy) {
+  kmp_trait_clause *clause = new kmp_trait_clause();
+  EXPECT_NE(clause, nullptr);
+  delete clause;
+}
+
+TEST(kmp_trait_clause_test, SetExprWithTrait) {
+  kmp_trait_clause *clause = new kmp_trait_clause();
+  clause->set_expr(new kmp_literal_trait(2));
+
+  // The trait is wrapped in kmp_trait_expr_single internally
+  kmp_trait_expr *expr = clause->get_expr();
+  EXPECT_NE(expr, nullptr);
+
+  delete clause;
+}
+
+TEST(kmp_trait_clause_test, SetExprWithExpr) {
+  kmp_trait_clause *clause = new kmp_trait_clause();
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+  group->add_expr(new kmp_literal_trait(1));
+  clause->set_expr(group);
+
+  EXPECT_EQ(clause->get_expr(), group);
+
+  delete clause;
+}
+
+TEST(kmp_trait_clause_test, Equality) {
+  kmp_trait_clause *c1 = new kmp_trait_clause();
+  kmp_trait_clause *c2 = new kmp_trait_clause();
+
+  c1->set_expr(new kmp_literal_trait(1));
+  c2->set_expr(new kmp_literal_trait(1));
+
+  EXPECT_TRUE(*c1 == *c2);
+
+  delete c1;
+  delete c2;
+}
+
+TEST(kmp_trait_clause_test, EqualityDifferentExprs) {
+  kmp_trait_clause *c1 = new kmp_trait_clause();
+  kmp_trait_clause *c2 = new kmp_trait_clause();
+
+  c1->set_expr(new kmp_literal_trait(1));
+  c2->set_expr(new kmp_literal_trait(2));
+
+  EXPECT_FALSE(*c1 == *c2);
+
+  delete c1;
+  delete c2;
+}
+
+//===----------------------------------------------------------------------===//
+// kmp_trait_context Tests
+//===----------------------------------------------------------------------===//
+
+TEST(kmp_trait_context_test, CreateAndDestroy) {
+  kmp_trait_context *context = new kmp_trait_context();
+  EXPECT_NE(context, nullptr);
+  delete context;
+}
+
+TEST(kmp_trait_context_test, AddClause) {
+  kmp_trait_context *context = new kmp_trait_context();
+  kmp_trait_clause *clause = new kmp_trait_clause();
+  clause->set_expr(new kmp_literal_trait(2));
+  context->add_clause(clause);
+
+  // Mock: 4 devices
+  context->set_num_devices([]() { return 4; });
+
+  EXPECT_TRUE(context->match(2));
+  EXPECT_FALSE(context->match(0));
+  // Out of range
+  EXPECT_FALSE(context->match(5));
+
+  delete context;
+}
+
+TEST(kmp_trait_context_test, MultipleClauses) {
+  kmp_trait_context *context = new kmp_trait_context();
+
+  kmp_trait_clause *c1 = new kmp_trait_clause();
+  c1->set_expr(new kmp_literal_trait(1));
+  context->add_clause(c1);
+
+  kmp_trait_clause *c2 = new kmp_trait_clause();
+  c2->set_expr(new kmp_literal_trait(2));
+  context->add_clause(c2);
+
+  kmp_trait_clause *c3 = new kmp_trait_clause();
+  c3->set_expr(new kmp_literal_trait(3));
+  context->add_clause(c3);
+
+  // Mock: 5 devices
+  context->set_num_devices([]() { return 5; });
+
+  // Context uses OR semantics between clauses
+  EXPECT_TRUE(context->match(1));
+  EXPECT_TRUE(context->match(2));
+  EXPECT_TRUE(context->match(3));
+  EXPECT_FALSE(context->match(0));
+  EXPECT_FALSE(context->match(4));
+
+  delete context;
+}
+
+TEST(kmp_trait_context_test, EmptyContextMatchesNothing) {
+  kmp_trait_context *context = new kmp_trait_context();
+
+  // Mock: 4 devices
+  context->set_num_devices([]() { return 4; });
+
+  EXPECT_FALSE(context->match(0));
+  EXPECT_FALSE(context->match(1));
+
+  delete context;
+}
+
+TEST(kmp_trait_context_test, WildcardClause) {
+  kmp_trait_context *context = new kmp_trait_context();
+  kmp_trait_clause *clause = new kmp_trait_clause();
+  clause->set_expr(new kmp_wildcard_trait());
+  context->add_clause(clause);
+
+  // Mock: 4 devices
+  context->set_num_devices([]() { return 4; });
+
+  // In-range devices match
+  EXPECT_TRUE(context->match(0));
+  EXPECT_TRUE(context->match(3));
+  // Out of range devices return false
+  EXPECT_FALSE(context->match(100));
+  EXPECT_FALSE(context->match(-1));
+
+  delete context;
+}
+
+TEST(kmp_trait_context_test, EvaluateWithMock) {
+  kmp_trait_context *context = new kmp_trait_context();
+
+  // Mock: 5 devices
+  context->set_num_devices([]() { return 5; });
+
+  kmp_trait_clause *c1 = new kmp_trait_clause();
+  c1->set_expr(new kmp_literal_trait(1));
+  context->add_clause(c1);
+
+  kmp_trait_clause *c2 = new kmp_trait_clause();
+  c2->set_expr(new kmp_literal_trait(3));
+  context->add_clause(c2);
+
+  kmp_vector<int> result = context->evaluate();
+  EXPECT_EQ(result.size(), 2u);
+  EXPECT_TRUE(result.contains(1));
+  EXPECT_TRUE(result.contains(3));
+  EXPECT_FALSE(result.contains(0));
+  EXPECT_FALSE(result.contains(2));
+  EXPECT_FALSE(result.contains(4));
+
+  delete context;
+}
+
+TEST(kmp_trait_context_test, Equality) {
+  kmp_trait_context *ctx1 = new kmp_trait_context();
+  kmp_trait_context *ctx2 = new kmp_trait_context();
+
+  kmp_trait_clause *c1 = new kmp_trait_clause();
+  c1->set_expr(new kmp_literal_trait(1));
+  ctx1->add_clause(c1);
+
+  kmp_trait_clause *c2 = new kmp_trait_clause();
+  c2->set_expr(new kmp_literal_trait(1));
+  ctx2->add_clause(c2);
+
+  EXPECT_TRUE(*ctx1 == *ctx2);
+
+  delete ctx1;
+  delete ctx2;
+}
+
+TEST(kmp_trait_context_test, EqualityDifferentClauses) {
+  kmp_trait_context *ctx1 = new kmp_trait_context();
+  kmp_trait_context *ctx2 = new kmp_trait_context();
+
+  kmp_trait_clause *c1 = new kmp_trait_clause();
+  c1->set_expr(new kmp_literal_trait(1));
+  ctx1->add_clause(c1);
+
+  kmp_trait_clause *c2 = new kmp_trait_clause();
+  c2->set_expr(new kmp_literal_trait(2));
+  ctx2->add_clause(c2);
+
+  EXPECT_FALSE(*ctx1 == *ctx2);
+
+  delete ctx1;
+  delete ctx2;
+}
+
+//===----------------------------------------------------------------------===//
+// kmp_trait_context Iterator Tests
+//===----------------------------------------------------------------------===//
+
+TEST(kmp_trait_context_test, IteratorRangeBasedFor) {
+  kmp_trait_context *context = new kmp_trait_context();
+
+  // Mock: 5 devices
+  context->set_num_devices([]() { return 5; });
+
+  kmp_trait_clause *c1 = new kmp_trait_clause();
+  c1->set_expr(new kmp_literal_trait(1));
+  context->add_clause(c1);
+
+  kmp_trait_clause *c2 = new kmp_trait_clause();
+  c2->set_expr(new kmp_literal_trait(3));
+  context->add_clause(c2);
+
+  // Use range-based for loop (should auto-evaluate)
+  kmp_vector<int> collected;
+  for (int d : *context) {
+    collected.push_back(d);
+  }
+
+  EXPECT_EQ(collected.size(), 2u);
+  EXPECT_TRUE(collected.contains(1));
+  EXPECT_TRUE(collected.contains(3));
+
+  delete context;
+}
+
+TEST(kmp_trait_context_test, IteratorAutoEvaluates) {
+  kmp_trait_context *context = new kmp_trait_context();
+
+  // Mock: 4 devices
+  context->set_num_devices([]() { return 4; });
+
+  kmp_trait_clause *clause = new kmp_trait_clause();
+  clause->set_expr(new kmp_wildcard_trait());
+  context->add_clause(clause);
+
+  // Directly use begin()/end() without calling evaluate() first
+  int count = 0;
+  for (const int *it = context->begin(); it != context->end(); ++it) {
+    EXPECT_GE(*it, 0);
+    EXPECT_LT(*it, 4);
+    count++;
+  }
+
+  EXPECT_EQ(count, 4);
+
+  delete context;
+}
+
+TEST(kmp_trait_context_test, IteratorEmptyContext) {
+  kmp_trait_context *context = new kmp_trait_context();
+
+  // Mock: 4 devices
+  context->set_num_devices([]() { return 4; });
+
+  // Empty context - no clauses added
+  int count = 0;
+  for (int d : *context) {
+    (void)d;
+    count++;
+  }
+
+  EXPECT_EQ(count, 0);
+  EXPECT_EQ(context->begin(), context->end());
+
+  delete context;
+}
+
+TEST(kmp_trait_context_test, IteratorBeginEnd) {
+  kmp_trait_context *context = new kmp_trait_context();
+
+  // Mock: 3 devices
+  context->set_num_devices([]() { return 3; });
+
+  kmp_trait_clause *clause = new kmp_trait_clause();
+  clause->set_expr(new kmp_literal_trait(2));
+  context->add_clause(clause);
+
+  // Test begin/end directly
+  const int *b = context->begin();
+  const int *e = context->end();
+
+  EXPECT_EQ(e - b, 1); // Should have exactly 1 element
+  EXPECT_EQ(*b, 2);
+
+  delete context;
+}
+
+TEST(kmp_trait_context_test, IteratorMultipleDevices) {
+  kmp_trait_context *context = new kmp_trait_context();
+
+  // Add clauses for devices 0, 2, 4
+  for (int i = 0; i < 6; i += 2) {
+    kmp_trait_clause *clause = new kmp_trait_clause();
+    clause->set_expr(new kmp_literal_trait(i));
+    context->add_clause(clause);
+  }
+
+  // Mock: 6 devices (must be set after adding clauses to propagate to them)
+  context->set_num_devices([]() { return 6; });
+
+  // Collect via iterator
+  kmp_vector<int> collected;
+  for (int d : *context) {
+    collected.push_back(d);
+  }
+
+  EXPECT_EQ(collected.size(), 3u);
+  EXPECT_TRUE(collected.contains(0));
+  EXPECT_TRUE(collected.contains(2));
+  EXPECT_TRUE(collected.contains(4));
+
+  delete context;
+}
+
+TEST(kmp_trait_context_test, IteratorConsistentWithEvaluate) {
+  kmp_trait_context *context = new kmp_trait_context();
+
+  // Mock: 5 devices
+  context->set_num_devices([]() { return 5; });
+
+  kmp_trait_clause *c1 = new kmp_trait_clause();
+  c1->set_expr(new kmp_literal_trait(1));
+  context->add_clause(c1);
+
+  kmp_trait_clause *c2 = new kmp_trait_clause();
+  c2->set_expr(new kmp_literal_trait(4));
+  context->add_clause(c2);
+
+  // Get result via evaluate()
+  const kmp_vector<int> &eval_result = context->evaluate();
+
+  // Collect via iterator
+  kmp_vector<int> iter_result;
+  for (int d : *context) {
+    iter_result.push_back(d);
+  }
+
+  // Both should give the same results
+  EXPECT_EQ(eval_result.size(), iter_result.size());
+  for (size_t i = 0; i < eval_result.size(); i++) {
+    EXPECT_EQ(eval_result[i], iter_result[i]);
+  }
+
+  delete context;
+}
+
+TEST(kmp_trait_context_test, EvaluateReturnsByReference) {
+  kmp_trait_context *context = new kmp_trait_context();
+
+  // Mock: 3 devices
+  context->set_num_devices([]() { return 3; });
+
+  kmp_trait_clause *clause = new kmp_trait_clause();
+  clause->set_expr(new kmp_wildcard_trait());
+  context->add_clause(clause);
+
+  // Multiple calls to evaluate() should return reference to the same data
+  const kmp_vector<int> &result1 = context->evaluate();
+  const kmp_vector<int> &result2 = context->evaluate();
+
+  EXPECT_EQ(&result1, &result2);
+
+  delete context;
+}
+
+//===----------------------------------------------------------------------===//
+// get_num_devices Propagation Tests
+//===----------------------------------------------------------------------===//
+
+TEST(kmp_trait_context_test, PropagationToClausesAddedAfterSetNumDevices) {
+  kmp_trait_context *context = new kmp_trait_context();
+
+  // Set mock BEFORE adding clauses - propagation should still work
+  context->set_num_devices([]() { return 6; });
+
+  // Add clauses for devices 0, 2, 4 (all require 6 devices to be in range)
+  for (int i = 0; i < 6; i += 2) {
+    kmp_trait_clause *clause = new kmp_trait_clause();
+    clause->set_expr(new kmp_literal_trait(i));
+    context->add_clause(clause);
+  }
+
+  // All three devices should match because propagation worked
+  kmp_vector<int> collected;
+  for (int d : *context) {
+    collected.push_back(d);
+  }
+
+  EXPECT_EQ(collected.size(), 3u);
+  EXPECT_TRUE(collected.contains(0));
+  EXPECT_TRUE(collected.contains(2));
+  EXPECT_TRUE(collected.contains(4));
+
+  delete context;
+}
+
+TEST(kmp_trait_context_test, PropagationToClausesAddedBeforeSetNumDevices) {
+  kmp_trait_context *context = new kmp_trait_context();
+
+  // Add clauses BEFORE setting mock
+  for (int i = 0; i < 6; i += 2) {
+    kmp_trait_clause *clause = new kmp_trait_clause();
+    clause->set_expr(new kmp_literal_trait(i));
+    context->add_clause(clause);
+  }
+
+  // Set mock AFTER adding clauses
+  context->set_num_devices([]() { return 6; });
+
+  // All three devices should match
+  kmp_vector<int> collected;
+  for (int d : *context) {
+    collected.push_back(d);
+  }
+
+  EXPECT_EQ(collected.size(), 3u);
+  EXPECT_TRUE(collected.contains(0));
+  EXPECT_TRUE(collected.contains(2));
+  EXPECT_TRUE(collected.contains(4));
+
+  delete context;
+}
+
+TEST(kmp_trait_expr_group_test, PropagationToExprsAddedAfterSetNumDevices) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+
+  // Set mock BEFORE adding expressions
+  group->set_num_devices([]() { return 8; });
+
+  // Add expressions for devices 5, 6, 7 (require 8 devices)
+  group->add_expr(new kmp_literal_trait(5));
+  group->add_expr(new kmp_literal_trait(6));
+  group->add_expr(new kmp_literal_trait(7));
+
+  // All should match
+  EXPECT_TRUE(group->match(5));
+  EXPECT_TRUE(group->match(6));
+  EXPECT_TRUE(group->match(7));
+  EXPECT_FALSE(group->match(4));
+
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, PropagationToExprsAddedBeforeSetNumDevices) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+
+  // Add expressions BEFORE setting mock
+  group->add_expr(new kmp_literal_trait(5));
+  group->add_expr(new kmp_literal_trait(6));
+  group->add_expr(new kmp_literal_trait(7));
+
+  // Set mock AFTER adding expressions
+  group->set_num_devices([]() { return 8; });
+
+  // All should match
+  EXPECT_TRUE(group->match(5));
+  EXPECT_TRUE(group->match(6));
+  EXPECT_TRUE(group->match(7));
+  EXPECT_FALSE(group->match(4));
+
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, PropagationToNestedGroups) {
+  kmp_trait_expr_group *outer = new kmp_trait_expr_group();
+  outer->set_group_type(kmp_trait_expr_group::OR);
+
+  // Set mock on outer group FIRST
+  outer->set_num_devices([]() { return 10; });
+
+  // Create inner group and add high-numbered devices
+  kmp_trait_expr_group *inner = new kmp_trait_expr_group();
+  inner->set_group_type(kmp_trait_expr_group::OR);
+  inner->add_expr(new kmp_literal_trait(8));
+  inner->add_expr(new kmp_literal_trait(9));
+
+  // Add inner to outer - should propagate mock to inner and its children
+  outer->add_expr(inner);
+
+  // Add another expression directly to outer
+  outer->add_expr(new kmp_literal_trait(7));
+
+  // All should match with 10 devices
+  EXPECT_TRUE(outer->match(7));
+  EXPECT_TRUE(outer->match(8));
+  EXPECT_TRUE(outer->match(9));
+  EXPECT_FALSE(outer->match(10)); // Out of range
+
+  delete outer;
+}
+
+TEST(kmp_trait_expr_group_test, PropagationToDeeplyNestedGroups) {
+  // Create a deeply nested structure: outer -> middle -> inner
+  kmp_trait_expr_group *outer = new kmp_trait_expr_group();
+  outer->set_group_type(kmp_trait_expr_group::OR);
+
+  // Set mock on outer
+  outer->set_num_devices([]() { return 12; });
+
+  kmp_trait_expr_group *middle = new kmp_trait_expr_group();
+  middle->set_group_type(kmp_trait_expr_group::OR);
+
+  kmp_trait_expr_group *inner = new kmp_trait_expr_group();
+  inner->set_group_type(kmp_trait_expr_group::OR);
+  inner->add_expr(new kmp_literal_trait(10));
+  inner->add_expr(new kmp_literal_trait(11));
+
+  middle->add_expr(inner);
+  middle->add_expr(new kmp_literal_trait(9));
+
+  outer->add_expr(middle);
+  outer->add_expr(new kmp_literal_trait(8));
+
+  // All devices 8-11 should match (requires 12 devices)
+  EXPECT_TRUE(outer->match(8));
+  EXPECT_TRUE(outer->match(9));
+  EXPECT_TRUE(outer->match(10));
+  EXPECT_TRUE(outer->match(11));
+  EXPECT_FALSE(outer->match(12)); // Out of range
+
+  delete outer;
+}
+
+TEST(kmp_trait_context_test, PropagationToNestedGroupsInClauses) {
+  kmp_trait_context *context = new kmp_trait_context();
+
+  // Set mock on context FIRST
+  context->set_num_devices([]() { return 10; });
+
+  // Create a group with nested structure
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+  group->set_group_type(kmp_trait_expr_group::OR);
+
+  kmp_trait_expr_group *inner = new kmp_trait_expr_group();
+  inner->set_group_type(kmp_trait_expr_group::OR);
+  inner->add_expr(new kmp_literal_trait(8));
+  inner->add_expr(new kmp_literal_trait(9));
+
+  group->add_expr(inner);
+  group->add_expr(new kmp_literal_trait(7));
+
+  // Create clause with the group
+  kmp_trait_clause *clause = new kmp_trait_clause();
+  clause->set_expr(group);
+
+  // Add clause to context - should propagate to group and inner
+  context->add_clause(clause);
+
+  // All should match
+  kmp_vector<int> collected;
+  for (int d : *context) {
+    collected.push_back(d);
+  }
+
+  EXPECT_EQ(collected.size(), 3u);
+  EXPECT_TRUE(collected.contains(7));
+  EXPECT_TRUE(collected.contains(8));
+  EXPECT_TRUE(collected.contains(9));
+
+  delete context;
+}
+
+TEST(kmp_trait_context_test, PropagationMixedOrder) {
+  // Test a complex scenario with mixed ordering
+  kmp_trait_context *context = new kmp_trait_context();
+
+  // Add first clause before set_num_devices
+  kmp_trait_clause *c1 = new kmp_trait_clause();
+  c1->set_expr(new kmp_literal_trait(5));
+  context->add_clause(c1);
+
+  // Set mock
+  context->set_num_devices([]() { return 8; });
+
+  // Add second clause after set_num_devices
+  kmp_trait_clause *c2 = new kmp_trait_clause();
+  c2->set_expr(new kmp_literal_trait(6));
+  context->add_clause(c2);
+
+  // Add third clause with a group
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+  group->add_expr(new kmp_literal_trait(7));
+
+  kmp_trait_clause *c3 = new kmp_trait_clause();
+  c3->set_expr(group);
+  context->add_clause(c3);
+
+  // All three should match
+  kmp_vector<int> collected;
+  for (int d : *context) {
+    collected.push_back(d);
+  }
+
+  EXPECT_EQ(collected.size(), 3u);
+  EXPECT_TRUE(collected.contains(5));
+  EXPECT_TRUE(collected.contains(6));
+  EXPECT_TRUE(collected.contains(7));
+
+  delete context;
+}
+
+} // namespace

>From fc74f598ee1bd8c8753c50f2011af80b39cf0838 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Thu, 18 Jun 2026 03:22:19 -0500
Subject: [PATCH 2/5] misc small fixes

---
 openmp/runtime/src/i18n/en_US.txt             |  1 -
 openmp/runtime/src/kmp_traits.cpp             |  9 +++++++-
 openmp/runtime/src/kmp_traits.h               |  8 ++++---
 .../unittests/Traits/TestOMPTraitParser.cpp   | 23 +++++++++++++++++++
 .../unittests/Traits/TestOMPTraits.cpp        | 19 +++++++++++++++
 5 files changed, 55 insertions(+), 5 deletions(-)

diff --git a/openmp/runtime/src/i18n/en_US.txt b/openmp/runtime/src/i18n/en_US.txt
index bc74ffaf23465..e492f66270d60 100644
--- a/openmp/runtime/src/i18n/en_US.txt
+++ b/openmp/runtime/src/i18n/en_US.txt
@@ -362,7 +362,6 @@ TopologyGeneric              "%1$s: %2$s (%3$d total cores)"
 AffGranularityBad            "%1$s: granularity setting: %2$s does not exist in topology.  Using granularity=%3$s instead."
 TopologyHybrid               "%1$s: hybrid core type detected: %2$d %3$s cores."
 TopologyHybridCoreEff        "%1$s:   %2$d with core efficiency %3$d."
-TraitParserInvalidDeviceKind "trait parser while parsing %1$s: invalid device kind (allowed: host/nohost/cpu/gpu/fpga)"
 TraitParserInvalidUID        "trait parser while parsing %1$s: invalid uid (%2$s)"
 TraitParserMaxRecursion      "trait parser while parsing %1$s: max recursion depth (%2$d) exceeded"
 TraitParserFailed            "trait parser while parsing %1$s: failed to parse trait specification (%2$s)"
diff --git a/openmp/runtime/src/kmp_traits.cpp b/openmp/runtime/src/kmp_traits.cpp
index 9d44ca2affe2b..710851b6b353a 100644
--- a/openmp/runtime/src/kmp_traits.cpp
+++ b/openmp/runtime/src/kmp_traits.cpp
@@ -297,9 +297,16 @@ int kmp_trait_context::parse_single_device(kmp_str_ref spec,
                                            int device_num_limit,
                                            const char *dbg_name) {
   int device_num;
+  kmp_str_ref orig_spec = spec;
   spec.skip_space();
   if (!spec.consume_integer(device_num))
-    KMP_FATAL(TraitParserFailed, dbg_name, spec.copy());
+    KMP_FATAL(TraitParserFailed, dbg_name, orig_spec.copy());
+  // Reject any trailing characters so that malformed input (e.g. "10abc" or
+  // "10,11") fails loudly instead of being silently truncated to the leading
+  // integer.
+  spec.skip_space();
+  if (!spec.empty())
+    KMP_FATAL(TraitParserFailed, dbg_name, orig_spec.copy());
   if (device_num > device_num_limit)
     KMP_FATAL(TraitParserValueTooLarge, dbg_name, device_num, device_num_limit);
   return device_num;
diff --git a/openmp/runtime/src/kmp_traits.h b/openmp/runtime/src/kmp_traits.h
index 3f93fa74f0b74..a82b886898ff6 100644
--- a/openmp/runtime/src/kmp_traits.h
+++ b/openmp/runtime/src/kmp_traits.h
@@ -262,9 +262,11 @@ class kmp_trait_expr_group final : public kmp_trait_expr {
       return false;
     const kmp_trait_expr_group &other_group =
         static_cast<const kmp_trait_expr_group &>(other);
-    return exprs.is_set_equal(
-        other_group.exprs, [](kmp_trait_expr *const &a,
-                              kmp_trait_expr *const &b) { return *a == *b; });
+    return type == other_group.type &&
+           exprs.is_set_equal(other_group.exprs, [](kmp_trait_expr *const &a,
+                                                    kmp_trait_expr *const &b) {
+             return *a == *b;
+           });
   }
 };
 
diff --git a/openmp/runtime/unittests/Traits/TestOMPTraitParser.cpp b/openmp/runtime/unittests/Traits/TestOMPTraitParser.cpp
index 7d78ca0d202cf..ab66899b66d2e 100644
--- a/openmp/runtime/unittests/Traits/TestOMPTraitParser.cpp
+++ b/openmp/runtime/unittests/Traits/TestOMPTraitParser.cpp
@@ -1035,4 +1035,27 @@ TEST(ParseSingleDeviceTest, LargeNumber) {
   EXPECT_EQ(result, 999999);
 }
 
+TEST(ParseSingleDeviceTest, TrailingGarbage) {
+  // Trailing non-integer characters must not be silently ignored.
+  ASSERT_DEATH(kmp_trait_context::parse_single_device(kmp_str_ref("10abc"), 100,
+                                                      "trailing_garbage"),
+               "OMP: Error #[0-9]+: trait parser while parsing "
+               "trailing_garbage: "
+               "failed to parse trait specification \\(10abc\\)");
+}
+
+TEST(ParseSingleDeviceTest, TrailingList) {
+  // Only a single device is accepted; a comma-separated list must fail.
+  ASSERT_DEATH(kmp_trait_context::parse_single_device(kmp_str_ref("10,11"), 100,
+                                                      "trailing_list"),
+               "OMP: Error #[0-9]+: trait parser while parsing trailing_list: "
+               "failed to parse trait specification \\(10,11\\)");
+}
+
+TEST(ParseSingleDeviceTest, TrailingSpaces) {
+  // Trailing whitespace is allowed.
+  int result = kmp_trait_context::parse_single_device(kmp_str_ref("7  "), 10);
+  EXPECT_EQ(result, 7);
+}
+
 } // namespace
diff --git a/openmp/runtime/unittests/Traits/TestOMPTraits.cpp b/openmp/runtime/unittests/Traits/TestOMPTraits.cpp
index 9af10db803cc3..12146f07089de 100644
--- a/openmp/runtime/unittests/Traits/TestOMPTraits.cpp
+++ b/openmp/runtime/unittests/Traits/TestOMPTraits.cpp
@@ -492,6 +492,25 @@ TEST(kmp_trait_expr_group_test, EqualityDifferentNegation) {
   delete g2;
 }
 
+TEST(kmp_trait_expr_group_test, EqualityDifferentGroupType) {
+  // Groups with the same sub-expressions but different AND/OR type are not
+  // equal.
+  kmp_trait_expr_group *g1 = new kmp_trait_expr_group();
+  kmp_trait_expr_group *g2 = new kmp_trait_expr_group();
+
+  g1->add_expr(new kmp_literal_trait(1));
+  g1->add_expr(new kmp_literal_trait(2));
+  g1->set_group_type(kmp_trait_expr_group::AND);
+  g2->add_expr(new kmp_literal_trait(1));
+  g2->add_expr(new kmp_literal_trait(2));
+  g2->set_group_type(kmp_trait_expr_group::OR);
+
+  EXPECT_FALSE(*g1 == *g2);
+
+  delete g1;
+  delete g2;
+}
+
 TEST(kmp_trait_expr_group_test, NestedGroups) {
   kmp_trait_expr_group *outer = new kmp_trait_expr_group();
   outer->set_group_type(kmp_trait_expr_group::OR);

>From 514190c570da99bda612a403b050de9369e038a7 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Thu, 18 Jun 2026 06:43:53 -0500
Subject: [PATCH 3/5] move parse_single_device to other PR

---
 openmp/runtime/src/i18n/en_US.txt             |  1 -
 openmp/runtime/src/kmp_traits.cpp             | 19 -----
 openmp/runtime/src/kmp_traits.h               |  7 --
 .../unittests/Traits/TestOMPTraitParser.cpp   | 80 -------------------
 4 files changed, 107 deletions(-)

diff --git a/openmp/runtime/src/i18n/en_US.txt b/openmp/runtime/src/i18n/en_US.txt
index e492f66270d60..f45b2e70d1547 100644
--- a/openmp/runtime/src/i18n/en_US.txt
+++ b/openmp/runtime/src/i18n/en_US.txt
@@ -365,7 +365,6 @@ TopologyHybridCoreEff        "%1$s:   %2$d with core efficiency %3$d."
 TraitParserInvalidUID        "trait parser while parsing %1$s: invalid uid (%2$s)"
 TraitParserMaxRecursion      "trait parser while parsing %1$s: max recursion depth (%2$d) exceeded"
 TraitParserFailed            "trait parser while parsing %1$s: failed to parse trait specification (%2$s)"
-TraitParserValueTooLarge     "trait parser while parsing %1$s: value %2$d above limit (%3$d)"
 
 # --- OpenMP errors detected at runtime ---
 #
diff --git a/openmp/runtime/src/kmp_traits.cpp b/openmp/runtime/src/kmp_traits.cpp
index 710851b6b353a..ad307833c6aac 100644
--- a/openmp/runtime/src/kmp_traits.cpp
+++ b/openmp/runtime/src/kmp_traits.cpp
@@ -292,22 +292,3 @@ kmp_trait_context *kmp_trait_context::parse_from_spec(kmp_str_ref spec,
     KMP_FATAL(TraitParserFailed, dbg_name, spec.copy());
   return context;
 }
-
-int kmp_trait_context::parse_single_device(kmp_str_ref spec,
-                                           int device_num_limit,
-                                           const char *dbg_name) {
-  int device_num;
-  kmp_str_ref orig_spec = spec;
-  spec.skip_space();
-  if (!spec.consume_integer(device_num))
-    KMP_FATAL(TraitParserFailed, dbg_name, orig_spec.copy());
-  // Reject any trailing characters so that malformed input (e.g. "10abc" or
-  // "10,11") fails loudly instead of being silently truncated to the leading
-  // integer.
-  spec.skip_space();
-  if (!spec.empty())
-    KMP_FATAL(TraitParserFailed, dbg_name, orig_spec.copy());
-  if (device_num > device_num_limit)
-    KMP_FATAL(TraitParserValueTooLarge, dbg_name, device_num, device_num_limit);
-  return device_num;
-}
diff --git a/openmp/runtime/src/kmp_traits.h b/openmp/runtime/src/kmp_traits.h
index a82b886898ff6..052017a66225f 100644
--- a/openmp/runtime/src/kmp_traits.h
+++ b/openmp/runtime/src/kmp_traits.h
@@ -361,13 +361,6 @@ class kmp_trait_context final {
   static kmp_trait_context *parse_from_spec(kmp_str_ref spec,
                                             const char *dbg_name = nullptr);
 
-  // Parse only a single device number from the spec.
-  // This is useful for backward compatibility with legacy code.
-  // If dbg_name is provided, it will be used in error messages to identify the
-  // source of the device number.
-  static int parse_single_device(kmp_str_ref spec, int device_num_limit,
-                                 const char *dbg_name = nullptr);
-
   void add_clause(kmp_trait_clause *clause) {
     assert(clause);
     clauses.push_back(clause);
diff --git a/openmp/runtime/unittests/Traits/TestOMPTraitParser.cpp b/openmp/runtime/unittests/Traits/TestOMPTraitParser.cpp
index ab66899b66d2e..08f19ad72e322 100644
--- a/openmp/runtime/unittests/Traits/TestOMPTraitParser.cpp
+++ b/openmp/runtime/unittests/Traits/TestOMPTraitParser.cpp
@@ -978,84 +978,4 @@ TEST_F(ParserTest, DoubleComma) {
                "failed to parse trait specification \\(,,uid\\(b\\)\\)");
 }
 
-//===----------------------------------------------------------------------===//
-// parse_single_device Tests
-//===----------------------------------------------------------------------===//
-
-TEST(ParseSingleDeviceTest, ValidSingleDigit) {
-  int result = kmp_trait_context::parse_single_device(kmp_str_ref("5"), 10);
-  EXPECT_EQ(result, 5);
-}
-
-TEST(ParseSingleDeviceTest, ValidMultiDigit) {
-  int result = kmp_trait_context::parse_single_device(kmp_str_ref("123"), 200);
-  EXPECT_EQ(result, 123);
-}
-
-TEST(ParseSingleDeviceTest, Zero) {
-  int result = kmp_trait_context::parse_single_device(kmp_str_ref("0"), 10);
-  EXPECT_EQ(result, 0);
-}
-
-TEST(ParseSingleDeviceTest, AtLimit) {
-  int result = kmp_trait_context::parse_single_device(kmp_str_ref("10"), 10);
-  EXPECT_EQ(result, 10);
-}
-
-TEST(ParseSingleDeviceTest, AboveLimit) {
-  ASSERT_DEATH(kmp_trait_context::parse_single_device(kmp_str_ref("11"), 10,
-                                                      "above_limit"),
-               "OMP: Error #[0-9]+: trait parser while parsing above_limit: "
-               "value 11 above limit \\(10\\)");
-}
-
-TEST(ParseSingleDeviceTest, NonInteger) {
-  ASSERT_DEATH(kmp_trait_context::parse_single_device(kmp_str_ref("abc"), 10,
-                                                      "non_integer"),
-               "OMP: Error #[0-9]+: trait parser while parsing non_integer: "
-               "failed to parse trait specification \\(abc\\)");
-}
-
-TEST(ParseSingleDeviceTest, EmptyString) {
-  ASSERT_DEATH(
-      kmp_trait_context::parse_single_device(kmp_str_ref(""), 10, "empty"),
-      "OMP: Error #[0-9]+: trait parser while parsing empty: "
-      "failed to parse trait specification \\(\\)");
-}
-
-TEST(ParseSingleDeviceTest, LeadingSpaces) {
-  // consume_integer skips leading spaces
-  int result = kmp_trait_context::parse_single_device(kmp_str_ref("  7"), 10);
-  EXPECT_EQ(result, 7);
-}
-
-TEST(ParseSingleDeviceTest, LargeNumber) {
-  int result =
-      kmp_trait_context::parse_single_device(kmp_str_ref("999999"), 1000000);
-  EXPECT_EQ(result, 999999);
-}
-
-TEST(ParseSingleDeviceTest, TrailingGarbage) {
-  // Trailing non-integer characters must not be silently ignored.
-  ASSERT_DEATH(kmp_trait_context::parse_single_device(kmp_str_ref("10abc"), 100,
-                                                      "trailing_garbage"),
-               "OMP: Error #[0-9]+: trait parser while parsing "
-               "trailing_garbage: "
-               "failed to parse trait specification \\(10abc\\)");
-}
-
-TEST(ParseSingleDeviceTest, TrailingList) {
-  // Only a single device is accepted; a comma-separated list must fail.
-  ASSERT_DEATH(kmp_trait_context::parse_single_device(kmp_str_ref("10,11"), 100,
-                                                      "trailing_list"),
-               "OMP: Error #[0-9]+: trait parser while parsing trailing_list: "
-               "failed to parse trait specification \\(10,11\\)");
-}
-
-TEST(ParseSingleDeviceTest, TrailingSpaces) {
-  // Trailing whitespace is allowed.
-  int result = kmp_trait_context::parse_single_device(kmp_str_ref("7  "), 10);
-  EXPECT_EQ(result, 7);
-}
-
 } // namespace

>From 0a69f164b47549ce04d6c2b771452c86104786c0 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Thu, 9 Jul 2026 10:57:38 -0500
Subject: [PATCH 4/5] add lexer; improve structure

---
 openmp/runtime/src/i18n/en_US.txt             |   3 +-
 openmp/runtime/src/kmp_traits.cpp             | 351 ++++++++++++------
 openmp/runtime/src/kmp_traits.h               |  67 ++++
 .../runtime/unittests/Traits/CMakeLists.txt   |   1 +
 .../unittests/Traits/TestOMPTraitLexer.cpp    | 350 +++++++++++++++++
 .../unittests/Traits/TestOMPTraitParser.cpp   | 101 ++++-
 6 files changed, 756 insertions(+), 117 deletions(-)
 create mode 100644 openmp/runtime/unittests/Traits/TestOMPTraitLexer.cpp

diff --git a/openmp/runtime/src/i18n/en_US.txt b/openmp/runtime/src/i18n/en_US.txt
index f45b2e70d1547..3b7187edaeb13 100644
--- a/openmp/runtime/src/i18n/en_US.txt
+++ b/openmp/runtime/src/i18n/en_US.txt
@@ -362,7 +362,8 @@ TopologyGeneric              "%1$s: %2$s (%3$d total cores)"
 AffGranularityBad            "%1$s: granularity setting: %2$s does not exist in topology.  Using granularity=%3$s instead."
 TopologyHybrid               "%1$s: hybrid core type detected: %2$d %3$s cores."
 TopologyHybridCoreEff        "%1$s:   %2$d with core efficiency %3$d."
-TraitParserInvalidUID        "trait parser while parsing %1$s: invalid uid (%2$s)"
+TraitParserError             "trait parser while parsing %1$s: error: %2$s"
+TraitParserInvalidTraitValue "trait parser while parsing %1$s: invalid value for trait '%2$s' (%3$s)"
 TraitParserMaxRecursion      "trait parser while parsing %1$s: max recursion depth (%2$d) exceeded"
 TraitParserFailed            "trait parser while parsing %1$s: failed to parse trait specification (%2$s)"
 
diff --git a/openmp/runtime/src/kmp_traits.cpp b/openmp/runtime/src/kmp_traits.cpp
index ad307833c6aac..6070f388ec1d2 100644
--- a/openmp/runtime/src/kmp_traits.cpp
+++ b/openmp/runtime/src/kmp_traits.cpp
@@ -57,11 +57,8 @@ using namespace kmp_traits;
 //       trait_expr
 //     | trait_expr {"&&" trait_expr}
 //     | trait_expr {"||" trait_expr}
-// trait_expr =
-//       trait_expr_single
-//     | trait_expr_group_paren
-// trait_expr_single = ["!"] trait
-// trait_expr_group_paren = ["!"] "(" trait_expr_group ")"
+// trait_expr = ["!"] (trait | trait_expr_group_paren)
+// trait_expr_group_paren = "(" trait_expr_group ")"
 // trait =
 //       "uid" "(" uid_value ")"
 // uid_value = (letter | digit0 | symbol) {letter | digit0 | symbol}
@@ -97,190 +94,322 @@ using namespace kmp_traits;
 // digit = "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9"
 // symbol = "-" | "_"
 
+// A character that can appear in a word (uid keyword / uid_value / integer),
+// i.e. a letter, a digit, or one of the symbols "-" / "_".
+static bool is_word_char(char c) {
+  return isalnum(static_cast<unsigned char>(c)) || c == '-' || c == '_';
+}
+
+static bool is_digit(char c) {
+  return static_cast<bool>(isdigit(static_cast<unsigned char>(c)));
+}
+
+namespace lexer {
+
+token kmp_lexer::lex() {
+  scan.skip_space();
+
+  const char *start = scan.begin();
+  if (scan.empty())
+    return {token_kind::END, kmp_str_ref(start, 0)};
+
+  // Two-character operators.
+  if (scan.consume_front("&&"))
+    return {token_kind::AND, kmp_str_ref(start, 2)};
+  if (scan.consume_front("||"))
+    return {token_kind::OR, kmp_str_ref(start, 2)};
+
+  // Word characters form a single WORD.
+  kmp_str_ref word = scan.take_while(is_word_char);
+  if (!word.empty()) {
+    scan.drop_front(word.length());
+    return {token_kind::WORD, word};
+  }
+
+  // Single-character tokens.
+  token_kind kind;
+  switch (*start) {
+  case ',':
+    kind = token_kind::COMMA;
+    break;
+  case '*':
+    kind = token_kind::STAR;
+    break;
+  case '!':
+    kind = token_kind::NOT;
+    break;
+  case '(':
+    kind = token_kind::L_PAREN;
+    break;
+  case ')':
+    kind = token_kind::R_PAREN;
+    break;
+  case '[':
+    kind = token_kind::L_BRACKET;
+    break;
+  case ']':
+    kind = token_kind::R_BRACKET;
+    break;
+  case ':':
+    kind = token_kind::COLON;
+    break;
+  default:
+    kind = token_kind::UNKNOWN;
+    break;
+  }
+  scan.drop_front(1);
+  return {kind, kmp_str_ref(start, 1)};
+}
+
+} // namespace lexer
+
 namespace parser {
 
-#define MAX_RECURSION_DEPTH 64
+constexpr int MAX_RECURSION_DEPTH = 64;
 
 using namespace kmp_traits;
+using namespace lexer;
 
-static kmp_str_ref consume_uid_value(kmp_str_ref &scan, const char *dbg_name) {
-  scan.skip_space();
-  kmp_str_ref uid = scan.take_while([](char c) {
-    return isalnum(static_cast<unsigned char>(c)) || c == '-' || c == '_';
-  });
-  scan.drop_front(uid.length());
-  if (uid.empty() || !scan.consume_front(")"))
-    KMP_FATAL(TraitParserInvalidUID, dbg_name, uid.copy());
+// Check whether a token is a WORD whose text equals the given keyword.
+static bool word_is(const token &tok, kmp_str_ref keyword) {
+  kmp_str_ref text = tok.text;
+  return tok.kind == token_kind::WORD && text.consume_front(keyword) &&
+         text.empty();
+}
+
+// Check whether a token is a WORD that has the shape of a device number, i.e.
+// device_number = ["-"] integer0 (an optional "-" followed by digits only).
+// The lexer emits every word-character run as a WORD, so it is the parser that
+// tells a device number apart from a name or uid_value (which may also contain
+// "-" and digits).
+static bool word_is_number(const token &tok) {
+  if (tok.kind != token_kind::WORD)
+    return false;
+  kmp_str_ref text = tok.text;
+  text.consume_front("-");
+  return !text.empty() && text.find_if_not(is_digit) == kmp_str_ref::npos;
+}
+
+// uid_value = (letter | digit0 | symbol) {letter | digit0 | symbol}
+// Consumes and returns a uid value.
+// An invalid uid value is a hard error.
+static kmp_str_ref consume_uid_value(kmp_lexer &lex, const char *dbg_name) {
+  const token &tok = lex.peek();
+  if (tok.kind != token_kind::WORD)
+    KMP_FATAL(TraitParserInvalidTraitValue, dbg_name, "uid", tok.text.copy());
+  kmp_str_ref uid = tok.text;
+  lex.next(); // consume the uid_value
   return uid;
 }
 
-static bool consume_trait(kmp_trait_expr_single &expr, kmp_str_ref &scan,
+// trait = "uid" "(" uid_value ")"
+// (more traits will be added as needed in the future)
+// Returns false without consuming anything if the next token is not a
+// recognized trait name.
+// Once a trait name is consumed we are committed to a trait expression, so a
+// missing "(" or ")" or an invalid trait value is a hard error.
+static bool consume_trait(kmp_trait_expr_single &expr, kmp_lexer &lex,
                           const char *dbg_name) {
-  scan.skip_space();
-  if (!scan.consume_front("uid("))
+  if (!word_is(lex.peek(), "uid"))
     return false;
-  kmp_str_ref uid = consume_uid_value(scan, dbg_name);
+  // Add more traits as needed in the future.
+
+  lex.next(); // consume trait name
+  if (lex.peek().kind != token_kind::L_PAREN)
+    KMP_FATAL(TraitParserError, dbg_name, "expected '(' after trait name");
+  lex.next(); // consume "("
+  kmp_str_ref uid = consume_uid_value(lex, dbg_name);
+  if (lex.peek().kind != token_kind::R_PAREN)
+    KMP_FATAL(TraitParserError, dbg_name, "expected ')' after trait value");
+  lex.next(); // consume ")"
   expr.set_trait(new kmp_uid_trait(uid));
   return true;
 }
 
-static bool consume_trait_expr_single(kmp_trait_expr_single &expr,
-                                      kmp_str_ref &scan, const char *dbg_name) {
-  kmp_str_ref orig_scan = scan;
-
-  scan.skip_space();
-  if (scan.consume_front("!"))
-    expr.set_negated();
-  if (consume_trait(expr, scan, dbg_name))
-    return true;
-  scan = orig_scan;
-  return false;
-}
-
 // forward declaration
 static bool consume_trait_expr_group(kmp_trait_expr_group &group,
-                                     kmp_str_ref &scan, int max_recursion,
+                                     kmp_lexer &lex, int max_recursion,
                                      const char *dbg_name);
 
+// trait_expr_group_paren = "(" trait_expr_group ")"
+// Returns false without consuming anything if the next token is not "(", so the
+// caller can try other alternatives.
+// Once "(" is consumed we are committed to a parenthesized group, so a missing
+// group or ")" is a hard error.
 static bool consume_trait_expr_group_paren(kmp_trait_expr_group &group,
-                                           kmp_str_ref &scan, int max_recursion,
+                                           bool negated, kmp_lexer &lex,
+                                           int max_recursion,
                                            const char *dbg_name) {
-  if (max_recursion-- <= 0)
-    KMP_FATAL(TraitParserMaxRecursion, dbg_name, MAX_RECURSION_DEPTH);
-  kmp_str_ref orig_scan = scan;
-
-  scan.skip_space();
-  if (scan.consume_front("!"))
-    group.set_negated();
-
-  scan.skip_space();
-  if (!scan.consume_front("(") ||
-      !consume_trait_expr_group(group, scan, max_recursion, dbg_name)) {
-    scan = orig_scan;
-    return false;
-  }
-
-  scan.skip_space();
-  if (!scan.consume_front(")")) {
-    scan = orig_scan;
+  if (lex.peek().kind != token_kind::L_PAREN)
     return false;
-  }
+  group.set_negated(negated);
+  lex.next(); // consume "("
+  if (!consume_trait_expr_group(group, lex, max_recursion, dbg_name))
+    KMP_FATAL(TraitParserError, dbg_name,
+              "expected trait expression after '('");
+  if (lex.peek().kind != token_kind::R_PAREN)
+    KMP_FATAL(TraitParserError, dbg_name,
+              "expected ')' after trait expression group");
+  lex.next(); // consume ")"
   return true;
 }
 
-static bool consume_trait_expr(kmp_trait_expr *&expr, kmp_str_ref &scan,
+// trait_expr = ["!"] (trait | trait_expr_group_paren)
+// Returns false without consuming anything if neither a parenthesized group nor
+// a single trait can be consumed.
+// Once an optional leading "!" has been consumed we are committed to a trait
+// expression, so a missing trait or parenthesized group is a hard error.
+static bool consume_trait_expr(kmp_trait_expr *&expr, kmp_lexer &lex,
                                int max_recursion, const char *dbg_name) {
   if (max_recursion-- <= 0)
     KMP_FATAL(TraitParserMaxRecursion, dbg_name, MAX_RECURSION_DEPTH);
 
-  // Parse a single trait expression
-  kmp_trait_expr_single *single_expr = new kmp_trait_expr_single();
-  if (consume_trait_expr_single(*single_expr, scan, dbg_name)) {
-    expr = single_expr;
-    return true;
-  }
-  delete single_expr;
+  // Consume the optional leading "!"; it applies to whatever follows.
+  bool negated = lex.peek().kind == token_kind::NOT;
+  if (negated)
+    lex.next();
 
-  // Parse a parenthesized group trait expression
-  kmp_trait_expr_group *group_expr = new kmp_trait_expr_group();
-  if (consume_trait_expr_group_paren(*group_expr, scan, max_recursion,
+  // Try a parenthesized group (starts with "(") ...
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+  if (consume_trait_expr_group_paren(*group, negated, lex, max_recursion,
                                      dbg_name)) {
-    expr = group_expr;
+    expr = group;
     return true;
   }
-  delete group_expr;
+  delete group;
 
+  // ... otherwise it must be a single trait.
+  kmp_trait_expr_single *single = new kmp_trait_expr_single();
+  single->set_negated(negated);
+  if (consume_trait(*single, lex, dbg_name)) {
+    expr = single;
+    return true;
+  }
+  delete single;
+
+  // A leading "!" has already committed us to a trait expression, so its
+  // absence is an error; without it, nothing was consumed and the caller can
+  // recover/report.
+  if (negated)
+    KMP_FATAL(TraitParserError, dbg_name,
+              "expected trait expression after '!'");
   return false;
 }
 
+// trait_expr_group =
+//       trait_expr
+//     | trait_expr {"&&" trait_expr}
+//     | trait_expr {"||" trait_expr}
+// Returns false without consuming anything if no trait expression can be
+// consumed.
+// Any other missing or invalid tokens are a hard error.
 static bool consume_trait_expr_group(kmp_trait_expr_group &group,
-                                     kmp_str_ref &scan, int max_recursion,
+                                     kmp_lexer &lex, int max_recursion,
                                      const char *dbg_name) {
   if (max_recursion-- <= 0)
     KMP_FATAL(TraitParserMaxRecursion, dbg_name, MAX_RECURSION_DEPTH);
 
   kmp_trait_expr *expr = nullptr;
-  if (!consume_trait_expr(expr, scan, max_recursion, dbg_name))
+  if (!consume_trait_expr(expr, lex, max_recursion, dbg_name))
     return false;
-
   group.add_expr(expr);
-  const char *op = nullptr;
 
-  scan.skip_space();
-  if (scan.consume_front("||")) {
+  token_kind op;
+  if (lex.peek().kind == token_kind::OR) {
     group.set_group_type(kmp_trait_expr_group::OR);
-    op = "||";
-  } else if (scan.consume_front("&&")) {
+    op = token_kind::OR;
+  } else if (lex.peek().kind == token_kind::AND) {
     group.set_group_type(kmp_trait_expr_group::AND);
-    op = "&&";
+    op = token_kind::AND;
   } else {
-    return true; // single trait expression, no group
+    return true; // single trait expression, no operator
   }
-
-  // Now that we got an operator, we need at least one more trait expr.
-  do {
-    if (!consume_trait_expr(expr, scan, max_recursion, dbg_name))
-      return false;
+  lex.next(); // consume the operator
+
+  // Having consumed an operator, we are committed: at least one more trait
+  // expression must follow, so its absence is a hard error.
+  while (true) {
+    if (!consume_trait_expr(expr, lex, max_recursion, dbg_name))
+      KMP_FATAL(TraitParserError, dbg_name,
+                "expected trait expression after operator");
     group.add_expr(expr);
-    scan.skip_space();
-  } while (scan.consume_front(op));
+    if (lex.peek().kind != op)
+      break;
+    lex.next(); // consume the operator
+  }
 
   return true;
 }
 
-static bool consume_clause(kmp_trait_clause &clause, kmp_str_ref &scan,
-                           const char *dbg_name) {
-  kmp_str_ref orig_scan = scan;
-  scan.skip_space();
+// device_number = ["-"] integer0
+// Returns false without consuming anything if the next token is not shaped like
+// a device number. A WORD that has the shape of a device number is treated as a
+// device number, so once it is recognized it is committed: a value that is not
+// a valid device index (negative or too large to fit an int) is a hard error.
+static bool consume_device_number(kmp_trait_clause &clause, kmp_lexer &lex,
+                                  const char *dbg_name) {
+  if (!word_is_number(lex.peek()))
+    return false;
+  kmp_str_ref number = lex.peek().text;
+  int value;
+  if (!number.consume_integer(value))
+    KMP_FATAL(TraitParserError, dbg_name, "device number out of range");
+  lex.next();
+  clause.set_expr(new kmp_literal_trait(value));
+  return true;
+}
 
+// clause =
+//       device_number
+//     | "*" [index_expr]
+//     | trait_expr_group
+//     | trait_expr index_expr
+// Returns false without consuming anything if no clause can be consumed.
+// Any other missing or invalid tokens are a hard error.
+static bool consume_clause(kmp_trait_clause &clause, kmp_lexer &lex,
+                           const char *dbg_name) {
   // Parse wildcard "trait"
-  if (scan.consume_front("*")) {
+  if (lex.peek().kind == token_kind::STAR) {
+    lex.next();
     clause.set_expr(new kmp_wildcard_trait());
     return true;
   }
 
-  // Parse a literal device number
-  int value;
-  if (scan.consume_integer(value)) {
-    clause.set_expr(new kmp_literal_trait(value));
+  // Parse a literal device number. A WORD that is not shaped like a number is
+  // not a device number and starts a trait expression group instead.
+  if (consume_device_number(clause, lex, dbg_name))
     return true;
-  }
 
   // Parse a trait expression group
   kmp_trait_expr_group *group = new kmp_trait_expr_group();
-  if (consume_trait_expr_group(*group, scan, MAX_RECURSION_DEPTH, dbg_name)) {
+  if (consume_trait_expr_group(*group, lex, MAX_RECURSION_DEPTH, dbg_name)) {
     clause.set_expr(group);
     return true;
   }
   delete group;
 
-  scan = orig_scan;
   return false;
 }
 
-static bool consume_list(kmp_trait_context &context, kmp_str_ref &scan,
+// list = [clause {',' clause}]
+static void consume_list(kmp_trait_context &context, kmp_lexer &lex,
                          const char *dbg_name) {
-  kmp_str_ref orig_scan = scan;
-  scan.skip_space();
+  kmp_str_ref lex_pos = lex.remaining();
 
-  while (!scan.empty()) {
+  while (lex.peek().kind != token_kind::END) {
     kmp_trait_clause *clause = new kmp_trait_clause();
-    if (!consume_clause(*clause, scan, dbg_name)) {
+    if (!consume_clause(*clause, lex, dbg_name)) {
       delete clause;
-      scan = orig_scan;
-      return false;
+      KMP_FATAL(TraitParserFailed, dbg_name, lex_pos.copy());
     }
     context.add_clause(clause);
-    orig_scan = scan;
 
-    scan.skip_space();
-    if (!scan.consume_front(",") && !scan.empty()) {
-      scan = orig_scan;
-      return false;
-    }
+    lex_pos = lex.remaining();
+    if (lex.peek().kind == token_kind::COMMA)
+      lex.next();
+    else if (lex.peek().kind != token_kind::END)
+      KMP_FATAL(TraitParserFailed, dbg_name, lex_pos.copy());
   }
-
-  return true;
 }
 
 } // namespace parser
@@ -288,7 +417,7 @@ static bool consume_list(kmp_trait_context &context, kmp_str_ref &scan,
 kmp_trait_context *kmp_trait_context::parse_from_spec(kmp_str_ref spec,
                                                       const char *dbg_name) {
   kmp_trait_context *context = new kmp_trait_context();
-  if (!parser::consume_list(*context, spec, dbg_name))
-    KMP_FATAL(TraitParserFailed, dbg_name, spec.copy());
+  lexer::kmp_lexer lex(spec);
+  parser::consume_list(*context, lex, dbg_name);
   return context;
 }
diff --git a/openmp/runtime/src/kmp_traits.h b/openmp/runtime/src/kmp_traits.h
index 052017a66225f..9d70a44277078 100644
--- a/openmp/runtime/src/kmp_traits.h
+++ b/openmp/runtime/src/kmp_traits.h
@@ -16,6 +16,73 @@
 #include "kmp.h"
 #include "kmp_adt.h"
 
+// Lexer for the OpenMP trait grammar (the grammar is documented in
+// kmp_traits.cpp).
+namespace lexer {
+
+enum class token_kind {
+  END, // end of the input
+  COMMA, // ,
+  STAR, // *
+  NOT, // !
+  L_PAREN, // (
+  R_PAREN, // )
+  L_BRACKET, // [
+  R_BRACKET, // ]
+  COLON, // :
+  AND, // &&
+  OR, // ||
+  WORD, // word characters
+  UNKNOWN, // an unrecognized character (e.g. a lone '&' or '|')
+};
+
+// A single token produced by the lexer. `text` is a non-owning reference into
+// the source string the token was lexed from.
+struct token {
+  token_kind kind = token_kind::UNKNOWN;
+  kmp_str_ref text = kmp_str_ref("");
+};
+
+class kmp_lexer final {
+  kmp_str_ref scan;
+  token lookahead{token_kind::END, kmp_str_ref("")};
+  bool has_lookahead = false;
+
+  // Lex the next token directly from the input, advancing past it.
+  token lex();
+
+public:
+  explicit kmp_lexer(kmp_str_ref source) : scan(source) {}
+
+  // Return the next token and advance past it.
+  token next() {
+    if (has_lookahead) {
+      has_lookahead = false;
+      return lookahead;
+    }
+    return lex();
+  }
+
+  // Return the next token without advancing.
+  token peek() {
+    if (!has_lookahead) {
+      lookahead = lex();
+      has_lookahead = true;
+    }
+    return lookahead;
+  }
+
+  // Return the yet-unconsumed portion of the source, with any whitespace before
+  // the next token skipped.
+  kmp_str_ref remaining() {
+    token t = peek();
+    return kmp_str_ref(t.text.begin(),
+                       static_cast<size_t>(scan.end() - t.text.begin()));
+  }
+};
+
+} // namespace lexer
+
 namespace kmp_traits {
 
 extern "C" int omp_get_num_devices();
diff --git a/openmp/runtime/unittests/Traits/CMakeLists.txt b/openmp/runtime/unittests/Traits/CMakeLists.txt
index d53db903c9f3a..aef30716bcf14 100644
--- a/openmp/runtime/unittests/Traits/CMakeLists.txt
+++ b/openmp/runtime/unittests/Traits/CMakeLists.txt
@@ -2,5 +2,6 @@ add_openmp_unittest(TraitsTests
   MockOMP.cpp
   TestOMPTraits.cpp
   TestOMPTraitParser.cpp
+  TestOMPTraitLexer.cpp
 )
 
diff --git a/openmp/runtime/unittests/Traits/TestOMPTraitLexer.cpp b/openmp/runtime/unittests/Traits/TestOMPTraitLexer.cpp
new file mode 100644
index 0000000000000..43c5821fc3b5b
--- /dev/null
+++ b/openmp/runtime/unittests/Traits/TestOMPTraitLexer.cpp
@@ -0,0 +1,350 @@
+//===- TestOMPTraitLexer.cpp - Tests for OMP Trait Lexer -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "kmp_traits.h"
+#include "gtest/gtest.h"
+
+#include <cstring>
+#include <string>
+
+using namespace lexer;
+
+namespace {
+
+//===----------------------------------------------------------------------===//
+// Helpers
+//===----------------------------------------------------------------------===//
+
+// Compare a token's text against a C string.
+static bool text_is(const token &tok, const char *expected) {
+  size_t len = strlen(expected);
+  if (tok.text.length() != len)
+    return false;
+  if (len == 0)
+    return true;
+  return memcmp(tok.text.begin(), expected, len) == 0;
+}
+
+// Lex the whole spec into a vector of tokens (excluding the trailing END).
+static kmp_vector<token> tokenize(const char *spec) {
+  kmp_lexer lex{kmp_str_ref(spec)};
+  kmp_vector<token> tokens;
+  for (token tok = lex.next(); tok.kind != token_kind::END; tok = lex.next())
+    tokens.push_back(tok);
+  return tokens;
+}
+
+// Assert a token has the expected kind and text.
+static void expect_token(const token &tok, token_kind kind, const char *text) {
+  EXPECT_EQ(tok.kind, kind);
+  EXPECT_TRUE(text_is(tok, text))
+      << "expected token text \"" << text << "\" but got \""
+      << std::string(tok.text.begin(), tok.text.length()) << "\"";
+}
+
+//===----------------------------------------------------------------------===//
+// Single Tokens
+//===----------------------------------------------------------------------===//
+
+TEST(LexerTest, EmptyInput) {
+  kmp_lexer lex{kmp_str_ref("")};
+  EXPECT_EQ(lex.next().kind, token_kind::END);
+}
+
+TEST(LexerTest, WhitespaceOnly) {
+  kmp_lexer lex{kmp_str_ref("   \t  ")};
+  EXPECT_EQ(lex.next().kind, token_kind::END);
+}
+
+TEST(LexerTest, EndIsSticky) {
+  // Once the input is exhausted, next() keeps returning END.
+  kmp_lexer lex{kmp_str_ref("*")};
+  EXPECT_EQ(lex.next().kind, token_kind::STAR);
+  EXPECT_EQ(lex.next().kind, token_kind::END);
+  EXPECT_EQ(lex.next().kind, token_kind::END);
+}
+
+TEST(LexerTest, Punctuation) {
+  kmp_vector<token> tokens = tokenize(",*!()[]:");
+
+  ASSERT_EQ(tokens.size(), 8u);
+  expect_token(tokens[0], token_kind::COMMA, ",");
+  expect_token(tokens[1], token_kind::STAR, "*");
+  expect_token(tokens[2], token_kind::NOT, "!");
+  expect_token(tokens[3], token_kind::L_PAREN, "(");
+  expect_token(tokens[4], token_kind::R_PAREN, ")");
+  expect_token(tokens[5], token_kind::L_BRACKET, "[");
+  expect_token(tokens[6], token_kind::R_BRACKET, "]");
+  expect_token(tokens[7], token_kind::COLON, ":");
+}
+
+TEST(LexerTest, DoubleColon) {
+  // "::" is lexed as two COLON tokens.
+  kmp_vector<token> tokens = tokenize("::");
+
+  ASSERT_EQ(tokens.size(), 2u);
+  expect_token(tokens[0], token_kind::COLON, ":");
+  expect_token(tokens[1], token_kind::COLON, ":");
+}
+
+TEST(LexerTest, AndOperator) {
+  kmp_vector<token> tokens = tokenize("&&");
+
+  ASSERT_EQ(tokens.size(), 1u);
+  expect_token(tokens[0], token_kind::AND, "&&");
+}
+
+TEST(LexerTest, OrOperator) {
+  kmp_vector<token> tokens = tokenize("||");
+
+  ASSERT_EQ(tokens.size(), 1u);
+  expect_token(tokens[0], token_kind::OR, "||");
+}
+
+//===----------------------------------------------------------------------===//
+// Numbers
+//
+// The lexer does not have a dedicated number token: a run of digits is just a
+// WORD. Recognizing a device number is the parser's job (see consume_clause).
+//===----------------------------------------------------------------------===//
+
+TEST(LexerTest, SingleDigitNumber) {
+  kmp_vector<token> tokens = tokenize("0");
+
+  ASSERT_EQ(tokens.size(), 1u);
+  expect_token(tokens[0], token_kind::WORD, "0");
+}
+
+TEST(LexerTest, MultiDigitNumber) {
+  kmp_vector<token> tokens = tokenize("12345");
+
+  ASSERT_EQ(tokens.size(), 1u);
+  expect_token(tokens[0], token_kind::WORD, "12345");
+}
+
+TEST(LexerTest, NumberList) {
+  kmp_vector<token> tokens = tokenize("1,2,3");
+
+  ASSERT_EQ(tokens.size(), 5u);
+  expect_token(tokens[0], token_kind::WORD, "1");
+  expect_token(tokens[1], token_kind::COMMA, ",");
+  expect_token(tokens[2], token_kind::WORD, "2");
+  expect_token(tokens[3], token_kind::COMMA, ",");
+  expect_token(tokens[4], token_kind::WORD, "3");
+}
+
+//===----------------------------------------------------------------------===//
+// Words
+//===----------------------------------------------------------------------===//
+
+TEST(LexerTest, SimpleWord) {
+  kmp_vector<token> tokens = tokenize("uid");
+
+  ASSERT_EQ(tokens.size(), 1u);
+  expect_token(tokens[0], token_kind::WORD, "uid");
+}
+
+TEST(LexerTest, WordWithDash) {
+  // A uid_value may contain '-'; it stays part of the word.
+  kmp_vector<token> tokens = tokenize("device-0");
+
+  ASSERT_EQ(tokens.size(), 1u);
+  expect_token(tokens[0], token_kind::WORD, "device-0");
+}
+
+TEST(LexerTest, WordWithUnderscore) {
+  kmp_vector<token> tokens = tokenize("my_device_123");
+
+  ASSERT_EQ(tokens.size(), 1u);
+  expect_token(tokens[0], token_kind::WORD, "my_device_123");
+}
+
+TEST(LexerTest, LeadingMinusIsWord) {
+  // The grammar allows a leading "-" for device numbers, but since '-' is also
+  // a uid_value symbol the lexer keeps the contiguous run together as a single
+  // WORD. Distinguishing a negative device number is left to the parser.
+  kmp_vector<token> tokens = tokenize("-5");
+
+  ASSERT_EQ(tokens.size(), 1u);
+  expect_token(tokens[0], token_kind::WORD, "-5");
+}
+
+TEST(LexerTest, WhitespaceBreaksWords) {
+  // A uid_value may not contain whitespace, so spaces split the run.
+  kmp_vector<token> tokens = tokenize("device - 0");
+
+  ASSERT_EQ(tokens.size(), 3u);
+  expect_token(tokens[0], token_kind::WORD, "device");
+  expect_token(tokens[1], token_kind::WORD, "-");
+  expect_token(tokens[2], token_kind::WORD, "0");
+}
+
+//===----------------------------------------------------------------------===//
+// Full trait expressions
+//===----------------------------------------------------------------------===//
+
+TEST(LexerTest, UidTrait) {
+  kmp_vector<token> tokens = tokenize("uid(device-0)");
+
+  ASSERT_EQ(tokens.size(), 4u);
+  expect_token(tokens[0], token_kind::WORD, "uid");
+  expect_token(tokens[1], token_kind::L_PAREN, "(");
+  expect_token(tokens[2], token_kind::WORD, "device-0");
+  expect_token(tokens[3], token_kind::R_PAREN, ")");
+}
+
+TEST(LexerTest, NegatedUidTrait) {
+  kmp_vector<token> tokens = tokenize("!uid(a)");
+
+  ASSERT_EQ(tokens.size(), 5u);
+  expect_token(tokens[0], token_kind::NOT, "!");
+  expect_token(tokens[1], token_kind::WORD, "uid");
+  expect_token(tokens[2], token_kind::L_PAREN, "(");
+  expect_token(tokens[3], token_kind::WORD, "a");
+  expect_token(tokens[4], token_kind::R_PAREN, ")");
+}
+
+TEST(LexerTest, AndGroup) {
+  kmp_vector<token> tokens = tokenize("(uid(a) && uid(b))");
+
+  ASSERT_EQ(tokens.size(), 11u);
+  expect_token(tokens[0], token_kind::L_PAREN, "(");
+  expect_token(tokens[1], token_kind::WORD, "uid");
+  expect_token(tokens[2], token_kind::L_PAREN, "(");
+  expect_token(tokens[3], token_kind::WORD, "a");
+  expect_token(tokens[4], token_kind::R_PAREN, ")");
+  expect_token(tokens[5], token_kind::AND, "&&");
+  expect_token(tokens[6], token_kind::WORD, "uid");
+  expect_token(tokens[7], token_kind::L_PAREN, "(");
+  expect_token(tokens[8], token_kind::WORD, "b");
+  expect_token(tokens[9], token_kind::R_PAREN, ")");
+  expect_token(tokens[10], token_kind::R_PAREN, ")");
+}
+
+TEST(LexerTest, OrGroup) {
+  kmp_vector<token> tokens = tokenize("uid(a)||uid(b)");
+
+  ASSERT_EQ(tokens.size(), 9u);
+  expect_token(tokens[0], token_kind::WORD, "uid");
+  expect_token(tokens[1], token_kind::L_PAREN, "(");
+  expect_token(tokens[2], token_kind::WORD, "a");
+  expect_token(tokens[3], token_kind::R_PAREN, ")");
+  expect_token(tokens[4], token_kind::OR, "||");
+  expect_token(tokens[5], token_kind::WORD, "uid");
+  expect_token(tokens[6], token_kind::L_PAREN, "(");
+  expect_token(tokens[7], token_kind::WORD, "b");
+  expect_token(tokens[8], token_kind::R_PAREN, ")");
+}
+
+TEST(LexerTest, WildcardWithLiterals) {
+  kmp_vector<token> tokens = tokenize("1, *, 3");
+
+  ASSERT_EQ(tokens.size(), 5u);
+  expect_token(tokens[0], token_kind::WORD, "1");
+  expect_token(tokens[1], token_kind::COMMA, ",");
+  expect_token(tokens[2], token_kind::STAR, "*");
+  expect_token(tokens[3], token_kind::COMMA, ",");
+  expect_token(tokens[4], token_kind::WORD, "3");
+}
+
+TEST(LexerTest, IndexExpression) {
+  kmp_vector<token> tokens = tokenize("[1:2:3]");
+
+  ASSERT_EQ(tokens.size(), 7u);
+  expect_token(tokens[0], token_kind::L_BRACKET, "[");
+  expect_token(tokens[1], token_kind::WORD, "1");
+  expect_token(tokens[2], token_kind::COLON, ":");
+  expect_token(tokens[3], token_kind::WORD, "2");
+  expect_token(tokens[4], token_kind::COLON, ":");
+  expect_token(tokens[5], token_kind::WORD, "3");
+  expect_token(tokens[6], token_kind::R_BRACKET, "]");
+}
+
+//===----------------------------------------------------------------------===//
+// Whitespace handling
+//===----------------------------------------------------------------------===//
+
+TEST(LexerTest, LeadingAndTrailingWhitespace) {
+  kmp_vector<token> tokens = tokenize("   uid ( a )   ");
+
+  ASSERT_EQ(tokens.size(), 4u);
+  expect_token(tokens[0], token_kind::WORD, "uid");
+  expect_token(tokens[1], token_kind::L_PAREN, "(");
+  expect_token(tokens[2], token_kind::WORD, "a");
+  expect_token(tokens[3], token_kind::R_PAREN, ")");
+}
+
+TEST(LexerTest, WhitespaceAroundOperators) {
+  kmp_vector<token> tokens = tokenize("uid(a)   &&   uid(b)");
+
+  ASSERT_EQ(tokens.size(), 9u);
+  expect_token(tokens[4], token_kind::AND, "&&");
+}
+
+//===----------------------------------------------------------------------===//
+// Unknown / error characters
+//===----------------------------------------------------------------------===//
+
+TEST(LexerTest, UnknownCharacter) {
+  kmp_vector<token> tokens = tokenize("@");
+
+  ASSERT_EQ(tokens.size(), 1u);
+  expect_token(tokens[0], token_kind::UNKNOWN, "@");
+}
+
+TEST(LexerTest, LoneAmpersandIsUnknown) {
+  kmp_vector<token> tokens = tokenize("&");
+
+  ASSERT_EQ(tokens.size(), 1u);
+  expect_token(tokens[0], token_kind::UNKNOWN, "&");
+}
+
+TEST(LexerTest, LonePipeIsUnknown) {
+  kmp_vector<token> tokens = tokenize("|");
+
+  ASSERT_EQ(tokens.size(), 1u);
+  expect_token(tokens[0], token_kind::UNKNOWN, "|");
+}
+
+TEST(LexerTest, TripleAmpersand) {
+  // "&&&" is "&&" followed by a lone (unknown) '&'.
+  kmp_vector<token> tokens = tokenize("&&&");
+
+  ASSERT_EQ(tokens.size(), 2u);
+  expect_token(tokens[0], token_kind::AND, "&&");
+  expect_token(tokens[1], token_kind::UNKNOWN, "&");
+}
+
+//===----------------------------------------------------------------------===//
+// peek()
+//===----------------------------------------------------------------------===//
+
+TEST(LexerTest, PeekDoesNotAdvance) {
+  kmp_lexer lex{kmp_str_ref("uid(a)")};
+
+  // Peek repeatedly returns the same token without consuming it.
+  EXPECT_EQ(lex.peek().kind, token_kind::WORD);
+  EXPECT_TRUE(text_is(lex.peek(), "uid"));
+  EXPECT_EQ(lex.peek().kind, token_kind::WORD);
+
+  // next() returns the peeked token, then advances.
+  token tok = lex.next();
+  expect_token(tok, token_kind::WORD, "uid");
+  EXPECT_EQ(lex.peek().kind, token_kind::L_PAREN);
+  expect_token(lex.next(), token_kind::L_PAREN, "(");
+}
+
+TEST(LexerTest, PeekAtEnd) {
+  kmp_lexer lex{kmp_str_ref("")};
+
+  EXPECT_EQ(lex.peek().kind, token_kind::END);
+  EXPECT_EQ(lex.next().kind, token_kind::END);
+  EXPECT_EQ(lex.peek().kind, token_kind::END);
+}
+
+} // namespace
diff --git a/openmp/runtime/unittests/Traits/TestOMPTraitParser.cpp b/openmp/runtime/unittests/Traits/TestOMPTraitParser.cpp
index 08f19ad72e322..59e2c4b07318a 100644
--- a/openmp/runtime/unittests/Traits/TestOMPTraitParser.cpp
+++ b/openmp/runtime/unittests/Traits/TestOMPTraitParser.cpp
@@ -870,6 +870,30 @@ TEST_F(ParserTest, NestedMixedWithSpaces) {
   check_result<false, 3>(context, result);
 }
 
+TEST_F(ParserTest, DeepNestingWithinLimit) {
+  // A moderate nesting depth (well under the recursion limit) parses fine and
+  // is transparent to matching.
+  constexpr int depth = 20;
+  const char *inner = "uid(device-0)";
+  char spec[64];
+  char *p = spec;
+  for (int i = 0; i < depth; ++i)
+    *p++ = '(';
+  for (const char *s = inner; *s; ++s)
+    *p++ = *s;
+  for (int i = 0; i < depth; ++i)
+    *p++ = ')';
+  *p = '\0';
+  parse(spec, "deep_nesting_within_limit");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 0>(context, result);
+  check_result<false, 1, 2, 3>(context, result);
+}
+
 //===----------------------------------------------------------------------===//
 // Empty Input
 //===----------------------------------------------------------------------===//
@@ -931,17 +955,34 @@ TEST_F(ParserTest, MixedOrAndSameLevel) {
 }
 
 TEST_F(ParserTest, InvalidUID) {
-  // Empty UID is not allowed
+  // Empty UID is not allowed; the offending token (the closing ')') is reported
+  // as the invalid trait value.
   ASSERT_DEATH(parse("uid()", "invalid_uid"),
                "OMP: Error #[0-9]+: trait parser while parsing invalid_uid: "
-               "invalid uid \\(\\)");
+               "invalid value for trait 'uid' \\(\\)\\)");
+}
+
+TEST_F(ParserTest, MissingOpenParenAfterTrait) {
+  // A trait name must be followed by "(".
+  ASSERT_DEATH(parse("uid", "missing_open_paren"),
+               "OMP: Error #[0-9]+: trait parser while parsing "
+               "missing_open_paren: error: expected '\\(' after trait name");
+}
+
+TEST_F(ParserTest, MissingCloseParenAfterTraitValue) {
+  // A trait value must be followed by ")".
+  ASSERT_DEATH(parse("uid(a", "missing_close_paren"),
+               "OMP: Error #[0-9]+: trait parser while parsing "
+               "missing_close_paren: error: expected '\\)' after trait value");
 }
 
 TEST_F(ParserTest, UnclosedParenthesis) {
+  // Once "(" is consumed we are committed to a group, so a missing ")" is a
+  // hard error rather than a recoverable parse failure.
   ASSERT_DEATH(
       parse("(uid(a)", "unclosed_parenthesis"),
       "OMP: Error #[0-9]+: trait parser while parsing unclosed_parenthesis: "
-      "failed to parse trait specification \\(\\(uid\\(a\\)\\)");
+      "error: expected '\\)' after trait expression group");
 }
 
 TEST_F(ParserTest, UnmatchedClosingParenthesis) {
@@ -952,17 +993,46 @@ TEST_F(ParserTest, UnmatchedClosingParenthesis) {
 }
 
 TEST_F(ParserTest, EmptyParentheses) {
+  // "(" commits to a group, so an empty group is a hard error.
   ASSERT_DEATH(
       parse("()", "empty_parentheses"),
       "OMP: Error #[0-9]+: trait parser while parsing empty_parentheses: "
-      "failed to parse trait specification \\(\\(\\)\\)");
+      "error: expected trait expression after '\\('");
 }
 
 TEST_F(ParserTest, TrailingOperator) {
+  // A consumed "&&"/"||" commits to another operand, so its absence is a hard
+  // error.
   ASSERT_DEATH(
       parse("uid(a) &&", "trailing_operator"),
       "OMP: Error #[0-9]+: trait parser while parsing trailing_operator: "
-      "failed to parse trait specification \\(uid\\(a\\) &&\\)");
+      "error: expected trait expression after operator");
+}
+
+TEST_F(ParserTest, NegationWithoutTrait) {
+  // A consumed "!" commits to a trait expression, so its absence is a hard
+  // error rather than a recoverable parse failure.
+  ASSERT_DEATH(
+      parse("!", "negation_without_trait"),
+      "OMP: Error #[0-9]+: trait parser while parsing negation_without_trait: "
+      "error: expected trait expression after '!'");
+}
+
+TEST_F(ParserTest, DeviceNumberOutOfRange) {
+  // A word shaped like a device number is treated as a device number, so one
+  // that does not fit an int is a hard error.
+  ASSERT_DEATH(parse("99999999999", "device_number_out_of_range"),
+               "OMP: Error #[0-9]+: trait parser while parsing "
+               "device_number_out_of_range: error: device number out of range");
+}
+
+TEST_F(ParserTest, NegativeDeviceNumber) {
+  // "-5" is lexed as a single WORD; the parser recognizes its numeric shape and
+  // rejects it as an out-of-range device number rather than treating it as a
+  // name/uid_value.
+  ASSERT_DEATH(parse("-5", "negative_device_number"),
+               "OMP: Error #[0-9]+: trait parser while parsing "
+               "negative_device_number: error: device number out of range");
 }
 
 TEST_F(ParserTest, LeadingOperator) {
@@ -978,4 +1048,25 @@ TEST_F(ParserTest, DoubleComma) {
                "failed to parse trait specification \\(,,uid\\(b\\)\\)");
 }
 
+TEST_F(ParserTest, UnrecognizedTrait) {
+  // "uid" is currently the only trait name; any other word is not a trait, so
+  // the clause cannot be parsed.
+  ASSERT_DEATH(
+      parse("foo", "unrecognized_trait"),
+      "OMP: Error #[0-9]+: trait parser while parsing unrecognized_trait: "
+      "failed to parse trait specification \\(foo\\)");
+}
+
+TEST_F(ParserTest, MaxRecursionExceeded) {
+  // Nesting parentheses deeper than the recursion limit is a hard error. The
+  // guard trips before the input is exhausted, so unbalanced "(" are enough.
+  char spec[101];
+  for (int i = 0; i < 100; ++i)
+    spec[i] = '(';
+  spec[100] = '\0';
+  ASSERT_DEATH(parse(spec, "max_recursion"),
+               "OMP: Error #[0-9]+: trait parser while parsing max_recursion: "
+               "max recursion depth \\(64\\) exceeded");
+}
+
 } // namespace

>From c872e47914791a4813b4bc4ba6ed53a920b52758 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Wed, 12 Aug 2026 11:02:33 -0500
Subject: [PATCH 5/5] use const pointer directly

---
 openmp/runtime/src/kmp_traits.h | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/openmp/runtime/src/kmp_traits.h b/openmp/runtime/src/kmp_traits.h
index 9d70a44277078..ed19ceccd8846 100644
--- a/openmp/runtime/src/kmp_traits.h
+++ b/openmp/runtime/src/kmp_traits.h
@@ -330,10 +330,9 @@ class kmp_trait_expr_group final : public kmp_trait_expr {
     const kmp_trait_expr_group &other_group =
         static_cast<const kmp_trait_expr_group &>(other);
     return type == other_group.type &&
-           exprs.is_set_equal(other_group.exprs, [](kmp_trait_expr *const &a,
-                                                    kmp_trait_expr *const &b) {
-             return *a == *b;
-           });
+           exprs.is_set_equal(other_group.exprs,
+                              [](const kmp_trait_expr *a,
+                                 const kmp_trait_expr *b) { return *a == *b; });
   }
 };
 


From openmp-commits at lists.llvm.org  Wed Aug 12 09:06:05 2026
From: openmp-commits at lists.llvm.org (Robert Imschweiler via Openmp-commits)
Date: Wed, 12 Aug 2026 09:06:05 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [libomp] OpenMP 6.0: Add device trait
	parser (PR #176164)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/176164@github.com>
Message-ID: <6a7c99ed.5fe1956f.37c983.5f44@mx.google.com>


https://github.com/ro-i auto_merge_enabled https://github.com/llvm/llvm-project/pull/176164

From openmp-commits at lists.llvm.org  Wed Aug 12 09:08:08 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Wed, 12 Aug 2026 09:08:08 -0700 (PDT)
Subject: [Openmp-commits] [openmp] e4bb0ac - [libomp] OpenMP 6.0: Add device
	trait parser (#176164)
Message-ID: <6a7c9a68.9edb5c9d.324415.2de5@mx.google.com>


Author: Robert Imschweiler
Date: 2026-08-12T16:08:00Z
New Revision: e4bb0acccb4fd033c6ac7a4e0200b62ed101aeab

URL: https://github.com/llvm/llvm-project/commit/e4bb0acccb4fd033c6ac7a4e0200b62ed101aeab
DIFF: https://github.com/llvm/llvm-project/commit/e4bb0acccb4fd033c6ac7a4e0200b62ed101aeab.diff

LOG: [libomp] OpenMP 6.0: Add device trait parser (#176164)

OpenMP 6.0 introduced a device trait specification language for the
environment variables OMP_AVAILABLE_DEVICES (4.3.7) and
OMP_DEFAULT_DEVICE (4.3.8).

This commit defines a grammar for that language and implements a parser
for a large part of this grammar.

Added: 
    openmp/runtime/src/kmp_traits.cpp
    openmp/runtime/src/kmp_traits.h
    openmp/runtime/unittests/Traits/CMakeLists.txt
    openmp/runtime/unittests/Traits/MockOMP.cpp
    openmp/runtime/unittests/Traits/TestOMPTraitLexer.cpp
    openmp/runtime/unittests/Traits/TestOMPTraitParser.cpp
    openmp/runtime/unittests/Traits/TestOMPTraits.cpp

Modified: 
    openmp/runtime/src/CMakeLists.txt
    openmp/runtime/src/i18n/en_US.txt
    openmp/runtime/unittests/CMakeLists.txt

Removed: 
    


################################################################################
diff  --git a/openmp/runtime/src/CMakeLists.txt b/openmp/runtime/src/CMakeLists.txt
index 42729674481e5..69def2b03b1b4 100644
--- a/openmp/runtime/src/CMakeLists.txt
+++ b/openmp/runtime/src/CMakeLists.txt
@@ -98,6 +98,7 @@ else()
     kmp_str.cpp
     kmp_tasking.cpp
     kmp_threadprivate.cpp
+    kmp_traits.cpp
     kmp_utility.cpp
     kmp_barrier.cpp
     kmp_wait_release.cpp

diff  --git a/openmp/runtime/src/i18n/en_US.txt b/openmp/runtime/src/i18n/en_US.txt
index b3699df6db65d..3b7187edaeb13 100644
--- a/openmp/runtime/src/i18n/en_US.txt
+++ b/openmp/runtime/src/i18n/en_US.txt
@@ -362,6 +362,10 @@ TopologyGeneric              "%1$s: %2$s (%3$d total cores)"
 AffGranularityBad            "%1$s: granularity setting: %2$s does not exist in topology.  Using granularity=%3$s instead."
 TopologyHybrid               "%1$s: hybrid core type detected: %2$d %3$s cores."
 TopologyHybridCoreEff        "%1$s:   %2$d with core efficiency %3$d."
+TraitParserError             "trait parser while parsing %1$s: error: %2$s"
+TraitParserInvalidTraitValue "trait parser while parsing %1$s: invalid value for trait '%2$s' (%3$s)"
+TraitParserMaxRecursion      "trait parser while parsing %1$s: max recursion depth (%2$d) exceeded"
+TraitParserFailed            "trait parser while parsing %1$s: failed to parse trait specification (%2$s)"
 
 # --- OpenMP errors detected at runtime ---
 #

diff  --git a/openmp/runtime/src/kmp_traits.cpp b/openmp/runtime/src/kmp_traits.cpp
new file mode 100644
index 0000000000000..6070f388ec1d2
--- /dev/null
+++ b/openmp/runtime/src/kmp_traits.cpp
@@ -0,0 +1,423 @@
+/*
+ * kmp_traits.cpp -- Handle OpenMP context traits
+ *
+ * OpenMP 6.0 specifies the following trait sets:
+ * - construct
+ * - device
+ * - target device
+ * - implementation
+ * - extension
+ * - dynamic
+ * Currently, the implementation in this file supports traits from the (target)
+ * device and implementation trait sets that are relevant for implementing the
+ * OMP_DEFAULT_DEVICE and OMP_AVAILABLE_DEVICES environment variables.
+ */
+
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "kmp_traits.h"
+#include "kmp_i18n.h"
+
+using namespace kmp_traits;
+
+// OpenMP trait grammar (in EBNF), currently used for parsing the
+// OMP_DEFAULT_DEVICE/OMP_AVAILABLE_DEVICES environment variables
+//
+// Notes about the grammar:
+// - Device traits are going to be translated into device numbers (aka integers)
+// later in the runtime. The parser handles device numbers as device traits that
+// have already been translated.
+// - "*" is also not a trait, strictly speaking. But it's also supported by the
+// parser and converted into a "match any" wildcard trait.
+// - OpenMP 6.0 explicitly excludes "&&" and "||" from appearing in the same
+// grouping level.
+// - This grammar currently only supports plain integers for array subsripts /
+// sections, no expressions.
+// - TODO:
+//   - Add support for more traits
+//
+// TODOs regarding the implementation (not the grammar):
+// - Implement array subscript/section parsing
+// - Implement grammar TODOs after they have been incorporated into the grammar
+//
+// list = [clause {',' clause}]
+// clause =
+//       device_number
+//     | "*" [index_expr]
+//     | trait_expr_group
+//     | trait_expr index_expr
+// device_number = ["-"] integer0
+// trait_expr_group =
+//       trait_expr
+//     | trait_expr {"&&" trait_expr}
+//     | trait_expr {"||" trait_expr}
+// trait_expr = ["!"] (trait | trait_expr_group_paren)
+// trait_expr_group_paren = "(" trait_expr_group ")"
+// trait =
+//       "uid" "(" uid_value ")"
+// uid_value = (letter | digit0 | symbol) {letter | digit0 | symbol}
+//
+// index_expr = "[" integer0 "]" | "[" array_section "]"
+// array_section =
+//       lower_bound ":" length ":" stride
+//     | lower_bound ":" length ":"
+//     | lower_bound ":" length
+//     | lower_bound "::" stride
+//     | lower_bound "::"
+//     | lower_bound ":"
+//     | ":" length ":" stride
+//     | ":" length ":"
+//     | ":" length
+//     | "::" stride
+//     | "::"
+//     | ":"
+// lower_bound = integer0
+// length = integer0
+// stride = integer
+//
+// integer0 = 0 | integer
+// integer = digit {digit0}
+//
+// letter =
+//       "A" | "B" | "C" | "D" | "E" | "F" | "G" | "H" | "I" | "J" | "K" | "L"
+//     | "M" | "N" | "O" | "P" | "Q" | "R" | "S" | "T" | "U" | "V" | "W" | "X"
+//     | "Y" | "Z" | "a" | "b" | "c" | "d" | "e" | "f" | "g" | "h" | "i" | "j"
+//     | "k" | "l" | "m" | "n" | "o" | "p" | "q" | "r" | "s" | "t" | "u" | "v"
+//     | "w" | "x" | "y" | "z"
+// digit0 = "0" | digit
+// digit = "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9"
+// symbol = "-" | "_"
+
+// A character that can appear in a word (uid keyword / uid_value / integer),
+// i.e. a letter, a digit, or one of the symbols "-" / "_".
+static bool is_word_char(char c) {
+  return isalnum(static_cast<unsigned char>(c)) || c == '-' || c == '_';
+}
+
+static bool is_digit(char c) {
+  return static_cast<bool>(isdigit(static_cast<unsigned char>(c)));
+}
+
+namespace lexer {
+
+token kmp_lexer::lex() {
+  scan.skip_space();
+
+  const char *start = scan.begin();
+  if (scan.empty())
+    return {token_kind::END, kmp_str_ref(start, 0)};
+
+  // Two-character operators.
+  if (scan.consume_front("&&"))
+    return {token_kind::AND, kmp_str_ref(start, 2)};
+  if (scan.consume_front("||"))
+    return {token_kind::OR, kmp_str_ref(start, 2)};
+
+  // Word characters form a single WORD.
+  kmp_str_ref word = scan.take_while(is_word_char);
+  if (!word.empty()) {
+    scan.drop_front(word.length());
+    return {token_kind::WORD, word};
+  }
+
+  // Single-character tokens.
+  token_kind kind;
+  switch (*start) {
+  case ',':
+    kind = token_kind::COMMA;
+    break;
+  case '*':
+    kind = token_kind::STAR;
+    break;
+  case '!':
+    kind = token_kind::NOT;
+    break;
+  case '(':
+    kind = token_kind::L_PAREN;
+    break;
+  case ')':
+    kind = token_kind::R_PAREN;
+    break;
+  case '[':
+    kind = token_kind::L_BRACKET;
+    break;
+  case ']':
+    kind = token_kind::R_BRACKET;
+    break;
+  case ':':
+    kind = token_kind::COLON;
+    break;
+  default:
+    kind = token_kind::UNKNOWN;
+    break;
+  }
+  scan.drop_front(1);
+  return {kind, kmp_str_ref(start, 1)};
+}
+
+} // namespace lexer
+
+namespace parser {
+
+constexpr int MAX_RECURSION_DEPTH = 64;
+
+using namespace kmp_traits;
+using namespace lexer;
+
+// Check whether a token is a WORD whose text equals the given keyword.
+static bool word_is(const token &tok, kmp_str_ref keyword) {
+  kmp_str_ref text = tok.text;
+  return tok.kind == token_kind::WORD && text.consume_front(keyword) &&
+         text.empty();
+}
+
+// Check whether a token is a WORD that has the shape of a device number, i.e.
+// device_number = ["-"] integer0 (an optional "-" followed by digits only).
+// The lexer emits every word-character run as a WORD, so it is the parser that
+// tells a device number apart from a name or uid_value (which may also contain
+// "-" and digits).
+static bool word_is_number(const token &tok) {
+  if (tok.kind != token_kind::WORD)
+    return false;
+  kmp_str_ref text = tok.text;
+  text.consume_front("-");
+  return !text.empty() && text.find_if_not(is_digit) == kmp_str_ref::npos;
+}
+
+// uid_value = (letter | digit0 | symbol) {letter | digit0 | symbol}
+// Consumes and returns a uid value.
+// An invalid uid value is a hard error.
+static kmp_str_ref consume_uid_value(kmp_lexer &lex, const char *dbg_name) {
+  const token &tok = lex.peek();
+  if (tok.kind != token_kind::WORD)
+    KMP_FATAL(TraitParserInvalidTraitValue, dbg_name, "uid", tok.text.copy());
+  kmp_str_ref uid = tok.text;
+  lex.next(); // consume the uid_value
+  return uid;
+}
+
+// trait = "uid" "(" uid_value ")"
+// (more traits will be added as needed in the future)
+// Returns false without consuming anything if the next token is not a
+// recognized trait name.
+// Once a trait name is consumed we are committed to a trait expression, so a
+// missing "(" or ")" or an invalid trait value is a hard error.
+static bool consume_trait(kmp_trait_expr_single &expr, kmp_lexer &lex,
+                          const char *dbg_name) {
+  if (!word_is(lex.peek(), "uid"))
+    return false;
+  // Add more traits as needed in the future.
+
+  lex.next(); // consume trait name
+  if (lex.peek().kind != token_kind::L_PAREN)
+    KMP_FATAL(TraitParserError, dbg_name, "expected '(' after trait name");
+  lex.next(); // consume "("
+  kmp_str_ref uid = consume_uid_value(lex, dbg_name);
+  if (lex.peek().kind != token_kind::R_PAREN)
+    KMP_FATAL(TraitParserError, dbg_name, "expected ')' after trait value");
+  lex.next(); // consume ")"
+  expr.set_trait(new kmp_uid_trait(uid));
+  return true;
+}
+
+// forward declaration
+static bool consume_trait_expr_group(kmp_trait_expr_group &group,
+                                     kmp_lexer &lex, int max_recursion,
+                                     const char *dbg_name);
+
+// trait_expr_group_paren = "(" trait_expr_group ")"
+// Returns false without consuming anything if the next token is not "(", so the
+// caller can try other alternatives.
+// Once "(" is consumed we are committed to a parenthesized group, so a missing
+// group or ")" is a hard error.
+static bool consume_trait_expr_group_paren(kmp_trait_expr_group &group,
+                                           bool negated, kmp_lexer &lex,
+                                           int max_recursion,
+                                           const char *dbg_name) {
+  if (lex.peek().kind != token_kind::L_PAREN)
+    return false;
+  group.set_negated(negated);
+  lex.next(); // consume "("
+  if (!consume_trait_expr_group(group, lex, max_recursion, dbg_name))
+    KMP_FATAL(TraitParserError, dbg_name,
+              "expected trait expression after '('");
+  if (lex.peek().kind != token_kind::R_PAREN)
+    KMP_FATAL(TraitParserError, dbg_name,
+              "expected ')' after trait expression group");
+  lex.next(); // consume ")"
+  return true;
+}
+
+// trait_expr = ["!"] (trait | trait_expr_group_paren)
+// Returns false without consuming anything if neither a parenthesized group nor
+// a single trait can be consumed.
+// Once an optional leading "!" has been consumed we are committed to a trait
+// expression, so a missing trait or parenthesized group is a hard error.
+static bool consume_trait_expr(kmp_trait_expr *&expr, kmp_lexer &lex,
+                               int max_recursion, const char *dbg_name) {
+  if (max_recursion-- <= 0)
+    KMP_FATAL(TraitParserMaxRecursion, dbg_name, MAX_RECURSION_DEPTH);
+
+  // Consume the optional leading "!"; it applies to whatever follows.
+  bool negated = lex.peek().kind == token_kind::NOT;
+  if (negated)
+    lex.next();
+
+  // Try a parenthesized group (starts with "(") ...
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+  if (consume_trait_expr_group_paren(*group, negated, lex, max_recursion,
+                                     dbg_name)) {
+    expr = group;
+    return true;
+  }
+  delete group;
+
+  // ... otherwise it must be a single trait.
+  kmp_trait_expr_single *single = new kmp_trait_expr_single();
+  single->set_negated(negated);
+  if (consume_trait(*single, lex, dbg_name)) {
+    expr = single;
+    return true;
+  }
+  delete single;
+
+  // A leading "!" has already committed us to a trait expression, so its
+  // absence is an error; without it, nothing was consumed and the caller can
+  // recover/report.
+  if (negated)
+    KMP_FATAL(TraitParserError, dbg_name,
+              "expected trait expression after '!'");
+  return false;
+}
+
+// trait_expr_group =
+//       trait_expr
+//     | trait_expr {"&&" trait_expr}
+//     | trait_expr {"||" trait_expr}
+// Returns false without consuming anything if no trait expression can be
+// consumed.
+// Any other missing or invalid tokens are a hard error.
+static bool consume_trait_expr_group(kmp_trait_expr_group &group,
+                                     kmp_lexer &lex, int max_recursion,
+                                     const char *dbg_name) {
+  if (max_recursion-- <= 0)
+    KMP_FATAL(TraitParserMaxRecursion, dbg_name, MAX_RECURSION_DEPTH);
+
+  kmp_trait_expr *expr = nullptr;
+  if (!consume_trait_expr(expr, lex, max_recursion, dbg_name))
+    return false;
+  group.add_expr(expr);
+
+  token_kind op;
+  if (lex.peek().kind == token_kind::OR) {
+    group.set_group_type(kmp_trait_expr_group::OR);
+    op = token_kind::OR;
+  } else if (lex.peek().kind == token_kind::AND) {
+    group.set_group_type(kmp_trait_expr_group::AND);
+    op = token_kind::AND;
+  } else {
+    return true; // single trait expression, no operator
+  }
+  lex.next(); // consume the operator
+
+  // Having consumed an operator, we are committed: at least one more trait
+  // expression must follow, so its absence is a hard error.
+  while (true) {
+    if (!consume_trait_expr(expr, lex, max_recursion, dbg_name))
+      KMP_FATAL(TraitParserError, dbg_name,
+                "expected trait expression after operator");
+    group.add_expr(expr);
+    if (lex.peek().kind != op)
+      break;
+    lex.next(); // consume the operator
+  }
+
+  return true;
+}
+
+// device_number = ["-"] integer0
+// Returns false without consuming anything if the next token is not shaped like
+// a device number. A WORD that has the shape of a device number is treated as a
+// device number, so once it is recognized it is committed: a value that is not
+// a valid device index (negative or too large to fit an int) is a hard error.
+static bool consume_device_number(kmp_trait_clause &clause, kmp_lexer &lex,
+                                  const char *dbg_name) {
+  if (!word_is_number(lex.peek()))
+    return false;
+  kmp_str_ref number = lex.peek().text;
+  int value;
+  if (!number.consume_integer(value))
+    KMP_FATAL(TraitParserError, dbg_name, "device number out of range");
+  lex.next();
+  clause.set_expr(new kmp_literal_trait(value));
+  return true;
+}
+
+// clause =
+//       device_number
+//     | "*" [index_expr]
+//     | trait_expr_group
+//     | trait_expr index_expr
+// Returns false without consuming anything if no clause can be consumed.
+// Any other missing or invalid tokens are a hard error.
+static bool consume_clause(kmp_trait_clause &clause, kmp_lexer &lex,
+                           const char *dbg_name) {
+  // Parse wildcard "trait"
+  if (lex.peek().kind == token_kind::STAR) {
+    lex.next();
+    clause.set_expr(new kmp_wildcard_trait());
+    return true;
+  }
+
+  // Parse a literal device number. A WORD that is not shaped like a number is
+  // not a device number and starts a trait expression group instead.
+  if (consume_device_number(clause, lex, dbg_name))
+    return true;
+
+  // Parse a trait expression group
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+  if (consume_trait_expr_group(*group, lex, MAX_RECURSION_DEPTH, dbg_name)) {
+    clause.set_expr(group);
+    return true;
+  }
+  delete group;
+
+  return false;
+}
+
+// list = [clause {',' clause}]
+static void consume_list(kmp_trait_context &context, kmp_lexer &lex,
+                         const char *dbg_name) {
+  kmp_str_ref lex_pos = lex.remaining();
+
+  while (lex.peek().kind != token_kind::END) {
+    kmp_trait_clause *clause = new kmp_trait_clause();
+    if (!consume_clause(*clause, lex, dbg_name)) {
+      delete clause;
+      KMP_FATAL(TraitParserFailed, dbg_name, lex_pos.copy());
+    }
+    context.add_clause(clause);
+
+    lex_pos = lex.remaining();
+    if (lex.peek().kind == token_kind::COMMA)
+      lex.next();
+    else if (lex.peek().kind != token_kind::END)
+      KMP_FATAL(TraitParserFailed, dbg_name, lex_pos.copy());
+  }
+}
+
+} // namespace parser
+
+kmp_trait_context *kmp_trait_context::parse_from_spec(kmp_str_ref spec,
+                                                      const char *dbg_name) {
+  kmp_trait_context *context = new kmp_trait_context();
+  lexer::kmp_lexer lex(spec);
+  parser::consume_list(*context, lex, dbg_name);
+  return context;
+}

diff  --git a/openmp/runtime/src/kmp_traits.h b/openmp/runtime/src/kmp_traits.h
new file mode 100644
index 0000000000000..ed19ceccd8846
--- /dev/null
+++ b/openmp/runtime/src/kmp_traits.h
@@ -0,0 +1,497 @@
+//===----------- Traits.h - OpenMP context traits -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Implementation of OpenMP context traits.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OPENMP_TRAITS_H
+#define OPENMP_TRAITS_H
+
+#include "kmp.h"
+#include "kmp_adt.h"
+
+// Lexer for the OpenMP trait grammar (the grammar is documented in
+// kmp_traits.cpp).
+namespace lexer {
+
+enum class token_kind {
+  END, // end of the input
+  COMMA, // ,
+  STAR, // *
+  NOT, // !
+  L_PAREN, // (
+  R_PAREN, // )
+  L_BRACKET, // [
+  R_BRACKET, // ]
+  COLON, // :
+  AND, // &&
+  OR, // ||
+  WORD, // word characters
+  UNKNOWN, // an unrecognized character (e.g. a lone '&' or '|')
+};
+
+// A single token produced by the lexer. `text` is a non-owning reference into
+// the source string the token was lexed from.
+struct token {
+  token_kind kind = token_kind::UNKNOWN;
+  kmp_str_ref text = kmp_str_ref("");
+};
+
+class kmp_lexer final {
+  kmp_str_ref scan;
+  token lookahead{token_kind::END, kmp_str_ref("")};
+  bool has_lookahead = false;
+
+  // Lex the next token directly from the input, advancing past it.
+  token lex();
+
+public:
+  explicit kmp_lexer(kmp_str_ref source) : scan(source) {}
+
+  // Return the next token and advance past it.
+  token next() {
+    if (has_lookahead) {
+      has_lookahead = false;
+      return lookahead;
+    }
+    return lex();
+  }
+
+  // Return the next token without advancing.
+  token peek() {
+    if (!has_lookahead) {
+      lookahead = lex();
+      has_lookahead = true;
+    }
+    return lookahead;
+  }
+
+  // Return the yet-unconsumed portion of the source, with any whitespace before
+  // the next token skipped.
+  kmp_str_ref remaining() {
+    token t = peek();
+    return kmp_str_ref(t.text.begin(),
+                       static_cast<size_t>(scan.end() - t.text.begin()));
+  }
+};
+
+} // namespace lexer
+
+namespace kmp_traits {
+
+extern "C" int omp_get_num_devices();
+extern "C" const char *omp_get_uid_from_device(int device_num);
+
+class kmp_trait {
+protected:
+  enum trait_type { WILDCARD_T, LITERAL_T, UID_T };
+  trait_type _type;
+
+  kmp_trait(trait_type type) : _type(type) {}
+
+public:
+  virtual ~kmp_trait() = default;
+
+  kmp_trait(const kmp_trait &) = delete;
+  kmp_trait(kmp_trait &&) = delete;
+  kmp_trait &operator=(const kmp_trait &) = delete;
+  kmp_trait &operator=(kmp_trait &&) = delete;
+
+  virtual bool match(int device) const = 0;
+
+  // Use KMP_INTERNAL_MALLOC/KMP_INTERNAL_FREE for memory management.
+  void *operator new(size_t size) { return KMP_INTERNAL_MALLOC(size); }
+  void operator delete(void *ptr) { KMP_INTERNAL_FREE(ptr); }
+
+  virtual bool operator==(const kmp_trait &other) const {
+    return _type == other._type;
+  }
+};
+
+/// Represents a wildcard trait that matches any device.
+class kmp_wildcard_trait final : public kmp_trait {
+public:
+  kmp_wildcard_trait() : kmp_trait(WILDCARD_T) {}
+
+  bool match([[maybe_unused]] int device) const override { return true; }
+
+  bool operator==(const kmp_trait &other) const override {
+    return kmp_trait::operator==(other);
+  }
+};
+
+/// Represents a specific device number.
+class kmp_literal_trait final : public kmp_trait {
+  int device_num;
+
+public:
+  kmp_literal_trait(int device_num)
+      : kmp_trait(LITERAL_T), device_num(device_num) {
+    assert(device_num >= 0 && "Device number must be non-negative");
+  }
+
+  bool match(int device) const override { return device_num == device; }
+
+  bool operator==(const kmp_trait &other) const override {
+    return kmp_trait::operator==(other) &&
+           device_num ==
+               static_cast<const kmp_literal_trait &>(other).device_num;
+  }
+};
+
+/// Represents a specific UID.
+/// UID is deliberately not resolved at construction time since libomptarget
+/// might not be initialized yet. This is why we delay calls to
+/// omp_get_uid_from_device / omp_get_device_from_uid until the trait is
+/// evaluated.
+class kmp_uid_trait final : public kmp_trait {
+  char *uid;
+  // Can be used by unit tests to mock omp_get_uid_from_device.
+  const char *(*get_uid_from_device)(int device) = omp_get_uid_from_device;
+
+public:
+  kmp_uid_trait(kmp_str_ref uid) : kmp_trait(UID_T), uid(uid.copy()) {}
+
+  ~kmp_uid_trait() override {
+    if (uid)
+      KMP_INTERNAL_FREE(uid);
+  }
+
+  bool match(int device) const override {
+    const char *device_uid = get_uid_from_device(device);
+    if (!device_uid || !uid)
+      return false;
+    return strcmp(device_uid, uid) == 0;
+  }
+
+  // For testing purposes only: set the function that returns the UID from a
+  // device.
+  void set_uid_from_device(const char *(*uid_from_device)(int)) {
+    get_uid_from_device = uid_from_device;
+  }
+
+  bool operator==(const kmp_trait &other) const override {
+    if (!kmp_trait::operator==(other))
+      return false;
+    const char *other_uid = static_cast<const kmp_uid_trait &>(other).uid;
+    return uid && other_uid ? strcmp(uid, other_uid) == 0 : uid == other_uid;
+  }
+};
+
+/// Abstract class representing either a single trait expression or a collection
+/// of trait expressions that are ANDed or ORed together.
+class kmp_trait_expr {
+protected:
+  enum expr_type { SINGLE_T, GROUP_T };
+  expr_type _type;
+  // Determines if the expression is negated (true) or not (false).
+  bool negated = false;
+  // Can be used by unit tests to mock omp_get_num_devices.
+  int (*get_num_devices)() = omp_get_num_devices;
+
+  kmp_trait_expr(expr_type type) : _type(type) {}
+  kmp_trait_expr(expr_type type, bool negated)
+      : _type(type), negated(negated) {}
+
+  virtual bool match_impl(int device, int num_devices) const = 0;
+
+public:
+  virtual ~kmp_trait_expr() = default;
+
+  kmp_trait_expr(const kmp_trait_expr &) = delete;
+  kmp_trait_expr(kmp_trait_expr &&) = delete;
+  kmp_trait_expr &operator=(const kmp_trait_expr &) = delete;
+  kmp_trait_expr &operator=(kmp_trait_expr &&) = delete;
+
+  bool is_negated() const { return negated; }
+
+  // Check if the device matches the expression.
+  bool match(int device, int num_devices = -1) const {
+    if (num_devices == -1)
+      num_devices = get_num_devices();
+    if (device < 0 || device >= num_devices)
+      return false;
+    return match_impl(device, num_devices);
+  }
+
+  void set_negated(bool neg = true) { negated = neg; }
+
+  // For testing purposes only: set the function that returns the number of
+  // devices.
+  void set_num_devices(int (*num_devices)()) { get_num_devices = num_devices; }
+
+  // Use KMP_INTERNAL_MALLOC/KMP_INTERNAL_FREE for memory management.
+  void *operator new(size_t size) { return KMP_INTERNAL_MALLOC(size); }
+  void operator delete(void *ptr) { KMP_INTERNAL_FREE(ptr); }
+
+  virtual bool operator==(const kmp_trait_expr &other) const {
+    return _type == other._type && negated == other.negated;
+  }
+};
+
+/// Represents a single (possibly negated) trait.
+class kmp_trait_expr_single final : public kmp_trait_expr {
+  kmp_trait *trait = nullptr;
+
+protected:
+  bool match_impl(int device, [[maybe_unused]] int num_devices) const override {
+    assert(trait);
+    bool result = trait->match(device);
+    return negated ? !result : result;
+  }
+
+public:
+  kmp_trait_expr_single() : kmp_trait_expr(SINGLE_T) {}
+  kmp_trait_expr_single(bool negated) : kmp_trait_expr(SINGLE_T, negated) {}
+  kmp_trait_expr_single(kmp_trait *trait)
+      : kmp_trait_expr(SINGLE_T), trait(trait) {
+    assert(trait && "kmp_trait_expr_single requires a non-null trait");
+  }
+  ~kmp_trait_expr_single() override { delete trait; }
+
+  void set_trait(kmp_trait *new_trait) {
+    assert(new_trait);
+    if (trait)
+      delete trait;
+    trait = new_trait;
+  }
+
+  bool operator==(const kmp_trait_expr &other) const override {
+    if (!kmp_trait_expr::operator==(other))
+      return false;
+    const kmp_trait_expr_single &other_single =
+        static_cast<const kmp_trait_expr_single &>(other);
+    return trait && other_single.trait ? *trait == *other_single.trait
+                                       : trait == other_single.trait;
+  }
+};
+
+/// Represents a (possibly negated) collection of traits that are either ANDed
+/// or ORed together.
+class kmp_trait_expr_group final : public kmp_trait_expr {
+public:
+  enum group_type { AND, OR };
+
+private:
+  kmp_vector<kmp_trait_expr *> exprs;
+  // Determines if all traits have to match (true) or any of them (false).
+  group_type type = OR;
+
+protected:
+  bool match_impl(int device, int num_devices) const override {
+    size_t matched = 0;
+    for (const kmp_trait_expr *expr : exprs) {
+      if (expr->match(device, num_devices))
+        matched++;
+    }
+    // Note: AND evaluates to true for an empty group.
+    bool result = type == AND ? matched == exprs.size() : matched > 0;
+    return negated ? !result : result;
+  }
+
+public:
+  kmp_trait_expr_group() : kmp_trait_expr(GROUP_T) {}
+  kmp_trait_expr_group(bool negated) : kmp_trait_expr(GROUP_T, negated) {}
+  ~kmp_trait_expr_group() override {
+    for (kmp_trait_expr *expr : exprs)
+      delete expr;
+  }
+
+  void add_expr(kmp_trait *trait) {
+    assert(trait);
+    add_expr(new kmp_trait_expr_single(trait));
+  }
+  void add_expr(kmp_trait_expr *expr) {
+    assert(expr);
+    exprs.push_back(expr);
+    // Propagate get_num_devices to the expression.
+    expr->set_num_devices(get_num_devices);
+  }
+
+  group_type get_group_type() const { return type; }
+
+  void set_group_type(group_type new_type) { type = new_type; }
+
+  void set_num_devices(int (*num_devices)()) {
+    kmp_trait_expr::set_num_devices(num_devices);
+    for (kmp_trait_expr *expr : exprs)
+      expr->set_num_devices(num_devices);
+  }
+
+  bool operator==(const kmp_trait_expr &other) const override {
+    if (!kmp_trait_expr::operator==(other))
+      return false;
+    const kmp_trait_expr_group &other_group =
+        static_cast<const kmp_trait_expr_group &>(other);
+    return type == other_group.type &&
+           exprs.is_set_equal(other_group.exprs,
+                              [](const kmp_trait_expr *a,
+                                 const kmp_trait_expr *b) { return *a == *b; });
+  }
+};
+
+class kmp_trait_clause final {
+  kmp_trait_expr *expr = nullptr;
+
+public:
+  kmp_trait_clause() = default;
+  ~kmp_trait_clause() { delete expr; }
+
+  kmp_trait_clause(const kmp_trait_clause &) = delete;
+  kmp_trait_clause(kmp_trait_clause &&) = delete;
+  kmp_trait_clause &operator=(const kmp_trait_clause &) = delete;
+  kmp_trait_clause &operator=(kmp_trait_clause &&) = delete;
+
+  kmp_trait_expr *get_expr() { return expr; }
+
+  bool match(int device, int num_devices = -1) const {
+    assert(expr);
+    return expr->match(device, num_devices);
+  }
+
+  void set_expr(kmp_trait *trait) {
+    assert(trait);
+    if (expr)
+      delete expr;
+    expr = new kmp_trait_expr_single(trait);
+  }
+  void set_expr(kmp_trait_expr *new_expr) {
+    assert(new_expr);
+    if (expr)
+      delete expr;
+    expr = new_expr;
+  }
+
+  // Use KMP_INTERNAL_MALLOC/KMP_INTERNAL_FREE for memory management.
+  void *operator new(size_t size) { return KMP_INTERNAL_MALLOC(size); }
+  void operator delete(void *ptr) { KMP_INTERNAL_FREE(ptr); }
+
+  bool operator==(const kmp_trait_clause &other) const {
+    return expr && other.expr ? *expr == *other.expr : expr == other.expr;
+  }
+};
+
+} // namespace kmp_traits
+
+class kmp_trait_context final {
+  using kmp_trait_clause = kmp_traits::kmp_trait_clause;
+  using kmp_trait_expr = kmp_traits::kmp_trait_expr;
+
+  kmp_vector<kmp_trait_clause *> clauses;
+  // List of devices that have been evaluated.
+  kmp_vector<int> devices;
+  bool evaluated = false;
+  // Can be used by unit tests to mock omp_get_num_devices.
+  int (*get_num_devices)() = kmp_traits::omp_get_num_devices;
+
+  void _evaluate() {
+    devices.clear();
+    for (int d = 0; d < get_num_devices(); ++d) {
+      if (_match(d))
+        devices.push_back(d);
+    }
+    evaluated = true;
+  }
+
+  bool _match(int device) const {
+    if (device < 0 || device >= get_num_devices())
+      return false;
+    for (kmp_trait_clause *clause : clauses) {
+      if (clause->match(device))
+        return true;
+    }
+    return false;
+  }
+
+public:
+  kmp_trait_context() = default;
+  ~kmp_trait_context() {
+    for (kmp_trait_clause *clause : clauses)
+      delete clause;
+  }
+
+  kmp_trait_context(const kmp_trait_context &) = delete;
+  kmp_trait_context(kmp_trait_context &&) = delete;
+  kmp_trait_context &operator=(const kmp_trait_context &) = delete;
+  kmp_trait_context &operator=(kmp_trait_context &&) = delete;
+
+  // Parse a trait specification from a string.
+  // If dbg_name is provided, it will be used in error messages to identify the
+  // source of the trait specification.
+  static kmp_trait_context *parse_from_spec(kmp_str_ref spec,
+                                            const char *dbg_name = nullptr);
+
+  void add_clause(kmp_trait_clause *clause) {
+    assert(clause);
+    clauses.push_back(clause);
+    // Propagate get_num_devices to the clause.
+    if (kmp_trait_expr *expr = clause->get_expr())
+      expr->set_num_devices(get_num_devices);
+  }
+
+  // Returns the list of devices that match the trait specification represented
+  // by the context. The list contains devices numbers forming a set and sorted
+  // in ascending order.
+  // Note to future developers: if we want to add an option to force
+  // re-evaluation, we need to consider that the devices vector and thus the
+  // context iterators are invalidated.
+  const kmp_vector<int> &evaluate() {
+    trigger_evaluation();
+    return devices;
+  }
+
+  const kmp_vector<int> &evaluate() const {
+    assert(evaluated && "kmp_trait_context not evaluated");
+    return devices;
+  }
+
+  // Check if the device matches the trait specification represented by the
+  // context.
+  bool match(int device) { return evaluate().contains(device); }
+
+  bool match(int device) const {
+    assert(evaluated && "kmp_trait_context not evaluated");
+    return devices.contains(device);
+  }
+
+  // For testing purposes only: set the function that returns the number of
+  // devices.
+  void set_num_devices(int (*num_devices)()) {
+    get_num_devices = num_devices;
+    for (kmp_trait_clause *clause : clauses) {
+      if (kmp_trait_expr *expr = clause->get_expr())
+        expr->set_num_devices(num_devices);
+    }
+  }
+
+  // Triggers lazy evaluation if not already evaluated.
+  void trigger_evaluation() {
+    if (!evaluated)
+      _evaluate();
+  }
+
+  // Use KMP_INTERNAL_MALLOC/KMP_INTERNAL_FREE for memory management.
+  void *operator new(size_t size) { return KMP_INTERNAL_MALLOC(size); }
+  void operator delete(void *ptr) { KMP_INTERNAL_FREE(ptr); }
+
+  bool operator==(const kmp_trait_context &other) const {
+    auto clause_comp = [](kmp_trait_clause *const &a,
+                          kmp_trait_clause *const &b) { return *a == *b; };
+    return clauses.is_set_equal(other.clauses, clause_comp);
+  }
+
+  // Iterator support (returns the iterators of the devices vector; triggers
+  // lazy evaluation if not already evaluated and if the context is not const).
+  const int *begin() { return evaluate().begin(); }
+  const int *end() { return evaluate().end(); }
+  const int *begin() const { return evaluate().begin(); }
+  const int *end() const { return evaluate().end(); }
+};
+
+#endif // OPENMP_TRAITS_H

diff  --git a/openmp/runtime/unittests/CMakeLists.txt b/openmp/runtime/unittests/CMakeLists.txt
index 8335c94cfb6d9..96884397f2b30 100644
--- a/openmp/runtime/unittests/CMakeLists.txt
+++ b/openmp/runtime/unittests/CMakeLists.txt
@@ -69,3 +69,4 @@ add_openmp_testsuite(
 
 add_subdirectory(ADT)
 add_subdirectory(String)
+add_subdirectory(Traits)

diff  --git a/openmp/runtime/unittests/Traits/CMakeLists.txt b/openmp/runtime/unittests/Traits/CMakeLists.txt
new file mode 100644
index 0000000000000..aef30716bcf14
--- /dev/null
+++ b/openmp/runtime/unittests/Traits/CMakeLists.txt
@@ -0,0 +1,7 @@
+add_openmp_unittest(TraitsTests
+  MockOMP.cpp
+  TestOMPTraits.cpp
+  TestOMPTraitParser.cpp
+  TestOMPTraitLexer.cpp
+)
+

diff  --git a/openmp/runtime/unittests/Traits/MockOMP.cpp b/openmp/runtime/unittests/Traits/MockOMP.cpp
new file mode 100644
index 0000000000000..2c8623798b7d7
--- /dev/null
+++ b/openmp/runtime/unittests/Traits/MockOMP.cpp
@@ -0,0 +1,24 @@
+//===- MockOMP.cpp - Mock OMP functions for testing ----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Mock implementations for OMP functions (libomptarget not available in tests).
+//
+//===----------------------------------------------------------------------===//
+
+// Mock device count for testing
+static int MockNumDevices = 4;
+static const char *MockDeviceUIDs[] = {"device-0", "device-1", "device-2",
+                                       "device-3"};
+
+extern "C" int omp_get_num_devices() { return MockNumDevices; }
+
+extern "C" const char *omp_get_uid_from_device(int DeviceNum) {
+  if (DeviceNum >= 0 && DeviceNum < MockNumDevices)
+    return MockDeviceUIDs[DeviceNum];
+  return "";
+}

diff  --git a/openmp/runtime/unittests/Traits/TestOMPTraitLexer.cpp b/openmp/runtime/unittests/Traits/TestOMPTraitLexer.cpp
new file mode 100644
index 0000000000000..43c5821fc3b5b
--- /dev/null
+++ b/openmp/runtime/unittests/Traits/TestOMPTraitLexer.cpp
@@ -0,0 +1,350 @@
+//===- TestOMPTraitLexer.cpp - Tests for OMP Trait Lexer -----------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "kmp_traits.h"
+#include "gtest/gtest.h"
+
+#include <cstring>
+#include <string>
+
+using namespace lexer;
+
+namespace {
+
+//===----------------------------------------------------------------------===//
+// Helpers
+//===----------------------------------------------------------------------===//
+
+// Compare a token's text against a C string.
+static bool text_is(const token &tok, const char *expected) {
+  size_t len = strlen(expected);
+  if (tok.text.length() != len)
+    return false;
+  if (len == 0)
+    return true;
+  return memcmp(tok.text.begin(), expected, len) == 0;
+}
+
+// Lex the whole spec into a vector of tokens (excluding the trailing END).
+static kmp_vector<token> tokenize(const char *spec) {
+  kmp_lexer lex{kmp_str_ref(spec)};
+  kmp_vector<token> tokens;
+  for (token tok = lex.next(); tok.kind != token_kind::END; tok = lex.next())
+    tokens.push_back(tok);
+  return tokens;
+}
+
+// Assert a token has the expected kind and text.
+static void expect_token(const token &tok, token_kind kind, const char *text) {
+  EXPECT_EQ(tok.kind, kind);
+  EXPECT_TRUE(text_is(tok, text))
+      << "expected token text \"" << text << "\" but got \""
+      << std::string(tok.text.begin(), tok.text.length()) << "\"";
+}
+
+//===----------------------------------------------------------------------===//
+// Single Tokens
+//===----------------------------------------------------------------------===//
+
+TEST(LexerTest, EmptyInput) {
+  kmp_lexer lex{kmp_str_ref("")};
+  EXPECT_EQ(lex.next().kind, token_kind::END);
+}
+
+TEST(LexerTest, WhitespaceOnly) {
+  kmp_lexer lex{kmp_str_ref("   \t  ")};
+  EXPECT_EQ(lex.next().kind, token_kind::END);
+}
+
+TEST(LexerTest, EndIsSticky) {
+  // Once the input is exhausted, next() keeps returning END.
+  kmp_lexer lex{kmp_str_ref("*")};
+  EXPECT_EQ(lex.next().kind, token_kind::STAR);
+  EXPECT_EQ(lex.next().kind, token_kind::END);
+  EXPECT_EQ(lex.next().kind, token_kind::END);
+}
+
+TEST(LexerTest, Punctuation) {
+  kmp_vector<token> tokens = tokenize(",*!()[]:");
+
+  ASSERT_EQ(tokens.size(), 8u);
+  expect_token(tokens[0], token_kind::COMMA, ",");
+  expect_token(tokens[1], token_kind::STAR, "*");
+  expect_token(tokens[2], token_kind::NOT, "!");
+  expect_token(tokens[3], token_kind::L_PAREN, "(");
+  expect_token(tokens[4], token_kind::R_PAREN, ")");
+  expect_token(tokens[5], token_kind::L_BRACKET, "[");
+  expect_token(tokens[6], token_kind::R_BRACKET, "]");
+  expect_token(tokens[7], token_kind::COLON, ":");
+}
+
+TEST(LexerTest, DoubleColon) {
+  // "::" is lexed as two COLON tokens.
+  kmp_vector<token> tokens = tokenize("::");
+
+  ASSERT_EQ(tokens.size(), 2u);
+  expect_token(tokens[0], token_kind::COLON, ":");
+  expect_token(tokens[1], token_kind::COLON, ":");
+}
+
+TEST(LexerTest, AndOperator) {
+  kmp_vector<token> tokens = tokenize("&&");
+
+  ASSERT_EQ(tokens.size(), 1u);
+  expect_token(tokens[0], token_kind::AND, "&&");
+}
+
+TEST(LexerTest, OrOperator) {
+  kmp_vector<token> tokens = tokenize("||");
+
+  ASSERT_EQ(tokens.size(), 1u);
+  expect_token(tokens[0], token_kind::OR, "||");
+}
+
+//===----------------------------------------------------------------------===//
+// Numbers
+//
+// The lexer does not have a dedicated number token: a run of digits is just a
+// WORD. Recognizing a device number is the parser's job (see consume_clause).
+//===----------------------------------------------------------------------===//
+
+TEST(LexerTest, SingleDigitNumber) {
+  kmp_vector<token> tokens = tokenize("0");
+
+  ASSERT_EQ(tokens.size(), 1u);
+  expect_token(tokens[0], token_kind::WORD, "0");
+}
+
+TEST(LexerTest, MultiDigitNumber) {
+  kmp_vector<token> tokens = tokenize("12345");
+
+  ASSERT_EQ(tokens.size(), 1u);
+  expect_token(tokens[0], token_kind::WORD, "12345");
+}
+
+TEST(LexerTest, NumberList) {
+  kmp_vector<token> tokens = tokenize("1,2,3");
+
+  ASSERT_EQ(tokens.size(), 5u);
+  expect_token(tokens[0], token_kind::WORD, "1");
+  expect_token(tokens[1], token_kind::COMMA, ",");
+  expect_token(tokens[2], token_kind::WORD, "2");
+  expect_token(tokens[3], token_kind::COMMA, ",");
+  expect_token(tokens[4], token_kind::WORD, "3");
+}
+
+//===----------------------------------------------------------------------===//
+// Words
+//===----------------------------------------------------------------------===//
+
+TEST(LexerTest, SimpleWord) {
+  kmp_vector<token> tokens = tokenize("uid");
+
+  ASSERT_EQ(tokens.size(), 1u);
+  expect_token(tokens[0], token_kind::WORD, "uid");
+}
+
+TEST(LexerTest, WordWithDash) {
+  // A uid_value may contain '-'; it stays part of the word.
+  kmp_vector<token> tokens = tokenize("device-0");
+
+  ASSERT_EQ(tokens.size(), 1u);
+  expect_token(tokens[0], token_kind::WORD, "device-0");
+}
+
+TEST(LexerTest, WordWithUnderscore) {
+  kmp_vector<token> tokens = tokenize("my_device_123");
+
+  ASSERT_EQ(tokens.size(), 1u);
+  expect_token(tokens[0], token_kind::WORD, "my_device_123");
+}
+
+TEST(LexerTest, LeadingMinusIsWord) {
+  // The grammar allows a leading "-" for device numbers, but since '-' is also
+  // a uid_value symbol the lexer keeps the contiguous run together as a single
+  // WORD. Distinguishing a negative device number is left to the parser.
+  kmp_vector<token> tokens = tokenize("-5");
+
+  ASSERT_EQ(tokens.size(), 1u);
+  expect_token(tokens[0], token_kind::WORD, "-5");
+}
+
+TEST(LexerTest, WhitespaceBreaksWords) {
+  // A uid_value may not contain whitespace, so spaces split the run.
+  kmp_vector<token> tokens = tokenize("device - 0");
+
+  ASSERT_EQ(tokens.size(), 3u);
+  expect_token(tokens[0], token_kind::WORD, "device");
+  expect_token(tokens[1], token_kind::WORD, "-");
+  expect_token(tokens[2], token_kind::WORD, "0");
+}
+
+//===----------------------------------------------------------------------===//
+// Full trait expressions
+//===----------------------------------------------------------------------===//
+
+TEST(LexerTest, UidTrait) {
+  kmp_vector<token> tokens = tokenize("uid(device-0)");
+
+  ASSERT_EQ(tokens.size(), 4u);
+  expect_token(tokens[0], token_kind::WORD, "uid");
+  expect_token(tokens[1], token_kind::L_PAREN, "(");
+  expect_token(tokens[2], token_kind::WORD, "device-0");
+  expect_token(tokens[3], token_kind::R_PAREN, ")");
+}
+
+TEST(LexerTest, NegatedUidTrait) {
+  kmp_vector<token> tokens = tokenize("!uid(a)");
+
+  ASSERT_EQ(tokens.size(), 5u);
+  expect_token(tokens[0], token_kind::NOT, "!");
+  expect_token(tokens[1], token_kind::WORD, "uid");
+  expect_token(tokens[2], token_kind::L_PAREN, "(");
+  expect_token(tokens[3], token_kind::WORD, "a");
+  expect_token(tokens[4], token_kind::R_PAREN, ")");
+}
+
+TEST(LexerTest, AndGroup) {
+  kmp_vector<token> tokens = tokenize("(uid(a) && uid(b))");
+
+  ASSERT_EQ(tokens.size(), 11u);
+  expect_token(tokens[0], token_kind::L_PAREN, "(");
+  expect_token(tokens[1], token_kind::WORD, "uid");
+  expect_token(tokens[2], token_kind::L_PAREN, "(");
+  expect_token(tokens[3], token_kind::WORD, "a");
+  expect_token(tokens[4], token_kind::R_PAREN, ")");
+  expect_token(tokens[5], token_kind::AND, "&&");
+  expect_token(tokens[6], token_kind::WORD, "uid");
+  expect_token(tokens[7], token_kind::L_PAREN, "(");
+  expect_token(tokens[8], token_kind::WORD, "b");
+  expect_token(tokens[9], token_kind::R_PAREN, ")");
+  expect_token(tokens[10], token_kind::R_PAREN, ")");
+}
+
+TEST(LexerTest, OrGroup) {
+  kmp_vector<token> tokens = tokenize("uid(a)||uid(b)");
+
+  ASSERT_EQ(tokens.size(), 9u);
+  expect_token(tokens[0], token_kind::WORD, "uid");
+  expect_token(tokens[1], token_kind::L_PAREN, "(");
+  expect_token(tokens[2], token_kind::WORD, "a");
+  expect_token(tokens[3], token_kind::R_PAREN, ")");
+  expect_token(tokens[4], token_kind::OR, "||");
+  expect_token(tokens[5], token_kind::WORD, "uid");
+  expect_token(tokens[6], token_kind::L_PAREN, "(");
+  expect_token(tokens[7], token_kind::WORD, "b");
+  expect_token(tokens[8], token_kind::R_PAREN, ")");
+}
+
+TEST(LexerTest, WildcardWithLiterals) {
+  kmp_vector<token> tokens = tokenize("1, *, 3");
+
+  ASSERT_EQ(tokens.size(), 5u);
+  expect_token(tokens[0], token_kind::WORD, "1");
+  expect_token(tokens[1], token_kind::COMMA, ",");
+  expect_token(tokens[2], token_kind::STAR, "*");
+  expect_token(tokens[3], token_kind::COMMA, ",");
+  expect_token(tokens[4], token_kind::WORD, "3");
+}
+
+TEST(LexerTest, IndexExpression) {
+  kmp_vector<token> tokens = tokenize("[1:2:3]");
+
+  ASSERT_EQ(tokens.size(), 7u);
+  expect_token(tokens[0], token_kind::L_BRACKET, "[");
+  expect_token(tokens[1], token_kind::WORD, "1");
+  expect_token(tokens[2], token_kind::COLON, ":");
+  expect_token(tokens[3], token_kind::WORD, "2");
+  expect_token(tokens[4], token_kind::COLON, ":");
+  expect_token(tokens[5], token_kind::WORD, "3");
+  expect_token(tokens[6], token_kind::R_BRACKET, "]");
+}
+
+//===----------------------------------------------------------------------===//
+// Whitespace handling
+//===----------------------------------------------------------------------===//
+
+TEST(LexerTest, LeadingAndTrailingWhitespace) {
+  kmp_vector<token> tokens = tokenize("   uid ( a )   ");
+
+  ASSERT_EQ(tokens.size(), 4u);
+  expect_token(tokens[0], token_kind::WORD, "uid");
+  expect_token(tokens[1], token_kind::L_PAREN, "(");
+  expect_token(tokens[2], token_kind::WORD, "a");
+  expect_token(tokens[3], token_kind::R_PAREN, ")");
+}
+
+TEST(LexerTest, WhitespaceAroundOperators) {
+  kmp_vector<token> tokens = tokenize("uid(a)   &&   uid(b)");
+
+  ASSERT_EQ(tokens.size(), 9u);
+  expect_token(tokens[4], token_kind::AND, "&&");
+}
+
+//===----------------------------------------------------------------------===//
+// Unknown / error characters
+//===----------------------------------------------------------------------===//
+
+TEST(LexerTest, UnknownCharacter) {
+  kmp_vector<token> tokens = tokenize("@");
+
+  ASSERT_EQ(tokens.size(), 1u);
+  expect_token(tokens[0], token_kind::UNKNOWN, "@");
+}
+
+TEST(LexerTest, LoneAmpersandIsUnknown) {
+  kmp_vector<token> tokens = tokenize("&");
+
+  ASSERT_EQ(tokens.size(), 1u);
+  expect_token(tokens[0], token_kind::UNKNOWN, "&");
+}
+
+TEST(LexerTest, LonePipeIsUnknown) {
+  kmp_vector<token> tokens = tokenize("|");
+
+  ASSERT_EQ(tokens.size(), 1u);
+  expect_token(tokens[0], token_kind::UNKNOWN, "|");
+}
+
+TEST(LexerTest, TripleAmpersand) {
+  // "&&&" is "&&" followed by a lone (unknown) '&'.
+  kmp_vector<token> tokens = tokenize("&&&");
+
+  ASSERT_EQ(tokens.size(), 2u);
+  expect_token(tokens[0], token_kind::AND, "&&");
+  expect_token(tokens[1], token_kind::UNKNOWN, "&");
+}
+
+//===----------------------------------------------------------------------===//
+// peek()
+//===----------------------------------------------------------------------===//
+
+TEST(LexerTest, PeekDoesNotAdvance) {
+  kmp_lexer lex{kmp_str_ref("uid(a)")};
+
+  // Peek repeatedly returns the same token without consuming it.
+  EXPECT_EQ(lex.peek().kind, token_kind::WORD);
+  EXPECT_TRUE(text_is(lex.peek(), "uid"));
+  EXPECT_EQ(lex.peek().kind, token_kind::WORD);
+
+  // next() returns the peeked token, then advances.
+  token tok = lex.next();
+  expect_token(tok, token_kind::WORD, "uid");
+  EXPECT_EQ(lex.peek().kind, token_kind::L_PAREN);
+  expect_token(lex.next(), token_kind::L_PAREN, "(");
+}
+
+TEST(LexerTest, PeekAtEnd) {
+  kmp_lexer lex{kmp_str_ref("")};
+
+  EXPECT_EQ(lex.peek().kind, token_kind::END);
+  EXPECT_EQ(lex.next().kind, token_kind::END);
+  EXPECT_EQ(lex.peek().kind, token_kind::END);
+}
+
+} // namespace

diff  --git a/openmp/runtime/unittests/Traits/TestOMPTraitParser.cpp b/openmp/runtime/unittests/Traits/TestOMPTraitParser.cpp
new file mode 100644
index 0000000000000..59e2c4b07318a
--- /dev/null
+++ b/openmp/runtime/unittests/Traits/TestOMPTraitParser.cpp
@@ -0,0 +1,1072 @@
+//===- TestOMPTraitParser.cpp - Tests for OMP Trait Parser ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "kmp_traits.h"
+#include "gtest/gtest.h"
+
+namespace {
+
+//===----------------------------------------------------------------------===//
+// Helper to parse and auto-cleanup
+//===----------------------------------------------------------------------===//
+
+class ParserTest : public ::testing::Test {
+protected:
+  kmp_trait_context *context = nullptr;
+
+  void parse(const char *spec, const char *dbg_name = nullptr) {
+    context = kmp_trait_context::parse_from_spec(kmp_str_ref(spec), dbg_name);
+  }
+
+  void TearDown() override {
+    if (context) {
+      delete context;
+      context = nullptr;
+    }
+  }
+};
+
+template <bool expected_result>
+static void check_result_single(kmp_trait_context *context,
+                                const kmp_vector<int> &result,
+                                int expected_device_num) {
+  EXPECT_EQ(context->match(expected_device_num), expected_result);
+  EXPECT_EQ(result.contains(expected_device_num), expected_result);
+}
+
+template <bool expected_result, int... device_nums>
+static void check_result(kmp_trait_context *context,
+                         const kmp_vector<int> &result) {
+  (check_result_single<expected_result>(context, result, device_nums), ...);
+}
+
+template <bool expected_result, int... device_nums>
+static void check_result(kmp_trait_context *context) {
+  const kmp_vector<int> &result = context->evaluate();
+  check_result<expected_result, device_nums...>(context, result);
+}
+
+//===----------------------------------------------------------------------===//
+// Literal Device Numbers
+//===----------------------------------------------------------------------===//
+
+TEST_F(ParserTest, SingleLiteral) {
+  parse("5");
+
+  ASSERT_NE(context, nullptr);
+  // Device 5 is out of range (mock has 4 devices: 0-3), so match returns false
+
+  EXPECT_EQ(context->evaluate().size(), 0u);
+  check_result<false, 5, 0, 4, 6>(context);
+}
+
+TEST_F(ParserTest, ZeroLiteral) {
+  parse("0");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 0>(context, result);
+  check_result<false, 1>(context, result);
+}
+
+TEST_F(ParserTest, MultipleLiterals) {
+  parse("1,2,3");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 3u);
+  check_result<true, 1, 2, 3>(context, result);
+  check_result<false, 0, 4>(context, result);
+}
+
+TEST_F(ParserTest, LiteralsWithSpaces) {
+  parse("1, 2, 3");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 3u);
+  check_result<true, 1, 2, 3>(context, result);
+  check_result<false, 0, 4>(context, result);
+}
+
+TEST_F(ParserTest, LiteralsWithLeadingSpaces) {
+  parse("  1,  2,  3");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 3u);
+  check_result<true, 1, 2, 3>(context, result);
+  check_result<false, 0, 4>(context, result);
+}
+
+TEST_F(ParserTest, LargeLiteral) {
+  parse("12345");
+
+  ASSERT_NE(context, nullptr);
+  // Device 12345 is out of range, so match returns false
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 0u);
+  check_result<false, 12345, 0>(context, result);
+}
+
+//===----------------------------------------------------------------------===//
+// Wildcard
+//===----------------------------------------------------------------------===//
+
+TEST_F(ParserTest, Wildcard) {
+  parse("*");
+
+  ASSERT_NE(context, nullptr);
+  // Wildcard matches all 4 mock devices
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+  check_result<false, 100>(context, result);
+}
+
+TEST_F(ParserTest, WildcardWithLiterals) {
+  parse("1, *, 3");
+
+  ASSERT_NE(context, nullptr);
+  // Wildcard makes all in-range devices match
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+  check_result<false, 100>(context, result);
+}
+
+//===----------------------------------------------------------------------===//
+// UID Traits
+//===----------------------------------------------------------------------===//
+
+TEST_F(ParserTest, UIDTrait) {
+  parse("uid(device-0)");
+
+  ASSERT_NE(context, nullptr);
+  // Uses mock: device-0 is at index 0
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 0>(context, result);
+  check_result<false, 1>(context, result);
+}
+
+TEST_F(ParserTest, UIDTraitWithUnderscore) {
+  parse("uid(my_device_123)");
+
+  ASSERT_NE(context, nullptr);
+  // This UID doesn't match any mock device
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 0u);
+  check_result<false, 0, 1>(context, result);
+}
+
+TEST_F(ParserTest, UIDTraitWithDash) {
+  parse("uid(device-2)");
+
+  ASSERT_NE(context, nullptr);
+  // Uses mock: device-2 is at index 2
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 2>(context, result);
+  check_result<false, 0, 1, 3>(context, result);
+}
+
+TEST_F(ParserTest, MultipleUIDTraits) {
+  parse("uid(device-1), uid(device-3)");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 2u);
+  check_result<true, 1, 3>(context, result);
+  check_result<false, 0, 2>(context, result);
+}
+
+TEST_F(ParserTest, MixedLiteralsAndUIDs) {
+  parse("0, uid(device-2), 1, uid(device-3)");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+}
+
+//===----------------------------------------------------------------------===//
+// Negation
+//===----------------------------------------------------------------------===//
+
+TEST_F(ParserTest, NegatedUID) {
+  parse("!uid(device-0)");
+
+  ASSERT_NE(context, nullptr);
+  // Negated: everything except device-0 matches
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 3u);
+  check_result<false, 0>(context, result);
+  check_result<true, 1, 2, 3>(context, result);
+}
+
+//===----------------------------------------------------------------------===//
+// Grouping with Parentheses
+//===----------------------------------------------------------------------===//
+
+TEST_F(ParserTest, SimpleGroup) {
+  parse("(uid(device-1))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<false, 0>(context, result);
+  check_result<true, 1>(context, result);
+}
+
+TEST_F(ParserTest, GroupWithOR) {
+  parse("(uid(device-0) || uid(device-2))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 2u);
+  check_result<true, 0, 2>(context, result);
+  check_result<false, 1, 3>(context, result);
+}
+
+TEST_F(ParserTest, GroupWithAND) {
+  parse("(uid(device-0) && uid(device-0))");
+
+  ASSERT_NE(context, nullptr);
+  // Both refer to same device, so AND passes for device 0
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 0>(context, result);
+  check_result<false, 1>(context, result);
+}
+
+TEST_F(ParserTest, NegatedGroup) {
+  parse("!(uid(device-0) || uid(device-1))");
+
+  ASSERT_NE(context, nullptr);
+  // Negated: matches devices NOT in {0, 1}
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 2u);
+  check_result<false, 0, 1>(context, result);
+  check_result<true, 2, 3>(context, result);
+}
+
+//===----------------------------------------------------------------------===//
+// Complex Expressions
+//===----------------------------------------------------------------------===//
+
+TEST_F(ParserTest, ComplexMixed) {
+  parse("0, 1, uid(device-2), *");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+  check_result<false, 100>(context, result);
+}
+
+TEST_F(ParserTest, MultipleORGroups) {
+  parse("(uid(device-0) || uid(device-1)), (uid(device-2) || uid(device-3))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+}
+
+//===----------------------------------------------------------------------===//
+// Complex Boolean Operators
+//===----------------------------------------------------------------------===//
+
+TEST_F(ParserTest, ThreeWayOR) {
+  // Three UIDs combined with OR
+  parse("(uid(device-0) || uid(device-1) || uid(device-2))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 3u);
+  check_result<true, 0, 1, 2>(context, result);
+  check_result<false, 3>(context, result);
+}
+
+TEST_F(ParserTest, FourWayOR) {
+  // All four mock devices via OR
+  parse("(uid(device-0) || uid(device-1) || uid(device-2) || uid(device-3))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, ThreeWayAND) {
+  // Three identical UIDs with AND - all must match same device
+  parse("(uid(device-1) && uid(device-1) && uid(device-1))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 1>(context, result);
+  check_result<false, 0, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, ANDWithDifferentUIDs) {
+  // AND with 
diff erent UIDs - can never match (device can't have two UIDs)
+  parse("(uid(device-0) && uid(device-1))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 0u);
+  check_result<false, 0, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, NegatedThreeWayOR) {
+  // Negate a group of three UIDs - matches devices NOT in {0, 1, 2}
+  parse("!(uid(device-0) || uid(device-1) || uid(device-2))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<false, 0, 1, 2>(context, result);
+  check_result<true, 3>(context, result);
+}
+
+TEST_F(ParserTest, NegatedAND) {
+  // Negate an AND group - since AND never matches, negation matches all
+  parse("!(uid(device-0) && uid(device-1))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, NegatedANDWithSameUID) {
+  // Negate an AND that matches device-0 - matches everything except 0
+  parse("!(uid(device-0) && uid(device-0))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 3u);
+  check_result<false, 0>(context, result);
+  check_result<true, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, NestedParensWithOR) {
+  // Nested parentheses around OR
+  parse("((uid(device-0) || uid(device-1)))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 2u);
+  check_result<true, 0, 1>(context, result);
+  check_result<false, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, NestedParensWithAND) {
+  // Nested parentheses around AND
+  parse("((uid(device-2) && uid(device-2)))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 2>(context, result);
+  check_result<false, 0, 1, 3>(context, result);
+}
+
+TEST_F(ParserTest, DoubleNegation) {
+  // Double negation: !!uid(device-0) should match device-0
+  parse("!(!uid(device-0))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 0>(context, result);
+  check_result<false, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, NegatedNestedOR) {
+  // Negate nested OR group
+  parse("!((uid(device-0) || uid(device-1)))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 2u);
+  check_result<false, 0, 1>(context, result);
+  check_result<true, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, MultipleNegatedExprs) {
+  // Multiple negated clauses - OR semantics between clauses
+  parse("!uid(device-0), !uid(device-1)");
+
+  ASSERT_NE(context, nullptr);
+  // First clause matches 1,2,3; Second clause matches 0,2,3
+  // OR between clauses: union = all devices
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, MixedNegatedAndNonNegated) {
+  // Mix negated and non-negated clauses
+  parse("uid(device-0), !uid(device-0)");
+
+  ASSERT_NE(context, nullptr);
+  // First matches 0, second matches 1,2,3 -> union = all
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, ComplexORGroupsInSeparateExprs) {
+  // Complex OR groups as separate clauses
+  parse("(uid(device-0) || uid(device-1)), (uid(device-2) || uid(device-3))");
+
+  ASSERT_NE(context, nullptr);
+  // First matches 0,1; Second matches 2,3 -> union = all
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, NegatedORGroupWithLiteral) {
+  // Negated OR group combined with literal in separate clauses
+  parse("!(uid(device-0) || uid(device-1)), 0");
+
+  ASSERT_NE(context, nullptr);
+  // First matches 2,3; Second matches 0 -> union = 0,2,3
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 3u);
+  check_result<true, 0, 2, 3>(context, result);
+  check_result<false, 1>(context, result);
+}
+
+TEST_F(ParserTest, DeeplyNestedWithOperators) {
+  // Deeply nested with operators
+  parse("(((uid(device-0) || uid(device-1))))");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 2u);
+  check_result<true, 0, 1>(context, result);
+  check_result<false, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, ORWithSpacesAroundOperators) {
+  // OR with lots of whitespace
+  parse("(  uid(device-0)   ||   uid(device-2)   ||   uid(device-3)  )");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 3u);
+  check_result<true, 0, 2, 3>(context, result);
+  check_result<false, 1>(context, result);
+}
+
+TEST_F(ParserTest, ANDWithSpacesAroundOperators) {
+  // AND with lots of whitespace
+  parse("(  uid(device-1)   &&   uid(device-1)  )");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 1>(context, result);
+  check_result<false, 0, 2, 3>(context, result);
+}
+
+//===----------------------------------------------------------------------===//
+// Mixed && and || (in separate clauses/groups)
+//===----------------------------------------------------------------------===//
+
+TEST_F(ParserTest, ORExprAndANDExpr) {
+  // OR group in first clause, AND group in second clause
+  parse("(uid(device-0) || uid(device-1)), (uid(device-2) && uid(device-2))");
+
+  ASSERT_NE(context, nullptr);
+  // First clause matches 0,1; Second clause matches 2 -> union = 0,1,2
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 3u);
+  check_result<true, 0, 1, 2>(context, result);
+  check_result<false, 3>(context, result);
+}
+
+TEST_F(ParserTest, ANDExprAndORExpr) {
+  // AND group first, OR group second
+  parse("(uid(device-0) && uid(device-0)), (uid(device-2) || uid(device-3))");
+
+  ASSERT_NE(context, nullptr);
+  // First clause matches 0; Second clause matches 2,3 -> union = 0,2,3
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 3u);
+  check_result<true, 0, 2, 3>(context, result);
+  check_result<false, 1>(context, result);
+}
+
+TEST_F(ParserTest, MultipleANDAndORExprs) {
+  // Multiple clauses alternating between AND and OR
+  parse("(uid(device-0) && uid(device-0)), (uid(device-1) || uid(device-2)), "
+        "(uid(device-3) && uid(device-3))");
+
+  ASSERT_NE(context, nullptr);
+  // Expr 1 matches 0; Expr 2 matches 1,2; Expr 3 matches 3 -> all
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, NegatedORWithAND) {
+  // Negated OR clause combined with AND clause
+  parse("!(uid(device-0) || uid(device-1)), (uid(device-0) && uid(device-0))");
+
+  ASSERT_NE(context, nullptr);
+  // First clause matches 2,3; Second clause matches 0 -> union = 0,2,3
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 3u);
+  check_result<true, 0, 2, 3>(context, result);
+  check_result<false, 1>(context, result);
+}
+
+TEST_F(ParserTest, NegatedANDWithOR) {
+  // Negated AND clause combined with OR clause
+  parse("!(uid(device-0) && uid(device-0)), (uid(device-0) || uid(device-1))");
+
+  ASSERT_NE(context, nullptr);
+  // First clause matches 1,2,3; Second clause matches 0,1 -> all
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, ComplexMixedOperators) {
+  // Complex mix: OR, AND, negated OR, literal
+  parse("(uid(device-0) || uid(device-1)), (uid(device-2) && uid(device-2)), "
+        "!(uid(device-0) || uid(device-1) || uid(device-2)), 0");
+
+  ASSERT_NE(context, nullptr);
+  // Expr 1: 0,1; Expr 2: 2; Expr 3: NOT(0,1,2) = 3; Expr 4: 0
+  // Union = all
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, ANDNeverMatchesWithOR) {
+  // AND that never matches combined with OR that does
+  parse("(uid(device-0) && uid(device-1)), (uid(device-2) || uid(device-3))");
+
+  ASSERT_NE(context, nullptr);
+  // First clause: never matches (
diff erent UIDs); Second: 2,3
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 2u);
+  check_result<false, 0, 1>(context, result);
+  check_result<true, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, ORNeverMatchesWithAND) {
+  // OR with non-existent UIDs combined with AND that matches
+  parse("(uid(nonexistent-a) || uid(nonexistent-b)), (uid(device-0) && "
+        "uid(device-0))");
+
+  ASSERT_NE(context, nullptr);
+  // First clause: no match; Second: 0
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 0>(context, result);
+  check_result<false, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, ThreeWayORAndThreeWayAND) {
+  // Three-way OR and three-way AND in separate clauses
+  parse("(uid(device-0) || uid(device-1) || uid(device-2)), (uid(device-3) && "
+        "uid(device-3) && uid(device-3))");
+
+  ASSERT_NE(context, nullptr);
+  // First: 0,1,2; Second: 3 -> all
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, NegatedMixedExprs) {
+  // Both clauses negated with 
diff erent operators
+  parse("!(uid(device-0) || uid(device-1)), !(uid(device-2) && uid(device-2))");
+
+  ASSERT_NE(context, nullptr);
+  // First: NOT(0,1) = 2,3; Second: NOT(2) = 0,1,3
+  // Union = all
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, LiteralsWithMixedOperatorExprs) {
+  // Literals combined with both OR and AND clauses
+  parse("0, (uid(device-1) || uid(device-2)), 3, (uid(device-0) && "
+        "uid(device-0))");
+
+  ASSERT_NE(context, nullptr);
+  // Literals: 0,3; OR clause: 1,2; AND clause: 0
+  // Union = all
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 4u);
+  check_result<true, 0, 1, 2, 3>(context, result);
+}
+
+//===----------------------------------------------------------------------===//
+// Nested Mixed Operators (|| and && at 
diff erent nesting levels)
+//===----------------------------------------------------------------------===//
+
+TEST_F(ParserTest, ORContainingANDGroup) {
+  // Outer OR with inner AND group: (A || (B && C))
+  // For (B && C) to match, both B and C must match same device
+  parse("(uid(device-0) || (uid(device-1) && uid(device-1)))");
+
+  ASSERT_NE(context, nullptr);
+  // device-0 matches via first operand
+  // device-1 matches via (device-1 && device-1)
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 2u);
+  check_result<true, 0, 1>(context, result);
+  check_result<false, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, ANDContainingORGroup) {
+  // Outer AND with inner OR group: (A && (B || C))
+  // Both the trait A and the group (B || C) must match
+  // Since A is uid(device-0), only device-0 can satisfy A
+  // (B || C) must also match device-0 for AND to succeed
+  parse("(uid(device-0) && (uid(device-0) || uid(device-1)))");
+
+  ASSERT_NE(context, nullptr);
+  // device-0: uid(device-0) matches AND (uid(device-0) || uid(device-1))
+  // matches -> true device-1: uid(device-0) doesn't match -> false
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 0>(context, result);
+  check_result<false, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, ORWithTwoANDGroups) {
+  // ((A && B) || (C && D)) - OR of two AND groups
+  parse(
+      "((uid(device-0) && uid(device-0)) || (uid(device-2) && uid(device-2)))");
+
+  ASSERT_NE(context, nullptr);
+  // First AND matches device-0; Second AND matches device-2
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 2u);
+  check_result<true, 0, 2>(context, result);
+  check_result<false, 1, 3>(context, result);
+}
+
+TEST_F(ParserTest, ANDWithTwoORGroups) {
+  // ((A || B) && (C || D)) - AND of two OR groups
+  // For a device to match: must match (A || B) AND must match (C || D)
+  parse(
+      "((uid(device-0) || uid(device-1)) && (uid(device-0) || uid(device-2)))");
+
+  ASSERT_NE(context, nullptr);
+  // device-0: matches (0||1) AND matches (0||2) -> true
+  // device-1: matches (0||1) but NOT (0||2) -> false
+  // device-2: NOT (0||1) -> false
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 0>(context, result);
+  check_result<false, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, ORWithNestedANDContainingOR) {
+  // (A || (B && (C || D))) - three levels of nesting
+  parse(
+      "(uid(device-3) || (uid(device-0) && (uid(device-0) || uid(device-1))))");
+
+  ASSERT_NE(context, nullptr);
+  // device-0: inner (0||1) matches, uid(device-0) matches -> AND matches; OR
+  // satisfied device-1: inner (0||1) matches, but uid(device-0) doesn't -> AND
+  // fails; outer uid(device-3) fails device-3: outer uid(device-3) matches
+  // directly
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 2u);
+  check_result<true, 0, 3>(context, result);
+  check_result<false, 1, 2>(context, result);
+}
+
+TEST_F(ParserTest, ANDWithNestedORContainingAND) {
+  // (A && (B || (C && D))) - three levels of nesting
+  parse(
+      "(uid(device-0) && (uid(device-0) || (uid(device-1) && uid(device-1))))");
+
+  ASSERT_NE(context, nullptr);
+  // device-0: uid(device-0) matches; inner (uid(device-0) || ...) matches ->
+  // AND satisfied device-1: uid(device-0) doesn't match -> AND fails
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 0>(context, result);
+  check_result<false, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, NegatedORContainingAND) {
+  // !(A || (B && C)) - negated complex expression
+  parse("!(uid(device-0) || (uid(device-1) && uid(device-1)))");
+
+  ASSERT_NE(context, nullptr);
+  // Without negation: matches 0, 1
+  // With negation: matches 2, 3
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 2u);
+  check_result<false, 0, 1>(context, result);
+  check_result<true, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, NegatedANDContainingOR) {
+  // !(A && (B || C)) - negated complex expression
+  parse("!(uid(device-0) && (uid(device-0) || uid(device-1)))");
+
+  ASSERT_NE(context, nullptr);
+  // Without negation: matches only 0
+  // With negation: matches 1, 2, 3
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 3u);
+  check_result<false, 0>(context, result);
+  check_result<true, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, ComplexNestedWithAllDevices) {
+  // ((A || B) && (C || D)) where union covers all but AND restricts
+  parse(
+      "((uid(device-0) || uid(device-1)) && (uid(device-1) || uid(device-2)))");
+
+  ASSERT_NE(context, nullptr);
+  // device-0: (0||1)=true, (1||2)=false -> AND=false
+  // device-1: (0||1)=true, (1||2)=true -> AND=true
+  // device-2: (0||1)=false -> AND=false
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 1>(context, result);
+  check_result<false, 0, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, TripleNestedMixedOperators) {
+  // (((A || B) && C) || D) - deeply nested with alternating operators
+  parse(
+      "(((uid(device-0) || uid(device-1)) && uid(device-0)) || uid(device-3))");
+
+  ASSERT_NE(context, nullptr);
+  // Inner (0||1): matches 0, 1
+  // Middle ((0||1) && 0): matches only 0
+  // Outer (... || 3): matches 0, 3
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 2u);
+  check_result<true, 0, 3>(context, result);
+  check_result<false, 1, 2>(context, result);
+}
+
+TEST_F(ParserTest, ANDChainWithNestedOR) {
+  // (A && (B || C) && D) - wait, this mixes operators at same level
+  // Actually: ((A && (B || C)) is valid - let's do that
+  // Let's do: (uid(device-0) && (uid(device-0) || uid(device-1)) &&
+  // uid(device-0)) This is three-way AND where middle operand is an OR group
+  parse("(uid(device-0) && (uid(device-0) || uid(device-1)) && uid(device-0))");
+
+  ASSERT_NE(context, nullptr);
+  // All three must match: uid(device-0), (0||1), uid(device-0)
+  // Only device-0 satisfies all
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 0>(context, result);
+  check_result<false, 1, 2, 3>(context, result);
+}
+
+TEST_F(ParserTest, ORChainWithNestedAND) {
+  // (A || (B && C) || D) - three-way OR where middle is AND group
+  parse("(uid(device-0) || (uid(device-1) && uid(device-1)) || uid(device-3))");
+
+  ASSERT_NE(context, nullptr);
+  // Any of: device-0, (device-1 && device-1), device-3
+  // Matches: 0, 1, 3
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 3u);
+  check_result<true, 0, 1, 3>(context, result);
+  check_result<false, 2>(context, result);
+}
+
+TEST_F(ParserTest, NestedMixedWithSpaces) {
+  // Nested mixed operators with lots of whitespace
+  parse("(  uid(device-0)  ||  ( uid(device-1)  &&  uid(device-1) )  ||  "
+        "uid(device-2)  )");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 3u);
+  check_result<true, 0, 1, 2>(context, result);
+  check_result<false, 3>(context, result);
+}
+
+TEST_F(ParserTest, DeepNestingWithinLimit) {
+  // A moderate nesting depth (well under the recursion limit) parses fine and
+  // is transparent to matching.
+  constexpr int depth = 20;
+  const char *inner = "uid(device-0)";
+  char spec[64];
+  char *p = spec;
+  for (int i = 0; i < depth; ++i)
+    *p++ = '(';
+  for (const char *s = inner; *s; ++s)
+    *p++ = *s;
+  for (int i = 0; i < depth; ++i)
+    *p++ = ')';
+  *p = '\0';
+  parse(spec, "deep_nesting_within_limit");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 1u);
+  check_result<true, 0>(context, result);
+  check_result<false, 1, 2, 3>(context, result);
+}
+
+//===----------------------------------------------------------------------===//
+// Empty Input
+//===----------------------------------------------------------------------===//
+
+TEST_F(ParserTest, EmptyString) {
+  parse("");
+
+  ASSERT_NE(context, nullptr);
+  // Empty context matches nothing
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 0u);
+  check_result<false, 0, 1>(context, result);
+}
+
+TEST_F(ParserTest, OnlyWhitespace) {
+  parse("   ");
+
+  ASSERT_NE(context, nullptr);
+  kmp_vector<int> result = context->evaluate();
+
+  EXPECT_EQ(result.size(), 0u);
+  check_result<false, 0>(context, result);
+}
+
+//===----------------------------------------------------------------------===//
+// Error Cases
+//===----------------------------------------------------------------------===//
+
+TEST_F(ParserTest, OnlyComma) {
+  ASSERT_DEATH(
+      parse(",", "test_only_comma"),
+      "OMP: Error #[0-9]+: trait parser while parsing test_only_comma: "
+      "failed to parse trait specification \\(,\\)");
+}
+
+TEST_F(ParserTest, OnlyCommaNullDbgName) {
+  ASSERT_DEATH(parse(","),
+               "OMP: Error #[0-9]+: trait parser while parsing \\(null\\): "
+               "failed to parse trait specification \\(,\\)");
+}
+
+TEST_F(ParserTest, MixedAndOrSameLevel) {
+  // OpenMP 6.0 explicitly excludes "&&" and "||" from appearing in the same
+  // grouping level.
+  ASSERT_DEATH(
+      parse("uid(a) && uid(b) || uid(c)", "mixed_and_or_same_level"),
+      "OMP: Error #[0-9]+: trait parser while parsing mixed_and_or_same_level: "
+      "failed to parse trait specification "
+      "\\(\\|\\| uid\\(c\\)\\)");
+}
+
+TEST_F(ParserTest, MixedOrAndSameLevel) {
+  ASSERT_DEATH(
+      parse("uid(a) || uid(b) && uid(c)", "mixed_or_and_same_level"),
+      "OMP: Error #[0-9]+: trait parser while parsing mixed_or_and_same_level: "
+      "failed to parse trait specification "
+      "\\(&& uid\\(c\\)\\)");
+}
+
+TEST_F(ParserTest, InvalidUID) {
+  // Empty UID is not allowed; the offending token (the closing ')') is reported
+  // as the invalid trait value.
+  ASSERT_DEATH(parse("uid()", "invalid_uid"),
+               "OMP: Error #[0-9]+: trait parser while parsing invalid_uid: "
+               "invalid value for trait 'uid' \\(\\)\\)");
+}
+
+TEST_F(ParserTest, MissingOpenParenAfterTrait) {
+  // A trait name must be followed by "(".
+  ASSERT_DEATH(parse("uid", "missing_open_paren"),
+               "OMP: Error #[0-9]+: trait parser while parsing "
+               "missing_open_paren: error: expected '\\(' after trait name");
+}
+
+TEST_F(ParserTest, MissingCloseParenAfterTraitValue) {
+  // A trait value must be followed by ")".
+  ASSERT_DEATH(parse("uid(a", "missing_close_paren"),
+               "OMP: Error #[0-9]+: trait parser while parsing "
+               "missing_close_paren: error: expected '\\)' after trait value");
+}
+
+TEST_F(ParserTest, UnclosedParenthesis) {
+  // Once "(" is consumed we are committed to a group, so a missing ")" is a
+  // hard error rather than a recoverable parse failure.
+  ASSERT_DEATH(
+      parse("(uid(a)", "unclosed_parenthesis"),
+      "OMP: Error #[0-9]+: trait parser while parsing unclosed_parenthesis: "
+      "error: expected '\\)' after trait expression group");
+}
+
+TEST_F(ParserTest, UnmatchedClosingParenthesis) {
+  ASSERT_DEATH(parse("uid(a))", "unmatched_closing_parenthesis"),
+               "OMP: Error #[0-9]+: trait parser while parsing "
+               "unmatched_closing_parenthesis: "
+               "failed to parse trait specification \\(\\)\\)");
+}
+
+TEST_F(ParserTest, EmptyParentheses) {
+  // "(" commits to a group, so an empty group is a hard error.
+  ASSERT_DEATH(
+      parse("()", "empty_parentheses"),
+      "OMP: Error #[0-9]+: trait parser while parsing empty_parentheses: "
+      "error: expected trait expression after '\\('");
+}
+
+TEST_F(ParserTest, TrailingOperator) {
+  // A consumed "&&"/"||" commits to another operand, so its absence is a hard
+  // error.
+  ASSERT_DEATH(
+      parse("uid(a) &&", "trailing_operator"),
+      "OMP: Error #[0-9]+: trait parser while parsing trailing_operator: "
+      "error: expected trait expression after operator");
+}
+
+TEST_F(ParserTest, NegationWithoutTrait) {
+  // A consumed "!" commits to a trait expression, so its absence is a hard
+  // error rather than a recoverable parse failure.
+  ASSERT_DEATH(
+      parse("!", "negation_without_trait"),
+      "OMP: Error #[0-9]+: trait parser while parsing negation_without_trait: "
+      "error: expected trait expression after '!'");
+}
+
+TEST_F(ParserTest, DeviceNumberOutOfRange) {
+  // A word shaped like a device number is treated as a device number, so one
+  // that does not fit an int is a hard error.
+  ASSERT_DEATH(parse("99999999999", "device_number_out_of_range"),
+               "OMP: Error #[0-9]+: trait parser while parsing "
+               "device_number_out_of_range: error: device number out of range");
+}
+
+TEST_F(ParserTest, NegativeDeviceNumber) {
+  // "-5" is lexed as a single WORD; the parser recognizes its numeric shape and
+  // rejects it as an out-of-range device number rather than treating it as a
+  // name/uid_value.
+  ASSERT_DEATH(parse("-5", "negative_device_number"),
+               "OMP: Error #[0-9]+: trait parser while parsing "
+               "negative_device_number: error: device number out of range");
+}
+
+TEST_F(ParserTest, LeadingOperator) {
+  ASSERT_DEATH(
+      parse("&& uid(a)", "leading_operator"),
+      "OMP: Error #[0-9]+: trait parser while parsing leading_operator: "
+      "failed to parse trait specification \\(&& uid\\(a\\)\\)");
+}
+
+TEST_F(ParserTest, DoubleComma) {
+  ASSERT_DEATH(parse("uid(a),,uid(b)", "double_comma"),
+               "OMP: Error #[0-9]+: trait parser while parsing double_comma: "
+               "failed to parse trait specification \\(,,uid\\(b\\)\\)");
+}
+
+TEST_F(ParserTest, UnrecognizedTrait) {
+  // "uid" is currently the only trait name; any other word is not a trait, so
+  // the clause cannot be parsed.
+  ASSERT_DEATH(
+      parse("foo", "unrecognized_trait"),
+      "OMP: Error #[0-9]+: trait parser while parsing unrecognized_trait: "
+      "failed to parse trait specification \\(foo\\)");
+}
+
+TEST_F(ParserTest, MaxRecursionExceeded) {
+  // Nesting parentheses deeper than the recursion limit is a hard error. The
+  // guard trips before the input is exhausted, so unbalanced "(" are enough.
+  char spec[101];
+  for (int i = 0; i < 100; ++i)
+    spec[i] = '(';
+  spec[100] = '\0';
+  ASSERT_DEATH(parse(spec, "max_recursion"),
+               "OMP: Error #[0-9]+: trait parser while parsing max_recursion: "
+               "max recursion depth \\(64\\) exceeded");
+}
+
+} // namespace

diff  --git a/openmp/runtime/unittests/Traits/TestOMPTraits.cpp b/openmp/runtime/unittests/Traits/TestOMPTraits.cpp
new file mode 100644
index 0000000000000..12146f07089de
--- /dev/null
+++ b/openmp/runtime/unittests/Traits/TestOMPTraits.cpp
@@ -0,0 +1,1151 @@
+//===- TestOMPTraits.cpp - Tests for OMP Trait classes -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "kmp_traits.h"
+#include "gtest/gtest.h"
+
+using namespace kmp_traits;
+
+namespace {
+
+//===----------------------------------------------------------------------===//
+// kmp_wildcard_trait Tests
+//===----------------------------------------------------------------------===//
+
+TEST(kmp_wildcard_trait_test, MatchesAnyDevice) {
+  kmp_wildcard_trait *trait = new kmp_wildcard_trait();
+
+  EXPECT_TRUE(trait->match(0));
+  EXPECT_TRUE(trait->match(1));
+  EXPECT_TRUE(trait->match(100));
+  EXPECT_TRUE(trait->match(-1));
+
+  delete trait;
+}
+
+TEST(kmp_wildcard_trait_test, Equality) {
+  kmp_wildcard_trait *t1 = new kmp_wildcard_trait();
+  kmp_wildcard_trait *t2 = new kmp_wildcard_trait();
+
+  EXPECT_TRUE(*t1 == *t2);
+
+  delete t1;
+  delete t2;
+}
+
+//===----------------------------------------------------------------------===//
+// kmp_literal_trait Tests
+//===----------------------------------------------------------------------===//
+
+TEST(kmp_literal_trait_test, MatchesExactDevice) {
+  kmp_literal_trait *trait = new kmp_literal_trait(5);
+
+  EXPECT_TRUE(trait->match(5));
+  EXPECT_FALSE(trait->match(0));
+  EXPECT_FALSE(trait->match(4));
+  EXPECT_FALSE(trait->match(6));
+
+  delete trait;
+}
+
+TEST(kmp_literal_trait_test, MatchesZero) {
+  kmp_literal_trait *trait = new kmp_literal_trait(0);
+
+  EXPECT_TRUE(trait->match(0));
+  EXPECT_FALSE(trait->match(1));
+
+  delete trait;
+}
+
+#ifndef NDEBUG
+TEST(kmp_literal_trait_test, MatchesNegative) {
+  EXPECT_DEATH(new kmp_literal_trait(-1), "Device number must be non-negative");
+}
+#endif
+
+TEST(kmp_literal_trait_test, EqualitySameValue) {
+  kmp_literal_trait *t1 = new kmp_literal_trait(42);
+  kmp_literal_trait *t2 = new kmp_literal_trait(42);
+
+  EXPECT_TRUE(*t1 == *t2);
+
+  delete t1;
+  delete t2;
+}
+
+TEST(kmp_literal_trait_test, EqualityDifferentValue) {
+  kmp_literal_trait *t1 = new kmp_literal_trait(1);
+  kmp_literal_trait *t2 = new kmp_literal_trait(2);
+
+  EXPECT_FALSE(*t1 == *t2);
+
+  delete t1;
+  delete t2;
+}
+
+//===----------------------------------------------------------------------===//
+// kmp_uid_trait Tests
+//===----------------------------------------------------------------------===//
+
+TEST(kmp_uid_trait_test, Construction) {
+  kmp_uid_trait *trait = new kmp_uid_trait(kmp_str_ref("test-uid"));
+
+  // Just verify it can be constructed without crashing
+  delete trait;
+}
+
+TEST(kmp_uid_trait_test, MatchWithMock) {
+  kmp_uid_trait *trait = new kmp_uid_trait(kmp_str_ref("device-0"));
+
+  // Uses the mock omp_get_uid_from_device
+  EXPECT_TRUE(trait->match(0)); // device-0 matches
+  EXPECT_FALSE(trait->match(1)); // device-1 doesn't match
+  EXPECT_FALSE(trait->match(2)); // device-2 doesn't match
+
+  delete trait;
+}
+
+TEST(kmp_uid_trait_test, MatchWithCustomMock) {
+  kmp_uid_trait *trait = new kmp_uid_trait(kmp_str_ref("custom-uid"));
+
+  // Set a custom mock function
+  trait->set_uid_from_device([](int device) -> const char * {
+    return device == 2 ? "custom-uid" : "other";
+  });
+
+  EXPECT_FALSE(trait->match(0));
+  EXPECT_FALSE(trait->match(1));
+  EXPECT_TRUE(trait->match(2)); // custom-uid matches device 2
+  EXPECT_FALSE(trait->match(3));
+
+  delete trait;
+}
+
+TEST(kmp_uid_trait_test, EqualitySameUID) {
+  kmp_uid_trait *t1 = new kmp_uid_trait(kmp_str_ref("my-device"));
+  kmp_uid_trait *t2 = new kmp_uid_trait(kmp_str_ref("my-device"));
+
+  EXPECT_TRUE(*t1 == *t2);
+
+  delete t1;
+  delete t2;
+}
+
+TEST(kmp_uid_trait_test, EqualityDifferentUID) {
+  kmp_uid_trait *t1 = new kmp_uid_trait(kmp_str_ref("device-a"));
+  kmp_uid_trait *t2 = new kmp_uid_trait(kmp_str_ref("device-b"));
+
+  EXPECT_FALSE(*t1 == *t2);
+
+  delete t1;
+  delete t2;
+}
+
+//===----------------------------------------------------------------------===//
+// kmp_trait_expr_single Tests
+//===----------------------------------------------------------------------===//
+
+TEST(kmp_trait_expr_single_test, CreateAndDestroy) {
+  kmp_trait_expr_single *expr = new kmp_trait_expr_single();
+  EXPECT_NE(expr, nullptr);
+  delete expr;
+}
+
+TEST(kmp_trait_expr_single_test, CreateWithTrait) {
+  kmp_trait_expr_single *expr =
+      new kmp_trait_expr_single(new kmp_literal_trait(2));
+
+  // Mock: 4 devices
+  expr->set_num_devices([]() { return 4; });
+
+  EXPECT_TRUE(expr->match(2));
+  EXPECT_FALSE(expr->match(0));
+  EXPECT_FALSE(expr->match(1));
+  EXPECT_FALSE(expr->match(5)); // Out of range
+
+  delete expr;
+}
+
+TEST(kmp_trait_expr_single_test, SetTrait) {
+  kmp_trait_expr_single *expr = new kmp_trait_expr_single();
+  expr->set_trait(new kmp_literal_trait(3));
+
+  // Mock: 4 devices
+  expr->set_num_devices([]() { return 4; });
+
+  EXPECT_TRUE(expr->match(3));
+  EXPECT_FALSE(expr->match(0));
+
+  delete expr;
+}
+
+TEST(kmp_trait_expr_single_test, DefaultNotNegated) {
+  kmp_trait_expr_single *expr = new kmp_trait_expr_single();
+
+  EXPECT_FALSE(expr->is_negated());
+
+  delete expr;
+}
+
+TEST(kmp_trait_expr_single_test, SetNegated) {
+  kmp_trait_expr_single *expr = new kmp_trait_expr_single();
+
+  expr->set_negated(true);
+  EXPECT_TRUE(expr->is_negated());
+
+  expr->set_negated(false);
+  EXPECT_FALSE(expr->is_negated());
+
+  delete expr;
+}
+
+TEST(kmp_trait_expr_single_test, MatchNegated) {
+  kmp_trait_expr_single *expr =
+      new kmp_trait_expr_single(new kmp_literal_trait(2));
+  expr->set_negated(true);
+
+  // Mock: 4 devices
+  expr->set_num_devices([]() { return 4; });
+
+  // Without negation: matches 2
+  // With negation: matches everything in-range except 2
+  EXPECT_FALSE(expr->match(2));
+  EXPECT_TRUE(expr->match(0));
+  EXPECT_TRUE(expr->match(1));
+  EXPECT_TRUE(expr->match(3));
+  // Out of range devices return false regardless of negation
+  EXPECT_FALSE(expr->match(5));
+
+  delete expr;
+}
+
+TEST(kmp_trait_expr_single_test, MatchWildcard) {
+  kmp_trait_expr_single *expr =
+      new kmp_trait_expr_single(new kmp_wildcard_trait());
+
+  // Mock: 4 devices
+  expr->set_num_devices([]() { return 4; });
+
+  // Wildcard matches any in-range device
+  EXPECT_TRUE(expr->match(0));
+  EXPECT_TRUE(expr->match(3));
+  // Out of range devices return false
+  EXPECT_FALSE(expr->match(100));
+
+  delete expr;
+}
+
+TEST(kmp_trait_expr_single_test, Equality) {
+  kmp_trait_expr_single *e1 =
+      new kmp_trait_expr_single(new kmp_literal_trait(1));
+  kmp_trait_expr_single *e2 =
+      new kmp_trait_expr_single(new kmp_literal_trait(1));
+
+  EXPECT_TRUE(*e1 == *e2);
+
+  delete e1;
+  delete e2;
+}
+
+TEST(kmp_trait_expr_single_test, EqualityDifferentTrait) {
+  kmp_trait_expr_single *e1 =
+      new kmp_trait_expr_single(new kmp_literal_trait(1));
+  kmp_trait_expr_single *e2 =
+      new kmp_trait_expr_single(new kmp_literal_trait(2));
+
+  EXPECT_FALSE(*e1 == *e2);
+
+  delete e1;
+  delete e2;
+}
+
+TEST(kmp_trait_expr_single_test, EqualityDifferentNegation) {
+  kmp_trait_expr_single *e1 =
+      new kmp_trait_expr_single(new kmp_literal_trait(1));
+  kmp_trait_expr_single *e2 =
+      new kmp_trait_expr_single(new kmp_literal_trait(1));
+  e2->set_negated(true);
+
+  EXPECT_FALSE(*e1 == *e2);
+
+  delete e1;
+  delete e2;
+}
+
+//===----------------------------------------------------------------------===//
+// kmp_trait_expr_group Tests
+//===----------------------------------------------------------------------===//
+
+TEST(kmp_trait_expr_group_test, CreateAndDestroy) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+  EXPECT_NE(group, nullptr);
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, DefaultTypeIsOR) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+
+  EXPECT_EQ(group->get_group_type(), kmp_trait_expr_group::OR);
+
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, SetTypeAND) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+
+  group->set_group_type(kmp_trait_expr_group::AND);
+  EXPECT_EQ(group->get_group_type(), kmp_trait_expr_group::AND);
+
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, DefaultNotNegated) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+
+  EXPECT_FALSE(group->is_negated());
+
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, SetNegated) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+
+  group->set_negated(true);
+  EXPECT_TRUE(group->is_negated());
+
+  group->set_negated(false);
+  EXPECT_FALSE(group->is_negated());
+
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, AddTraitDirectly) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+
+  group->add_expr(new kmp_wildcard_trait());
+
+  // Mock: 4 devices
+  group->set_num_devices([]() { return 4; });
+
+  // Wildcard matches any in-range device
+  EXPECT_TRUE(group->match(0));
+  EXPECT_TRUE(group->match(3));
+  // Out of range devices return false
+  EXPECT_FALSE(group->match(100));
+
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, AddExpr) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+
+  group->add_expr(new kmp_trait_expr_single(new kmp_literal_trait(2)));
+
+  // Mock: 4 devices
+  group->set_num_devices([]() { return 4; });
+
+  EXPECT_TRUE(group->match(2));
+  EXPECT_FALSE(group->match(0));
+  EXPECT_FALSE(group->match(5)); // Out of range
+
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, MatchORSemantics) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+  group->set_group_type(kmp_trait_expr_group::OR);
+
+  group->add_expr(new kmp_literal_trait(1));
+  group->add_expr(new kmp_literal_trait(2));
+  group->add_expr(new kmp_literal_trait(3));
+
+  // Mock: 5 devices
+  group->set_num_devices([]() { return 5; });
+
+  // OR: matches if ANY trait matches
+  EXPECT_TRUE(group->match(1));
+  EXPECT_TRUE(group->match(2));
+  EXPECT_TRUE(group->match(3));
+  EXPECT_FALSE(group->match(0));
+  EXPECT_FALSE(group->match(4));
+
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, MatchANDSemantics) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+  group->set_group_type(kmp_trait_expr_group::AND);
+
+  // For AND to pass, ALL traits must match the same device
+  // A single literal only matches one device
+  group->add_expr(new kmp_literal_trait(2));
+
+  // Mock: 4 devices
+  group->set_num_devices([]() { return 4; });
+
+  EXPECT_TRUE(group->match(2));
+  EXPECT_FALSE(group->match(0));
+  // Out of range
+  EXPECT_FALSE(group->match(5));
+
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, MatchANDWithWildcard) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+  group->set_group_type(kmp_trait_expr_group::AND);
+
+  group->add_expr(new kmp_wildcard_trait());
+  group->add_expr(new kmp_literal_trait(2));
+
+  // Mock: 4 devices
+  group->set_num_devices([]() { return 4; });
+
+  // Wildcard matches all, literal matches 2
+  // AND: both must match
+  EXPECT_TRUE(group->match(2));
+  EXPECT_FALSE(group->match(0));
+  // Out of range
+  EXPECT_FALSE(group->match(5));
+
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, MatchNegated) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+
+  group->add_expr(new kmp_literal_trait(2));
+  group->set_negated(true);
+
+  // Mock: 4 devices
+  group->set_num_devices([]() { return 4; });
+
+  // Without negation: matches 2
+  // With negation: matches everything in-range except 2
+  EXPECT_FALSE(group->match(2));
+  EXPECT_TRUE(group->match(0));
+  EXPECT_TRUE(group->match(1));
+  EXPECT_TRUE(group->match(3));
+  // Out of range devices return false regardless of negation
+  EXPECT_FALSE(group->match(5));
+
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, MatchEmptyGroupOR) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+  group->set_group_type(kmp_trait_expr_group::OR);
+
+  // Mock: 4 devices
+  group->set_num_devices([]() { return 4; });
+
+  // Empty OR: no traits match, so result is false
+  EXPECT_FALSE(group->match(0));
+  EXPECT_FALSE(group->match(1));
+
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, MatchEmptyGroupAND) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+  group->set_group_type(kmp_trait_expr_group::AND);
+
+  // Mock: 4 devices
+  group->set_num_devices([]() { return 4; });
+
+  // Empty AND: vacuously true (0 out of 0 traits match)
+  EXPECT_TRUE(group->match(0));
+  EXPECT_TRUE(group->match(1));
+
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, Equality) {
+  kmp_trait_expr_group *g1 = new kmp_trait_expr_group();
+  kmp_trait_expr_group *g2 = new kmp_trait_expr_group();
+
+  g1->add_expr(new kmp_literal_trait(1));
+  g2->add_expr(new kmp_literal_trait(1));
+
+  EXPECT_TRUE(*g1 == *g2);
+
+  delete g1;
+  delete g2;
+}
+
+TEST(kmp_trait_expr_group_test, EqualityDifferentNegation) {
+  kmp_trait_expr_group *g1 = new kmp_trait_expr_group();
+  kmp_trait_expr_group *g2 = new kmp_trait_expr_group();
+
+  g1->add_expr(new kmp_literal_trait(1));
+  g2->add_expr(new kmp_literal_trait(1));
+  g2->set_negated(true);
+
+  EXPECT_FALSE(*g1 == *g2);
+
+  delete g1;
+  delete g2;
+}
+
+TEST(kmp_trait_expr_group_test, EqualityDifferentGroupType) {
+  // Groups with the same sub-expressions but 
diff erent AND/OR type are not
+  // equal.
+  kmp_trait_expr_group *g1 = new kmp_trait_expr_group();
+  kmp_trait_expr_group *g2 = new kmp_trait_expr_group();
+
+  g1->add_expr(new kmp_literal_trait(1));
+  g1->add_expr(new kmp_literal_trait(2));
+  g1->set_group_type(kmp_trait_expr_group::AND);
+  g2->add_expr(new kmp_literal_trait(1));
+  g2->add_expr(new kmp_literal_trait(2));
+  g2->set_group_type(kmp_trait_expr_group::OR);
+
+  EXPECT_FALSE(*g1 == *g2);
+
+  delete g1;
+  delete g2;
+}
+
+TEST(kmp_trait_expr_group_test, NestedGroups) {
+  kmp_trait_expr_group *outer = new kmp_trait_expr_group();
+  outer->set_group_type(kmp_trait_expr_group::OR);
+
+  kmp_trait_expr_group *inner = new kmp_trait_expr_group();
+  inner->set_group_type(kmp_trait_expr_group::AND);
+  inner->add_expr(new kmp_literal_trait(1));
+  inner->add_expr(new kmp_wildcard_trait());
+
+  outer->add_expr(inner);
+  outer->add_expr(new kmp_literal_trait(2));
+
+  // Mock: 4 devices
+  outer->set_num_devices([]() { return 4; });
+
+  // Inner matches device 1 (literal 1 AND wildcard)
+  // Outer matches 1 OR 2
+  EXPECT_TRUE(outer->match(1));
+  EXPECT_TRUE(outer->match(2));
+  EXPECT_FALSE(outer->match(0));
+  EXPECT_FALSE(outer->match(3));
+
+  delete outer;
+}
+
+//===----------------------------------------------------------------------===//
+// kmp_trait_clause Tests
+//===----------------------------------------------------------------------===//
+
+TEST(kmp_trait_clause_test, CreateAndDestroy) {
+  kmp_trait_clause *clause = new kmp_trait_clause();
+  EXPECT_NE(clause, nullptr);
+  delete clause;
+}
+
+TEST(kmp_trait_clause_test, SetExprWithTrait) {
+  kmp_trait_clause *clause = new kmp_trait_clause();
+  clause->set_expr(new kmp_literal_trait(2));
+
+  // The trait is wrapped in kmp_trait_expr_single internally
+  kmp_trait_expr *expr = clause->get_expr();
+  EXPECT_NE(expr, nullptr);
+
+  delete clause;
+}
+
+TEST(kmp_trait_clause_test, SetExprWithExpr) {
+  kmp_trait_clause *clause = new kmp_trait_clause();
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+  group->add_expr(new kmp_literal_trait(1));
+  clause->set_expr(group);
+
+  EXPECT_EQ(clause->get_expr(), group);
+
+  delete clause;
+}
+
+TEST(kmp_trait_clause_test, Equality) {
+  kmp_trait_clause *c1 = new kmp_trait_clause();
+  kmp_trait_clause *c2 = new kmp_trait_clause();
+
+  c1->set_expr(new kmp_literal_trait(1));
+  c2->set_expr(new kmp_literal_trait(1));
+
+  EXPECT_TRUE(*c1 == *c2);
+
+  delete c1;
+  delete c2;
+}
+
+TEST(kmp_trait_clause_test, EqualityDifferentExprs) {
+  kmp_trait_clause *c1 = new kmp_trait_clause();
+  kmp_trait_clause *c2 = new kmp_trait_clause();
+
+  c1->set_expr(new kmp_literal_trait(1));
+  c2->set_expr(new kmp_literal_trait(2));
+
+  EXPECT_FALSE(*c1 == *c2);
+
+  delete c1;
+  delete c2;
+}
+
+//===----------------------------------------------------------------------===//
+// kmp_trait_context Tests
+//===----------------------------------------------------------------------===//
+
+TEST(kmp_trait_context_test, CreateAndDestroy) {
+  kmp_trait_context *context = new kmp_trait_context();
+  EXPECT_NE(context, nullptr);
+  delete context;
+}
+
+TEST(kmp_trait_context_test, AddClause) {
+  kmp_trait_context *context = new kmp_trait_context();
+  kmp_trait_clause *clause = new kmp_trait_clause();
+  clause->set_expr(new kmp_literal_trait(2));
+  context->add_clause(clause);
+
+  // Mock: 4 devices
+  context->set_num_devices([]() { return 4; });
+
+  EXPECT_TRUE(context->match(2));
+  EXPECT_FALSE(context->match(0));
+  // Out of range
+  EXPECT_FALSE(context->match(5));
+
+  delete context;
+}
+
+TEST(kmp_trait_context_test, MultipleClauses) {
+  kmp_trait_context *context = new kmp_trait_context();
+
+  kmp_trait_clause *c1 = new kmp_trait_clause();
+  c1->set_expr(new kmp_literal_trait(1));
+  context->add_clause(c1);
+
+  kmp_trait_clause *c2 = new kmp_trait_clause();
+  c2->set_expr(new kmp_literal_trait(2));
+  context->add_clause(c2);
+
+  kmp_trait_clause *c3 = new kmp_trait_clause();
+  c3->set_expr(new kmp_literal_trait(3));
+  context->add_clause(c3);
+
+  // Mock: 5 devices
+  context->set_num_devices([]() { return 5; });
+
+  // Context uses OR semantics between clauses
+  EXPECT_TRUE(context->match(1));
+  EXPECT_TRUE(context->match(2));
+  EXPECT_TRUE(context->match(3));
+  EXPECT_FALSE(context->match(0));
+  EXPECT_FALSE(context->match(4));
+
+  delete context;
+}
+
+TEST(kmp_trait_context_test, EmptyContextMatchesNothing) {
+  kmp_trait_context *context = new kmp_trait_context();
+
+  // Mock: 4 devices
+  context->set_num_devices([]() { return 4; });
+
+  EXPECT_FALSE(context->match(0));
+  EXPECT_FALSE(context->match(1));
+
+  delete context;
+}
+
+TEST(kmp_trait_context_test, WildcardClause) {
+  kmp_trait_context *context = new kmp_trait_context();
+  kmp_trait_clause *clause = new kmp_trait_clause();
+  clause->set_expr(new kmp_wildcard_trait());
+  context->add_clause(clause);
+
+  // Mock: 4 devices
+  context->set_num_devices([]() { return 4; });
+
+  // In-range devices match
+  EXPECT_TRUE(context->match(0));
+  EXPECT_TRUE(context->match(3));
+  // Out of range devices return false
+  EXPECT_FALSE(context->match(100));
+  EXPECT_FALSE(context->match(-1));
+
+  delete context;
+}
+
+TEST(kmp_trait_context_test, EvaluateWithMock) {
+  kmp_trait_context *context = new kmp_trait_context();
+
+  // Mock: 5 devices
+  context->set_num_devices([]() { return 5; });
+
+  kmp_trait_clause *c1 = new kmp_trait_clause();
+  c1->set_expr(new kmp_literal_trait(1));
+  context->add_clause(c1);
+
+  kmp_trait_clause *c2 = new kmp_trait_clause();
+  c2->set_expr(new kmp_literal_trait(3));
+  context->add_clause(c2);
+
+  kmp_vector<int> result = context->evaluate();
+  EXPECT_EQ(result.size(), 2u);
+  EXPECT_TRUE(result.contains(1));
+  EXPECT_TRUE(result.contains(3));
+  EXPECT_FALSE(result.contains(0));
+  EXPECT_FALSE(result.contains(2));
+  EXPECT_FALSE(result.contains(4));
+
+  delete context;
+}
+
+TEST(kmp_trait_context_test, Equality) {
+  kmp_trait_context *ctx1 = new kmp_trait_context();
+  kmp_trait_context *ctx2 = new kmp_trait_context();
+
+  kmp_trait_clause *c1 = new kmp_trait_clause();
+  c1->set_expr(new kmp_literal_trait(1));
+  ctx1->add_clause(c1);
+
+  kmp_trait_clause *c2 = new kmp_trait_clause();
+  c2->set_expr(new kmp_literal_trait(1));
+  ctx2->add_clause(c2);
+
+  EXPECT_TRUE(*ctx1 == *ctx2);
+
+  delete ctx1;
+  delete ctx2;
+}
+
+TEST(kmp_trait_context_test, EqualityDifferentClauses) {
+  kmp_trait_context *ctx1 = new kmp_trait_context();
+  kmp_trait_context *ctx2 = new kmp_trait_context();
+
+  kmp_trait_clause *c1 = new kmp_trait_clause();
+  c1->set_expr(new kmp_literal_trait(1));
+  ctx1->add_clause(c1);
+
+  kmp_trait_clause *c2 = new kmp_trait_clause();
+  c2->set_expr(new kmp_literal_trait(2));
+  ctx2->add_clause(c2);
+
+  EXPECT_FALSE(*ctx1 == *ctx2);
+
+  delete ctx1;
+  delete ctx2;
+}
+
+//===----------------------------------------------------------------------===//
+// kmp_trait_context Iterator Tests
+//===----------------------------------------------------------------------===//
+
+TEST(kmp_trait_context_test, IteratorRangeBasedFor) {
+  kmp_trait_context *context = new kmp_trait_context();
+
+  // Mock: 5 devices
+  context->set_num_devices([]() { return 5; });
+
+  kmp_trait_clause *c1 = new kmp_trait_clause();
+  c1->set_expr(new kmp_literal_trait(1));
+  context->add_clause(c1);
+
+  kmp_trait_clause *c2 = new kmp_trait_clause();
+  c2->set_expr(new kmp_literal_trait(3));
+  context->add_clause(c2);
+
+  // Use range-based for loop (should auto-evaluate)
+  kmp_vector<int> collected;
+  for (int d : *context) {
+    collected.push_back(d);
+  }
+
+  EXPECT_EQ(collected.size(), 2u);
+  EXPECT_TRUE(collected.contains(1));
+  EXPECT_TRUE(collected.contains(3));
+
+  delete context;
+}
+
+TEST(kmp_trait_context_test, IteratorAutoEvaluates) {
+  kmp_trait_context *context = new kmp_trait_context();
+
+  // Mock: 4 devices
+  context->set_num_devices([]() { return 4; });
+
+  kmp_trait_clause *clause = new kmp_trait_clause();
+  clause->set_expr(new kmp_wildcard_trait());
+  context->add_clause(clause);
+
+  // Directly use begin()/end() without calling evaluate() first
+  int count = 0;
+  for (const int *it = context->begin(); it != context->end(); ++it) {
+    EXPECT_GE(*it, 0);
+    EXPECT_LT(*it, 4);
+    count++;
+  }
+
+  EXPECT_EQ(count, 4);
+
+  delete context;
+}
+
+TEST(kmp_trait_context_test, IteratorEmptyContext) {
+  kmp_trait_context *context = new kmp_trait_context();
+
+  // Mock: 4 devices
+  context->set_num_devices([]() { return 4; });
+
+  // Empty context - no clauses added
+  int count = 0;
+  for (int d : *context) {
+    (void)d;
+    count++;
+  }
+
+  EXPECT_EQ(count, 0);
+  EXPECT_EQ(context->begin(), context->end());
+
+  delete context;
+}
+
+TEST(kmp_trait_context_test, IteratorBeginEnd) {
+  kmp_trait_context *context = new kmp_trait_context();
+
+  // Mock: 3 devices
+  context->set_num_devices([]() { return 3; });
+
+  kmp_trait_clause *clause = new kmp_trait_clause();
+  clause->set_expr(new kmp_literal_trait(2));
+  context->add_clause(clause);
+
+  // Test begin/end directly
+  const int *b = context->begin();
+  const int *e = context->end();
+
+  EXPECT_EQ(e - b, 1); // Should have exactly 1 element
+  EXPECT_EQ(*b, 2);
+
+  delete context;
+}
+
+TEST(kmp_trait_context_test, IteratorMultipleDevices) {
+  kmp_trait_context *context = new kmp_trait_context();
+
+  // Add clauses for devices 0, 2, 4
+  for (int i = 0; i < 6; i += 2) {
+    kmp_trait_clause *clause = new kmp_trait_clause();
+    clause->set_expr(new kmp_literal_trait(i));
+    context->add_clause(clause);
+  }
+
+  // Mock: 6 devices (must be set after adding clauses to propagate to them)
+  context->set_num_devices([]() { return 6; });
+
+  // Collect via iterator
+  kmp_vector<int> collected;
+  for (int d : *context) {
+    collected.push_back(d);
+  }
+
+  EXPECT_EQ(collected.size(), 3u);
+  EXPECT_TRUE(collected.contains(0));
+  EXPECT_TRUE(collected.contains(2));
+  EXPECT_TRUE(collected.contains(4));
+
+  delete context;
+}
+
+TEST(kmp_trait_context_test, IteratorConsistentWithEvaluate) {
+  kmp_trait_context *context = new kmp_trait_context();
+
+  // Mock: 5 devices
+  context->set_num_devices([]() { return 5; });
+
+  kmp_trait_clause *c1 = new kmp_trait_clause();
+  c1->set_expr(new kmp_literal_trait(1));
+  context->add_clause(c1);
+
+  kmp_trait_clause *c2 = new kmp_trait_clause();
+  c2->set_expr(new kmp_literal_trait(4));
+  context->add_clause(c2);
+
+  // Get result via evaluate()
+  const kmp_vector<int> &eval_result = context->evaluate();
+
+  // Collect via iterator
+  kmp_vector<int> iter_result;
+  for (int d : *context) {
+    iter_result.push_back(d);
+  }
+
+  // Both should give the same results
+  EXPECT_EQ(eval_result.size(), iter_result.size());
+  for (size_t i = 0; i < eval_result.size(); i++) {
+    EXPECT_EQ(eval_result[i], iter_result[i]);
+  }
+
+  delete context;
+}
+
+TEST(kmp_trait_context_test, EvaluateReturnsByReference) {
+  kmp_trait_context *context = new kmp_trait_context();
+
+  // Mock: 3 devices
+  context->set_num_devices([]() { return 3; });
+
+  kmp_trait_clause *clause = new kmp_trait_clause();
+  clause->set_expr(new kmp_wildcard_trait());
+  context->add_clause(clause);
+
+  // Multiple calls to evaluate() should return reference to the same data
+  const kmp_vector<int> &result1 = context->evaluate();
+  const kmp_vector<int> &result2 = context->evaluate();
+
+  EXPECT_EQ(&result1, &result2);
+
+  delete context;
+}
+
+//===----------------------------------------------------------------------===//
+// get_num_devices Propagation Tests
+//===----------------------------------------------------------------------===//
+
+TEST(kmp_trait_context_test, PropagationToClausesAddedAfterSetNumDevices) {
+  kmp_trait_context *context = new kmp_trait_context();
+
+  // Set mock BEFORE adding clauses - propagation should still work
+  context->set_num_devices([]() { return 6; });
+
+  // Add clauses for devices 0, 2, 4 (all require 6 devices to be in range)
+  for (int i = 0; i < 6; i += 2) {
+    kmp_trait_clause *clause = new kmp_trait_clause();
+    clause->set_expr(new kmp_literal_trait(i));
+    context->add_clause(clause);
+  }
+
+  // All three devices should match because propagation worked
+  kmp_vector<int> collected;
+  for (int d : *context) {
+    collected.push_back(d);
+  }
+
+  EXPECT_EQ(collected.size(), 3u);
+  EXPECT_TRUE(collected.contains(0));
+  EXPECT_TRUE(collected.contains(2));
+  EXPECT_TRUE(collected.contains(4));
+
+  delete context;
+}
+
+TEST(kmp_trait_context_test, PropagationToClausesAddedBeforeSetNumDevices) {
+  kmp_trait_context *context = new kmp_trait_context();
+
+  // Add clauses BEFORE setting mock
+  for (int i = 0; i < 6; i += 2) {
+    kmp_trait_clause *clause = new kmp_trait_clause();
+    clause->set_expr(new kmp_literal_trait(i));
+    context->add_clause(clause);
+  }
+
+  // Set mock AFTER adding clauses
+  context->set_num_devices([]() { return 6; });
+
+  // All three devices should match
+  kmp_vector<int> collected;
+  for (int d : *context) {
+    collected.push_back(d);
+  }
+
+  EXPECT_EQ(collected.size(), 3u);
+  EXPECT_TRUE(collected.contains(0));
+  EXPECT_TRUE(collected.contains(2));
+  EXPECT_TRUE(collected.contains(4));
+
+  delete context;
+}
+
+TEST(kmp_trait_expr_group_test, PropagationToExprsAddedAfterSetNumDevices) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+
+  // Set mock BEFORE adding expressions
+  group->set_num_devices([]() { return 8; });
+
+  // Add expressions for devices 5, 6, 7 (require 8 devices)
+  group->add_expr(new kmp_literal_trait(5));
+  group->add_expr(new kmp_literal_trait(6));
+  group->add_expr(new kmp_literal_trait(7));
+
+  // All should match
+  EXPECT_TRUE(group->match(5));
+  EXPECT_TRUE(group->match(6));
+  EXPECT_TRUE(group->match(7));
+  EXPECT_FALSE(group->match(4));
+
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, PropagationToExprsAddedBeforeSetNumDevices) {
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+
+  // Add expressions BEFORE setting mock
+  group->add_expr(new kmp_literal_trait(5));
+  group->add_expr(new kmp_literal_trait(6));
+  group->add_expr(new kmp_literal_trait(7));
+
+  // Set mock AFTER adding expressions
+  group->set_num_devices([]() { return 8; });
+
+  // All should match
+  EXPECT_TRUE(group->match(5));
+  EXPECT_TRUE(group->match(6));
+  EXPECT_TRUE(group->match(7));
+  EXPECT_FALSE(group->match(4));
+
+  delete group;
+}
+
+TEST(kmp_trait_expr_group_test, PropagationToNestedGroups) {
+  kmp_trait_expr_group *outer = new kmp_trait_expr_group();
+  outer->set_group_type(kmp_trait_expr_group::OR);
+
+  // Set mock on outer group FIRST
+  outer->set_num_devices([]() { return 10; });
+
+  // Create inner group and add high-numbered devices
+  kmp_trait_expr_group *inner = new kmp_trait_expr_group();
+  inner->set_group_type(kmp_trait_expr_group::OR);
+  inner->add_expr(new kmp_literal_trait(8));
+  inner->add_expr(new kmp_literal_trait(9));
+
+  // Add inner to outer - should propagate mock to inner and its children
+  outer->add_expr(inner);
+
+  // Add another expression directly to outer
+  outer->add_expr(new kmp_literal_trait(7));
+
+  // All should match with 10 devices
+  EXPECT_TRUE(outer->match(7));
+  EXPECT_TRUE(outer->match(8));
+  EXPECT_TRUE(outer->match(9));
+  EXPECT_FALSE(outer->match(10)); // Out of range
+
+  delete outer;
+}
+
+TEST(kmp_trait_expr_group_test, PropagationToDeeplyNestedGroups) {
+  // Create a deeply nested structure: outer -> middle -> inner
+  kmp_trait_expr_group *outer = new kmp_trait_expr_group();
+  outer->set_group_type(kmp_trait_expr_group::OR);
+
+  // Set mock on outer
+  outer->set_num_devices([]() { return 12; });
+
+  kmp_trait_expr_group *middle = new kmp_trait_expr_group();
+  middle->set_group_type(kmp_trait_expr_group::OR);
+
+  kmp_trait_expr_group *inner = new kmp_trait_expr_group();
+  inner->set_group_type(kmp_trait_expr_group::OR);
+  inner->add_expr(new kmp_literal_trait(10));
+  inner->add_expr(new kmp_literal_trait(11));
+
+  middle->add_expr(inner);
+  middle->add_expr(new kmp_literal_trait(9));
+
+  outer->add_expr(middle);
+  outer->add_expr(new kmp_literal_trait(8));
+
+  // All devices 8-11 should match (requires 12 devices)
+  EXPECT_TRUE(outer->match(8));
+  EXPECT_TRUE(outer->match(9));
+  EXPECT_TRUE(outer->match(10));
+  EXPECT_TRUE(outer->match(11));
+  EXPECT_FALSE(outer->match(12)); // Out of range
+
+  delete outer;
+}
+
+TEST(kmp_trait_context_test, PropagationToNestedGroupsInClauses) {
+  kmp_trait_context *context = new kmp_trait_context();
+
+  // Set mock on context FIRST
+  context->set_num_devices([]() { return 10; });
+
+  // Create a group with nested structure
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+  group->set_group_type(kmp_trait_expr_group::OR);
+
+  kmp_trait_expr_group *inner = new kmp_trait_expr_group();
+  inner->set_group_type(kmp_trait_expr_group::OR);
+  inner->add_expr(new kmp_literal_trait(8));
+  inner->add_expr(new kmp_literal_trait(9));
+
+  group->add_expr(inner);
+  group->add_expr(new kmp_literal_trait(7));
+
+  // Create clause with the group
+  kmp_trait_clause *clause = new kmp_trait_clause();
+  clause->set_expr(group);
+
+  // Add clause to context - should propagate to group and inner
+  context->add_clause(clause);
+
+  // All should match
+  kmp_vector<int> collected;
+  for (int d : *context) {
+    collected.push_back(d);
+  }
+
+  EXPECT_EQ(collected.size(), 3u);
+  EXPECT_TRUE(collected.contains(7));
+  EXPECT_TRUE(collected.contains(8));
+  EXPECT_TRUE(collected.contains(9));
+
+  delete context;
+}
+
+TEST(kmp_trait_context_test, PropagationMixedOrder) {
+  // Test a complex scenario with mixed ordering
+  kmp_trait_context *context = new kmp_trait_context();
+
+  // Add first clause before set_num_devices
+  kmp_trait_clause *c1 = new kmp_trait_clause();
+  c1->set_expr(new kmp_literal_trait(5));
+  context->add_clause(c1);
+
+  // Set mock
+  context->set_num_devices([]() { return 8; });
+
+  // Add second clause after set_num_devices
+  kmp_trait_clause *c2 = new kmp_trait_clause();
+  c2->set_expr(new kmp_literal_trait(6));
+  context->add_clause(c2);
+
+  // Add third clause with a group
+  kmp_trait_expr_group *group = new kmp_trait_expr_group();
+  group->add_expr(new kmp_literal_trait(7));
+
+  kmp_trait_clause *c3 = new kmp_trait_clause();
+  c3->set_expr(group);
+  context->add_clause(c3);
+
+  // All three should match
+  kmp_vector<int> collected;
+  for (int d : *context) {
+    collected.push_back(d);
+  }
+
+  EXPECT_EQ(collected.size(), 3u);
+  EXPECT_TRUE(collected.contains(5));
+  EXPECT_TRUE(collected.contains(6));
+  EXPECT_TRUE(collected.contains(7));
+
+  delete context;
+}
+
+} // namespace


        

From openmp-commits at lists.llvm.org  Wed Aug 12 09:08:11 2026
From: openmp-commits at lists.llvm.org (Robert Imschweiler via Openmp-commits)
Date: Wed, 12 Aug 2026 09:08:11 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [libomp] OpenMP 6.0: Add device trait
	parser (PR #176164)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/176164@github.com>
Message-ID: <6a7c9a6b.d681cab9.37a3e3.afc7@mx.google.com>


https://github.com/ro-i closed https://github.com/llvm/llvm-project/pull/176164

From openmp-commits at lists.llvm.org  Wed Aug 12 09:08:12 2026
From: openmp-commits at lists.llvm.org (Robert Imschweiler via Openmp-commits)
Date: Wed, 12 Aug 2026 09:08:12 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [libomp] Parse OMP_DEFAULT_DEVICE with
 new device trait parser (PR #176166)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/176166@github.com>
Message-ID: <6a7c9a6c.c5504292.2f6b98.647b@mx.google.com>


https://github.com/ro-i edited https://github.com/llvm/llvm-project/pull/176166

From openmp-commits at lists.llvm.org  Wed Aug 12 11:49:44 2026
From: openmp-commits at lists.llvm.org (Nick Sarnie via Openmp-commits)
Date: Wed, 12 Aug 2026 11:49:44 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [Offload] Add logger in Plugin API
 functions for OpenMP (PR #215582)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/215582@github.com>
Message-ID: <6a7cc048.4a342cbc.d98ad.5b77@mx.google.com>


================
@@ -0,0 +1,167 @@
+//===-- Shared/APITrace.h - Tracing of offload API calls --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Prints one line per offload API call while tracing is enabled:
+//
+//   ---> init_device(.DeviceId = 0)-> OFFLOAD_SUCCESS (1053 us)
+//
+// Tracing is off until the runtime above the plugins turns it on, see
+// GenericPluginTy::set_api_trace.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OMPTARGET_SHARED_API_TRACE_H
+#define OMPTARGET_SHARED_API_TRACE_H
+
+#include "APITypes.h"
+#include "omptarget.h"
+
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <atomic>
+#include <chrono>
+#include <type_traits>
+#include <utility>
+
+namespace llvm::offload::trace {
+
+inline std::atomic<bool> &getTraceEnabledFlag() {
+  static std::atomic<bool> Enabled = false;
----------------
sarnex wrote:

is it safe to have this definition in the header? inlining seems required for correctness here

https://github.com/llvm/llvm-project/pull/215582

From openmp-commits at lists.llvm.org  Wed Aug 12 11:49:45 2026
From: openmp-commits at lists.llvm.org (Nick Sarnie via Openmp-commits)
Date: Wed, 12 Aug 2026 11:49:45 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [Offload] Add logger in Plugin API
 functions for OpenMP (PR #215582)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/215582@github.com>
Message-ID: <6a7cc049.e5907290.4f8b4.37ce@mx.google.com>


================
@@ -0,0 +1,167 @@
+//===-- Shared/APITrace.h - Tracing of offload API calls --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Prints one line per offload API call while tracing is enabled:
+//
+//   ---> init_device(.DeviceId = 0)-> OFFLOAD_SUCCESS (1053 us)
+//
+// Tracing is off until the runtime above the plugins turns it on, see
+// GenericPluginTy::set_api_trace.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OMPTARGET_SHARED_API_TRACE_H
+#define OMPTARGET_SHARED_API_TRACE_H
+
+#include "APITypes.h"
+#include "omptarget.h"
+
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <atomic>
+#include <chrono>
+#include <type_traits>
+#include <utility>
+
+namespace llvm::offload::trace {
+
+inline std::atomic<bool> &getTraceEnabledFlag() {
+  static std::atomic<bool> Enabled = false;
+  return Enabled;
+}
+
+/// The flag is read on every call so that tracing can be toggled around a
+/// region of interest.
+inline bool isTraceEnabled() {
+  return getTraceEnabledFlag().load(std::memory_order_relaxed);
+}
+
+inline void setTraceEnabled(bool Enable) {
+  getTraceEnabledFlag().store(Enable, std::memory_order_relaxed);
+}
+
+inline raw_ostream &operator<<(raw_ostream &OS, __tgt_device_binary Binary) {
+  return OS << reinterpret_cast<void *>(Binary.handle);
+}
+
+inline raw_ostream &operator<<(raw_ostream &OS, OffloadStatusTy Status) {
+  switch (Status) {
+  case OFFLOAD_SUCCESS:
+    return OS << "OFFLOAD_SUCCESS";
+  case OFFLOAD_FAIL:
+    return OS << "OFFLOAD_FAIL";
+  }
+  return OS << static_cast<int32_t>(Status);
+}
+
+inline raw_ostream &operator<<(raw_ostream &OS, TargetAllocTy Kind) {
+  switch (Kind) {
+  case TARGET_ALLOC_DEVICE:
+    return OS << "TARGET_ALLOC_DEVICE";
+  case TARGET_ALLOC_HOST:
+    return OS << "TARGET_ALLOC_HOST";
+  case TARGET_ALLOC_SHARED:
+    return OS << "TARGET_ALLOC_SHARED";
+  case TARGET_ALLOC_DEFAULT:
+    return OS << "TARGET_ALLOC_DEFAULT";
+  }
+  return OS << static_cast<int32_t>(Kind);
+}
+
+template <typename Ty, typename = void> struct IsPrintable : std::false_type {};
+template <typename Ty>
+struct IsPrintable<Ty, std::void_t<decltype(std::declval<raw_ostream &>()
+                                            << std::declval<const Ty &>())>>
+    : std::true_type {};
+
+/// Types with no raw_ostream overload are shown as an address, so that adding a
+/// parameter to a traced entry point never breaks the build.
+template <typename Ty> void printArg(raw_ostream &OS, const Ty &Arg) {
+  using DecayedTy = std::decay_t<Ty>;
----------------
sarnex wrote:

nit: we could add on a `std::remove_const` and then be able to do a simpler check in the next `if`

https://github.com/llvm/llvm-project/pull/215582

From openmp-commits at lists.llvm.org  Wed Aug 12 11:49:45 2026
From: openmp-commits at lists.llvm.org (Nick Sarnie via Openmp-commits)
Date: Wed, 12 Aug 2026 11:49:45 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [Offload] Add logger in Plugin API
 functions for OpenMP (PR #215582)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/215582@github.com>
Message-ID: <6a7cc049.8e75e0cb.2dff9e.5b01@mx.google.com>


https://github.com/sarnex edited https://github.com/llvm/llvm-project/pull/215582

From openmp-commits at lists.llvm.org  Wed Aug 12 11:49:45 2026
From: openmp-commits at lists.llvm.org (Nick Sarnie via Openmp-commits)
Date: Wed, 12 Aug 2026 11:49:45 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [Offload] Add logger in Plugin API
 functions for OpenMP (PR #215582)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/215582@github.com>
Message-ID: <6a7cc049.6c8668f2.8d125.a495@mx.google.com>


https://github.com/sarnex commented:

not really familiar with this area but seems reasonable

https://github.com/llvm/llvm-project/pull/215582

From openmp-commits at lists.llvm.org  Wed Aug 12 13:03:59 2026
From: openmp-commits at lists.llvm.org (Kelvin Li via Openmp-commits)
Date: Wed, 12 Aug 2026 13:03:59 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OMP] Remove omp_get_ancestor_thread_num
 and omp_get_team_size in ompx.h (PR #215841)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/215841@github.com>
Message-ID: <6a7cd1af.4b9e17a8.34cba1.6dee@mx.google.com>


kkwli wrote:

https://github.com/llvm/llvm-project/pull/215882 to fix the build failure

https://github.com/llvm/llvm-project/pull/215841

From openmp-commits at lists.llvm.org  Wed Aug 12 20:28:20 2026
From: openmp-commits at lists.llvm.org (Eugene Epshteyn via Openmp-commits)
Date: Wed, 12 Aug 2026 20:28:20 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [libomp] OpenMP 6.0: Add device trait
	parser (PR #176164)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/176164@github.com>
Message-ID: <6a7d39d4.f4d142a5.1eeedd.a79b@mx.google.com>


eugeneepshteyn wrote:

This change seems to be the one causing failures building in debug build: https://github.com/llvm/llvm-project/issues/215937

https://github.com/llvm/llvm-project/pull/176164

From openmp-commits at lists.llvm.org  Wed Aug 12 23:17:56 2026
From: openmp-commits at lists.llvm.org (Robert Imschweiler via Openmp-commits)
Date: Wed, 12 Aug 2026 23:17:56 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [libomp] Fix debug build after #176164
	(PR #215961)
Message-ID: <llvm.org/llvm/llvm-project/pull/215961@github.com>


https://github.com/ro-i created https://github.com/llvm/llvm-project/pull/215961

Fixes https://github.com/llvm/llvm-project/issues/215937.

>From 0b4cdc19f7d05daf219473e11c422794a8d90e4d Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Thu, 13 Aug 2026 01:15:59 -0500
Subject: [PATCH] [libomp] Fix debug build after #176164

Fixes https://github.com/llvm/llvm-project/issues/215937.
---
 openmp/runtime/src/kmp_traits.h | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/openmp/runtime/src/kmp_traits.h b/openmp/runtime/src/kmp_traits.h
index ed19ceccd8846..66913bd9db22c 100644
--- a/openmp/runtime/src/kmp_traits.h
+++ b/openmp/runtime/src/kmp_traits.h
@@ -103,7 +103,12 @@ class kmp_trait {
   kmp_trait &operator=(const kmp_trait &) = delete;
   kmp_trait &operator=(kmp_trait &&) = delete;
 
-  virtual bool match(int device) const = 0;
+  // Not pure virtual to avoid a dependency on __cxa_pure_virtual, which lives
+  // in libsupc++ and is not linked into libomp. Derived classes must override.
+  virtual bool match([[maybe_unused]] int device) const {
+    KMP_ASSERT2(0, "kmp_trait::match() must be overridden");
+    return false;
+  }
 
   // Use KMP_INTERNAL_MALLOC/KMP_INTERNAL_FREE for memory management.
   void *operator new(size_t size) { return KMP_INTERNAL_MALLOC(size); }
@@ -199,7 +204,13 @@ class kmp_trait_expr {
   kmp_trait_expr(expr_type type, bool negated)
       : _type(type), negated(negated) {}
 
-  virtual bool match_impl(int device, int num_devices) const = 0;
+  // Not pure virtual to avoid a dependency on __cxa_pure_virtual, which lives
+  // in libsupc++ and is not linked into libomp. Derived classes must override.
+  virtual bool match_impl([[maybe_unused]] int device,
+                          [[maybe_unused]] int num_devices) const {
+    KMP_ASSERT2(0, "kmp_trait_expr::match_impl() must be overridden");
+    return false;
+  }
 
 public:
   virtual ~kmp_trait_expr() = default;


From openmp-commits at lists.llvm.org  Thu Aug 13 01:16:30 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Thu, 13 Aug 2026 01:16:30 -0700 (PDT)
Subject: [Openmp-commits] [openmp] 2664668 - [libomp] Fix debug build after
	#176164 (#215961)
Message-ID: <6a7d7d5e.3493695c.2d6521.4240@mx.google.com>


Author: Robert Imschweiler
Date: 2026-08-13T10:16:25+02:00
New Revision: 26646686161b302f7f53776637f1d3cc1eca4b01

URL: https://github.com/llvm/llvm-project/commit/26646686161b302f7f53776637f1d3cc1eca4b01
DIFF: https://github.com/llvm/llvm-project/commit/26646686161b302f7f53776637f1d3cc1eca4b01.diff

LOG: [libomp] Fix debug build after #176164 (#215961)

Fixes https://github.com/llvm/llvm-project/issues/215937.

Added: 
    

Modified: 
    openmp/runtime/src/kmp_traits.h

Removed: 
    


################################################################################
diff  --git a/openmp/runtime/src/kmp_traits.h b/openmp/runtime/src/kmp_traits.h
index ed19ceccd8846..66913bd9db22c 100644
--- a/openmp/runtime/src/kmp_traits.h
+++ b/openmp/runtime/src/kmp_traits.h
@@ -103,7 +103,12 @@ class kmp_trait {
   kmp_trait &operator=(const kmp_trait &) = delete;
   kmp_trait &operator=(kmp_trait &&) = delete;
 
-  virtual bool match(int device) const = 0;
+  // Not pure virtual to avoid a dependency on __cxa_pure_virtual, which lives
+  // in libsupc++ and is not linked into libomp. Derived classes must override.
+  virtual bool match([[maybe_unused]] int device) const {
+    KMP_ASSERT2(0, "kmp_trait::match() must be overridden");
+    return false;
+  }
 
   // Use KMP_INTERNAL_MALLOC/KMP_INTERNAL_FREE for memory management.
   void *operator new(size_t size) { return KMP_INTERNAL_MALLOC(size); }
@@ -199,7 +204,13 @@ class kmp_trait_expr {
   kmp_trait_expr(expr_type type, bool negated)
       : _type(type), negated(negated) {}
 
-  virtual bool match_impl(int device, int num_devices) const = 0;
+  // Not pure virtual to avoid a dependency on __cxa_pure_virtual, which lives
+  // in libsupc++ and is not linked into libomp. Derived classes must override.
+  virtual bool match_impl([[maybe_unused]] int device,
+                          [[maybe_unused]] int num_devices) const {
+    KMP_ASSERT2(0, "kmp_trait_expr::match_impl() must be overridden");
+    return false;
+  }
 
 public:
   virtual ~kmp_trait_expr() = default;


        

From openmp-commits at lists.llvm.org  Thu Aug 13 01:16:32 2026
From: openmp-commits at lists.llvm.org (Robert Imschweiler via Openmp-commits)
Date: Thu, 13 Aug 2026 01:16:32 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [libomp] Fix debug build after #176164
	(PR #215961)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/215961@github.com>
Message-ID: <6a7d7d60.2a9f4fc7.6a2c7.1ae5@mx.google.com>


https://github.com/ro-i closed https://github.com/llvm/llvm-project/pull/215961

From openmp-commits at lists.llvm.org  Thu Aug 13 01:17:07 2026
From: openmp-commits at lists.llvm.org (Robert Imschweiler via Openmp-commits)
Date: Thu, 13 Aug 2026 01:17:07 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [libomp] Fix debug build after #176164
	(PR #215961)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/215961@github.com>
Message-ID: <6a7d7d83.6edd682e.39b076.1e4f@mx.google.com>


ro-i wrote:

I'm already merging this trivial fix to unbreak the issue. In case of any post-commit review, I'll do a follow-up

https://github.com/llvm/llvm-project/pull/215961

From openmp-commits at lists.llvm.org  Thu Aug 13 01:17:28 2026
From: openmp-commits at lists.llvm.org (Robert Imschweiler via Openmp-commits)
Date: Thu, 13 Aug 2026 01:17:28 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [libomp] OpenMP 6.0: Add device trait
	parser (PR #176164)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/176164@github.com>
Message-ID: <6a7d7d98.4a46f23c.f503c.6492@mx.google.com>


ro-i wrote:

thanks for reporting. fixed.

https://github.com/llvm/llvm-project/pull/176164

From openmp-commits at lists.llvm.org  Thu Aug 13 03:12:51 2026
From: openmp-commits at lists.llvm.org (Eugene Epshteyn via Openmp-commits)
Date: Thu, 13 Aug 2026 03:12:51 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [libomp] Fix debug build after #176164
	(PR #215961)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/215961@github.com>
Message-ID: <6a7d98a3.b1fa726d.131073.31d2@mx.google.com>


https://github.com/eugeneepshteyn commented:

Post-commit LGTM. Thank you for the quick fix!

https://github.com/llvm/llvm-project/pull/215961

From openmp-commits at lists.llvm.org  Thu Aug 13 05:09:13 2026
From: openmp-commits at lists.llvm.org (Alex Duran via Openmp-commits)
Date: Thu, 13 Aug 2026 05:09:13 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [Offload] Add logger in Plugin API
 functions for OpenMP (PR #215582)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/215582@github.com>
Message-ID: <6a7db3e9.f6bcba31.2aaada.7611@mx.google.com>


================
@@ -0,0 +1,162 @@
+//===-- Shared/APITrace.h - Tracing of offload API calls --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Prints one line per offload API call when the OMP_INFOTYPE_API_TRACE bit of
+// LIBOMPTARGET_INFO is set:
+//
+//   ---> init_device(.DeviceId = 0)-> OFFLOAD_SUCCESS (1053 us)
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef OMPTARGET_SHARED_API_TRACE_H
+#define OMPTARGET_SHARED_API_TRACE_H
+
+#include "APITypes.h"
+#include "Debug.h"
+#include "omptarget.h"
+
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include <chrono>
+#include <type_traits>
+#include <utility>
+
+namespace llvm::offload::trace {
+
+/// The level is read on every call so that __tgt_set_info_flag can turn tracing
+/// on and off around a region of interest.
+inline bool isTraceEnabled() { return getInfoLevel() & OMP_INFOTYPE_API_TRACE; }
----------------
adurang wrote:

I think it's ok here as it is but could we have a way to disable this at compile-time?
```
#if LIBOFFLOAD_TRACE_PLUGINS
...
#else
inline bool isTraceEnabled() { return false; }
#endif 
```
I think some builds might want to have this only when debug is enabled but if we don't want to gate it together, at least would be nice to be able to control whether it should be enabled or not.

https://github.com/llvm/llvm-project/pull/215582

From openmp-commits at lists.llvm.org  Thu Aug 13 06:58:00 2026
From: openmp-commits at lists.llvm.org (Marcos Maronas via Openmp-commits)
Date: Thu, 13 Aug 2026 06:58:00 -0700 (PDT)
Subject: [Openmp-commits] [clang] [libc] [openmp] [Clang] Remove the old
 LLVM offloading driver entirely (PR #211924)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/211924@github.com>
Message-ID: <6a7dcd68.b1b3e0a1.12cbad.7f66@mx.google.com>


================
@@ -51,7 +51,7 @@
 
 // RUN: rm -rf %t && mkdir %t
 // RUN: touch %t/a.o %t/b.o
-// RUN: %clang -### -target x86_64-linux-gnu --offload-new-driver -fgpu-rdc \
+// RUN: %clang -### -target x86_64-linux-gnu --offload-link -fgpu-rdc \
----------------
maarquitos14 wrote:

Same question than above.

https://github.com/llvm/llvm-project/pull/211924

From openmp-commits at lists.llvm.org  Thu Aug 13 06:58:00 2026
From: openmp-commits at lists.llvm.org (Marcos Maronas via Openmp-commits)
Date: Thu, 13 Aug 2026 06:58:00 -0700 (PDT)
Subject: [Openmp-commits] [clang] [libc] [openmp] [Clang] Remove the old
 LLVM offloading driver entirely (PR #211924)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/211924@github.com>
Message-ID: <6a7dcd68.af6ba3ed.291f6b.6072@mx.google.com>


https://github.com/maarquitos14 edited https://github.com/llvm/llvm-project/pull/211924

From openmp-commits at lists.llvm.org  Thu Aug 13 06:58:01 2026
From: openmp-commits at lists.llvm.org (Marcos Maronas via Openmp-commits)
Date: Thu, 13 Aug 2026 06:58:01 -0700 (PDT)
Subject: [Openmp-commits] [clang] [libc] [openmp] [Clang] Remove the old
 LLVM offloading driver entirely (PR #211924)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/211924@github.com>
Message-ID: <6a7dcd69.7d702ed2.2f4a41.b2c6@mx.google.com>


https://github.com/maarquitos14 approved this pull request.

HIPSPV has been using the new offloading driver for a while now, and I'm not aware of any issues with it, nor any use of the old offloading driver, so I'm happy to proceed with this.

https://github.com/llvm/llvm-project/pull/211924

From openmp-commits at lists.llvm.org  Thu Aug 13 06:58:01 2026
From: openmp-commits at lists.llvm.org (Marcos Maronas via Openmp-commits)
Date: Thu, 13 Aug 2026 06:58:01 -0700 (PDT)
Subject: [Openmp-commits] [clang] [libc] [openmp] [Clang] Remove the old
 LLVM offloading driver entirely (PR #211924)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/211924@github.com>
Message-ID: <6a7dcd69.c5504292.14b477.78d4@mx.google.com>


================
@@ -29,7 +13,7 @@
 // RUN: cp %t/tu0.o %t/tu1.o
 // RUN: llvm-ar cr %t/libSDL2.a %t/tu1.o
 
-// RUN: %clang -### --target=x86_64-linux-gnu --offload-new-driver -fgpu-rdc \
+// RUN: %clang -### --target=x86_64-linux-gnu --offload-link -fgpu-rdc \
----------------
maarquitos14 wrote:

Why is `--offload-link` replacing `--offload-new-driver` here? Is this intended?

https://github.com/llvm/llvm-project/pull/211924

From openmp-commits at lists.llvm.org  Thu Aug 13 07:00:42 2026
From: openmp-commits at lists.llvm.org (Joseph Huber via Openmp-commits)
Date: Thu, 13 Aug 2026 07:00:42 -0700 (PDT)
Subject: [Openmp-commits] [clang] [libc] [openmp] [Clang] Remove the old
 LLVM offloading driver entirely (PR #211924)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/211924@github.com>
Message-ID: <6a7dce0a.adb4cce5.25c603.7ff2@mx.google.com>


================
@@ -29,7 +13,7 @@
 // RUN: cp %t/tu0.o %t/tu1.o
 // RUN: llvm-ar cr %t/libSDL2.a %t/tu1.o
 
-// RUN: %clang -### --target=x86_64-linux-gnu --offload-new-driver -fgpu-rdc \
+// RUN: %clang -### --target=x86_64-linux-gnu --offload-link -fgpu-rdc \
----------------
jhuber6 wrote:

Yes, `--offload-new-driver` used to imply the offloading toolchain, but now that it's going away we use `--offload-link` to enable it. That was always the intended mechanism for what `--offload-new-driver` used to be doing here.

https://github.com/llvm/llvm-project/pull/211924

From openmp-commits at lists.llvm.org  Thu Aug 13 08:03:16 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Thu, 13 Aug 2026 08:03:16 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP][OMPT] Make omp_control_tool
 enums available in omp-tools.h (PR #216076)
Message-ID: <llvm.org/llvm/llvm-project/pull/216076@github.com>


https://github.com/jprotze created https://github.com/llvm/llvm-project/pull/216076

OpenMP 6.1 will introduce this change: the enums must be available by including either header file, but including both must also be valid.

>From 4f8b0c3c18943ef53aae4e968817e68620a0d137 Mon Sep 17 00:00:00 2001
From: Joachim Jenke <jenke at itc.rwth-aachen.de>
Date: Thu, 13 Aug 2026 16:55:32 +0200
Subject: [PATCH] [OpenMP][OMPT] Make omp_control_tool enums available in
 omp-tools.h

OpenMP 6.1 will introduce this change: the enums must be available by
including either header file, but including both must also be valid.
---
 openmp/runtime/src/include/omp-tools.h.var | 18 ++++++++++++++++++
 openmp/runtime/src/include/omp.h.var       |  4 ++++
 2 files changed, 22 insertions(+)

diff --git a/openmp/runtime/src/include/omp-tools.h.var b/openmp/runtime/src/include/omp-tools.h.var
index 46643dff5feab..88031930a279a 100644
--- a/openmp/runtime/src/include/omp-tools.h.var
+++ b/openmp/runtime/src/include/omp-tools.h.var
@@ -302,6 +302,24 @@ typedef uint64_t ompt_device_time_t;
 
 typedef uint64_t ompt_buffer_cursor_t;
 
+    /* OpenMP 6.1: Define in omp.h and omp-tools.h */
+#ifndef OMPCONTROLENUM
+#define OMPCONTROLENUM
+typedef enum omp_control_tool_result_t {
+  omp_control_tool_notool             = -2,
+  omp_control_tool_nocallback         = -1,
+  omp_control_tool_success            = 0,
+  omp_control_tool_ignored            = 1
+} omp_control_tool_result_t;
+
+typedef enum omp_control_tool_t {
+  omp_control_tool_start              = 1,
+  omp_control_tool_pause              = 2,
+  omp_control_tool_flush              = 3,
+  omp_control_tool_end                = 4
+} omp_control_tool_t;
+#endif OMPCONTROLENUM
+
 typedef enum ompt_thread_t {
   ompt_thread_initial                 = 1,
   ompt_thread_worker                  = 2,
diff --git a/openmp/runtime/src/include/omp.h.var b/openmp/runtime/src/include/omp.h.var
index b05074198ca3f..f568a952350c8 100644
--- a/openmp/runtime/src/include/omp.h.var
+++ b/openmp/runtime/src/include/omp.h.var
@@ -312,6 +312,9 @@
     extern void   __KAI_KMPC_CONVENTION  kmp_set_warnings_off(void);
 
     /* OpenMP 5.0 Tool Control */
+    /* OpenMP 6.1: Define in omp.h and omp-tools.h */
+#ifndef OMPCONTROLENUM
+#define OMPCONTROLENUM
     typedef enum omp_control_tool_result_t {
         omp_control_tool_notool = -2,
         omp_control_tool_nocallback = -1,
@@ -325,6 +328,7 @@
         omp_control_tool_flush = 3,
         omp_control_tool_end = 4
     } omp_control_tool_t;
+#endif OMPCONTROLENUM
 
     extern int __KAI_KMPC_CONVENTION omp_control_tool(int, int, void*);
 


From openmp-commits at lists.llvm.org  Thu Aug 13 08:04:48 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Thu, 13 Aug 2026 08:04:48 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP][OMPT] Mark dispatch callback as
 sometimes on registration (PR #212991)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/212991@github.com>
Message-ID: <6a7ddd10.5d357cd9.1c7010.6cd3@mx.google.com>


https://github.com/jprotze approved this pull request.

Lgtm

https://github.com/llvm/llvm-project/pull/212991

From openmp-commits at lists.llvm.org  Thu Aug 13 09:53:31 2026
From: openmp-commits at lists.llvm.org (Spencer Bryngelson via Openmp-commits)
Date: Thu, 13 Aug 2026 09:53:31 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMP][DeviceRTL] Fix the
 chunked static distribution dropping iterations (PR #216117)
Message-ID: <llvm.org/llvm/llvm-project/pull/216117@github.com>


https://github.com/sbryngelson created https://github.com/llvm/llvm-project/pull/216117

`StaticLoopChunker` drops loop iterations whenever a non-zero thread chunk is used, so a chunked static distribution silently computes the wrong answer.

Two things are wrong. `NormalizedLoopNestChunked` starts a thread at `BId * BlockChunk + TId`, which does not account for the chunk, so with `ThreadChunk = C` the threads start one index apart and then each run `C` consecutive iterations: they overlap at the front and leave the rest of the space unvisited. And `DistributeFor` defaults `BlockChunk` to `NumThreads`, which cannot hold one chunk of `C` iterations for each of `NumThreads` threads, so every thread past the block chunk gets nothing at all.

The path is only reachable with a non-zero thread chunk. clang lowers worksharing loops to `__kmpc_for_static_init_4` and never reaches these entries, and flang passes a hardcoded `0` for both chunks, so nothing in tree exercises it today and no in-tree behaviour changes. It becomes reachable as soon as a frontend forwards a `schedule(static, C)` chunk, which is what #214303 needs.

### Testing

New `offload/test/offloading/distribute_for_thread_chunk.c` records which thread ran each iteration for `NumIters=32`, `NumThreads=8`, `ThreadChunk=4`, and checks both that no iteration is skipped and that the mapping is the one `schedule(static,4)` prescribes. It calls the entry directly for the same reason as `distribute_for_no_loop_num_threads.c`: no C or C++ construct reaches it.

On gfx90a, thread recorded per iteration:

| | result |
|---|---|
| expected | `0 0 0 0 1 1 1 1 2 2 2 2 ... 7 7 7 7` |
| before | 17 of 32 iterations unwritten, 10 misplaced |
| after | 0 unwritten, 0 misplaced, mapping exactly as expected |

The default path is unchanged: with `ThreadChunk = 0` the mapping is identical before and after, and `distribute_for_no_loop_num_threads.c` still reports `unwritten: 0`.

`__kmpc_for_static_loop` reaches the same helper with `BlockChunk` hardcoded to `0`, which makes `KernelIteration` zero so the induction variable never advances. That entry is left alone here; it is a separate defect and no frontend passes it a chunk either.

The analysis, the reduced test case and this fix were produced with Claude; I reviewed and verified them.


>From df13b38b517c76a6045c7831ac4cd3d4ce36e4ae Mon Sep 17 00:00:00 2001
From: Spencer Bryngelson <sbryngelson at gmail.com>
Date: Thu, 13 Aug 2026 11:52:57 -0500
Subject: [PATCH] [OpenMP][DeviceRTL] Fix the chunked static distribution
 dropping iterations

NormalizedLoopNestChunked starts thread T at BId * BlockChunk + TId, so with a
thread chunk the threads start one index apart and then run consecutive
chunks: they overlap at the front and leave the rest of the space unvisited.
The start index has to account for the chunk.

DistributeFor also defaults BlockChunk to NumThreads, which cannot hold one
chunk of ThreadChunk iterations for each of NumThreads threads, so the threads
past the block chunk get nothing.

The path is only reachable with a non-zero thread chunk, which no frontend
passes today, so nothing in tree changes behaviour.

Assisted-by: Claude <noreply at anthropic.com>
---
 .../offloading/distribute_for_thread_chunk.c  | 75 +++++++++++++++++++
 openmp/device/src/Workshare.cpp               | 12 +--
 2 files changed, 82 insertions(+), 5 deletions(-)
 create mode 100644 offload/test/offloading/distribute_for_thread_chunk.c

diff --git a/offload/test/offloading/distribute_for_thread_chunk.c b/offload/test/offloading/distribute_for_thread_chunk.c
new file mode 100644
index 0000000000000..bd3e8afcde31f
--- /dev/null
+++ b/offload/test/offloading/distribute_for_thread_chunk.c
@@ -0,0 +1,75 @@
+// Chunked static distribution: thread T owns iterations [T*chunk, T*chunk+chunk)
+// within each block chunk. The index used to start each thread has to account
+// for the chunk, and the block chunk has to cover one chunk per thread.
+//
+// The entry is called directly because no C/C++ construct reaches it: clang
+// emits __kmpc_for_static_init_4, only flang emits this one.
+//
+// RUN: %libomptarget-compile-run-and-check-generic
+// REQUIRES: gpu
+
+#include <omp.h>
+#include <stdio.h>
+
+#define N 32
+#define NT 8
+#define CHUNK 4
+
+struct Args {
+  int *Tid;
+};
+
+#pragma omp begin declare target
+extern void __kmpc_distribute_for_static_loop_4u(
+    void *Loc, void (*Fn)(unsigned, void *), void *Arg, unsigned NumIters,
+    unsigned NumThreads, unsigned BlockChunk, unsigned ThreadChunk,
+    unsigned char OneIterationPerThread);
+
+__attribute__((noinline)) static void body(unsigned I, void *A) {
+  ((struct Args *)A)->Tid[I] = omp_get_thread_num();
+}
+#pragma omp end declare target
+
+// For the host fallback copy only; the device uses the runtime's definition.
+#ifndef __AMDGCN__
+void __kmpc_distribute_for_static_loop_4u(
+    void *Loc, void (*Fn)(unsigned, void *), void *Arg, unsigned NumIters,
+    unsigned NumThreads, unsigned BlockChunk, unsigned ThreadChunk,
+    unsigned char OneIterationPerThread) {
+  for (unsigned I = 0; I < NumIters; ++I)
+    Fn(I, Arg);
+}
+#endif
+
+int main(void) {
+  static int Tid[N];
+  for (int I = 0; I < N; ++I)
+    Tid[I] = -1;
+
+#pragma omp target teams map(tofrom : Tid[0 : N]) num_teams(1) thread_limit(NT)
+  {
+#pragma omp parallel num_threads(NT)
+    {
+      struct Args A;
+      A.Tid = Tid;
+      __kmpc_distribute_for_static_loop_4u(0, body, &A, N, NT,
+                                           /*BlockChunk=*/0,
+                                           /*ThreadChunk=*/CHUNK,
+                                           /*OneIterationPerThread=*/0);
+    }
+  }
+
+  int Unwritten = 0, Misplaced = 0;
+  for (int I = 0; I < N; ++I) {
+    if (Tid[I] == -1)
+      ++Unwritten;
+    else if (Tid[I] != (I / CHUNK) % NT)
+      ++Misplaced;
+  }
+
+  // CHECK: unwritten: 0
+  // CHECK: misplaced: 0
+  printf("unwritten: %d\n", Unwritten);
+  printf("misplaced: %d\n", Misplaced);
+  return 0;
+}
diff --git a/openmp/device/src/Workshare.cpp b/openmp/device/src/Workshare.cpp
index a16ef93707989..fdb64c01eb41f 100644
--- a/openmp/device/src/Workshare.cpp
+++ b/openmp/device/src/Workshare.cpp
@@ -732,7 +732,7 @@ template <typename Ty> class StaticLoopChunker {
     Ty KernelIteration = NumBlocks * BlockChunk;
 
     // Start index in the chunked space.
-    Ty IV = BId * BlockChunk + TId;
+    Ty IV = BId * BlockChunk + TId * ThreadChunk;
     ASSERT(IV >= 0, "Bad index");
 
     // Cover the entire iteration space, assumptions in the caller might allow
@@ -900,14 +900,16 @@ template <typename Ty> class StaticLoopChunker {
     if (OneIterationPerThread)
       NumThreads = static_cast<Ty>(mapping::getMaxTeamThreads());
 
-    // If the block chunk is not specified we pick a default now.
-    if (BlockChunk == 0)
-      BlockChunk = NumThreads;
-
     // If the thread chunk is not specified we pick a default now.
     if (ThreadChunk == 0)
       ThreadChunk = 1;
 
+    // If the block chunk is not specified we pick a default now. It has to
+    // cover one chunk for every thread, otherwise the threads past the block
+    // chunk get no iterations at all.
+    if (BlockChunk == 0)
+      BlockChunk = NumThreads * ThreadChunk;
+
     // If we know we have more threads (across all blocks) than iterations we
     // can indicate that to avoid an outer loop.
     if (OneIterationPerThread)


From openmp-commits at lists.llvm.org  Thu Aug 13 09:54:10 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Thu, 13 Aug 2026 09:54:10 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMP][DeviceRTL] Fix the
 chunked static distribution dropping iterations (PR #216117)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/216117@github.com>
Message-ID: <6a7df6b2.4987c784.4462e.ef8c@mx.google.com>


llvmorg-github-actions[bot] wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-offload

Author: Spencer Bryngelson (sbryngelson)

<details>
<summary>Changes</summary>

`StaticLoopChunker` drops loop iterations whenever a non-zero thread chunk is used, so a chunked static distribution silently computes the wrong answer.

Two things are wrong. `NormalizedLoopNestChunked` starts a thread at `BId * BlockChunk + TId`, which does not account for the chunk, so with `ThreadChunk = C` the threads start one index apart and then each run `C` consecutive iterations: they overlap at the front and leave the rest of the space unvisited. And `DistributeFor` defaults `BlockChunk` to `NumThreads`, which cannot hold one chunk of `C` iterations for each of `NumThreads` threads, so every thread past the block chunk gets nothing at all.

The path is only reachable with a non-zero thread chunk. clang lowers worksharing loops to `__kmpc_for_static_init_4` and never reaches these entries, and flang passes a hardcoded `0` for both chunks, so nothing in tree exercises it today and no in-tree behaviour changes. It becomes reachable as soon as a frontend forwards a `schedule(static, C)` chunk, which is what #<!-- -->214303 needs.

### Testing

New `offload/test/offloading/distribute_for_thread_chunk.c` records which thread ran each iteration for `NumIters=32`, `NumThreads=8`, `ThreadChunk=4`, and checks both that no iteration is skipped and that the mapping is the one `schedule(static,4)` prescribes. It calls the entry directly for the same reason as `distribute_for_no_loop_num_threads.c`: no C or C++ construct reaches it.

On gfx90a, thread recorded per iteration:

| | result |
|---|---|
| expected | `0 0 0 0 1 1 1 1 2 2 2 2 ... 7 7 7 7` |
| before | 17 of 32 iterations unwritten, 10 misplaced |
| after | 0 unwritten, 0 misplaced, mapping exactly as expected |

The default path is unchanged: with `ThreadChunk = 0` the mapping is identical before and after, and `distribute_for_no_loop_num_threads.c` still reports `unwritten: 0`.

`__kmpc_for_static_loop` reaches the same helper with `BlockChunk` hardcoded to `0`, which makes `KernelIteration` zero so the induction variable never advances. That entry is left alone here; it is a separate defect and no frontend passes it a chunk either.

The analysis, the reduced test case and this fix were produced with Claude; I reviewed and verified them.


---
Full diff: https://github.com/llvm/llvm-project/pull/216117.diff


2 Files Affected:

- (added) offload/test/offloading/distribute_for_thread_chunk.c (+75) 
- (modified) openmp/device/src/Workshare.cpp (+7-5) 


``````````diff
diff --git a/offload/test/offloading/distribute_for_thread_chunk.c b/offload/test/offloading/distribute_for_thread_chunk.c
new file mode 100644
index 0000000000000..bd3e8afcde31f
--- /dev/null
+++ b/offload/test/offloading/distribute_for_thread_chunk.c
@@ -0,0 +1,75 @@
+// Chunked static distribution: thread T owns iterations [T*chunk, T*chunk+chunk)
+// within each block chunk. The index used to start each thread has to account
+// for the chunk, and the block chunk has to cover one chunk per thread.
+//
+// The entry is called directly because no C/C++ construct reaches it: clang
+// emits __kmpc_for_static_init_4, only flang emits this one.
+//
+// RUN: %libomptarget-compile-run-and-check-generic
+// REQUIRES: gpu
+
+#include <omp.h>
+#include <stdio.h>
+
+#define N 32
+#define NT 8
+#define CHUNK 4
+
+struct Args {
+  int *Tid;
+};
+
+#pragma omp begin declare target
+extern void __kmpc_distribute_for_static_loop_4u(
+    void *Loc, void (*Fn)(unsigned, void *), void *Arg, unsigned NumIters,
+    unsigned NumThreads, unsigned BlockChunk, unsigned ThreadChunk,
+    unsigned char OneIterationPerThread);
+
+__attribute__((noinline)) static void body(unsigned I, void *A) {
+  ((struct Args *)A)->Tid[I] = omp_get_thread_num();
+}
+#pragma omp end declare target
+
+// For the host fallback copy only; the device uses the runtime's definition.
+#ifndef __AMDGCN__
+void __kmpc_distribute_for_static_loop_4u(
+    void *Loc, void (*Fn)(unsigned, void *), void *Arg, unsigned NumIters,
+    unsigned NumThreads, unsigned BlockChunk, unsigned ThreadChunk,
+    unsigned char OneIterationPerThread) {
+  for (unsigned I = 0; I < NumIters; ++I)
+    Fn(I, Arg);
+}
+#endif
+
+int main(void) {
+  static int Tid[N];
+  for (int I = 0; I < N; ++I)
+    Tid[I] = -1;
+
+#pragma omp target teams map(tofrom : Tid[0 : N]) num_teams(1) thread_limit(NT)
+  {
+#pragma omp parallel num_threads(NT)
+    {
+      struct Args A;
+      A.Tid = Tid;
+      __kmpc_distribute_for_static_loop_4u(0, body, &A, N, NT,
+                                           /*BlockChunk=*/0,
+                                           /*ThreadChunk=*/CHUNK,
+                                           /*OneIterationPerThread=*/0);
+    }
+  }
+
+  int Unwritten = 0, Misplaced = 0;
+  for (int I = 0; I < N; ++I) {
+    if (Tid[I] == -1)
+      ++Unwritten;
+    else if (Tid[I] != (I / CHUNK) % NT)
+      ++Misplaced;
+  }
+
+  // CHECK: unwritten: 0
+  // CHECK: misplaced: 0
+  printf("unwritten: %d\n", Unwritten);
+  printf("misplaced: %d\n", Misplaced);
+  return 0;
+}
diff --git a/openmp/device/src/Workshare.cpp b/openmp/device/src/Workshare.cpp
index a16ef93707989..fdb64c01eb41f 100644
--- a/openmp/device/src/Workshare.cpp
+++ b/openmp/device/src/Workshare.cpp
@@ -732,7 +732,7 @@ template <typename Ty> class StaticLoopChunker {
     Ty KernelIteration = NumBlocks * BlockChunk;
 
     // Start index in the chunked space.
-    Ty IV = BId * BlockChunk + TId;
+    Ty IV = BId * BlockChunk + TId * ThreadChunk;
     ASSERT(IV >= 0, "Bad index");
 
     // Cover the entire iteration space, assumptions in the caller might allow
@@ -900,14 +900,16 @@ template <typename Ty> class StaticLoopChunker {
     if (OneIterationPerThread)
       NumThreads = static_cast<Ty>(mapping::getMaxTeamThreads());
 
-    // If the block chunk is not specified we pick a default now.
-    if (BlockChunk == 0)
-      BlockChunk = NumThreads;
-
     // If the thread chunk is not specified we pick a default now.
     if (ThreadChunk == 0)
       ThreadChunk = 1;
 
+    // If the block chunk is not specified we pick a default now. It has to
+    // cover one chunk for every thread, otherwise the threads past the block
+    // chunk get no iterations at all.
+    if (BlockChunk == 0)
+      BlockChunk = NumThreads * ThreadChunk;
+
     // If we know we have more threads (across all blocks) than iterations we
     // can indicate that to avoid an outer loop.
     if (OneIterationPerThread)

``````````

</details>


https://github.com/llvm/llvm-project/pull/216117

From openmp-commits at lists.llvm.org  Thu Aug 13 12:51:00 2026
From: openmp-commits at lists.llvm.org (Spencer Bryngelson via Openmp-commits)
Date: Thu, 13 Aug 2026 12:51:00 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMP][DeviceRTL] Fix the
 chunked static distribution dropping iterations (PR #216117)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/216117@github.com>
Message-ID: <6a7e2024.436e7ae1.306a64.8913@mx.google.com>


sbryngelson wrote:

Yes, Fortran: #214303, where `schedule(static,C)` is ignored on device.

A pragma test cannot reach this code. Compiled for gfx90a with current main (53a702813473), these three

```c
#pragma omp target teams distribute parallel for schedule(static, 4)
#pragma omp target teams distribute parallel for
#pragma omp target parallel for schedule(static, 4)
```

emit `__kmpc_for_static_init_4` and `__kmpc_distribute_static_init_4`. No `*_static_loop_*` entry appears at all; clang uses the `*_static_init_*` family. Only flang emits `__kmpc_distribute_for_static_loop_*`, so a C test would compile and pass without entering the code being fixed.

The file reference was too terse, sorry: `offload/test/offloading/distribute_for_no_loop_num_threads.c`, added by 9c317bbbdbea (#214073) two days ago, so it may predate your checkout. Same problem, same approach: it calls the entry directly and says why at the top.

On existing coverage: the `*_static_loop_*` entries are referenced by six MLIR lit tests and an OMPIRBuilder unit test, but those check emitted IR. That offload test is the only one that executes them, and it passes `ThreadChunk = 0`. No test in tree passes a non-zero chunk, so this would not have been caught regardless of @DominikAdamski's tests.

If you would rather avoid a direct-call test, the alternative is to land this with the flang side of #214303 and test through `!$omp ... schedule(static,4)`. That gives a source-level test, at the cost of coupling a runtime fix to a frontend change.


https://github.com/llvm/llvm-project/pull/216117

From openmp-commits at lists.llvm.org  Thu Aug 13 13:06:20 2026
From: openmp-commits at lists.llvm.org (Spencer Bryngelson via Openmp-commits)
Date: Thu, 13 Aug 2026 13:06:20 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMP][DeviceRTL] Fix the
 chunked static distribution dropping iterations (PR #216117)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/216117@github.com>
Message-ID: <6a7e23bc.d76d81af.1c0491.5b0c@mx.google.com>


https://github.com/sbryngelson edited https://github.com/llvm/llvm-project/pull/216117

From openmp-commits at lists.llvm.org  Thu Aug 13 21:32:08 2026
From: openmp-commits at lists.llvm.org (Anchu Rajendran S via Openmp-commits)
Date: Thu, 13 Aug 2026 21:32:08 -0700 (PDT)
Subject: [Openmp-commits] [flang] [mlir] [openmp] [MLIR][OpenMP] Add scan
 reduction lowering to llvm (PR #167031)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/167031@github.com>
Message-ID: <6a7e9a48.7d702ed2.24827e.da82@mx.google.com>


https://github.com/anchuraj closed https://github.com/llvm/llvm-project/pull/167031

From openmp-commits at lists.llvm.org  Thu Aug 13 21:32:40 2026
From: openmp-commits at lists.llvm.org (Anchu Rajendran S via Openmp-commits)
Date: Thu, 13 Aug 2026 21:32:40 -0700 (PDT)
Subject: [Openmp-commits] [flang] [mlir] [openmp] [MLIR][OpenMP] Add scan
 reduction lowering to llvm (PR #167031)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/167031@github.com>
Message-ID: <6a7e9a68.d1496011.2f876a.66e6@mx.google.com>


anchuraj wrote:

Is under active development in https://github.com/llvm/llvm-project/pull/206747

https://github.com/llvm/llvm-project/pull/167031

From openmp-commits at lists.llvm.org  Fri Aug 14 01:46:30 2026
From: openmp-commits at lists.llvm.org (Amilendra Kodithuwakku via Openmp-commits)
Date: Fri, 14 Aug 2026 01:46:30 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP][OMPT] Fix warning : extra tokens
 at end of #endif directive (PR #216279)
Message-ID: <llvm.org/llvm/llvm-project/pull/216279@github.com>


https://github.com/amilendra created https://github.com/llvm/llvm-project/pull/216279

None

>From 724b300b4a9ec9803a87006967eca793253e001f Mon Sep 17 00:00:00 2001
From: Amilendra Kodithuwakku <amilendra.kodithuwakku at arm.com>
Date: Fri, 14 Aug 2026 09:14:04 +0100
Subject: [PATCH] [OpenMP][OMPT] Fix warning : extra tokens at end of #endif
 directive

---
 openmp/runtime/src/include/omp-tools.h.var | 2 +-
 openmp/runtime/src/include/omp.h.var       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/openmp/runtime/src/include/omp-tools.h.var b/openmp/runtime/src/include/omp-tools.h.var
index 88031930a279a..3f1ff5997a0ac 100644
--- a/openmp/runtime/src/include/omp-tools.h.var
+++ b/openmp/runtime/src/include/omp-tools.h.var
@@ -318,7 +318,7 @@ typedef enum omp_control_tool_t {
   omp_control_tool_flush              = 3,
   omp_control_tool_end                = 4
 } omp_control_tool_t;
-#endif OMPCONTROLENUM
+#endif /* OMPCONTROLENUM */
 
 typedef enum ompt_thread_t {
   ompt_thread_initial                 = 1,
diff --git a/openmp/runtime/src/include/omp.h.var b/openmp/runtime/src/include/omp.h.var
index f568a952350c8..edea72235f8e4 100644
--- a/openmp/runtime/src/include/omp.h.var
+++ b/openmp/runtime/src/include/omp.h.var
@@ -328,7 +328,7 @@
         omp_control_tool_flush = 3,
         omp_control_tool_end = 4
     } omp_control_tool_t;
-#endif OMPCONTROLENUM
+#endif /* OMPCONTROLENUM */
 
     extern int __KAI_KMPC_CONVENTION omp_control_tool(int, int, void*);
 


From openmp-commits at lists.llvm.org  Fri Aug 14 01:47:43 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Fri, 14 Aug 2026 01:47:43 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP][OMPT] Fix warning : extra tokens
 at end of #endif directive (PR #216279)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/216279@github.com>
Message-ID: <6a7ed62f.b1b3e0a1.5664a.d9c9@mx.google.com>


https://github.com/jprotze approved this pull request.


https://github.com/llvm/llvm-project/pull/216279

From openmp-commits at lists.llvm.org  Fri Aug 14 01:48:22 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Fri, 14 Aug 2026 01:48:22 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP][OMPT] Fix warning : extra tokens
 at end of #endif directive (PR #216279)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/216279@github.com>
Message-ID: <6a7ed656.642c3f8d.364216.36d2@mx.google.com>


https://github.com/jprotze auto_merge_enabled https://github.com/llvm/llvm-project/pull/216279

From openmp-commits at lists.llvm.org  Fri Aug 14 01:48:59 2026
From: openmp-commits at lists.llvm.org (Amilendra Kodithuwakku via Openmp-commits)
Date: Fri, 14 Aug 2026 01:48:59 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP][OMPT] Fix warning : extra tokens
 at end of #endif directive (PR #216279)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/216279@github.com>
Message-ID: <6a7ed67b.4987c784.4462e.79ac@mx.google.com>


https://github.com/amilendra edited https://github.com/llvm/llvm-project/pull/216279

From openmp-commits at lists.llvm.org  Fri Aug 14 01:50:38 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Fri, 14 Aug 2026 01:50:38 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP][OMPT] Fix warning : extra tokens
 at end of #endif directive (PR #216279)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/216279@github.com>
Message-ID: <6a7ed6de.c5504292.3277c9.361a@mx.google.com>


jprotze wrote:

I'm wondering why my compiler did not complain. :shrug: 

https://github.com/llvm/llvm-project/pull/216279

From openmp-commits at lists.llvm.org  Fri Aug 14 02:03:47 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Fri, 14 Aug 2026 02:03:47 -0700 (PDT)
Subject: [Openmp-commits] [openmp] 16fe1b6 - [OpenMP][OMPT] Fix warning :
 extra tokens at end of #endif directive (#216279)
Message-ID: <6a7ed9f3.9edb5c9d.ad79f.044b@mx.google.com>


Author: Amilendra Kodithuwakku
Date: 2026-08-14T09:03:41Z
New Revision: 16fe1b642d3b8e8c5a7ba3c55965434f4d31da0f

URL: https://github.com/llvm/llvm-project/commit/16fe1b642d3b8e8c5a7ba3c55965434f4d31da0f
DIFF: https://github.com/llvm/llvm-project/commit/16fe1b642d3b8e8c5a7ba3c55965434f4d31da0f.diff

LOG: [OpenMP][OMPT] Fix warning : extra tokens at end of #endif directive (#216279)

Added: 
    

Modified: 
    openmp/runtime/src/include/omp-tools.h.var
    openmp/runtime/src/include/omp.h.var

Removed: 
    


################################################################################
diff  --git a/openmp/runtime/src/include/omp-tools.h.var b/openmp/runtime/src/include/omp-tools.h.var
index 88031930a279a..3f1ff5997a0ac 100644
--- a/openmp/runtime/src/include/omp-tools.h.var
+++ b/openmp/runtime/src/include/omp-tools.h.var
@@ -318,7 +318,7 @@ typedef enum omp_control_tool_t {
   omp_control_tool_flush              = 3,
   omp_control_tool_end                = 4
 } omp_control_tool_t;
-#endif OMPCONTROLENUM
+#endif /* OMPCONTROLENUM */
 
 typedef enum ompt_thread_t {
   ompt_thread_initial                 = 1,

diff  --git a/openmp/runtime/src/include/omp.h.var b/openmp/runtime/src/include/omp.h.var
index f568a952350c8..edea72235f8e4 100644
--- a/openmp/runtime/src/include/omp.h.var
+++ b/openmp/runtime/src/include/omp.h.var
@@ -328,7 +328,7 @@
         omp_control_tool_flush = 3,
         omp_control_tool_end = 4
     } omp_control_tool_t;
-#endif OMPCONTROLENUM
+#endif /* OMPCONTROLENUM */
 
     extern int __KAI_KMPC_CONVENTION omp_control_tool(int, int, void*);
 


        

From openmp-commits at lists.llvm.org  Fri Aug 14 02:03:48 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Fri, 14 Aug 2026 02:03:48 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP][OMPT] Fix warning : extra tokens
 at end of #endif directive (PR #216279)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/216279@github.com>
Message-ID: <6a7ed9f4.d1496011.60cc2.a766@mx.google.com>


https://github.com/jprotze closed https://github.com/llvm/llvm-project/pull/216279

From openmp-commits at lists.llvm.org  Fri Aug 14 02:10:23 2026
From: openmp-commits at lists.llvm.org (LLVM Continuous Integration via Openmp-commits)
Date: Fri, 14 Aug 2026 02:10:23 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP][OMPT] Fix warning : extra tokens
 at end of #endif directive (PR #216279)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/216279@github.com>
Message-ID: <6a7edb7f.ed8a271b.38c9a9.e35e@mx.google.com>


llvm-ci wrote:

LLVM Buildbot has detected a new failure on builder `amdgpu-clang-flang` running on `AMD-bb-w-05` while building `openmp` at step 3 "annotate".

Full details are available at: https://lab.llvm.org/buildbot/#/builders/234/builds/2219

<details>
<summary>Here is the relevant piece of the build log for the reference</summary>

```
Step 3 (annotate) failure: 'python ../llvm.src/offload/ci/openmp-offload-amdgpu-clang-flang.py ...' (failure)
...
PASS: libomptarget :: amdgpu-amd-amdhsa :: unified_shared_memory/api.c (927 of 1398)
PASS: libomptarget :: amdgpu-amd-amdhsa :: unified_shared_memory/associate_ptr.c (928 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/present/target_data_at_exit.c (929 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/present/target_enter_data.c (930 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/present/target_exit_data_delete.c (931 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/present/target_exit_data_release.c (932 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/present/unified_shared_memory.c (933 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/present/zero_length_array_section.c (934 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/firstprivate_aligned.cpp (935 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/has_device_addr.cpp (936 of 1398)
FAIL: libomptarget :: amdgpu-amd-amdhsa :: ompt/veccopy_no_device_init.c (937 of 1398)
******************** TEST 'libomptarget :: amdgpu-amd-amdhsa :: ompt/veccopy_no_device_init.c' FAILED ********************
Exit Code: 2

Command Output (stdout):
--
# RUN: at line 2
/home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./bin/clang -fopenmp    -I /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test -I /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/runtimes/runtimes-bins/openmp/runtime/src -I /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test/../languages/include -I /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test/../languages/include/cuda -I /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test/../languages/include/hip -I /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test/../languages/kernel/include -L /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib/x86_64-unknown-linux-gnu -L /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib/x86_64-unknown-linux-gnu -L /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib -L /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/runtimes/runtimes-bins/openmp/runtime/src  -Wl,-rpath,/home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib/x86_64-unknown-linux-gnu -Wl,-rpath,/home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/runtimes/runtimes-bins/openmp/runtime/src -Wl,-rpath,/home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib/x86_64-unknown-linux-gnu -Wl,-rpath,/home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib -Xoffload-linker -L/home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib/amdgpu-amd-amdhsa -Xoffload-linker -L/home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib/clang/24/lib/amdgpu-amd-amdhsa  -fopenmp-targets=amdgpu-amd-amdhsa /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test/ompt/veccopy_no_device_init.c -o /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/runtimes/runtimes-bins/offload/test/amdgpu-amd-amdhsa/ompt/Output/veccopy_no_device_init.c.tmp -Xoffload-linker -lompdevice && /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/runtimes/runtimes-bins/offload/test/amdgpu-amd-amdhsa/ompt/Output/veccopy_no_device_init.c.tmp | /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./bin/FileCheck /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test/ompt/veccopy_no_device_init.c
# executed command: /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./bin/clang -fopenmp -I /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test -I /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/runtimes/runtimes-bins/openmp/runtime/src -I /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test/../languages/include -I /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test/../languages/include/cuda -I /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test/../languages/include/hip -I /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test/../languages/kernel/include -L /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib/x86_64-unknown-linux-gnu -L /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib/x86_64-unknown-linux-gnu -L /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib -L /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/runtimes/runtimes-bins/openmp/runtime/src -Wl,-rpath,/home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib/x86_64-unknown-linux-gnu -Wl,-rpath,/home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/runtimes/runtimes-bins/openmp/runtime/src -Wl,-rpath,/home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib/x86_64-unknown-linux-gnu -Wl,-rpath,/home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib -Xoffload-linker -L/home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib/amdgpu-amd-amdhsa -Xoffload-linker -L/home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib/clang/24/lib/amdgpu-amd-amdhsa -fopenmp-targets=amdgpu-amd-amdhsa /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test/ompt/veccopy_no_device_init.c -o /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/runtimes/runtimes-bins/offload/test/amdgpu-amd-amdhsa/ompt/Output/veccopy_no_device_init.c.tmp -Xoffload-linker -lompdevice
# note: command had no output on stdout or stderr
# executed command: /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/runtimes/runtimes-bins/offload/test/amdgpu-amd-amdhsa/ompt/Output/veccopy_no_device_init.c.tmp
# .---command stderr------------
# | Display last 2 kernels launched:
# | Kernel 0: 'omp target in main @ 37 (__omp_offloading_fc00_4b23161_main_l37)'
# | Kernel 1: 'omp target in main @ 31 (__omp_offloading_fc00_4b23161_main_l31)'
# | Use 'OFFLOAD_TRACK_NUM_KERNEL_LAUNCH_TRACES=<num>' to adjust the number of shown stack traces (0 now, up to 8)
# | OFFLOAD ERROR: memory access fault by GPU 2 (agent 0x631b2a939dd0) at virtual address 0x1000. Reasons: Unknown (0)
# | Use 'OFFLOAD_TRACK_ALLOCATION_TRACES=true' to track device allocations
# `-----------------------------
# error: command failed with exit status: -6
# executed command: /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./bin/FileCheck /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test/ompt/veccopy_no_device_init.c
# .---command stderr------------
# | FileCheck error: '<stdin>' is empty.
# | FileCheck command line:  /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./bin/FileCheck /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test/ompt/veccopy_no_device_init.c
# `-----------------------------
# error: command failed with exit status: 2

--

********************
PASS: libomptarget :: amdgpu-amd-amdhsa :: ompt/veccopy_wrong_return.c (938 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/present/zero_length_array_section_exit.c (939 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/private_mapping.c (940 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/is_device_ptr.cpp (941 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/lambda_by_value.cpp (942 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/map_back_race.cpp (943 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/map_both_pointer_pointee.c (944 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/ptr_and_obj_motion.c (945 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/reduction_implicit_map.cpp (946 of 1398)
Step 9 (Add check check-offload) failure: Add check check-offload (failure)
...
PASS: libomptarget :: amdgpu-amd-amdhsa :: unified_shared_memory/api.c (927 of 1398)
PASS: libomptarget :: amdgpu-amd-amdhsa :: unified_shared_memory/associate_ptr.c (928 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/present/target_data_at_exit.c (929 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/present/target_enter_data.c (930 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/present/target_exit_data_delete.c (931 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/present/target_exit_data_release.c (932 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/present/unified_shared_memory.c (933 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/present/zero_length_array_section.c (934 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/firstprivate_aligned.cpp (935 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/has_device_addr.cpp (936 of 1398)
FAIL: libomptarget :: amdgpu-amd-amdhsa :: ompt/veccopy_no_device_init.c (937 of 1398)
******************** TEST 'libomptarget :: amdgpu-amd-amdhsa :: ompt/veccopy_no_device_init.c' FAILED ********************
Exit Code: 2

Command Output (stdout):
--
# RUN: at line 2
/home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./bin/clang -fopenmp    -I /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test -I /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/runtimes/runtimes-bins/openmp/runtime/src -I /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test/../languages/include -I /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test/../languages/include/cuda -I /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test/../languages/include/hip -I /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test/../languages/kernel/include -L /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib/x86_64-unknown-linux-gnu -L /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib/x86_64-unknown-linux-gnu -L /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib -L /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/runtimes/runtimes-bins/openmp/runtime/src  -Wl,-rpath,/home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib/x86_64-unknown-linux-gnu -Wl,-rpath,/home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/runtimes/runtimes-bins/openmp/runtime/src -Wl,-rpath,/home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib/x86_64-unknown-linux-gnu -Wl,-rpath,/home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib -Xoffload-linker -L/home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib/amdgpu-amd-amdhsa -Xoffload-linker -L/home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib/clang/24/lib/amdgpu-amd-amdhsa  -fopenmp-targets=amdgpu-amd-amdhsa /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test/ompt/veccopy_no_device_init.c -o /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/runtimes/runtimes-bins/offload/test/amdgpu-amd-amdhsa/ompt/Output/veccopy_no_device_init.c.tmp -Xoffload-linker -lompdevice && /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/runtimes/runtimes-bins/offload/test/amdgpu-amd-amdhsa/ompt/Output/veccopy_no_device_init.c.tmp | /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./bin/FileCheck /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test/ompt/veccopy_no_device_init.c
# executed command: /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./bin/clang -fopenmp -I /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test -I /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/runtimes/runtimes-bins/openmp/runtime/src -I /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test/../languages/include -I /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test/../languages/include/cuda -I /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test/../languages/include/hip -I /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test/../languages/kernel/include -L /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib/x86_64-unknown-linux-gnu -L /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib/x86_64-unknown-linux-gnu -L /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib -L /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/runtimes/runtimes-bins/openmp/runtime/src -Wl,-rpath,/home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib/x86_64-unknown-linux-gnu -Wl,-rpath,/home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/runtimes/runtimes-bins/openmp/runtime/src -Wl,-rpath,/home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib/x86_64-unknown-linux-gnu -Wl,-rpath,/home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib -Xoffload-linker -L/home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib/amdgpu-amd-amdhsa -Xoffload-linker -L/home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./lib/clang/24/lib/amdgpu-amd-amdhsa -fopenmp-targets=amdgpu-amd-amdhsa /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test/ompt/veccopy_no_device_init.c -o /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/runtimes/runtimes-bins/offload/test/amdgpu-amd-amdhsa/ompt/Output/veccopy_no_device_init.c.tmp -Xoffload-linker -lompdevice
# note: command had no output on stdout or stderr
# executed command: /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/runtimes/runtimes-bins/offload/test/amdgpu-amd-amdhsa/ompt/Output/veccopy_no_device_init.c.tmp
# .---command stderr------------
# | Display last 2 kernels launched:
# | Kernel 0: 'omp target in main @ 37 (__omp_offloading_fc00_4b23161_main_l37)'
# | Kernel 1: 'omp target in main @ 31 (__omp_offloading_fc00_4b23161_main_l31)'
# | Use 'OFFLOAD_TRACK_NUM_KERNEL_LAUNCH_TRACES=<num>' to adjust the number of shown stack traces (0 now, up to 8)
# | OFFLOAD ERROR: memory access fault by GPU 2 (agent 0x631b2a939dd0) at virtual address 0x1000. Reasons: Unknown (0)
# | Use 'OFFLOAD_TRACK_ALLOCATION_TRACES=true' to track device allocations
# `-----------------------------
# error: command failed with exit status: -6
# executed command: /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./bin/FileCheck /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test/ompt/veccopy_no_device_init.c
# .---command stderr------------
# | FileCheck error: '<stdin>' is empty.
# | FileCheck command line:  /home/botworker/bbot/amdgpu-clang-flang/build/llvm.build/./bin/FileCheck /home/botworker/bbot/amdgpu-clang-flang/llvm.src/offload/test/ompt/veccopy_no_device_init.c
# `-----------------------------
# error: command failed with exit status: 2

--

********************
PASS: libomptarget :: amdgpu-amd-amdhsa :: ompt/veccopy_wrong_return.c (938 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/present/zero_length_array_section_exit.c (939 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/private_mapping.c (940 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/is_device_ptr.cpp (941 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/lambda_by_value.cpp (942 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/map_back_race.cpp (943 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/map_both_pointer_pointee.c (944 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/ptr_and_obj_motion.c (945 of 1398)
PASS: libomptarget :: x86_64-unknown-linux-gnu :: mapping/reduction_implicit_map.cpp (946 of 1398)

```

</details>

https://github.com/llvm/llvm-project/pull/216279

From openmp-commits at lists.llvm.org  Fri Aug 14 04:39:06 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Fri, 14 Aug 2026 04:39:06 -0700 (PDT)
Subject: [Openmp-commits] [flang] [llvm] [openmp] [OpenMP][Flang] Add
 integer-kind wrappers to omp_lib (PR #213505)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213505@github.com>
Message-ID: <6a7efe5a.9edb5c9d.ad79f.8375@mx.google.com>


https://github.com/shivaramaarao updated https://github.com/llvm/llvm-project/pull/213505

>From 15cd461073fc94d73eefd082334be213434e21d3 Mon Sep 17 00:00:00 2001
From: Shivarama Rao <shivarama.rao at amd.com>
Date: Sun, 2 Aug 2026 10:20:39 +0530
Subject: [PATCH] [OpenMP][Flang] Add integer-kind wrappers to omp_lib

Many OpenMP routines in omp_lib only accept integer(kind=omp_integer_kind),
which is c_int. Calls with integer(1), integer(2), or integer(8) arguments
therefore fail Flang's type checking. These wrapper functions are added
to omp_lib module file to resolve this issue.

fixes issue #123948

Co-authored-by: Cursor <cursoragent at cursor.com>
---
 flang-rt/lib/runtime/CMakeLists.txt           |   12 +
 .../Lower/OpenMP/omp-lib-integer-wrappers.f90 |  237 ++++
 .../test/Lower/OpenMP/omp-lib-num-threads.f90 |    2 +-
 openmp/module/omp_lib.F90.var                 | 1080 ++++++++++++++++-
 4 files changed, 1283 insertions(+), 48 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/omp-lib-integer-wrappers.f90

diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index ae9a6e7fb7188..4cd3090113af3 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -322,6 +322,13 @@ else ()
   set(sources ${supported_sources} ${host_sources} ${f128_sources})
 endif ()
 
+# Include the Fortran OpenMP module implementation (omp_lib.o) in
+# libflang_rt.runtime when the OpenMP runtime is also being built.
+# Skip for GPU-default targets: omp_lib is a host-side Fortran module.
+if (TARGET libomp-mod AND NOT "${LLVM_DEFAULT_TARGET_TRIPLE}" MATCHES "^amdgcn|^nvptx")
+  list(APPEND sources "$<TARGET_OBJECTS:libomp-mod>")
+endif ()
+
 
 if (NOT WIN32)
   add_flangrt_library(flang_rt.runtime STATIC SHARED
@@ -333,6 +340,11 @@ if (NOT WIN32)
 
   enable_cuda_compilation(flang_rt.runtime "${sources}")
 
+  if (TARGET libomp-mod AND TARGET flang_rt.runtime.shared AND TARGET omp)
+    # Resolve bind(c) references from omp_lib when building the shared runtime.
+    target_link_libraries(flang_rt.runtime.shared PUBLIC omp)
+  endif ()
+
   # Select a default runtime, which is used for unit and regression tests.
   if (TARGET flang_rt.runtime.default)
     get_target_property(default_target flang_rt.runtime.default ALIASED_TARGET)
diff --git a/flang/test/Lower/OpenMP/omp-lib-integer-wrappers.f90 b/flang/test/Lower/OpenMP/omp-lib-integer-wrappers.f90
new file mode 100644
index 0000000000000..9881fe56261fa
--- /dev/null
+++ b/flang/test/Lower/OpenMP/omp-lib-integer-wrappers.f90
@@ -0,0 +1,237 @@
+! REQUIRES: openmp_runtime
+
+! RUN: %flang_fc1 -emit-hlfir %openmp_flags %s -o - 2>&1 | FileCheck %s
+! RUN: bbc %openmp_flags -emit-hlfir -o - %s 2>&1 | FileCheck %s
+! RUN: %flang_fc1 -emit-fir %openmp_flags %s -o - 2>&1 | FileCheck %s
+! RUN: bbc -emit-fir %openmp_flags -o - %s 2>&1 | FileCheck %s
+!
+! Test that omp_lib integer-kind wrapper procedures lower to module
+! procedures instead of failing intrinsic resolution.
+
+program main
+  use omp_lib
+  integer(1) :: i1 = 1_1
+  integer(2) :: i2 = 2_2
+  integer(4) :: i4 = 4_4
+  integer(8) :: i8 = 8_8
+  integer(omp_integer_kind) :: ires
+  integer(omp_sched_kind) :: sched_kind
+  integer(kmp_affinity_mask_kind) :: mask
+
+  call omp_set_num_threads(i1)
+  call omp_set_num_threads(i2)
+  call omp_set_num_threads(i4)
+  call omp_set_num_threads(i8)
+
+  call omp_set_max_active_levels(i1)
+  call omp_set_max_active_levels(i2)
+  call omp_set_max_active_levels(i4)
+  call omp_set_max_active_levels(i8)
+
+  call omp_set_default_device(i1)
+  call omp_set_default_device(i2)
+  call omp_set_default_device(i4)
+  call omp_set_default_device(i8)
+
+  call omp_set_num_teams(i1)
+  call omp_set_num_teams(i2)
+  call omp_set_num_teams(i4)
+  call omp_set_num_teams(i8)
+
+  call omp_set_teams_thread_limit(i1)
+  call omp_set_teams_thread_limit(i2)
+  call omp_set_teams_thread_limit(i4)
+  call omp_set_teams_thread_limit(i8)
+
+  call kmp_set_stacksize(i1)
+  call kmp_set_stacksize(i2)
+  call kmp_set_stacksize(i4)
+  call kmp_set_stacksize(i8)
+
+  call kmp_set_blocktime(i1)
+  call kmp_set_blocktime(i2)
+  call kmp_set_blocktime(i4)
+  call kmp_set_blocktime(i8)
+
+  call kmp_set_library(i1)
+  call kmp_set_library(i2)
+  call kmp_set_library(i4)
+  call kmp_set_library(i8)
+
+  call kmp_set_disp_num_buffers(i1)
+  call kmp_set_disp_num_buffers(i2)
+  call kmp_set_disp_num_buffers(i4)
+  call kmp_set_disp_num_buffers(i8)
+
+  ires = omp_get_ancestor_thread_num(i1)
+  ires = omp_get_ancestor_thread_num(i2)
+  ires = omp_get_ancestor_thread_num(i4)
+  ires = omp_get_ancestor_thread_num(i8)
+
+  ires = omp_get_team_size(i1)
+  ires = omp_get_team_size(i2)
+  ires = omp_get_team_size(i4)
+  ires = omp_get_team_size(i8)
+
+  ires = omp_get_place_num_procs(i1)
+  ires = omp_get_place_num_procs(i2)
+  ires = omp_get_place_num_procs(i4)
+  ires = omp_get_place_num_procs(i8)
+
+  call omp_set_schedule(omp_sched_static, i1)
+  call omp_set_schedule(omp_sched_static, i2)
+  call omp_set_schedule(omp_sched_static, i4)
+  call omp_set_schedule(omp_sched_static, i8)
+
+  call omp_get_schedule(sched_kind, i1)
+  call omp_get_schedule(sched_kind, i2)
+  call omp_get_schedule(sched_kind, i4)
+  call omp_get_schedule(sched_kind, i8)
+
+  ires = omp_pause_resource(omp_pause_soft, i1)
+  ires = omp_pause_resource(omp_pause_soft, i2)
+  ires = omp_pause_resource(omp_pause_soft, i4)
+  ires = omp_pause_resource(omp_pause_soft, i8)
+
+  call kmp_create_affinity_mask(mask)
+  ires = kmp_set_affinity_mask_proc(i1, mask)
+  ires = kmp_set_affinity_mask_proc(i2, mask)
+  ires = kmp_set_affinity_mask_proc(i4, mask)
+  ires = kmp_set_affinity_mask_proc(i8, mask)
+
+  ires = kmp_unset_affinity_mask_proc(i1, mask)
+  ires = kmp_unset_affinity_mask_proc(i2, mask)
+  ires = kmp_unset_affinity_mask_proc(i4, mask)
+  ires = kmp_unset_affinity_mask_proc(i8, mask)
+
+  ires = kmp_get_affinity_mask_proc(i1, mask)
+  ires = kmp_get_affinity_mask_proc(i2, mask)
+  ires = kmp_get_affinity_mask_proc(i4, mask)
+  ires = kmp_get_affinity_mask_proc(i8, mask)
+
+  block
+    integer(4) :: ids4(1)
+    integer(8) :: ids8(1)
+    integer(4) :: parts4(1)
+    integer(8) :: parts8(1)
+    call omp_get_place_proc_ids(i4, ids4)
+    call omp_get_place_proc_ids(i8, ids8)
+    call omp_get_partition_place_nums(parts4)
+    call omp_get_partition_place_nums(parts8)
+  end block
+end program
+
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_max_active_levels
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_default_device
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_num_teams
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_teams_thread_limit
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_stacksize
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_blocktime
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_library
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_disp_num_buffers
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_ancestor_thread_num
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_team_size
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_place_num_procs
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_schedule
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_schedule
+!CHECK-NOT: not yet implemented: intrinsic: omp_pause_resource
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_affinity_mask_proc
+!CHECK-NOT: not yet implemented: intrinsic: kmp_unset_affinity_mask_proc
+!CHECK-NOT: not yet implemented: intrinsic: kmp_get_affinity_mask_proc
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_place_proc_ids
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_partition_place_nums
+
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i1
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i2
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i4
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_max_active_levels_i1
+!CHECK: fir.call @_QMomp_libPomp_set_max_active_levels_i2
+!CHECK: fir.call @_QMomp_libPomp_set_max_active_levels_i4
+!CHECK: fir.call @_QMomp_libPomp_set_max_active_levels_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_default_device_i1
+!CHECK: fir.call @_QMomp_libPomp_set_default_device_i2
+!CHECK: fir.call @_QMomp_libPomp_set_default_device_i4
+!CHECK: fir.call @_QMomp_libPomp_set_default_device_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_num_teams_i1
+!CHECK: fir.call @_QMomp_libPomp_set_num_teams_i2
+!CHECK: fir.call @_QMomp_libPomp_set_num_teams_i4
+!CHECK: fir.call @_QMomp_libPomp_set_num_teams_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_teams_thread_limit_i1
+!CHECK: fir.call @_QMomp_libPomp_set_teams_thread_limit_i2
+!CHECK: fir.call @_QMomp_libPomp_set_teams_thread_limit_i4
+!CHECK: fir.call @_QMomp_libPomp_set_teams_thread_limit_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_stacksize_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_stacksize_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_stacksize_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_stacksize_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_blocktime_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_blocktime_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_blocktime_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_blocktime_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_library_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_library_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_library_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_library_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_disp_num_buffers_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_disp_num_buffers_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_disp_num_buffers_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_disp_num_buffers_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_ancestor_thread_num_i1
+!CHECK: fir.call @_QMomp_libPomp_get_ancestor_thread_num_i2
+!CHECK: fir.call @_QMomp_libPomp_get_ancestor_thread_num_i4
+!CHECK: fir.call @_QMomp_libPomp_get_ancestor_thread_num_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_team_size_i1
+!CHECK: fir.call @_QMomp_libPomp_get_team_size_i2
+!CHECK: fir.call @_QMomp_libPomp_get_team_size_i4
+!CHECK: fir.call @_QMomp_libPomp_get_team_size_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_place_num_procs_i1
+!CHECK: fir.call @_QMomp_libPomp_get_place_num_procs_i2
+!CHECK: fir.call @_QMomp_libPomp_get_place_num_procs_i4
+!CHECK: fir.call @_QMomp_libPomp_get_place_num_procs_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_schedule_i1
+!CHECK: fir.call @_QMomp_libPomp_set_schedule_i2
+!CHECK: fir.call @_QMomp_libPomp_set_schedule_i4
+!CHECK: fir.call @_QMomp_libPomp_set_schedule_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_schedule_i1
+!CHECK: fir.call @_QMomp_libPomp_get_schedule_i2
+!CHECK: fir.call @_QMomp_libPomp_get_schedule_i4
+!CHECK: fir.call @_QMomp_libPomp_get_schedule_i8
+
+!CHECK: fir.call @_QMomp_libPomp_pause_resource_i1
+!CHECK: fir.call @_QMomp_libPomp_pause_resource_i2
+!CHECK: fir.call @_QMomp_libPomp_pause_resource_i4
+!CHECK: fir.call @_QMomp_libPomp_pause_resource_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_affinity_mask_proc_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_affinity_mask_proc_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_affinity_mask_proc_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_affinity_mask_proc_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_unset_affinity_mask_proc_i1
+!CHECK: fir.call @_QMomp_libPkmp_unset_affinity_mask_proc_i2
+!CHECK: fir.call @_QMomp_libPkmp_unset_affinity_mask_proc_i4
+!CHECK: fir.call @_QMomp_libPkmp_unset_affinity_mask_proc_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_get_affinity_mask_proc_i1
+!CHECK: fir.call @_QMomp_libPkmp_get_affinity_mask_proc_i2
+!CHECK: fir.call @_QMomp_libPkmp_get_affinity_mask_proc_i4
+!CHECK: fir.call @_QMomp_libPkmp_get_affinity_mask_proc_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_place_proc_ids_i4
+!CHECK: fir.call @_QMomp_libPomp_get_place_proc_ids_i8
+!CHECK: fir.call @_QMomp_libPomp_get_partition_place_nums_i4
+!CHECK: fir.call @_QMomp_libPomp_get_partition_place_nums_i8
diff --git a/flang/test/Lower/OpenMP/omp-lib-num-threads.f90 b/flang/test/Lower/OpenMP/omp-lib-num-threads.f90
index 6e76dce473057..a2ec031a62d88 100644
--- a/flang/test/Lower/OpenMP/omp-lib-num-threads.f90
+++ b/flang/test/Lower/OpenMP/omp-lib-num-threads.f90
@@ -21,5 +21,5 @@ program main
 
 !CHECK-NOT: not yet implemented: intrinsic: omp_set_num_threads
 !CHECK-NOT: not yet implemented: intrinsic: omp_get_num_threads
-!CHECK: fir.call @omp_set_num_threads
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i4
 !CHECK: fir.call @omp_get_num_threads
diff --git a/openmp/module/omp_lib.F90.var b/openmp/module/omp_lib.F90.var
index a41b2faad9d3d..f91a8f2a4d7ac 100644
--- a/openmp/module/omp_lib.F90.var
+++ b/openmp/module/omp_lib.F90.var
@@ -63,6 +63,27 @@
         ! to be exported by this MODULE.
         private
 
+        private :: kmp_get_affinity_mask_proc_c
+        private :: kmp_set_affinity_mask_proc_c
+        private :: kmp_set_blocktime_c
+        private :: kmp_set_disp_num_buffers_c
+        private :: kmp_set_library_c
+        private :: kmp_set_stacksize_c
+        private :: kmp_unset_affinity_mask_proc_c
+        private :: omp_get_ancestor_thread_num_c
+        private :: omp_get_partition_place_nums_c
+        private :: omp_get_place_num_procs_c
+        private :: omp_get_place_proc_ids_c
+        private :: omp_get_schedule_c
+        private :: omp_get_team_size_c
+        private :: omp_pause_resource_c
+        private :: omp_set_default_device_c
+        private :: omp_set_max_active_levels_c
+        private :: omp_set_num_teams_c
+        private :: omp_set_num_threads_c
+        private :: omp_set_schedule_c
+        private :: omp_set_teams_thread_limit_c
+
         ! Re-export definitions in omp_lib_kinds
         public :: omp_integer_kind
         public :: omp_logical_kind
@@ -250,10 +271,11 @@
 !         *** omp_* entry points
 !         ***
 
-          subroutine omp_set_num_threads(num_threads) bind(c)
+          subroutine omp_set_num_threads_c(num_threads) bind(c,                &
+              name='omp_set_num_threads')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: num_threads
-          end subroutine omp_set_num_threads
+          end subroutine omp_set_num_threads_c
 
           subroutine omp_set_dynamic(dynamic_threads) bind(c)
             use omp_lib_kinds
@@ -310,10 +332,11 @@
             integer (kind=omp_integer_kind) omp_get_thread_limit
           end function omp_get_thread_limit
 
-          subroutine omp_set_max_active_levels(max_levels) bind(c)
+          subroutine omp_set_max_active_levels_c(max_levels) bind(c,           &
+              name='omp_set_max_active_levels')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: max_levels
-          end subroutine omp_set_max_active_levels
+          end subroutine omp_set_max_active_levels_c
 
           function omp_get_max_active_levels() bind(c)
             use omp_lib_kinds
@@ -330,29 +353,32 @@
             integer (kind=omp_integer_kind) omp_get_active_level
           end function omp_get_active_level
 
-          function omp_get_ancestor_thread_num(level) bind(c)
+          function omp_get_ancestor_thread_num_c(level) bind(c,                &
+              name='omp_get_ancestor_thread_num')
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) omp_get_ancestor_thread_num
+            integer (kind=omp_integer_kind) omp_get_ancestor_thread_num_c
             integer (kind=omp_integer_kind), value :: level
-          end function omp_get_ancestor_thread_num
+          end function omp_get_ancestor_thread_num_c
 
-          function omp_get_team_size(level) bind(c)
+          function omp_get_team_size_c(level) bind(c, name='omp_get_team_size')
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) omp_get_team_size
+            integer (kind=omp_integer_kind) omp_get_team_size_c
             integer (kind=omp_integer_kind), value :: level
-          end function omp_get_team_size
+          end function omp_get_team_size_c
 
-          subroutine omp_set_schedule(kind, chunk_size) bind(c)
+          subroutine omp_set_schedule_c(kind, chunk_size) bind(c,              &
+              name='omp_set_schedule')
             use omp_lib_kinds
             integer (kind=omp_sched_kind), value :: kind
             integer (kind=omp_integer_kind), value :: chunk_size
-          end subroutine omp_set_schedule
+          end subroutine omp_set_schedule_c
 
-          subroutine omp_get_schedule(kind, chunk_size) bind(c)
+          subroutine omp_get_schedule_c(kind, chunk_size) bind(c,              &
+              name='omp_get_schedule')
             use omp_lib_kinds
             integer (kind=omp_sched_kind) kind
             integer (kind=omp_integer_kind) chunk_size
-          end subroutine omp_get_schedule
+          end subroutine omp_get_schedule_c
 
           function omp_get_proc_bind() bind(c)
             use omp_lib_kinds
@@ -364,17 +390,19 @@
             integer (kind=omp_integer_kind) omp_get_num_places
           end function omp_get_num_places
 
-          function omp_get_place_num_procs(place_num) bind(c)
+          function omp_get_place_num_procs_c(place_num) bind(c,                &
+              name='omp_get_place_num_procs')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: place_num
-            integer (kind=omp_integer_kind) omp_get_place_num_procs
-          end function omp_get_place_num_procs
+            integer (kind=omp_integer_kind) omp_get_place_num_procs_c
+          end function omp_get_place_num_procs_c
 
-          subroutine omp_get_place_proc_ids(place_num, ids) bind(c)
+          subroutine omp_get_place_proc_ids_c(place_num, ids) bind(c,          &
+              name='omp_get_place_proc_ids')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: place_num
             integer (kind=omp_integer_kind) ids(*)
-          end subroutine omp_get_place_proc_ids
+          end subroutine omp_get_place_proc_ids_c
 
           function omp_get_place_num() bind(c)
             use omp_lib_kinds
@@ -386,10 +414,11 @@
             integer (kind=omp_integer_kind) omp_get_partition_num_places
           end function omp_get_partition_num_places
 
-          subroutine omp_get_partition_place_nums(place_nums) bind(c)
+          subroutine omp_get_partition_place_nums_c(place_nums) bind(c,        &
+              name='omp_get_partition_place_nums')
             use omp_lib_kinds
             integer (kind=omp_integer_kind) place_nums(*)
-          end subroutine omp_get_partition_place_nums
+          end subroutine omp_get_partition_place_nums_c
 
           function omp_get_wtime() bind(c)
             use omp_lib_kinds
@@ -406,10 +435,11 @@
             integer (kind=omp_integer_kind) omp_get_default_device
           end function omp_get_default_device
 
-          subroutine omp_set_default_device(device_num) bind(c)
+          subroutine omp_set_default_device_c(device_num) bind(c,              &
+              name='omp_set_default_device')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: device_num
-          end subroutine omp_set_default_device
+          end subroutine omp_set_default_device_c
 
           function omp_get_num_devices() bind(c)
             use omp_lib_kinds
@@ -446,12 +476,13 @@
             integer (kind=omp_integer_kind) omp_get_device_num
           end function omp_get_device_num
 
-          function omp_pause_resource(kind, device_num) bind(c)
+          function omp_pause_resource_c(kind, device_num) bind(c,              &
+              name='omp_pause_resource')
             use omp_lib_kinds
             integer (kind=omp_pause_resource_kind), value :: kind
             integer (kind=omp_integer_kind), value :: device_num
-            integer (kind=omp_integer_kind) omp_pause_resource
-          end function omp_pause_resource
+            integer (kind=omp_integer_kind) omp_pause_resource_c
+          end function omp_pause_resource_c
 
           function omp_pause_resource_all(kind) bind(c)
             use omp_lib_kinds
@@ -640,20 +671,22 @@
             integer (kind=kmp_size_t_kind) :: omp_capture_affinity
           end function omp_capture_affinity
 
-          subroutine omp_set_num_teams(num_teams) bind(c)
+          subroutine omp_set_num_teams_c(num_teams) bind(c,                    &
+              name='omp_set_num_teams')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: num_teams
-          end subroutine omp_set_num_teams
+          end subroutine omp_set_num_teams_c
 
           function omp_get_max_teams() bind(c)
             use omp_lib_kinds
             integer (kind=omp_integer_kind) omp_get_max_teams
           end function omp_get_max_teams
 
-          subroutine omp_set_teams_thread_limit(thread_limit) bind(c)
+          subroutine omp_set_teams_thread_limit_c(thread_limit) bind(c,        &
+              name='omp_set_teams_thread_limit')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: thread_limit
-          end subroutine omp_set_teams_thread_limit
+          end subroutine omp_set_teams_thread_limit_c
 
           function omp_get_teams_thread_limit() bind(c)
             use omp_lib_kinds
@@ -937,20 +970,20 @@
 !         *** kmp_* entry points
 !         ***
 
-          subroutine kmp_set_stacksize(size) bind(c)
+          subroutine kmp_set_stacksize_c(size) bind(c, name='kmp_set_stacksize')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: size
-          end subroutine kmp_set_stacksize
+          end subroutine kmp_set_stacksize_c
 
           subroutine kmp_set_stacksize_s(size) bind(c)
             use omp_lib_kinds
             integer (kind=kmp_size_t_kind), value :: size
           end subroutine kmp_set_stacksize_s
 
-          subroutine kmp_set_blocktime(msec) bind(c)
+          subroutine kmp_set_blocktime_c(msec) bind(c, name='kmp_set_blocktime')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: msec
-          end subroutine kmp_set_blocktime
+          end subroutine kmp_set_blocktime_c
 
           subroutine kmp_set_library_serial() bind(c)
           end subroutine kmp_set_library_serial
@@ -961,10 +994,10 @@
           subroutine kmp_set_library_throughput() bind(c)
           end subroutine kmp_set_library_throughput
 
-          subroutine kmp_set_library(libnum) bind(c)
+          subroutine kmp_set_library_c(libnum) bind(c, name='kmp_set_library')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: libnum
-          end subroutine kmp_set_library
+          end subroutine kmp_set_library_c
 
           subroutine kmp_set_defaults(string)
             character (len=*) :: string
@@ -990,10 +1023,11 @@
             integer (kind=omp_integer_kind) kmp_get_library
           end function kmp_get_library
 
-          subroutine kmp_set_disp_num_buffers(num) bind(c)
+          subroutine kmp_set_disp_num_buffers_c(num) bind(c,                   &
+              name='kmp_set_disp_num_buffers')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: num
-          end subroutine kmp_set_disp_num_buffers
+          end subroutine kmp_set_disp_num_buffers_c
 
           function kmp_set_affinity(mask) bind(c)
             use omp_lib_kinds
@@ -1022,26 +1056,29 @@
             integer (kind=kmp_affinity_mask_kind) mask
           end subroutine kmp_destroy_affinity_mask
 
-          function kmp_set_affinity_mask_proc(proc, mask) bind(c)
+          function kmp_set_affinity_mask_proc_c(proc, mask) bind(c,            &
+              name='kmp_set_affinity_mask_proc')
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) kmp_set_affinity_mask_proc
+            integer (kind=omp_integer_kind) kmp_set_affinity_mask_proc_c
             integer (kind=omp_integer_kind), value :: proc
             integer (kind=kmp_affinity_mask_kind) mask
-          end function kmp_set_affinity_mask_proc
+          end function kmp_set_affinity_mask_proc_c
 
-          function kmp_unset_affinity_mask_proc(proc, mask) bind(c)
+          function kmp_unset_affinity_mask_proc_c(proc, mask) bind(c,          &
+              name='kmp_unset_affinity_mask_proc')
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) kmp_unset_affinity_mask_proc
+            integer (kind=omp_integer_kind) kmp_unset_affinity_mask_proc_c
             integer (kind=omp_integer_kind), value :: proc
             integer (kind=kmp_affinity_mask_kind) mask
-          end function kmp_unset_affinity_mask_proc
+          end function kmp_unset_affinity_mask_proc_c
 
-          function kmp_get_affinity_mask_proc(proc, mask) bind(c)
+          function kmp_get_affinity_mask_proc_c(proc, mask) bind(c,            &
+              name='kmp_get_affinity_mask_proc')
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) kmp_get_affinity_mask_proc
+            integer (kind=omp_integer_kind) kmp_get_affinity_mask_proc_c
             integer (kind=omp_integer_kind), value :: proc
             integer (kind=kmp_affinity_mask_kind) mask
-          end function kmp_get_affinity_mask_proc
+          end function kmp_get_affinity_mask_proc_c
 
           function kmp_malloc(size) bind(c)
             use omp_lib_kinds
@@ -1150,6 +1187,98 @@
 
         end interface
 
+        ! Generic interfaces for multiple integer kinds
+        interface omp_set_num_threads
+          module procedure omp_set_num_threads_i1, omp_set_num_threads_i2,     &
+              omp_set_num_threads_i4, omp_set_num_threads_i8
+        end interface omp_set_num_threads
+        interface omp_set_max_active_levels
+          module procedure omp_set_max_active_levels_i1,                       &
+              omp_set_max_active_levels_i2, omp_set_max_active_levels_i4,      &
+              omp_set_max_active_levels_i8
+        end interface omp_set_max_active_levels
+        interface omp_set_default_device
+          module procedure omp_set_default_device_i1,                          &
+              omp_set_default_device_i2, omp_set_default_device_i4,            &
+              omp_set_default_device_i8
+        end interface omp_set_default_device
+        interface omp_set_num_teams
+          module procedure omp_set_num_teams_i1, omp_set_num_teams_i2,         &
+              omp_set_num_teams_i4, omp_set_num_teams_i8
+        end interface omp_set_num_teams
+        interface omp_set_teams_thread_limit
+          module procedure omp_set_teams_thread_limit_i1,                      &
+              omp_set_teams_thread_limit_i2, omp_set_teams_thread_limit_i4,    &
+              omp_set_teams_thread_limit_i8
+        end interface omp_set_teams_thread_limit
+        interface kmp_set_stacksize
+          module procedure kmp_set_stacksize_i1, kmp_set_stacksize_i2,         &
+              kmp_set_stacksize_i4, kmp_set_stacksize_i8
+        end interface kmp_set_stacksize
+        interface kmp_set_blocktime
+          module procedure kmp_set_blocktime_i1, kmp_set_blocktime_i2,         &
+              kmp_set_blocktime_i4, kmp_set_blocktime_i8
+        end interface kmp_set_blocktime
+        interface kmp_set_library
+          module procedure kmp_set_library_i1, kmp_set_library_i2,             &
+              kmp_set_library_i4, kmp_set_library_i8
+        end interface kmp_set_library
+        interface kmp_set_disp_num_buffers
+          module procedure kmp_set_disp_num_buffers_i1,                        &
+              kmp_set_disp_num_buffers_i2, kmp_set_disp_num_buffers_i4,        &
+              kmp_set_disp_num_buffers_i8
+        end interface kmp_set_disp_num_buffers
+        interface omp_get_ancestor_thread_num
+          module procedure omp_get_ancestor_thread_num_i1,                     &
+              omp_get_ancestor_thread_num_i2, omp_get_ancestor_thread_num_i4,  &
+              omp_get_ancestor_thread_num_i8
+        end interface omp_get_ancestor_thread_num
+        interface omp_get_team_size
+          module procedure omp_get_team_size_i1, omp_get_team_size_i2,         &
+              omp_get_team_size_i4, omp_get_team_size_i8
+        end interface omp_get_team_size
+        interface omp_get_place_num_procs
+          module procedure omp_get_place_num_procs_i1,                         &
+              omp_get_place_num_procs_i2, omp_get_place_num_procs_i4,          &
+              omp_get_place_num_procs_i8
+        end interface omp_get_place_num_procs
+        interface omp_set_schedule
+          module procedure omp_set_schedule_i1, omp_set_schedule_i2,           &
+              omp_set_schedule_i4, omp_set_schedule_i8
+        end interface omp_set_schedule
+        interface omp_get_schedule
+          module procedure omp_get_schedule_i1, omp_get_schedule_i2,           &
+              omp_get_schedule_i4, omp_get_schedule_i8
+        end interface omp_get_schedule
+        interface omp_pause_resource
+          module procedure omp_pause_resource_i1, omp_pause_resource_i2,       &
+              omp_pause_resource_i4, omp_pause_resource_i8
+        end interface omp_pause_resource
+        interface omp_get_place_proc_ids
+          module procedure omp_get_place_proc_ids_i4, omp_get_place_proc_ids_i8
+        end interface omp_get_place_proc_ids
+        interface omp_get_partition_place_nums
+          module procedure omp_get_partition_place_nums_i4,                    &
+              omp_get_partition_place_nums_i8
+        end interface omp_get_partition_place_nums
+        interface kmp_set_affinity_mask_proc
+          module procedure kmp_set_affinity_mask_proc_i1,                      &
+              kmp_set_affinity_mask_proc_i2, kmp_set_affinity_mask_proc_i4,    &
+              kmp_set_affinity_mask_proc_i8
+        end interface kmp_set_affinity_mask_proc
+        interface kmp_unset_affinity_mask_proc
+          module procedure kmp_unset_affinity_mask_proc_i1,                    &
+              kmp_unset_affinity_mask_proc_i2,                                 &
+              kmp_unset_affinity_mask_proc_i4,                                 &
+              kmp_unset_affinity_mask_proc_i8
+        end interface kmp_unset_affinity_mask_proc
+        interface kmp_get_affinity_mask_proc
+          module procedure kmp_get_affinity_mask_proc_i1,                      &
+              kmp_get_affinity_mask_proc_i2, kmp_get_affinity_mask_proc_i4,    &
+              kmp_get_affinity_mask_proc_i8
+        end interface kmp_get_affinity_mask_proc
+
+
         ! make the above routine definitions public
         public :: omp_set_num_threads
         public :: omp_set_dynamic
@@ -1277,4 +1406,861 @@
         public :: omp_get_interop_type_desc
         public :: omp_get_interop_rc_desc
 
+
+
+        ! Submodule procedure declarations for integer-kind wrappers.
+        interface
+          module subroutine omp_set_num_threads_i1(num_threads)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: num_threads
+          end subroutine omp_set_num_threads_i1
+          module subroutine omp_set_num_threads_i2(num_threads)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: num_threads
+          end subroutine omp_set_num_threads_i2
+          module subroutine omp_set_num_threads_i4(num_threads)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: num_threads
+          end subroutine omp_set_num_threads_i4
+          module subroutine omp_set_num_threads_i8(num_threads)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: num_threads
+          end subroutine omp_set_num_threads_i8
+          module subroutine omp_set_max_active_levels_i1(max_levels)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: max_levels
+          end subroutine omp_set_max_active_levels_i1
+          module subroutine omp_set_max_active_levels_i2(max_levels)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: max_levels
+          end subroutine omp_set_max_active_levels_i2
+          module subroutine omp_set_max_active_levels_i4(max_levels)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: max_levels
+          end subroutine omp_set_max_active_levels_i4
+          module subroutine omp_set_max_active_levels_i8(max_levels)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: max_levels
+          end subroutine omp_set_max_active_levels_i8
+          module subroutine omp_set_default_device_i1(device_num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: device_num
+          end subroutine omp_set_default_device_i1
+          module subroutine omp_set_default_device_i2(device_num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: device_num
+          end subroutine omp_set_default_device_i2
+          module subroutine omp_set_default_device_i4(device_num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: device_num
+          end subroutine omp_set_default_device_i4
+          module subroutine omp_set_default_device_i8(device_num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: device_num
+          end subroutine omp_set_default_device_i8
+          module subroutine omp_set_num_teams_i1(num_teams)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: num_teams
+          end subroutine omp_set_num_teams_i1
+          module subroutine omp_set_num_teams_i2(num_teams)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: num_teams
+          end subroutine omp_set_num_teams_i2
+          module subroutine omp_set_num_teams_i4(num_teams)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: num_teams
+          end subroutine omp_set_num_teams_i4
+          module subroutine omp_set_num_teams_i8(num_teams)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: num_teams
+          end subroutine omp_set_num_teams_i8
+          module subroutine omp_set_teams_thread_limit_i1(thread_limit)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: thread_limit
+          end subroutine omp_set_teams_thread_limit_i1
+          module subroutine omp_set_teams_thread_limit_i2(thread_limit)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: thread_limit
+          end subroutine omp_set_teams_thread_limit_i2
+          module subroutine omp_set_teams_thread_limit_i4(thread_limit)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: thread_limit
+          end subroutine omp_set_teams_thread_limit_i4
+          module subroutine omp_set_teams_thread_limit_i8(thread_limit)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: thread_limit
+          end subroutine omp_set_teams_thread_limit_i8
+          module subroutine kmp_set_stacksize_i1(size)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: size
+          end subroutine kmp_set_stacksize_i1
+          module subroutine kmp_set_stacksize_i2(size)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: size
+          end subroutine kmp_set_stacksize_i2
+          module subroutine kmp_set_stacksize_i4(size)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: size
+          end subroutine kmp_set_stacksize_i4
+          module subroutine kmp_set_stacksize_i8(size)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: size
+          end subroutine kmp_set_stacksize_i8
+          module subroutine kmp_set_blocktime_i1(msec)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: msec
+          end subroutine kmp_set_blocktime_i1
+          module subroutine kmp_set_blocktime_i2(msec)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: msec
+          end subroutine kmp_set_blocktime_i2
+          module subroutine kmp_set_blocktime_i4(msec)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: msec
+          end subroutine kmp_set_blocktime_i4
+          module subroutine kmp_set_blocktime_i8(msec)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: msec
+          end subroutine kmp_set_blocktime_i8
+          module subroutine kmp_set_library_i1(libnum)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: libnum
+          end subroutine kmp_set_library_i1
+          module subroutine kmp_set_library_i2(libnum)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: libnum
+          end subroutine kmp_set_library_i2
+          module subroutine kmp_set_library_i4(libnum)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: libnum
+          end subroutine kmp_set_library_i4
+          module subroutine kmp_set_library_i8(libnum)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: libnum
+          end subroutine kmp_set_library_i8
+          module subroutine kmp_set_disp_num_buffers_i1(num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: num
+          end subroutine kmp_set_disp_num_buffers_i1
+          module subroutine kmp_set_disp_num_buffers_i2(num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: num
+          end subroutine kmp_set_disp_num_buffers_i2
+          module subroutine kmp_set_disp_num_buffers_i4(num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: num
+          end subroutine kmp_set_disp_num_buffers_i4
+          module subroutine kmp_set_disp_num_buffers_i8(num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: num
+          end subroutine kmp_set_disp_num_buffers_i8
+          module function omp_get_ancestor_thread_num_i1(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_ancestor_thread_num_i1
+          end function omp_get_ancestor_thread_num_i1
+          module function omp_get_ancestor_thread_num_i2(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_ancestor_thread_num_i2
+          end function omp_get_ancestor_thread_num_i2
+          module function omp_get_ancestor_thread_num_i4(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_ancestor_thread_num_i4
+          end function omp_get_ancestor_thread_num_i4
+          module function omp_get_ancestor_thread_num_i8(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_ancestor_thread_num_i8
+          end function omp_get_ancestor_thread_num_i8
+          module function omp_get_team_size_i1(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_team_size_i1
+          end function omp_get_team_size_i1
+          module function omp_get_team_size_i2(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_team_size_i2
+          end function omp_get_team_size_i2
+          module function omp_get_team_size_i4(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_team_size_i4
+          end function omp_get_team_size_i4
+          module function omp_get_team_size_i8(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_team_size_i8
+          end function omp_get_team_size_i8
+          module function omp_get_place_num_procs_i1(place_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: place_num
+          integer(kind=omp_integer_kind) :: omp_get_place_num_procs_i1
+          end function omp_get_place_num_procs_i1
+          module function omp_get_place_num_procs_i2(place_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: place_num
+          integer(kind=omp_integer_kind) :: omp_get_place_num_procs_i2
+          end function omp_get_place_num_procs_i2
+          module function omp_get_place_num_procs_i4(place_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: place_num
+          integer(kind=omp_integer_kind) :: omp_get_place_num_procs_i4
+          end function omp_get_place_num_procs_i4
+          module function omp_get_place_num_procs_i8(place_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: place_num
+          integer(kind=omp_integer_kind) :: omp_get_place_num_procs_i8
+          end function omp_get_place_num_procs_i8
+          module subroutine omp_set_schedule_i1(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(kind=omp_sched_kind), value :: kind
+          integer(c_int8_t), value :: chunk_size
+          end subroutine omp_set_schedule_i1
+          module subroutine omp_set_schedule_i2(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(kind=omp_sched_kind), value :: kind
+          integer(c_int16_t), value :: chunk_size
+          end subroutine omp_set_schedule_i2
+          module subroutine omp_set_schedule_i4(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(kind=omp_sched_kind), value :: kind
+          integer(c_int), value :: chunk_size
+          end subroutine omp_set_schedule_i4
+          module subroutine omp_set_schedule_i8(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(kind=omp_sched_kind), value :: kind
+          integer(c_int64_t), value :: chunk_size
+          end subroutine omp_set_schedule_i8
+          module subroutine omp_get_schedule_i1(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(kind=omp_sched_kind), intent(out) :: kind
+          integer(c_int8_t), intent(out) :: chunk_size
+          end subroutine omp_get_schedule_i1
+          module subroutine omp_get_schedule_i2(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(kind=omp_sched_kind), intent(out) :: kind
+          integer(c_int16_t), intent(out) :: chunk_size
+          end subroutine omp_get_schedule_i2
+          module subroutine omp_get_schedule_i4(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(kind=omp_sched_kind), intent(out) :: kind
+          integer(c_int), intent(out) :: chunk_size
+          end subroutine omp_get_schedule_i4
+          module subroutine omp_get_schedule_i8(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(kind=omp_sched_kind), intent(out) :: kind
+          integer(c_int64_t), intent(out) :: chunk_size
+          end subroutine omp_get_schedule_i8
+          module function omp_pause_resource_i1(kind, device_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(kind=omp_pause_resource_kind), value :: kind
+          integer(c_int8_t), value :: device_num
+          integer(kind=omp_integer_kind) :: omp_pause_resource_i1
+          end function omp_pause_resource_i1
+          module function omp_pause_resource_i2(kind, device_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(kind=omp_pause_resource_kind), value :: kind
+          integer(c_int16_t), value :: device_num
+          integer(kind=omp_integer_kind) :: omp_pause_resource_i2
+          end function omp_pause_resource_i2
+          module function omp_pause_resource_i4(kind, device_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(kind=omp_pause_resource_kind), value :: kind
+          integer(c_int), value :: device_num
+          integer(kind=omp_integer_kind) :: omp_pause_resource_i4
+          end function omp_pause_resource_i4
+          module function omp_pause_resource_i8(kind, device_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(kind=omp_pause_resource_kind), value :: kind
+          integer(c_int64_t), value :: device_num
+          integer(kind=omp_integer_kind) :: omp_pause_resource_i8
+          end function omp_pause_resource_i8
+          module function kmp_set_affinity_mask_proc_i1(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_set_affinity_mask_proc_i1
+          end function kmp_set_affinity_mask_proc_i1
+          module function kmp_set_affinity_mask_proc_i2(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_set_affinity_mask_proc_i2
+          end function kmp_set_affinity_mask_proc_i2
+          module function kmp_set_affinity_mask_proc_i4(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_set_affinity_mask_proc_i4
+          end function kmp_set_affinity_mask_proc_i4
+          module function kmp_set_affinity_mask_proc_i8(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_set_affinity_mask_proc_i8
+          end function kmp_set_affinity_mask_proc_i8
+          module function kmp_unset_affinity_mask_proc_i1(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_unset_affinity_mask_proc_i1
+          end function kmp_unset_affinity_mask_proc_i1
+          module function kmp_unset_affinity_mask_proc_i2(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_unset_affinity_mask_proc_i2
+          end function kmp_unset_affinity_mask_proc_i2
+          module function kmp_unset_affinity_mask_proc_i4(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_unset_affinity_mask_proc_i4
+          end function kmp_unset_affinity_mask_proc_i4
+          module function kmp_unset_affinity_mask_proc_i8(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_unset_affinity_mask_proc_i8
+          end function kmp_unset_affinity_mask_proc_i8
+          module function kmp_get_affinity_mask_proc_i1(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_get_affinity_mask_proc_i1
+          end function kmp_get_affinity_mask_proc_i1
+          module function kmp_get_affinity_mask_proc_i2(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_get_affinity_mask_proc_i2
+          end function kmp_get_affinity_mask_proc_i2
+          module function kmp_get_affinity_mask_proc_i4(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_get_affinity_mask_proc_i4
+          end function kmp_get_affinity_mask_proc_i4
+          module function kmp_get_affinity_mask_proc_i8(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_get_affinity_mask_proc_i8
+          end function kmp_get_affinity_mask_proc_i8
+          module subroutine omp_get_place_proc_ids_i4(place_num, ids)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: place_num
+          integer(c_int), intent(out) :: ids(*)
+          end subroutine omp_get_place_proc_ids_i4
+          module subroutine omp_get_place_proc_ids_i8(place_num, ids)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: place_num
+          integer(c_int64_t), intent(out) :: ids(*)
+          end subroutine omp_get_place_proc_ids_i8
+          module subroutine omp_get_partition_place_nums_i4(place_nums)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), intent(out) :: place_nums(*)
+          end subroutine omp_get_partition_place_nums_i4
+          module subroutine omp_get_partition_place_nums_i8(place_nums)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), intent(out) :: place_nums(*)
+          end subroutine omp_get_partition_place_nums_i8
+        end interface
       end module omp_lib
+
+      submodule (omp_lib) omp_lib_impl
+        use omp_lib_kinds
+      contains
+
+        module procedure omp_set_num_threads_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_num_threads_c(int(num_threads, kind=c_int))
+        end procedure omp_set_num_threads_i1
+
+        module procedure omp_set_num_threads_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_num_threads_c(int(num_threads, kind=c_int))
+        end procedure omp_set_num_threads_i2
+
+        module procedure omp_set_num_threads_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_num_threads_c(int(num_threads, kind=c_int))
+        end procedure omp_set_num_threads_i4
+
+        module procedure omp_set_num_threads_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_num_threads_c(int(num_threads, kind=c_int))
+        end procedure omp_set_num_threads_i8
+
+        module procedure omp_set_max_active_levels_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_max_active_levels_c(int(max_levels, kind=c_int))
+        end procedure omp_set_max_active_levels_i1
+
+        module procedure omp_set_max_active_levels_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_max_active_levels_c(int(max_levels, kind=c_int))
+        end procedure omp_set_max_active_levels_i2
+
+        module procedure omp_set_max_active_levels_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_max_active_levels_c(int(max_levels, kind=c_int))
+        end procedure omp_set_max_active_levels_i4
+
+        module procedure omp_set_max_active_levels_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_max_active_levels_c(int(max_levels, kind=c_int))
+        end procedure omp_set_max_active_levels_i8
+
+        module procedure omp_set_default_device_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_default_device_c(int(device_num, kind=c_int))
+        end procedure omp_set_default_device_i1
+
+        module procedure omp_set_default_device_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_default_device_c(int(device_num, kind=c_int))
+        end procedure omp_set_default_device_i2
+
+        module procedure omp_set_default_device_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_default_device_c(int(device_num, kind=c_int))
+        end procedure omp_set_default_device_i4
+
+        module procedure omp_set_default_device_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_default_device_c(int(device_num, kind=c_int))
+        end procedure omp_set_default_device_i8
+
+        module procedure omp_set_num_teams_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_num_teams_c(int(num_teams, kind=c_int))
+        end procedure omp_set_num_teams_i1
+
+        module procedure omp_set_num_teams_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_num_teams_c(int(num_teams, kind=c_int))
+        end procedure omp_set_num_teams_i2
+
+        module procedure omp_set_num_teams_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_num_teams_c(int(num_teams, kind=c_int))
+        end procedure omp_set_num_teams_i4
+
+        module procedure omp_set_num_teams_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_num_teams_c(int(num_teams, kind=c_int))
+        end procedure omp_set_num_teams_i8
+
+        module procedure omp_set_teams_thread_limit_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_teams_thread_limit_c(int(thread_limit, kind=c_int))
+        end procedure omp_set_teams_thread_limit_i1
+
+        module procedure omp_set_teams_thread_limit_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_teams_thread_limit_c(int(thread_limit, kind=c_int))
+        end procedure omp_set_teams_thread_limit_i2
+
+        module procedure omp_set_teams_thread_limit_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_teams_thread_limit_c(int(thread_limit, kind=c_int))
+        end procedure omp_set_teams_thread_limit_i4
+
+        module procedure omp_set_teams_thread_limit_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_teams_thread_limit_c(int(thread_limit, kind=c_int))
+        end procedure omp_set_teams_thread_limit_i8
+
+        module procedure kmp_set_stacksize_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_stacksize_c(int(size, kind=c_int))
+        end procedure kmp_set_stacksize_i1
+
+        module procedure kmp_set_stacksize_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_stacksize_c(int(size, kind=c_int))
+        end procedure kmp_set_stacksize_i2
+
+        module procedure kmp_set_stacksize_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_stacksize_c(int(size, kind=c_int))
+        end procedure kmp_set_stacksize_i4
+
+        module procedure kmp_set_stacksize_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_stacksize_c(int(size, kind=c_int))
+        end procedure kmp_set_stacksize_i8
+
+        module procedure kmp_set_blocktime_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_blocktime_c(int(msec, kind=c_int))
+        end procedure kmp_set_blocktime_i1
+
+        module procedure kmp_set_blocktime_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_blocktime_c(int(msec, kind=c_int))
+        end procedure kmp_set_blocktime_i2
+
+        module procedure kmp_set_blocktime_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_blocktime_c(int(msec, kind=c_int))
+        end procedure kmp_set_blocktime_i4
+
+        module procedure kmp_set_blocktime_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_blocktime_c(int(msec, kind=c_int))
+        end procedure kmp_set_blocktime_i8
+
+        module procedure kmp_set_library_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_library_c(int(libnum, kind=c_int))
+        end procedure kmp_set_library_i1
+
+        module procedure kmp_set_library_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_library_c(int(libnum, kind=c_int))
+        end procedure kmp_set_library_i2
+
+        module procedure kmp_set_library_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_library_c(int(libnum, kind=c_int))
+        end procedure kmp_set_library_i4
+
+        module procedure kmp_set_library_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_library_c(int(libnum, kind=c_int))
+        end procedure kmp_set_library_i8
+
+        module procedure kmp_set_disp_num_buffers_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_disp_num_buffers_c(int(num, kind=c_int))
+        end procedure kmp_set_disp_num_buffers_i1
+
+        module procedure kmp_set_disp_num_buffers_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_disp_num_buffers_c(int(num, kind=c_int))
+        end procedure kmp_set_disp_num_buffers_i2
+
+        module procedure kmp_set_disp_num_buffers_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_disp_num_buffers_c(int(num, kind=c_int))
+        end procedure kmp_set_disp_num_buffers_i4
+
+        module procedure kmp_set_disp_num_buffers_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_disp_num_buffers_c(int(num, kind=c_int))
+        end procedure kmp_set_disp_num_buffers_i8
+
+        module procedure omp_get_ancestor_thread_num_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_ancestor_thread_num_i1 =                                     &
+          omp_get_ancestor_thread_num_c(int(level, kind=c_int))
+        end procedure omp_get_ancestor_thread_num_i1
+
+        module procedure omp_get_ancestor_thread_num_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_ancestor_thread_num_i2 =                                     &
+          omp_get_ancestor_thread_num_c(int(level, kind=c_int))
+        end procedure omp_get_ancestor_thread_num_i2
+
+        module procedure omp_get_ancestor_thread_num_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_ancestor_thread_num_i4 =                                     &
+          omp_get_ancestor_thread_num_c(int(level, kind=c_int))
+        end procedure omp_get_ancestor_thread_num_i4
+
+        module procedure omp_get_ancestor_thread_num_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_ancestor_thread_num_i8 =                                     &
+          omp_get_ancestor_thread_num_c(int(level, kind=c_int))
+        end procedure omp_get_ancestor_thread_num_i8
+
+        module procedure omp_get_team_size_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_team_size_i1 = omp_get_team_size_c(int(level, kind=c_int))
+        end procedure omp_get_team_size_i1
+
+        module procedure omp_get_team_size_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_team_size_i2 = omp_get_team_size_c(int(level, kind=c_int))
+        end procedure omp_get_team_size_i2
+
+        module procedure omp_get_team_size_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_team_size_i4 = omp_get_team_size_c(int(level, kind=c_int))
+        end procedure omp_get_team_size_i4
+
+        module procedure omp_get_team_size_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_team_size_i8 = omp_get_team_size_c(int(level, kind=c_int))
+        end procedure omp_get_team_size_i8
+
+        module procedure omp_get_place_num_procs_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_place_num_procs_i1 =                                         &
+          omp_get_place_num_procs_c(int(place_num, kind=c_int))
+        end procedure omp_get_place_num_procs_i1
+
+        module procedure omp_get_place_num_procs_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_place_num_procs_i2 =                                         &
+          omp_get_place_num_procs_c(int(place_num, kind=c_int))
+        end procedure omp_get_place_num_procs_i2
+
+        module procedure omp_get_place_num_procs_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_place_num_procs_i4 =                                         &
+          omp_get_place_num_procs_c(int(place_num, kind=c_int))
+        end procedure omp_get_place_num_procs_i4
+
+        module procedure omp_get_place_num_procs_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_place_num_procs_i8 =                                         &
+          omp_get_place_num_procs_c(int(place_num, kind=c_int))
+        end procedure omp_get_place_num_procs_i8
+
+        module procedure omp_set_schedule_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_schedule_c(kind, int(chunk_size, kind=c_int))
+        end procedure omp_set_schedule_i1
+
+        module procedure omp_set_schedule_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_schedule_c(kind, int(chunk_size, kind=c_int))
+        end procedure omp_set_schedule_i2
+
+        module procedure omp_set_schedule_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_schedule_c(kind, int(chunk_size, kind=c_int))
+        end procedure omp_set_schedule_i4
+
+        module procedure omp_set_schedule_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_schedule_c(kind, int(chunk_size, kind=c_int))
+        end procedure omp_set_schedule_i8
+
+        module procedure omp_get_schedule_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          integer(c_int) :: kind_c, chunk_c
+          call omp_get_schedule_c(kind_c, chunk_c)
+          kind = kind_c
+          chunk_size = chunk_c
+        end procedure omp_get_schedule_i1
+
+        module procedure omp_get_schedule_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          integer(c_int) :: kind_c, chunk_c
+          call omp_get_schedule_c(kind_c, chunk_c)
+          kind = kind_c
+          chunk_size = chunk_c
+        end procedure omp_get_schedule_i2
+
+        module procedure omp_get_schedule_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          integer(c_int) :: kind_c, chunk_c
+          call omp_get_schedule_c(kind_c, chunk_c)
+          kind = kind_c
+          chunk_size = chunk_c
+        end procedure omp_get_schedule_i4
+
+        module procedure omp_get_schedule_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          integer(c_int) :: kind_c, chunk_c
+          call omp_get_schedule_c(kind_c, chunk_c)
+          kind = kind_c
+          chunk_size = chunk_c
+        end procedure omp_get_schedule_i8
+
+        module procedure omp_pause_resource_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_pause_resource_i1 =                                              &
+          omp_pause_resource_c(kind, int(device_num, kind=c_int))
+        end procedure omp_pause_resource_i1
+
+        module procedure omp_pause_resource_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_pause_resource_i2 =                                              &
+          omp_pause_resource_c(kind, int(device_num, kind=c_int))
+        end procedure omp_pause_resource_i2
+
+        module procedure omp_pause_resource_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_pause_resource_i4 =                                              &
+          omp_pause_resource_c(kind, int(device_num, kind=c_int))
+        end procedure omp_pause_resource_i4
+
+        module procedure omp_pause_resource_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_pause_resource_i8 =                                              &
+          omp_pause_resource_c(kind, int(device_num, kind=c_int))
+        end procedure omp_pause_resource_i8
+
+        module procedure kmp_set_affinity_mask_proc_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_set_affinity_mask_proc_i1 =                                      &
+          kmp_set_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_set_affinity_mask_proc_i1
+
+        module procedure kmp_set_affinity_mask_proc_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_set_affinity_mask_proc_i2 =                                      &
+          kmp_set_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_set_affinity_mask_proc_i2
+
+        module procedure kmp_set_affinity_mask_proc_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_set_affinity_mask_proc_i4 =                                      &
+          kmp_set_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_set_affinity_mask_proc_i4
+
+        module procedure kmp_set_affinity_mask_proc_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_set_affinity_mask_proc_i8 =                                      &
+          kmp_set_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_set_affinity_mask_proc_i8
+
+        module procedure kmp_unset_affinity_mask_proc_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_unset_affinity_mask_proc_i1 =                                    &
+          kmp_unset_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_unset_affinity_mask_proc_i1
+
+        module procedure kmp_unset_affinity_mask_proc_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_unset_affinity_mask_proc_i2 =                                    &
+          kmp_unset_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_unset_affinity_mask_proc_i2
+
+        module procedure kmp_unset_affinity_mask_proc_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_unset_affinity_mask_proc_i4 =                                    &
+          kmp_unset_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_unset_affinity_mask_proc_i4
+
+        module procedure kmp_unset_affinity_mask_proc_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_unset_affinity_mask_proc_i8 =                                    &
+          kmp_unset_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_unset_affinity_mask_proc_i8
+
+        module procedure kmp_get_affinity_mask_proc_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_get_affinity_mask_proc_i1 =                                      &
+          kmp_get_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_get_affinity_mask_proc_i1
+
+        module procedure kmp_get_affinity_mask_proc_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_get_affinity_mask_proc_i2 =                                      &
+          kmp_get_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_get_affinity_mask_proc_i2
+
+        module procedure kmp_get_affinity_mask_proc_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_get_affinity_mask_proc_i4 =                                      &
+          kmp_get_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_get_affinity_mask_proc_i4
+
+        module procedure kmp_get_affinity_mask_proc_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_get_affinity_mask_proc_i8 =                                      &
+          kmp_get_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_get_affinity_mask_proc_i8
+
+        module procedure omp_get_place_proc_ids_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          integer(c_int) :: n, i
+          integer(c_int), allocatable :: ids_c(:)
+          n = omp_get_place_num_procs_c(int(place_num, kind=c_int))
+          allocate(ids_c(n))
+          call omp_get_place_proc_ids_c(int(place_num, kind=c_int), ids_c)
+          do i = 1, n
+          ids(i) = ids_c(i)
+          end do
+          deallocate(ids_c)
+        end procedure omp_get_place_proc_ids_i4
+
+        module procedure omp_get_place_proc_ids_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          integer(c_int) :: n, i
+          integer(c_int), allocatable :: ids_c(:)
+          n = omp_get_place_num_procs_c(int(place_num, kind=c_int))
+          allocate(ids_c(n))
+          call omp_get_place_proc_ids_c(int(place_num, kind=c_int), ids_c)
+          do i = 1, n
+          ids(i) = ids_c(i)
+          end do
+          deallocate(ids_c)
+        end procedure omp_get_place_proc_ids_i8
+
+        module procedure omp_get_partition_place_nums_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          integer(c_int) :: n, i
+          integer(c_int), allocatable :: place_nums_c(:)
+          n = omp_get_partition_num_places()
+          allocate(place_nums_c(n))
+          call omp_get_partition_place_nums_c(place_nums_c)
+          do i = 1, n
+          place_nums(i) = place_nums_c(i)
+          end do
+          deallocate(place_nums_c)
+        end procedure omp_get_partition_place_nums_i4
+
+        module procedure omp_get_partition_place_nums_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          integer(c_int) :: n, i
+          integer(c_int), allocatable :: place_nums_c(:)
+          n = omp_get_partition_num_places()
+          allocate(place_nums_c(n))
+          call omp_get_partition_place_nums_c(place_nums_c)
+          do i = 1, n
+          place_nums(i) = place_nums_c(i)
+          end do
+          deallocate(place_nums_c)
+        end procedure omp_get_partition_place_nums_i8
+
+      end submodule omp_lib_impl


From openmp-commits at lists.llvm.org  Fri Aug 14 05:29:52 2026
From: openmp-commits at lists.llvm.org (Manuel Carrasco via Openmp-commits)
Date: Fri, 14 Aug 2026 05:29:52 -0700 (PDT)
Subject: [Openmp-commits] [clang] [libc] [openmp] [Clang] Remove the old
 LLVM offloading driver entirely (PR #211924)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/211924@github.com>
Message-ID: <6a7f0a40.88a20ebf.84826.f18e@mx.google.com>


mgcarrasco wrote:

> HIPSPV has been using the new offloading driver for a while now, and I'm not aware of any issues with it, nor any use of the old offloading driver, so I'm happy to proceed with this.

Same. I also checked comgr and the flag is not used.

https://github.com/llvm/llvm-project/pull/211924

From openmp-commits at lists.llvm.org  Fri Aug 14 07:40:43 2026
From: openmp-commits at lists.llvm.org (Julian Brown via Openmp-commits)
Date: Fri, 14 Aug 2026 07:40:43 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [OpenMP] OpenMP 6.0
 taskgraph support (WIP) (PR #188765)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/188765@github.com>
Message-ID: <6a7f28eb.642c3f8d.364216.7a43@mx.google.com>


https://github.com/jtb20 updated https://github.com/llvm/llvm-project/pull/188765

>From 555a3a17a031bc06adfc82e4038aa1271e651d89 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Fri, 27 Mar 2026 11:23:09 -0500
Subject: [PATCH 01/26] [OpenMP] Mostly remove experimental taskgraph support
 from runtime

Because the changes to runtime support for taskgraph record/replay
are quite extensive, they will likely be more comprehensible if the
existing parts are removed before the new parts are added.  This is the
removal part.

commit-id:ebbd0706

Pull Request: https://github.com/llvm/llvm-project/pull/194046
---
 openmp/runtime/src/kmp.h                      |  72 ---
 openmp/runtime/src/kmp_global.cpp             |  12 -
 openmp/runtime/src/kmp_settings.cpp           |  28 --
 openmp/runtime/src/kmp_taskdeps.cpp           | 128 +----
 openmp/runtime/src/kmp_taskdeps.h             |  25 +-
 openmp/runtime/src/kmp_tasking.cpp            | 451 +-----------------
 .../test/tasking/omp_record_replay.cpp        |  48 --
 .../test/tasking/omp_record_replay_deps.cpp   |  63 ---
 .../omp_record_replay_deps_multi_succ.cpp     |  56 ---
 .../tasking/omp_record_replay_multiTDGs.cpp   |  76 ---
 .../tasking/omp_record_replay_print_dot.cpp   |  80 ----
 .../tasking/omp_record_replay_taskloop.cpp    |  50 --
 12 files changed, 6 insertions(+), 1083 deletions(-)
 delete mode 100644 openmp/runtime/test/tasking/omp_record_replay.cpp
 delete mode 100644 openmp/runtime/test/tasking/omp_record_replay_deps.cpp
 delete mode 100644 openmp/runtime/test/tasking/omp_record_replay_deps_multi_succ.cpp
 delete mode 100644 openmp/runtime/test/tasking/omp_record_replay_multiTDGs.cpp
 delete mode 100644 openmp/runtime/test/tasking/omp_record_replay_print_dot.cpp
 delete mode 100644 openmp/runtime/test/tasking/omp_record_replay_taskloop.cpp

diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index 818e07c2bb23f..00a56afa90b52 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -2619,26 +2619,6 @@ typedef struct {
 } kmp_event_t;
 
 #if OMP_TASKGRAPH_EXPERIMENTAL
-// Initial number of allocated nodes while recording
-#define INIT_MAPSIZE 50
-
-typedef struct kmp_taskgraph_flags { /*This needs to be exactly 32 bits */
-  unsigned nowait : 1;
-  unsigned re_record : 1;
-  unsigned reserved : 30;
-} kmp_taskgraph_flags_t;
-
-/// Represents a TDG node
-typedef struct kmp_node_info {
-  kmp_task_t *task; // Pointer to the actual task
-  kmp_int32 *successors; // Array of the succesors ids
-  kmp_int32 nsuccessors; // Number of succesors of the node
-  std::atomic<kmp_int32>
-      npredecessors_counter; // Number of predessors on the fly
-  kmp_int32 npredecessors; // Total number of predecessors
-  kmp_int32 successors_size; // Number of allocated succesors ids
-  kmp_taskdata_t *parent_task; // Parent implicit task
-} kmp_node_info_t;
 
 /// Represent a TDG's current status
 typedef enum kmp_tdg_status {
@@ -2647,43 +2627,12 @@ typedef enum kmp_tdg_status {
   KMP_TDG_READY = 2
 } kmp_tdg_status_t;
 
-/// Structure that contains a TDG
-typedef struct kmp_tdg_info {
-  kmp_int32 tdg_id; // Unique idenfifier of the TDG
-  kmp_taskgraph_flags_t tdg_flags; // Flags related to a TDG
-  kmp_int32 map_size; // Number of allocated TDG nodes
-  kmp_int32 num_roots; // Number of roots tasks int the TDG
-  kmp_int32 *root_tasks; // Array of tasks identifiers that are roots
-  kmp_node_info_t *record_map; // Array of TDG nodes
-  kmp_tdg_status_t tdg_status =
-      KMP_TDG_NONE; // Status of the TDG (recording, ready...)
-  std::atomic<kmp_int32> num_tasks; // Number of TDG nodes
-  kmp_bootstrap_lock_t
-      graph_lock; // Protect graph attributes when updated via taskloop_recur
-  // Taskloop reduction related
-  void *rec_taskred_data; // Data to pass to __kmpc_task_reduction_init or
-                          // __kmpc_taskred_init
-  kmp_int32 rec_num_taskred;
-} kmp_tdg_info_t;
-
-extern int __kmp_tdg_dot;
-extern kmp_int32 __kmp_max_tdgs;
-extern kmp_tdg_info_t **__kmp_global_tdgs;
-extern kmp_int32 __kmp_curr_tdg_idx;
-extern kmp_int32 __kmp_successors_size;
-extern std::atomic<kmp_int32> __kmp_tdg_task_id;
-extern kmp_int32 __kmp_num_tdg;
 #endif
 
 typedef struct kmp_tasking_flags { /* Total struct must be exactly 32 bits */
 #if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
   /* Same fields as in the #else branch, but in reverse order */
-#if OMP_TASKGRAPH_EXPERIMENTAL
-  unsigned reserved31 : 4;
-  unsigned onced : 1;
-#else
   unsigned reserved31 : 5;
-#endif
   unsigned hidden_helper : 1;
   unsigned target : 1;
   unsigned native : 1;
@@ -2739,13 +2688,8 @@ typedef struct kmp_tasking_flags { /* Total struct must be exactly 32 bits */
   unsigned native : 1; /* 1==gcc-compiled task, 0==intel */
   unsigned target : 1;
   unsigned hidden_helper : 1; /* 1 == hidden helper task */
-#if OMP_TASKGRAPH_EXPERIMENTAL
-  unsigned onced : 1; /* 1==ran once already, 0==never ran, record & replay purposes */
-  unsigned reserved31 : 4; /* reserved for library use */
-#else
   unsigned reserved31 : 5; /* reserved for library use */
 #endif
-#endif
 } kmp_tasking_flags_t;
 
 typedef struct kmp_target_data {
@@ -2795,9 +2739,6 @@ struct kmp_taskdata { /* aligned during dynamic allocation       */
   ompt_task_info_t ompt_task_info;
 #endif
 #if OMP_TASKGRAPH_EXPERIMENTAL
-  bool is_taskgraph = 0; // whether the task is within a TDG
-  kmp_tdg_info_t *tdg; // used to associate task with a TDG
-  kmp_int32 td_tdg_task_id; // local task id in its TDG
 #endif
   kmp_target_data_t td_target_data;
 }; // struct kmp_taskdata
@@ -4372,20 +4313,7 @@ KMP_EXPORT void __kmpc_init_lock_with_hint(ident_t *loc, kmp_int32 gtid,
 KMP_EXPORT void __kmpc_init_nest_lock_with_hint(ident_t *loc, kmp_int32 gtid,
                                                 void **user_lock,
                                                 uintptr_t hint);
-
 #if OMP_TASKGRAPH_EXPERIMENTAL
-// Taskgraph's Record & Replay mechanism
-// __kmp_tdg_is_recording: check whether a given TDG is recording
-// status: the tdg's current status
-static inline bool __kmp_tdg_is_recording(kmp_tdg_status_t status) {
-  return status == KMP_TDG_RECORDING;
-}
-
-KMP_EXPORT kmp_int32 __kmpc_start_record_task(ident_t *loc, kmp_int32 gtid,
-                                              kmp_int32 input_flags,
-                                              kmp_int32 tdg_id);
-KMP_EXPORT void __kmpc_end_record_task(ident_t *loc, kmp_int32 gtid,
-                                       kmp_int32 input_flags, kmp_int32 tdg_id);
 #endif
 /* Interface to fast scalable reduce methods routines */
 
diff --git a/openmp/runtime/src/kmp_global.cpp b/openmp/runtime/src/kmp_global.cpp
index 15b9babfaf0ba..3df46baa57544 100644
--- a/openmp/runtime/src/kmp_global.cpp
+++ b/openmp/runtime/src/kmp_global.cpp
@@ -553,17 +553,5 @@ int __kmp_nesting_mode = 0;
 int __kmp_nesting_mode_nlevels = 1;
 int *__kmp_nesting_nth_level;
 
-#if OMP_TASKGRAPH_EXPERIMENTAL
-// TDG record & replay
-int __kmp_tdg_dot = 0;
-kmp_int32 __kmp_max_tdgs = 100;
-kmp_tdg_info_t **__kmp_global_tdgs = NULL;
-kmp_int32 __kmp_curr_tdg_idx =
-    0; // Id of the current TDG being recorded or executed
-kmp_int32 __kmp_num_tdg = 0;
-kmp_int32 __kmp_successors_size = 10; // Initial succesor size list for
-                                      // recording
-std::atomic<kmp_int32> __kmp_tdg_task_id = 0;
-#endif
 // end of file //
 
diff --git a/openmp/runtime/src/kmp_settings.cpp b/openmp/runtime/src/kmp_settings.cpp
index 66ef6f8097dce..615058af9705f 100644
--- a/openmp/runtime/src/kmp_settings.cpp
+++ b/openmp/runtime/src/kmp_settings.cpp
@@ -1266,28 +1266,6 @@ static void __kmp_stg_parse_num_threads(char const *name, char const *value,
   K_DIAG(1, ("__kmp_dflt_team_nth == %d\n", __kmp_dflt_team_nth));
 } // __kmp_stg_parse_num_threads
 
-#if OMP_TASKGRAPH_EXPERIMENTAL
-static void __kmp_stg_parse_max_tdgs(char const *name, char const *value,
-                                     void *data) {
-  __kmp_stg_parse_int(name, value, 0, INT_MAX, &__kmp_max_tdgs);
-} // __kmp_stg_parse_max_tdgs
-
-static void __kmp_std_print_max_tdgs(kmp_str_buf_t *buffer, char const *name,
-                                     void *data) {
-  __kmp_stg_print_int(buffer, name, __kmp_max_tdgs);
-} // __kmp_std_print_max_tdgs
-
-static void __kmp_stg_parse_tdg_dot(char const *name, char const *value,
-                                   void *data) {
-  __kmp_stg_parse_bool(name, value, &__kmp_tdg_dot);
-} // __kmp_stg_parse_tdg_dot
-
-static void __kmp_stg_print_tdg_dot(kmp_str_buf_t *buffer, char const *name,
-                                   void *data) {
-  __kmp_stg_print_bool(buffer, name, __kmp_tdg_dot);
-} // __kmp_stg_print_tdg_dot
-#endif
-
 static void __kmp_stg_parse_num_hidden_helper_threads(char const *name,
                                                       char const *value,
                                                       void *data) {
@@ -5742,12 +5720,6 @@ static kmp_setting_t __kmp_stg_table[] = {
     {"LIBOMP_NUM_HIDDEN_HELPER_THREADS",
      __kmp_stg_parse_num_hidden_helper_threads,
      __kmp_stg_print_num_hidden_helper_threads, NULL, 0, 0},
-#if OMP_TASKGRAPH_EXPERIMENTAL
-    {"KMP_MAX_TDGS", __kmp_stg_parse_max_tdgs, __kmp_std_print_max_tdgs, NULL,
-     0, 0},
-    {"KMP_TDG_DOT", __kmp_stg_parse_tdg_dot, __kmp_stg_print_tdg_dot, NULL, 0,
-     0},
-#endif
 
 #if OMPT_SUPPORT
     {"OMP_TOOL", __kmp_stg_parse_omp_tool, __kmp_stg_print_omp_tool, NULL, 0,
diff --git a/openmp/runtime/src/kmp_taskdeps.cpp b/openmp/runtime/src/kmp_taskdeps.cpp
index b1a0848fc722f..6884cd144f89e 100644
--- a/openmp/runtime/src/kmp_taskdeps.cpp
+++ b/openmp/runtime/src/kmp_taskdeps.cpp
@@ -222,47 +222,6 @@ static kmp_depnode_list_t *__kmp_add_node(kmp_info_t *thread,
 static inline void __kmp_track_dependence(kmp_int32 gtid, kmp_depnode_t *source,
                                           kmp_depnode_t *sink,
                                           kmp_task_t *sink_task) {
-#if OMP_TASKGRAPH_EXPERIMENTAL
-  kmp_taskdata_t *task_source = KMP_TASK_TO_TASKDATA(source->dn.task);
-  kmp_taskdata_t *task_sink = KMP_TASK_TO_TASKDATA(sink_task);
-  if (source->dn.task && sink_task) {
-    // Not supporting dependency between two tasks that one is within the TDG
-    // and the other is not
-    KMP_ASSERT(task_source->is_taskgraph == task_sink->is_taskgraph);
-  }
-  if (task_sink->is_taskgraph &&
-      __kmp_tdg_is_recording(task_sink->tdg->tdg_status)) {
-    kmp_node_info_t *source_info =
-        &task_sink->tdg->record_map[task_source->td_tdg_task_id];
-    bool exists = false;
-    for (int i = 0; i < source_info->nsuccessors; i++) {
-      if (source_info->successors[i] == task_sink->td_tdg_task_id) {
-        exists = true;
-        break;
-      }
-    }
-    if (!exists) {
-      if (source_info->nsuccessors >= source_info->successors_size) {
-        kmp_uint old_size = source_info->successors_size;
-        source_info->successors_size = 2 * source_info->successors_size;
-        kmp_int32 *old_succ_ids = source_info->successors;
-        kmp_int32 *new_succ_ids = (kmp_int32 *)__kmp_allocate(
-            source_info->successors_size * sizeof(kmp_int32));
-        KMP_MEMCPY(new_succ_ids, old_succ_ids, old_size * sizeof(kmp_int32));
-        source_info->successors = new_succ_ids;
-        __kmp_free(old_succ_ids);
-      }
-
-      source_info->successors[source_info->nsuccessors] =
-          task_sink->td_tdg_task_id;
-      source_info->nsuccessors++;
-
-      kmp_node_info_t *sink_info =
-          &(task_sink->tdg->record_map[task_sink->td_tdg_task_id]);
-      sink_info->npredecessors++;
-    }
-  }
-#endif
 #ifdef KMP_SUPPORT_GRAPH_OUTPUT
   kmp_taskdata_t *task_source = KMP_TASK_TO_TASKDATA(source->dn.task);
   // do not use sink->dn.task as that is only filled after the dependences
@@ -311,24 +270,11 @@ __kmp_depnode_link_successor(kmp_int32 gtid, kmp_info_t *thread,
   // link node as successor of list elements
   for (kmp_depnode_list_t *p = plist; p; p = p->next) {
     kmp_depnode_t *dep = p->node;
-#if OMP_TASKGRAPH_EXPERIMENTAL
-    kmp_tdg_status tdg_status = KMP_TDG_NONE;
-    if (task) {
-      kmp_taskdata_t *td = KMP_TASK_TO_TASKDATA(task);
-      if (td->is_taskgraph)
-        tdg_status = KMP_TASK_TO_TASKDATA(task)->tdg->tdg_status;
-      if (__kmp_tdg_is_recording(tdg_status))
-        __kmp_track_dependence(gtid, dep, node, task);
-    }
-#endif
     if (dep->dn.task) {
       KMP_ACQUIRE_DEPNODE(gtid, dep);
       if (dep->dn.task) {
         if (!dep->dn.successors || dep->dn.successors->node != node) {
-#if OMP_TASKGRAPH_EXPERIMENTAL
-          if (!(__kmp_tdg_is_recording(tdg_status)) && task)
-#endif
-            __kmp_track_dependence(gtid, dep, node, task);
+          __kmp_track_dependence(gtid, dep, node, task);
           dep->dn.successors = __kmp_add_node(thread, dep->dn.successors, node);
           KA_TRACE(40, ("__kmp_process_deps: T#%d adding dependence from %p to "
                         "%p\n",
@@ -352,44 +298,18 @@ static inline kmp_int32 __kmp_depnode_link_successor(kmp_int32 gtid,
   if (!sink)
     return 0;
   kmp_int32 npredecessors = 0;
-#if OMP_TASKGRAPH_EXPERIMENTAL
-  kmp_tdg_status tdg_status = KMP_TDG_NONE;
-  kmp_taskdata_t *td = KMP_TASK_TO_TASKDATA(task);
-  if (task) {
-    if (td->is_taskgraph)
-      tdg_status = KMP_TASK_TO_TASKDATA(task)->tdg->tdg_status;
-    if (__kmp_tdg_is_recording(tdg_status) && sink->dn.task)
-      __kmp_track_dependence(gtid, sink, source, task);
-  }
-#endif
   if (sink->dn.task) {
     // synchronously add source to sink' list of successors
     KMP_ACQUIRE_DEPNODE(gtid, sink);
     if (sink->dn.task) {
       if (!sink->dn.successors || sink->dn.successors->node != source) {
-#if OMP_TASKGRAPH_EXPERIMENTAL
-        if (!(__kmp_tdg_is_recording(tdg_status)) && task)
-#endif
-          __kmp_track_dependence(gtid, sink, source, task);
+        __kmp_track_dependence(gtid, sink, source, task);
         sink->dn.successors = __kmp_add_node(thread, sink->dn.successors, source);
         KA_TRACE(40, ("__kmp_process_deps: T#%d adding dependence from %p to "
                     "%p\n",
                     gtid, KMP_TASK_TO_TASKDATA(sink->dn.task),
                     KMP_TASK_TO_TASKDATA(task)));
-#if OMP_TASKGRAPH_EXPERIMENTAL
-        if (__kmp_tdg_is_recording(tdg_status)) {
-          kmp_taskdata_t *tdd = KMP_TASK_TO_TASKDATA(sink->dn.task);
-          if (tdd->is_taskgraph) {
-            if (tdd->td_flags.onced)
-              // decrement npredecessors if sink->dn.task belongs to a taskgraph
-              // and
-              //  1) the task is reset to its initial state (by kmp_free_task) or
-              //  2) the task is complete but not yet reset
-              npredecessors--;
-          }
-        }
-#endif
-      npredecessors++;
+        npredecessors++;
       }
     }
     KMP_RELEASE_DEPNODE(gtid, sink);
@@ -694,48 +614,6 @@ kmp_int32 __kmpc_omp_task_with_deps(ident_t *loc_ref, kmp_int32 gtid,
   kmp_info_t *thread = __kmp_threads[gtid];
   kmp_taskdata_t *current_task = thread->th.th_current_task;
 
-#if OMP_TASKGRAPH_EXPERIMENTAL
-  // record TDG with deps
-  if (new_taskdata->is_taskgraph &&
-      __kmp_tdg_is_recording(new_taskdata->tdg->tdg_status)) {
-    kmp_tdg_info_t *tdg = new_taskdata->tdg;
-    // extend record_map if needed
-    if (new_taskdata->td_tdg_task_id >= tdg->map_size) {
-      __kmp_acquire_bootstrap_lock(&tdg->graph_lock);
-      if (new_taskdata->td_tdg_task_id >= tdg->map_size) {
-        kmp_uint old_size = tdg->map_size;
-        kmp_uint new_size = old_size * 2;
-        kmp_node_info_t *old_record = tdg->record_map;
-        kmp_node_info_t *new_record = (kmp_node_info_t *)__kmp_allocate(
-            new_size * sizeof(kmp_node_info_t));
-        KMP_MEMCPY(new_record, tdg->record_map,
-                   old_size * sizeof(kmp_node_info_t));
-        tdg->record_map = new_record;
-
-        __kmp_free(old_record);
-
-        for (kmp_uint i = old_size; i < new_size; i++) {
-          kmp_int32 *successorsList = (kmp_int32 *)__kmp_allocate(
-              __kmp_successors_size * sizeof(kmp_int32));
-          new_record[i].task = nullptr;
-          new_record[i].successors = successorsList;
-          new_record[i].nsuccessors = 0;
-          new_record[i].npredecessors = 0;
-          new_record[i].successors_size = __kmp_successors_size;
-          KMP_ATOMIC_ST_REL(&new_record[i].npredecessors_counter, 0);
-        }
-        // update the size at the end, so that we avoid other
-        // threads use old_record while map_size is already updated
-        tdg->map_size = new_size;
-      }
-      __kmp_release_bootstrap_lock(&tdg->graph_lock);
-    }
-    tdg->record_map[new_taskdata->td_tdg_task_id].task = new_task;
-    tdg->record_map[new_taskdata->td_tdg_task_id].parent_task =
-        new_taskdata->td_parent;
-    KMP_ATOMIC_INC(&tdg->num_tasks);
-  }
-#endif
 #if OMPT_SUPPORT
   if (ompt_enabled.enabled) {
     if (!current_task->ompt_task_info.frame.enter_frame.ptr)
diff --git a/openmp/runtime/src/kmp_taskdeps.h b/openmp/runtime/src/kmp_taskdeps.h
index 0792baf67f162..71e8e69d44593 100644
--- a/openmp/runtime/src/kmp_taskdeps.h
+++ b/openmp/runtime/src/kmp_taskdeps.h
@@ -95,23 +95,6 @@ static inline void __kmp_dephash_free(kmp_info_t *thread, kmp_dephash_t *h) {
 extern void __kmpc_give_task(kmp_task_t *ptask, kmp_int32 start);
 
 static inline void __kmp_release_deps(kmp_int32 gtid, kmp_taskdata_t *task) {
-
-#if OMP_TASKGRAPH_EXPERIMENTAL
-  if (task->is_taskgraph && !(__kmp_tdg_is_recording(task->tdg->tdg_status))) {
-    kmp_node_info_t *TaskInfo = &(task->tdg->record_map[task->td_tdg_task_id]);
-
-    for (int i = 0; i < TaskInfo->nsuccessors; i++) {
-      kmp_int32 successorNumber = TaskInfo->successors[i];
-      kmp_node_info_t *successor = &(task->tdg->record_map[successorNumber]);
-      kmp_int32 npredecessors = KMP_ATOMIC_DEC(&successor->npredecessors_counter) - 1;
-      if (successor->task != nullptr && npredecessors == 0) {
-        __kmp_omp_task(gtid, successor->task, false);
-      }
-    }
-    return;
-  }
-#endif
-
   kmp_info_t *thread = __kmp_threads[gtid];
   kmp_depnode_t *node = task->td_depnode;
 
@@ -140,12 +123,8 @@ static inline void __kmp_release_deps(kmp_int32 gtid, kmp_taskdata_t *task) {
                 gtid, task));
 
   KMP_ACQUIRE_DEPNODE(gtid, node);
-#if OMP_TASKGRAPH_EXPERIMENTAL
-  if (!task->is_taskgraph ||
-      (task->is_taskgraph && !__kmp_tdg_is_recording(task->tdg->tdg_status)))
-#endif
-    node->dn.task =
-        NULL; // mark this task as finished, so no new dependencies are generated
+  node->dn.task =
+      NULL; // mark this task as finished, so no new dependencies are generated
   KMP_RELEASE_DEPNODE(gtid, node);
 
   kmp_depnode_list_t *next;
diff --git a/openmp/runtime/src/kmp_tasking.cpp b/openmp/runtime/src/kmp_tasking.cpp
index be42ed6f76abc..23092927babc1 100644
--- a/openmp/runtime/src/kmp_tasking.cpp
+++ b/openmp/runtime/src/kmp_tasking.cpp
@@ -37,10 +37,6 @@ static void __kmp_alloc_task_deque(kmp_info_t *thread,
 static int __kmp_realloc_task_threads_data(kmp_info_t *thread,
                                            kmp_task_team_t *task_team);
 static void __kmp_bottom_half_finish_proxy(kmp_int32 gtid, kmp_task_t *ptask);
-#if OMP_TASKGRAPH_EXPERIMENTAL
-static kmp_tdg_info_t *__kmp_find_tdg(kmp_int32 tdg_id);
-int __kmp_taskloop_task(int gtid, void *ptask);
-#endif
 
 // returns 1 if new task is allowed to execute, 0 otherwise
 // checks Task Scheduling constraint (if requested) and
@@ -70,11 +66,7 @@ static bool __kmp_task_is_allowed(int gtid, const kmp_int32 is_constrained,
   }
   // Check mutexinoutset dependencies, acquire locks
   kmp_depnode_t *node = tasknew->td_depnode;
-#if OMP_TASKGRAPH_EXPERIMENTAL
-  if (!tasknew->is_taskgraph && UNLIKELY(node && (node->dn.mtx_num_locks > 0))) {
-#else
   if (UNLIKELY(node && (node->dn.mtx_num_locks > 0))) {
-#endif
     for (int i = 0; i < node->dn.mtx_num_locks; ++i) {
       KMP_DEBUG_ASSERT(node->dn.mtx_locks[i] != NULL);
       if (__kmp_test_lock(node->dn.mtx_locks[i], gtid))
@@ -665,33 +657,12 @@ static void __kmp_free_task(kmp_int32 gtid, kmp_taskdata_t *taskdata,
   task->data2.priority = 0;
 
   taskdata->td_flags.freed = 1;
-#if OMP_TASKGRAPH_EXPERIMENTAL
-  // do not free tasks in taskgraph
-  if (!taskdata->is_taskgraph) {
-#endif
 // deallocate the taskdata and shared variable blocks associated with this task
 #if USE_FAST_MEMORY
   __kmp_fast_free(thread, taskdata);
 #else /* ! USE_FAST_MEMORY */
   __kmp_thread_free(thread, taskdata);
 #endif
-#if OMP_TASKGRAPH_EXPERIMENTAL
-  } else {
-    taskdata->td_flags.complete = 0;
-    taskdata->td_flags.started = 0;
-    taskdata->td_flags.freed = 0;
-    taskdata->td_flags.executing = 0;
-    taskdata->td_flags.task_serial =
-        (taskdata->td_parent->td_flags.final ||
-          taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser);
-
-    // taskdata->td_allow_completion_event.pending_events_count = 1;
-    KMP_ATOMIC_ST_RLX(&taskdata->td_untied_count, 0);
-    KMP_ATOMIC_ST_RLX(&taskdata->td_incomplete_child_tasks, 0);
-    // start at one because counts current task and children
-    KMP_ATOMIC_ST_RLX(&taskdata->td_allocated_child_tasks, 1);
-  }
-#endif
 
   KA_TRACE(20, ("__kmp_free_task: T#%d freed task %p\n", gtid, taskdata));
 }
@@ -779,10 +750,6 @@ static bool __kmp_track_children_task(kmp_taskdata_t *taskdata) {
         flags.detachable == TASK_DETACHABLE || flags.hidden_helper;
   ret = ret ||
         KMP_ATOMIC_LD_ACQ(&taskdata->td_parent->td_incomplete_child_tasks) > 0;
-#if OMP_TASKGRAPH_EXPERIMENTAL
-  if (taskdata->td_taskgroup && taskdata->is_taskgraph)
-    ret = ret || KMP_ATOMIC_LD_ACQ(&taskdata->td_taskgroup->count) > 0;
-#endif
   return ret;
 }
 
@@ -802,10 +769,6 @@ static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task,
   kmp_info_t *thread = __kmp_threads[gtid];
   kmp_task_team_t *task_team =
       thread->th.th_task_team; // might be NULL for serial teams...
-#if OMP_TASKGRAPH_EXPERIMENTAL
-  // to avoid seg fault when we need to access taskdata->td_flags after free when using vanilla taskloop
-  bool is_taskgraph;
-#endif
 #if KMP_DEBUG
   kmp_int32 children = 0;
 #endif
@@ -815,10 +778,6 @@ static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task,
 
   KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT);
 
-#if OMP_TASKGRAPH_EXPERIMENTAL
-  is_taskgraph = taskdata->is_taskgraph;
-#endif
-
   if (UNLIKELY(taskdata->td_flags.tiedness == TASK_UNTIED)) {
     // untied task needs to check the counter so that the task structure is not
     // freed prematurely
@@ -923,9 +882,6 @@ static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task,
 
   if (completed) {
     taskdata->td_flags.complete = 1; // mark the task as completed
-#if OMP_TASKGRAPH_EXPERIMENTAL
-    taskdata->td_flags.onced = 1; // mark the task as ran once already
-#endif
 
 #if OMPT_SUPPORT
     // This is not a detached task, we are done here
@@ -942,11 +898,7 @@ static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task,
 #endif
           KMP_ATOMIC_DEC(&taskdata->td_parent->td_incomplete_child_tasks);
       KMP_DEBUG_ASSERT(children >= 0);
-#if OMP_TASKGRAPH_EXPERIMENTAL
-      if (taskdata->td_taskgroup && !taskdata->is_taskgraph)
-#else
       if (taskdata->td_taskgroup)
-#endif
         KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count);
     } else if (task_team && (task_team->tt.tt_found_proxy_tasks ||
                              task_team->tt.tt_hidden_helper_task_encountered)) {
@@ -985,19 +937,6 @@ static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task,
   // KMP_DEBUG_ASSERT( resumed_task->td_flags.executing == 0 );
   resumed_task->td_flags.executing = 1; // resume previous task
 
-#if OMP_TASKGRAPH_EXPERIMENTAL
-  if (is_taskgraph && __kmp_track_children_task(taskdata) &&
-      taskdata->td_taskgroup) {
-    // TDG: we only release taskgroup barrier here because
-    // free_task_and_ancestors will call
-    // __kmp_free_task, which resets all task parameters such as
-    // taskdata->started, etc. If we release the barrier earlier, these
-    // parameters could be read before being reset. This is not an issue for
-    // non-TDG implementation because we never reuse a task(data) structure
-    KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count);
-  }
-#endif
-
   KA_TRACE(
       10, ("__kmp_task_finish(exit): T#%d finished task %p, resuming task %p\n",
            gtid, taskdata, resumed_task));
@@ -1113,9 +1052,6 @@ void __kmp_init_implicit_task(ident_t *loc_ref, kmp_info_t *this_thr,
   task->td_flags.executing = 1;
   task->td_flags.complete = 0;
   task->td_flags.freed = 0;
-#if OMP_TASKGRAPH_EXPERIMENTAL
-  task->td_flags.onced = 0;
-#endif
 
   task->td_depnode = NULL;
   task->td_last_tied = task;
@@ -1159,9 +1095,6 @@ void __kmp_finish_implicit_task(kmp_info_t *thread) {
   if (task->td_dephash) {
     int children;
     task->td_flags.complete = 1;
-#if OMP_TASKGRAPH_EXPERIMENTAL
-    task->td_flags.onced = 1;
-#endif
     children = KMP_ATOMIC_LD_ACQ(&task->td_incomplete_child_tasks);
     kmp_tasking_flags_t flags_old = task->td_flags;
     if (children == 0 && flags_old.complete == 1) {
@@ -1390,11 +1323,6 @@ kmp_task_t *__kmp_task_alloc(ident_t *loc_ref, kmp_int32 gtid,
   taskdata->td_flags.executing = 0;
   taskdata->td_flags.complete = 0;
   taskdata->td_flags.freed = 0;
-#if OMP_TASKGRAPH_EXPERIMENTAL
-  taskdata->td_flags.onced = 0;
-  taskdata->is_taskgraph = 0;
-  taskdata->tdg = nullptr;
-#endif
   KMP_ATOMIC_ST_RLX(&taskdata->td_incomplete_child_tasks, 0);
   // start at one because counts current task and children
   KMP_ATOMIC_ST_RLX(&taskdata->td_allocated_child_tasks, 1);
@@ -1430,16 +1358,6 @@ kmp_task_t *__kmp_task_alloc(ident_t *loc_ref, kmp_int32 gtid,
     }
   }
 
-#if OMP_TASKGRAPH_EXPERIMENTAL
-  kmp_tdg_info_t *tdg = __kmp_find_tdg(__kmp_curr_tdg_idx);
-  if (tdg && __kmp_tdg_is_recording(tdg->tdg_status) &&
-      (task_entry != (kmp_routine_entry_t)__kmp_taskloop_task)) {
-    taskdata->is_taskgraph = 1;
-    taskdata->tdg = __kmp_global_tdgs[__kmp_curr_tdg_idx];
-    taskdata->td_task_id = KMP_GEN_TASK_ID();
-    taskdata->td_tdg_task_id = KMP_ATOMIC_INC(&__kmp_tdg_task_id);
-  }
-#endif
   KA_TRACE(20, ("__kmp_task_alloc(exit): T#%d created task %p parent=%p\n",
                 gtid, taskdata, taskdata->td_parent));
 
@@ -1807,53 +1725,6 @@ kmp_int32 __kmp_omp_task(kmp_int32 gtid, kmp_task_t *new_task,
                          bool serialize_immediate) {
   kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task);
 
-#if OMP_TASKGRAPH_EXPERIMENTAL
-  if (new_taskdata->is_taskgraph &&
-      __kmp_tdg_is_recording(new_taskdata->tdg->tdg_status)) {
-    kmp_tdg_info_t *tdg = new_taskdata->tdg;
-    // extend the record_map if needed
-    if (new_taskdata->td_tdg_task_id >= new_taskdata->tdg->map_size) {
-      __kmp_acquire_bootstrap_lock(&tdg->graph_lock);
-      // map_size could have been updated by another thread if recursive
-      // taskloop
-      if (new_taskdata->td_tdg_task_id >= tdg->map_size) {
-        kmp_uint old_size = tdg->map_size;
-        kmp_uint new_size = old_size * 2;
-        kmp_node_info_t *old_record = tdg->record_map;
-        kmp_node_info_t *new_record = (kmp_node_info_t *)__kmp_allocate(
-            new_size * sizeof(kmp_node_info_t));
-
-        KMP_MEMCPY(new_record, old_record, old_size * sizeof(kmp_node_info_t));
-        tdg->record_map = new_record;
-
-        __kmp_free(old_record);
-
-        for (kmp_uint i = old_size; i < new_size; i++) {
-          kmp_int32 *successorsList = (kmp_int32 *)__kmp_allocate(
-              __kmp_successors_size * sizeof(kmp_int32));
-          new_record[i].task = nullptr;
-          new_record[i].successors = successorsList;
-          new_record[i].nsuccessors = 0;
-          new_record[i].npredecessors = 0;
-          new_record[i].successors_size = __kmp_successors_size;
-          KMP_ATOMIC_ST_REL(&new_record[i].npredecessors_counter, 0);
-        }
-        // update the size at the end, so that we avoid other
-        // threads use old_record while map_size is already updated
-        tdg->map_size = new_size;
-      }
-      __kmp_release_bootstrap_lock(&tdg->graph_lock);
-    }
-    // record a task
-    if (tdg->record_map[new_taskdata->td_tdg_task_id].task == nullptr) {
-      tdg->record_map[new_taskdata->td_tdg_task_id].task = new_task;
-      tdg->record_map[new_taskdata->td_tdg_task_id].parent_task =
-          new_taskdata->td_parent;
-      KMP_ATOMIC_INC(&tdg->num_tasks);
-    }
-  }
-#endif
-
   /* Should we execute the new task or queue it? For now, let's just always try
      to queue it.  If the queue fills up, then we'll execute it.  */
   if (new_taskdata->td_flags.proxy == TASK_PROXY ||
@@ -2376,17 +2247,6 @@ the reduction either does not use omp_orig object, or the omp_orig is accessible
 without help of the runtime library.
 */
 void *__kmpc_task_reduction_init(int gtid, int num, void *data) {
-#if OMP_TASKGRAPH_EXPERIMENTAL
-  kmp_tdg_info_t *tdg = __kmp_find_tdg(__kmp_curr_tdg_idx);
-  if (tdg && __kmp_tdg_is_recording(tdg->tdg_status)) {
-    kmp_tdg_info_t *this_tdg = __kmp_global_tdgs[__kmp_curr_tdg_idx];
-    this_tdg->rec_taskred_data =
-        __kmp_allocate(sizeof(kmp_task_red_input_t) * num);
-    this_tdg->rec_num_taskred = num;
-    KMP_MEMCPY(this_tdg->rec_taskred_data, data,
-               sizeof(kmp_task_red_input_t) * num);
-  }
-#endif
   return __kmp_task_reduction_init(gtid, num, (kmp_task_red_input_t *)data);
 }
 
@@ -2403,17 +2263,6 @@ Note: this entry supposes the optional compiler-generated initializer routine
 has two parameters, pointer to object to be initialized and pointer to omp_orig
 */
 void *__kmpc_taskred_init(int gtid, int num, void *data) {
-#if OMP_TASKGRAPH_EXPERIMENTAL
-  kmp_tdg_info_t *tdg = __kmp_find_tdg(__kmp_curr_tdg_idx);
-  if (tdg && __kmp_tdg_is_recording(tdg->tdg_status)) {
-    kmp_tdg_info_t *this_tdg = __kmp_global_tdgs[__kmp_curr_tdg_idx];
-    this_tdg->rec_taskred_data =
-        __kmp_allocate(sizeof(kmp_task_red_input_t) * num);
-    this_tdg->rec_num_taskred = num;
-    KMP_MEMCPY(this_tdg->rec_taskred_data, data,
-               sizeof(kmp_task_red_input_t) * num);
-  }
-#endif
   return __kmp_task_reduction_init(gtid, num, (kmp_taskred_input_t *)data);
 }
 
@@ -2460,18 +2309,6 @@ void *__kmpc_task_reduction_get_th_data(int gtid, void *tskgrp, void *data) {
   kmp_int32 num;
   kmp_int32 tid = thread->th.th_info.ds.ds_tid;
 
-#if OMP_TASKGRAPH_EXPERIMENTAL
-  if ((thread->th.th_current_task->is_taskgraph) &&
-      (!__kmp_tdg_is_recording(
-          __kmp_global_tdgs[__kmp_curr_tdg_idx]->tdg_status))) {
-    tg = thread->th.th_current_task->td_taskgroup;
-    KMP_ASSERT(tg != NULL);
-    KMP_ASSERT(tg->reduce_data != NULL);
-    arr = (kmp_taskred_data_t *)(tg->reduce_data);
-    num = tg->reduce_num_data;
-  }
-#endif
-
   KMP_ASSERT(data != NULL);
   while (tg != NULL) {
     arr = (kmp_taskred_data_t *)(tg->reduce_data);
@@ -4238,9 +4075,6 @@ static void __kmp_first_top_half_finish_proxy(kmp_taskdata_t *taskdata) {
   KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0);
 
   taskdata->td_flags.complete = 1; // mark the task as completed
-#if OMP_TASKGRAPH_EXPERIMENTAL
-  taskdata->td_flags.onced = 1;
-#endif
 
   if (taskdata->td_taskgroup)
     KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count);
@@ -4442,12 +4276,7 @@ void __kmp_fulfill_event(kmp_event_t *event) {
 // taskloop_recur: used only when dealing with taskgraph,
 //      indicating whether we need to update task->td_task_id
 // returns:  a pointer to the allocated kmp_task_t structure (task).
-kmp_task_t *__kmp_task_dup_alloc(kmp_info_t *thread, kmp_task_t *task_src
-#if OMP_TASKGRAPH_EXPERIMENTAL
-                                 ,
-                                 int taskloop_recur
-#endif
-) {
+kmp_task_t *__kmp_task_dup_alloc(kmp_info_t *thread, kmp_task_t *task_src) {
   kmp_task_t *task;
   kmp_taskdata_t *taskdata;
   kmp_taskdata_t *taskdata_src = KMP_TASK_TO_TASKDATA(task_src);
@@ -4475,11 +4304,6 @@ kmp_task_t *__kmp_task_dup_alloc(kmp_info_t *thread, kmp_task_t *task_src
   task = KMP_TASKDATA_TO_TASK(taskdata);
 
   // Initialize new task (only specific fields not affected by memcpy)
-#if OMP_TASKGRAPH_EXPERIMENTAL
-  if (taskdata->is_taskgraph && !taskloop_recur &&
-      __kmp_tdg_is_recording(taskdata_src->tdg->tdg_status))
-    taskdata->td_tdg_task_id = KMP_ATOMIC_INC(&__kmp_tdg_task_id);
-#endif
   taskdata->td_task_id = KMP_GEN_TASK_ID();
   if (task->shareds != NULL) { // need setup shareds pointer
     shareds_offset = (char *)task_src->shareds - (char *)taskdata_src;
@@ -4708,11 +4532,7 @@ void __kmp_taskloop_linear(ident_t *loc, int gtid, kmp_task_t *task,
       }
     }
 
-#if OMP_TASKGRAPH_EXPERIMENTAL
-    next_task = __kmp_task_dup_alloc(thread, task, /* taskloop_recur */ 0);
-#else
     next_task = __kmp_task_dup_alloc(thread, task); // allocate new task
-#endif
 
     kmp_taskdata_t *next_taskdata = KMP_TASK_TO_TASKDATA(next_task);
     kmp_taskloop_bounds_t next_task_bounds =
@@ -4910,12 +4730,7 @@ void __kmp_taskloop_recur(ident_t *loc, int gtid, kmp_task_t *task,
   lb1 = ub0 + st;
 
   // create pattern task for 2nd half of the loop
-#if OMP_TASKGRAPH_EXPERIMENTAL
-  next_task = __kmp_task_dup_alloc(thread, task,
-                                   /* taskloop_recur */ 1);
-#else
   next_task = __kmp_task_dup_alloc(thread, task); // duplicate the task
-#endif
   // adjust lower bound (upper bound is not changed) for the 2nd half
   *(kmp_uint64 *)((char *)next_task + lower_offset) = lb1;
   if (ptask_dup != NULL) // construct firstprivates, etc.
@@ -4948,12 +4763,6 @@ void __kmp_taskloop_recur(ident_t *loc, int gtid, kmp_task_t *task,
   p->codeptr_ra = codeptr_ra;
 #endif
 
-#if OMP_TASKGRAPH_EXPERIMENTAL
-  kmp_taskdata_t *new_task_data = KMP_TASK_TO_TASKDATA(new_task);
-  new_task_data->tdg = taskdata->tdg;
-  new_task_data->is_taskgraph = 0;
-#endif
-
 #if OMPT_SUPPORT
   // schedule new task with correct return address for OMPT events
   __kmp_omp_taskloop_task(NULL, gtid, new_task, codeptr_ra);
@@ -4992,10 +4801,6 @@ static void __kmp_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
 #endif
     __kmpc_taskgroup(loc, gtid);
   }
-
-#if OMP_TASKGRAPH_EXPERIMENTAL
-  KMP_ATOMIC_DEC(&__kmp_tdg_task_id);
-#endif
   // =========================================================================
   // calculate loop parameters
   kmp_taskloop_bounds_t task_bounds(task, lb, ub);
@@ -5243,257 +5048,3 @@ bool __kmpc_omp_has_task_team(kmp_int32 gtid) {
 
   return taskdata->td_task_team != NULL;
 }
-
-#if OMP_TASKGRAPH_EXPERIMENTAL
-// __kmp_find_tdg: identify a TDG through its ID
-// tdg_id: ID of the TDG
-// returns: If a TDG corresponding to this ID is found and not
-// its initial state, return the pointer to it, otherwise nullptr
-static kmp_tdg_info_t *__kmp_find_tdg(kmp_int32 tdg_id) {
-  kmp_tdg_info_t *res = nullptr;
-  if (__kmp_max_tdgs == 0)
-    return res;
-
-  if (__kmp_global_tdgs == NULL)
-    __kmp_global_tdgs = (kmp_tdg_info_t **)__kmp_allocate(
-        sizeof(kmp_tdg_info_t *) * __kmp_max_tdgs);
-
-  if ((__kmp_global_tdgs[tdg_id]) &&
-      (__kmp_global_tdgs[tdg_id]->tdg_status != KMP_TDG_NONE))
-    res = __kmp_global_tdgs[tdg_id];
-  return res;
-}
-
-// __kmp_print_tdg_dot: prints the TDG to a dot file
-// tdg:    ID of the TDG
-// gtid:   Global Thread ID
-void __kmp_print_tdg_dot(kmp_tdg_info_t *tdg, kmp_int32 gtid) {
-  kmp_int32 tdg_id = tdg->tdg_id;
-  KA_TRACE(10, ("__kmp_print_tdg_dot(enter): T#%d tdg_id=%d \n", gtid, tdg_id));
-
-  char file_name[20];
-  sprintf(file_name, "tdg_%d.dot", tdg_id);
-  kmp_safe_raii_file_t tdg_file(file_name, "w");
-
-  kmp_int32 num_tasks = KMP_ATOMIC_LD_RLX(&tdg->num_tasks);
-  fprintf(tdg_file,
-          "digraph TDG {\n"
-          "   compound=true\n"
-          "   subgraph cluster {\n"
-          "      label=TDG_%d\n",
-          tdg_id);
-  for (kmp_int32 i = 0; i < num_tasks; i++) {
-    fprintf(tdg_file, "      %d[style=bold]\n", i);
-  }
-  fprintf(tdg_file, "   }\n");
-  for (kmp_int32 i = 0; i < num_tasks; i++) {
-    kmp_int32 nsuccessors = tdg->record_map[i].nsuccessors;
-    kmp_int32 *successors = tdg->record_map[i].successors;
-    if (nsuccessors > 0) {
-      for (kmp_int32 j = 0; j < nsuccessors; j++)
-        fprintf(tdg_file, "   %d -> %d \n", i, successors[j]);
-    }
-  }
-  fprintf(tdg_file, "}");
-  KA_TRACE(10, ("__kmp_print_tdg_dot(exit): T#%d tdg_id=%d \n", gtid, tdg_id));
-}
-
-// __kmp_exec_tdg: launch the execution of a previous
-// recorded TDG
-// gtid:   Global Thread ID
-// tdg:    ID of the TDG
-void __kmp_exec_tdg(kmp_int32 gtid, kmp_tdg_info_t *tdg) {
-  KMP_DEBUG_ASSERT(tdg->tdg_status == KMP_TDG_READY);
-  KA_TRACE(10, ("__kmp_exec_tdg(enter): T#%d tdg_id=%d num_roots=%d\n", gtid,
-                tdg->tdg_id, tdg->num_roots));
-  kmp_node_info_t *this_record_map = tdg->record_map;
-  kmp_int32 *this_root_tasks = tdg->root_tasks;
-  kmp_int32 this_num_roots = tdg->num_roots;
-  kmp_int32 this_num_tasks = KMP_ATOMIC_LD_RLX(&tdg->num_tasks);
-
-  kmp_info_t *thread = __kmp_threads[gtid];
-  kmp_taskdata_t *parent_task = thread->th.th_current_task;
-
-  if (tdg->rec_taskred_data) {
-    __kmpc_taskred_init(gtid, tdg->rec_num_taskred, tdg->rec_taskred_data);
-  }
-
-  for (kmp_int32 j = 0; j < this_num_tasks; j++) {
-    kmp_taskdata_t *td = KMP_TASK_TO_TASKDATA(this_record_map[j].task);
-
-    td->td_parent = parent_task;
-    this_record_map[j].parent_task = parent_task;
-
-    kmp_taskgroup_t *parent_taskgroup =
-        this_record_map[j].parent_task->td_taskgroup;
-
-    KMP_ATOMIC_ST_RLX(&this_record_map[j].npredecessors_counter,
-                      this_record_map[j].npredecessors);
-    KMP_ATOMIC_INC(&this_record_map[j].parent_task->td_incomplete_child_tasks);
-
-    if (parent_taskgroup) {
-      KMP_ATOMIC_INC(&parent_taskgroup->count);
-      // The taskgroup is different so we must update it
-      td->td_taskgroup = parent_taskgroup;
-    } else if (td->td_taskgroup != nullptr) {
-      // If the parent doesnt have a taskgroup, remove it from the task
-      td->td_taskgroup = nullptr;
-    }
-    if (this_record_map[j].parent_task->td_flags.tasktype == TASK_EXPLICIT)
-      KMP_ATOMIC_INC(&this_record_map[j].parent_task->td_allocated_child_tasks);
-  }
-
-  for (kmp_int32 j = 0; j < this_num_roots; ++j) {
-    __kmp_omp_task(gtid, this_record_map[this_root_tasks[j]].task, true);
-  }
-  KA_TRACE(10, ("__kmp_exec_tdg(exit): T#%d tdg_id=%d num_roots=%d\n", gtid,
-                tdg->tdg_id, tdg->num_roots));
-}
-
-// __kmp_start_record: set up a TDG structure and turn the
-// recording flag to true
-// gtid:        Global Thread ID of the encountering thread
-// input_flags: Flags associated with the TDG
-// tdg_id:      ID of the TDG to record
-static inline void __kmp_start_record(kmp_int32 gtid,
-                                      kmp_taskgraph_flags_t *flags,
-                                      kmp_int32 tdg_id) {
-  kmp_tdg_info_t *tdg =
-      (kmp_tdg_info_t *)__kmp_allocate(sizeof(kmp_tdg_info_t));
-  __kmp_global_tdgs[__kmp_curr_tdg_idx] = tdg;
-  // Initializing the TDG structure
-  tdg->tdg_id = tdg_id;
-  tdg->map_size = INIT_MAPSIZE;
-  tdg->num_roots = -1;
-  tdg->root_tasks = nullptr;
-  tdg->tdg_status = KMP_TDG_RECORDING;
-  tdg->rec_num_taskred = 0;
-  tdg->rec_taskred_data = nullptr;
-  KMP_ATOMIC_ST_RLX(&tdg->num_tasks, 0);
-
-  // Initializing the list of nodes in this TDG
-  kmp_node_info_t *this_record_map =
-      (kmp_node_info_t *)__kmp_allocate(INIT_MAPSIZE * sizeof(kmp_node_info_t));
-  for (kmp_int32 i = 0; i < INIT_MAPSIZE; i++) {
-    kmp_int32 *successorsList =
-        (kmp_int32 *)__kmp_allocate(__kmp_successors_size * sizeof(kmp_int32));
-    this_record_map[i].task = nullptr;
-    this_record_map[i].successors = successorsList;
-    this_record_map[i].nsuccessors = 0;
-    this_record_map[i].npredecessors = 0;
-    this_record_map[i].successors_size = __kmp_successors_size;
-    KMP_ATOMIC_ST_RLX(&this_record_map[i].npredecessors_counter, 0);
-  }
-
-  __kmp_global_tdgs[__kmp_curr_tdg_idx]->record_map = this_record_map;
-}
-
-// __kmpc_start_record_task: Wrapper around __kmp_start_record to mark
-// the beginning of the record process of a task region
-// loc_ref:     Location of TDG, not used yet
-// gtid:        Global Thread ID of the encountering thread
-// input_flags: Flags associated with the TDG
-// tdg_id:      ID of the TDG to record, for now, incremental integer
-// returns:     1 if we record, otherwise, 0
-kmp_int32 __kmpc_start_record_task(ident_t *loc_ref, kmp_int32 gtid,
-                                   kmp_int32 input_flags, kmp_int32 tdg_id) {
-
-  kmp_int32 res;
-  kmp_taskgraph_flags_t *flags = (kmp_taskgraph_flags_t *)&input_flags;
-  KA_TRACE(10,
-           ("__kmpc_start_record_task(enter): T#%d loc=%p flags=%d tdg_id=%d\n",
-            gtid, loc_ref, input_flags, tdg_id));
-
-  if (__kmp_max_tdgs == 0) {
-    KA_TRACE(
-        10,
-        ("__kmpc_start_record_task(abandon): T#%d loc=%p flags=%d tdg_id = %d, "
-         "__kmp_max_tdgs = 0\n",
-         gtid, loc_ref, input_flags, tdg_id));
-    return 1;
-  }
-
-  __kmpc_taskgroup(loc_ref, gtid);
-  if (kmp_tdg_info_t *tdg = __kmp_find_tdg(tdg_id)) {
-    // TODO: use re_record flag
-    __kmp_exec_tdg(gtid, tdg);
-    res = 0;
-  } else {
-    __kmp_curr_tdg_idx = tdg_id;
-    KMP_DEBUG_ASSERT(__kmp_curr_tdg_idx < __kmp_max_tdgs);
-    __kmp_start_record(gtid, flags, tdg_id);
-    __kmp_num_tdg++;
-    res = 1;
-  }
-  KA_TRACE(10, ("__kmpc_start_record_task(exit): T#%d TDG %d starts to %s\n",
-                gtid, tdg_id, res ? "record" : "execute"));
-  return res;
-}
-
-// __kmp_end_record: set up a TDG after recording it
-// gtid:   Global thread ID
-// tdg:    Pointer to the TDG
-void __kmp_end_record(kmp_int32 gtid, kmp_tdg_info_t *tdg) {
-  // Store roots
-  kmp_node_info_t *this_record_map = tdg->record_map;
-  kmp_int32 this_num_tasks = KMP_ATOMIC_LD_RLX(&tdg->num_tasks);
-  kmp_int32 *this_root_tasks =
-      (kmp_int32 *)__kmp_allocate(this_num_tasks * sizeof(kmp_int32));
-  kmp_int32 this_map_size = tdg->map_size;
-  kmp_int32 this_num_roots = 0;
-  kmp_info_t *thread = __kmp_threads[gtid];
-
-  for (kmp_int32 i = 0; i < this_num_tasks; i++) {
-    if (this_record_map[i].npredecessors == 0) {
-      this_root_tasks[this_num_roots++] = i;
-    }
-  }
-
-  // Update with roots info and mapsize
-  tdg->map_size = this_map_size;
-  tdg->num_roots = this_num_roots;
-  tdg->root_tasks = this_root_tasks;
-  KMP_DEBUG_ASSERT(tdg->tdg_status == KMP_TDG_RECORDING);
-  tdg->tdg_status = KMP_TDG_READY;
-
-  if (thread->th.th_current_task->td_dephash) {
-    __kmp_dephash_free(thread, thread->th.th_current_task->td_dephash);
-    thread->th.th_current_task->td_dephash = NULL;
-  }
-
-  // Reset predecessor counter
-  for (kmp_int32 i = 0; i < this_num_tasks; i++) {
-    KMP_ATOMIC_ST_RLX(&this_record_map[i].npredecessors_counter,
-                      this_record_map[i].npredecessors);
-  }
-  KMP_ATOMIC_ST_RLX(&__kmp_tdg_task_id, 0);
-
-  if (__kmp_tdg_dot)
-    __kmp_print_tdg_dot(tdg, gtid);
-}
-
-// __kmpc_end_record_task: wrapper around __kmp_end_record to mark
-// the end of recording phase
-//
-// loc_ref:      Source location information
-// gtid:         Global thread ID
-// input_flags:  Flags attached to the graph
-// tdg_id:       ID of the TDG just finished recording
-void __kmpc_end_record_task(ident_t *loc_ref, kmp_int32 gtid,
-                            kmp_int32 input_flags, kmp_int32 tdg_id) {
-  kmp_tdg_info_t *tdg = __kmp_find_tdg(tdg_id);
-
-  KA_TRACE(10, ("__kmpc_end_record_task(enter): T#%d loc=%p finishes recording"
-                " tdg=%d with flags=%d\n",
-                gtid, loc_ref, tdg_id, input_flags));
-  if (__kmp_max_tdgs) {
-    // TODO: use input_flags->nowait
-    __kmpc_end_taskgroup(loc_ref, gtid);
-    if (__kmp_tdg_is_recording(tdg->tdg_status))
-      __kmp_end_record(gtid, tdg);
-  }
-  KA_TRACE(10, ("__kmpc_end_record_task(exit): T#%d loc=%p finished recording"
-                " tdg=%d, its status is now READY\n",
-                gtid, loc_ref, tdg_id));
-}
-#endif
diff --git a/openmp/runtime/test/tasking/omp_record_replay.cpp b/openmp/runtime/test/tasking/omp_record_replay.cpp
deleted file mode 100644
index 4fea22e081da9..0000000000000
--- a/openmp/runtime/test/tasking/omp_record_replay.cpp
+++ /dev/null
@@ -1,48 +0,0 @@
-// REQUIRES: omp_taskgraph_experimental
-// RUN: %libomp-cxx-compile-and-run
-#include <iostream>
-#include <cassert>
-#define NT 100
-
-// Compiler-generated code (emulation)
-typedef struct ident {
-    void* dummy;
-} ident_t;
-
-
-#ifdef __cplusplus
-extern "C" {
-  int __kmpc_global_thread_num(ident_t *);
-  int __kmpc_start_record_task(ident_t *, int, int, int);
-  void __kmpc_end_record_task(ident_t *, int, int , int);
-}
-#endif
-
-void func(int *num_exec) {
-  (*num_exec)++;
-}
-
-int main() {
-  int num_exec = 0;
-  int num_tasks = 0;
-  int x=0;
-  #pragma omp parallel
-  #pragma omp single
-  for (int iter = 0; iter < NT; ++iter) {
-    int gtid = __kmpc_global_thread_num(nullptr);
-    int res =  __kmpc_start_record_task(nullptr, gtid, /* kmp_tdg_flags */ 0, /* tdg_id */0);
-    if (res) {
-      num_tasks++;
-      #pragma omp task 
-      func(&num_exec);
-    }
-    __kmpc_end_record_task(nullptr, gtid, /* kmp_tdg_flags */0, /* tdg_id */0);
-  }
-
-  assert(num_tasks==1);
-  assert(num_exec==NT);
-
-  std::cout << "Passed" << std::endl;
-  return 0;
-}
-// CHECK: Passed
diff --git a/openmp/runtime/test/tasking/omp_record_replay_deps.cpp b/openmp/runtime/test/tasking/omp_record_replay_deps.cpp
deleted file mode 100644
index 4c06ae3f7b273..0000000000000
--- a/openmp/runtime/test/tasking/omp_record_replay_deps.cpp
+++ /dev/null
@@ -1,63 +0,0 @@
-// REQUIRES: omp_taskgraph_experimental
-// RUN: %libomp-cxx-compile-and-run
-#include <iostream>
-#include <cassert>
-#define NT 100
-#define MULTIPLIER 100
-#define DECREMENT 5
-
-int val;
-// Compiler-generated code (emulation)
-typedef struct ident {
-    void* dummy;
-} ident_t;
-
-
-#ifdef __cplusplus
-extern "C" {
-  int __kmpc_global_thread_num(ident_t *);
-  int __kmpc_start_record_task(ident_t *, int, int, int);
-  void __kmpc_end_record_task(ident_t *, int, int, int);
-}
-#endif
-
-void sub() {
-  #pragma omp atomic
-  val -= DECREMENT;
-}
-
-void add() {
-  #pragma omp atomic
-  val += DECREMENT;
-}
-
-void mult() {
-  // no atomicity needed, can only be executed by 1 thread
-  // and no concurrency with other tasks possible
-  val *= MULTIPLIER;
-}
-
-int main() {
-  val = 0;
-  int *x, *y;
-  #pragma omp parallel
-  #pragma omp single
-  for (int iter = 0; iter < NT; ++iter) {
-    int gtid = __kmpc_global_thread_num(nullptr);
-    int res =  __kmpc_start_record_task(nullptr, gtid, /* kmp_tdg_flags */0, /* tdg_id */0);
-    if (res) {
-      #pragma omp task depend(out:y)
-      add();
-      #pragma omp task depend(out:x)
-      sub();
-      #pragma omp task depend(in:x,y)
-      mult();
-    }
-    __kmpc_end_record_task(nullptr, gtid, /* kmp_tdg_flags */0, /* tdg_id */0);
-  }
-  assert(val==0);
-
-  std::cout << "Passed" << std::endl;
-  return 0;
-}
-// CHECK: Passed
diff --git a/openmp/runtime/test/tasking/omp_record_replay_deps_multi_succ.cpp b/openmp/runtime/test/tasking/omp_record_replay_deps_multi_succ.cpp
deleted file mode 100644
index 6bcd3dee56030..0000000000000
--- a/openmp/runtime/test/tasking/omp_record_replay_deps_multi_succ.cpp
+++ /dev/null
@@ -1,56 +0,0 @@
-// REQUIRES: omp_taskgraph_experimental
-// RUN: %libomp-cxx-compile-and-run
-#include <omp.h>
-#include <cassert>
-#include <vector>
-
-constexpr const int TASKS_SIZE = 12;
-
-typedef struct ident ident_t;
-
-extern "C" {
-int __kmpc_global_thread_num(ident_t *);
-int __kmpc_start_record_task(ident_t *, int, int, int);
-void __kmpc_end_record_task(ident_t *, int, int, int);
-}
-
-void init(int &A, int val) { A = val; }
-
-void update(int &A, int &B, int val) { A = B + val; }
-
-void test(int nb, std::vector<std::vector<int>> &Ah) {
-#pragma omp parallel
-#pragma omp single
-  {
-    int gtid = __kmpc_global_thread_num(nullptr);
-    int res = __kmpc_start_record_task(nullptr, gtid, 0, 0);
-    if (res) {
-      for (int k = 0; k < nb; ++k) {
-#pragma omp task depend(inout : Ah[k][0])
-        init(Ah[k][0], k);
-
-        for (int i = 1; i < nb; ++i) {
-#pragma omp task depend(in : Ah[k][0]) depend(out : Ah[k][i])
-          update(Ah[k][i], Ah[k][0], 1);
-        }
-      }
-    }
-    __kmpc_end_record_task(nullptr, gtid, 0, 0);
-  }
-}
-
-int main() {
-  std::vector<std::vector<int>> matrix(TASKS_SIZE,
-                                       std::vector<int>(TASKS_SIZE, 0));
-
-  test(TASKS_SIZE, matrix);
-  test(TASKS_SIZE, matrix);
-
-  for (int k = 0; k < TASKS_SIZE; ++k) {
-    assert(matrix[k][0] == k);
-    for (int i = 1; i < TASKS_SIZE; ++i) {
-      assert(matrix[k][i] == k + 1);
-    }
-  }
-  return 0;
-}
diff --git a/openmp/runtime/test/tasking/omp_record_replay_multiTDGs.cpp b/openmp/runtime/test/tasking/omp_record_replay_multiTDGs.cpp
deleted file mode 100644
index 1864d5d89cc70..0000000000000
--- a/openmp/runtime/test/tasking/omp_record_replay_multiTDGs.cpp
+++ /dev/null
@@ -1,76 +0,0 @@
-// REQUIRES: omp_taskgraph_experimental
-// RUN: %libomp-cxx-compile-and-run
-#include <iostream>
-#include <cassert>
-#define NT 20
-#define MULTIPLIER 100
-#define DECREMENT 5
-
-// Compiler-generated code (emulation)
-typedef struct ident {
-    void* dummy;
-} ident_t;
-
-int val;
-#ifdef __cplusplus
-extern "C" {
-  int __kmpc_global_thread_num(ident_t *);
-  int __kmpc_start_record_task(ident_t *, int, int, int);
-  void __kmpc_end_record_task(ident_t *, int, int , int);
-}
-#endif
-
-void sub() {
-  #pragma omp atomic
-  val -= DECREMENT;
-}
-
-void add() {
-  #pragma omp atomic
-  val += DECREMENT;
-}
-
-void mult() {
-  // no atomicity needed, can only be executed by 1 thread
-  // and no concurrency with other tasks possible
-  val *= MULTIPLIER;
-}
-
-int main() {
-  int num_tasks = 0;
-  int *x, *y;
-  #pragma omp parallel
-  #pragma omp single
-  for (int iter = 0; iter < NT; ++iter) {
-    int gtid = __kmpc_global_thread_num(nullptr);
-    int res =  __kmpc_start_record_task(nullptr, gtid, /* kmp_tdg_flags */ 0, /* tdg_id */0);
-    if (res) {
-      num_tasks++;
-      #pragma omp task depend(out:y)
-      add();
-      #pragma omp task depend(out:x)
-      sub();
-      #pragma omp task depend(in:x,y)
-      mult();
-    }
-    __kmpc_end_record_task(nullptr, gtid, /* kmp_tdg_flags */0, /* tdg_id */0);
-    res =  __kmpc_start_record_task(nullptr, gtid, /* kmp_tdg_flags */ 0, /* tdg_id */1);
-    if (res) {
-      num_tasks++;
-      #pragma omp task depend(out:y)
-      add();
-      #pragma omp task depend(out:x)
-      sub();
-      #pragma omp task depend(in:x,y)
-      mult();
-    }
-    __kmpc_end_record_task(nullptr, gtid, /* kmp_tdg_flags */0, /* tdg_id */1);
-  }
-
-  assert(num_tasks==2);
-  assert(val==0);
-
-  std::cout << "Passed" << std::endl;
-  return 0;
-}
-// CHECK: Passed
diff --git a/openmp/runtime/test/tasking/omp_record_replay_print_dot.cpp b/openmp/runtime/test/tasking/omp_record_replay_print_dot.cpp
deleted file mode 100644
index 7f1f5ccd77d37..0000000000000
--- a/openmp/runtime/test/tasking/omp_record_replay_print_dot.cpp
+++ /dev/null
@@ -1,80 +0,0 @@
-// REQUIRES: omp_taskgraph_experimental
-// RUN: %libomp-cxx-compile-and-run
-#include <iostream>
-#include <fstream>
-#include <sstream>
-#include <cassert>
-
-// Compiler-generated code (emulation)
-typedef struct ident {
-    void* dummy;
-} ident_t;
-
-#ifdef __cplusplus
-extern "C" {
-  int __kmpc_global_thread_num(ident_t *);
-  int __kmpc_start_record_task(ident_t *, int, int, int);
-  void __kmpc_end_record_task(ident_t *, int, int , int);
-}
-#endif
-
-void func(int *num_exec) {
-  #pragma omp atomic
-  (*num_exec)++;
-}
-
-std::string tdg_string= "digraph TDG {\n"
-"   compound=true\n"
-"   subgraph cluster {\n"
-"      label=TDG_0\n"
-"      0[style=bold]\n"
-"      1[style=bold]\n"
-"      2[style=bold]\n"
-"      3[style=bold]\n"
-"   }\n"
-"   0 -> 1 \n"
-"   1 -> 2 \n"
-"   1 -> 3 \n"
-"}";
-
-int main() {
-  int num_exec = 0;
-  int x, y;
-
-  setenv("KMP_TDG_DOT","TRUE",1);
-  remove("tdg_0.dot");
-
-  #pragma omp parallel
-  #pragma omp single
-  {
-    int gtid = __kmpc_global_thread_num(nullptr);
-    int res = __kmpc_start_record_task(nullptr, gtid, /* kmp_tdg_flags */ 0, /* tdg_id */ 0);
-    if (res) {
-      #pragma omp task depend(out : x)
-      func(&num_exec);
-      #pragma omp task depend(in : x) depend(out : y)
-      func(&num_exec);
-      #pragma omp task depend(in : y)
-      func(&num_exec);
-      #pragma omp task depend(in : y)
-      func(&num_exec);
-    }
-
-    __kmpc_end_record_task(nullptr, gtid, /* kmp_tdg_flags */ 0, /* tdg_id */ 0);
-  }
-
-  assert(num_exec == 4);
-
-  std::ifstream tdg_file("tdg_0.dot");
-  assert(tdg_file.is_open());
-
-  std::stringstream tdg_file_stream;
-  tdg_file_stream << tdg_file.rdbuf();
-  int equal = tdg_string.compare(tdg_file_stream.str());
-
-  assert(equal == 0);
-
-  std::cout << "Passed" << std::endl;
-  return 0;
-}
-// CHECK: Passed
diff --git a/openmp/runtime/test/tasking/omp_record_replay_taskloop.cpp b/openmp/runtime/test/tasking/omp_record_replay_taskloop.cpp
deleted file mode 100644
index 163a1b4192d85..0000000000000
--- a/openmp/runtime/test/tasking/omp_record_replay_taskloop.cpp
+++ /dev/null
@@ -1,50 +0,0 @@
-// REQUIRES: omp_taskgraph_experimental
-// RUN: %libomp-cxx-compile-and-run
-#include <iostream>
-#include <cassert>
-
-#define NT 20
-#define N 128*128
-
-typedef struct ident {
-    void* dummy;
-} ident_t;
-
-
-#ifdef __cplusplus
-extern "C" {
-  int __kmpc_global_thread_num(ident_t *);
-  int __kmpc_start_record_task(ident_t *, int, int, int);
-  void __kmpc_end_record_task(ident_t *, int, int , int);
-}
-#endif
-
-int main() {
-  int num_tasks = 0;
-
-  int array[N];
-  for (int i = 0; i < N; ++i)
-    array[i] = 1;
-
-  long sum = 0;
-  #pragma omp parallel
-  #pragma omp single
-  for (int iter = 0; iter < NT; ++iter) {
-    int gtid = __kmpc_global_thread_num(nullptr);
-    int res =  __kmpc_start_record_task(nullptr, gtid, /* kmp_tdg_flags */0,  /* tdg_id */0);
-    if (res) {
-      num_tasks++;
-      #pragma omp taskloop reduction(+:sum) num_tasks(4096)
-      for (int i = 0; i < N; ++i) {
-        sum += array[i];
-      }
-    }
-    __kmpc_end_record_task(nullptr, gtid, /* kmp_tdg_flags */0,  /* tdg_id */0);
-  }
-  assert(sum==N*NT);
-  assert(num_tasks==1);
-
-  std::cout << "Passed" << std::endl;
-  return 0;
-}
-// CHECK: Passed

>From 9231bcf46a7f4a4e123031be203e9de533c61b75 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Fri, 27 Mar 2026 11:56:28 -0500
Subject: [PATCH 02/26] [OpenMP] New taskgraph runtime implementation

This patch contains the bulk of the new runtime support for taskgraph
record/replay.  Relative to previously-posted versions, this version
substantially reworks irreducible graph handling, and also reimplements
exec descriptor-based host replay.

Key points are as follows.

The task/taskdata structures and the dependencies between them are
duplicated whilst recording a taskgraph, keeping the existing runtime
dependency handling unaffected by the taskgraph implementation --
e.g. during runtime execution, it is valid for output dependencies to
be dropped as soon as the producing task completes.  This separation
is intended to eliminate race conditions when tasks which complete
unpredictably might or might not be marked as having a subsequent task
depend on them.

The dependencies between tasks in a taskgraph are processed by static
analysis: the high-level process is akin to turning data dependencies
between tasks into control-flow dependencies.  This is done by building
a set of successors and predecessors for each recorded task, then
decomposing the resulting DAG into parallel and sequential regions.
In the (presumed relatively unlikely, in real-world code) case that the
graph is irreducible, an algorithm is used to find the subgraph that
cannot be reduced, and make it an irreducible region (with children that
may be nodes, parallel or sequential regions, or even other irreducible
regions).

For irreducible regions in the output graph, the immediate children
*retain* their "intra" predecessor and successor lists.  Deps pointing
in or out of the irreducible region, or those for other region types
(parallel, sequential, etc.) are dropped.

The algorithm to find and collapse irreducible regions works as follows.
Firstly, the graph is decomposed into single-entry, single-exit (SESE)
regions, using calculated dominator and postdominator edges.  SESE regions
may be nested -- the top-level graph forms a SESE region itself, from the
entry point to the exit point, and there may also be subgraphs which form
SESE regions.  SESE regions thus form a tree, rooted at the outermost
level of the graph.  The leaves of that tree are taken and collapsed
into irreducible regions in our output graph.  We already know such
leaf regions cannot be reduced by parallel-sequential decomposition,
because the SESE-region finding code runs after the point where that
previous phase can do no further work.

The output of this process is a set of nested kmp_taskgraph_region
structures -- parallel, sequential or irreducible (each with some number
of children), or nodes representing a single task.  The two phases --
parallel/sequential decomposition and irreducible-region handling --
alternate until we obtain a single, top-level region.

This nested structure is then used for both host-based taskgraph replay
and (TBD) offload-based taskgraph replay.  It is hoped that the high-level
semantics will be useful when mapping to e.g. other graph APIs.  Already
for host-based execution, reductions only work with parallel regions: much
of the complexity here is to allow that to work reliably, particularly
where reduction results obtained at some point in the replay of a
taskgraph might be used within that *same* taskgraph replay instance.

Replaying a taskgraph processed into this nested region tree
on the CPU involves another set of linked structures, of type
kmp_taskgraph_exec_descr. These form a kind of trace of a traversal
over the kmp_taskgraph_region structure, so that a pointer to a
kmp_taskgraph_exec_descr is somewhat equivalent to a "program counter".

The bulk of the processing here is done by __kmp_build_exec_descrs_1: we
create descr structures for task nodes, for wait nodes, and for
parallel regions.  Each descr has a pointer to the originating region,
an in-degree, and a successor list.  Host-based replay is then quite
straightforward: __kmp_taskgraph_exec_descr_start looks for reduction
metadata on parallel regions, or invokes recorded tasks for task-node
regions.

Recorded taskgraphs are located directly by using a handle passed
in from the user's compiled program, rather than using a linked list
or hashtable to find taskgraph records to replay keyed by an index:
the compiler generates a static pointer per-lexical taskgraph construct
(instantiation in the case of templated code), and the address of that is
passed to the runtimeto be written back with the address of the processed
taskgraph data.

Pull Request: https://github.com/llvm/llvm-project/pull/216324
---
 openmp/runtime/src/kmp.h                      |  232 +-
 openmp/runtime/src/kmp_taskdeps.cpp           | 2834 ++++++++++++++++-
 openmp/runtime/src/kmp_taskdeps.h             |   27 +-
 openmp/runtime/src/kmp_tasking.cpp            | 1159 ++++++-
 .../tasking/omp_taskgraph_irreducible.cpp     |   84 +
 5 files changed, 4189 insertions(+), 147 deletions(-)
 create mode 100644 openmp/runtime/test/tasking/omp_taskgraph_irreducible.cpp

diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index 00a56afa90b52..c2fdab3de64bf 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -2497,6 +2497,13 @@ typedef struct kmp_task { /* GEH: Shouldn't this be aligned somehow? */
 @}
 */
 
+#if OMP_TASKGRAPH_EXPERIMENTAL
+typedef struct kmp_taskgraph_reduce_input_data {
+  void *reduce_data;
+  kmp_int32 reduce_num_data;
+} kmp_taskgraph_reduce_input_data_t;
+#endif
+
 typedef struct kmp_taskgroup {
   std::atomic<kmp_int32> count; // number of allocated and incomplete tasks
   std::atomic<kmp_int32>
@@ -2506,6 +2513,17 @@ typedef struct kmp_taskgroup {
   void *reduce_data; // reduction related info
   kmp_int32 reduce_num_data; // number of data items to reduce
   uintptr_t *gomp_data; // gomp reduction data
+#if OMP_TASKGRAPH_EXPERIMENTAL
+  struct {
+    // Points to taskgraph that tasks in this taskgroup are being recorded to.
+    std::atomic<struct kmp_taskgraph_record *> recording;
+    // Temporary holding place for input data for reductions for this taskgroup
+    // during taskgraph recording.  This is passed over to the first
+    // kmp_taskgraph_node we encounter inside the taskgroup.  We'll have to
+    // watch out for potential race conditions here.
+    kmp_taskgraph_reduce_input_data_t *reduce_input;
+  } taskgraph;
+#endif
 } kmp_taskgroup_t;
 
 // forward declarations
@@ -2557,6 +2575,14 @@ struct kmp_depnode_list {
 // Max number of mutexinoutset dependencies per node
 #define MAX_MTX_DEPS 4
 
+#if OMP_TASKGRAPH_EXPERIMENTAL
+struct kmp_taskgraph_node;
+struct kmp_taskgraph_region;
+struct kmp_taskgraph_record;
+
+struct kmp_bitset;
+#endif
+
 typedef struct kmp_base_depnode {
   kmp_depnode_list_t *successors; /* used under lock */
   kmp_task_t *task; /* non-NULL if depnode is active, used under lock */
@@ -2568,6 +2594,9 @@ typedef struct kmp_base_depnode {
 #endif
   std::atomic<kmp_int32> npredecessors;
   std::atomic<kmp_int32> nrefs;
+#if OMP_TASKGRAPH_EXPERIMENTAL
+  struct kmp_bitset *set_membership;
+#endif
 } kmp_base_depnode_t;
 
 union KMP_ALIGN_CACHE kmp_depnode {
@@ -2582,7 +2611,14 @@ struct kmp_dephash_entry {
   kmp_depnode_list_t *last_set;
   kmp_depnode_list_t *prev_set;
   kmp_uint8 last_flag;
+#if OMP_TASKGRAPH_EXPERIMENTAL
+  union {
+    kmp_lock_t *mtx_lock; /* is referenced by depnodes w/mutexinoutset dep */
+    kmp_int32 set_num;
+  };
+#else
   kmp_lock_t *mtx_lock; /* is referenced by depnodes w/mutexinoutset dep */
+#endif
   kmp_dephash_entry_t *next_in_bucket;
 };
 
@@ -2620,12 +2656,169 @@ typedef struct {
 
 #if OMP_TASKGRAPH_EXPERIMENTAL
 
-/// Represent a TDG's current status
-typedef enum kmp_tdg_status {
+/// Represent a task dependency graph's current status
+typedef enum kmp_taskgraph_status {
   KMP_TDG_NONE = 0,
   KMP_TDG_RECORDING = 1,
   KMP_TDG_READY = 2
-} kmp_tdg_status_t;
+} kmp_taskgraph_status_t;
+
+enum kmp_taskgraph_mark {
+  TASKGRAPH_UNMARKED,
+  TASKGRAPH_TEMP_MARK,
+  TASKGRAPH_PERMANENT_MARK,
+  TASKGRAPH_COMBINED,
+  TASKGRAPH_DELETED
+};
+
+typedef struct kmp_taskgraph_region_dep {
+  struct kmp_taskgraph_region *region;
+  struct kmp_taskgraph_region_dep *next;
+} kmp_taskgraph_region_dep_t;
+
+enum kmp_taskgraph_region_type {
+  TASKGRAPH_REGION_ENTRY,
+  TASKGRAPH_REGION_EXIT,
+  TASKGRAPH_REGION_NODE,
+  TASKGRAPH_REGION_WAIT,
+  TASKGRAPH_REGION_PARALLEL,
+  TASKGRAPH_REGION_EXCLUSIVE,
+  TASKGRAPH_REGION_SEQUENTIAL,
+  TASKGRAPH_REGION_IRREDUCIBLE
+};
+
+typedef struct kmp_taskgraph_node {
+  kmp_task_t *task;
+  bool taskloop_task;
+  kmp_taskgraph_reduce_input_data_t *reduce_input;
+  union {
+    // Valid when KMP_TDG_RECORDING in parent taskgraph record.
+    struct {
+      kmp_depend_info_t *dep_list;
+      kmp_int32 ndeps;
+      // This is a control dependency.  If not -1, it is the index of the
+      // taskgraph node which succeeds this one in an array of taskgraph nodes.
+      kmp_int32 cfg_successor;
+    } unresolved;
+
+  } u;
+} kmp_taskgraph_node_t;
+
+typedef struct kmp_taskgraph_region {
+  struct kmp_taskgraph_record *owner = nullptr;
+  // Initially, the lexical "next" region (which doesn't have to be a
+  // successor). Subsequently, a pointer to the next item in the worklist.
+  struct kmp_taskgraph_region *next = nullptr;
+  // The parent taskgraph for this one.  Initially nullptr.
+  struct kmp_taskgraph_region *parent = nullptr;
+  kmp_taskgraph_region_dep_t *successors = nullptr;
+  kmp_taskgraph_region_dep_t *predecessors = nullptr;
+  // Only valid while building the exec descr structure.  This could probably
+  // share storage with one of the other fields if we wanted to save space.
+  struct kmp_taskgraph_exec_descr *exec_descr = nullptr;
+  // The next allocated block.
+  struct kmp_taskgraph_region *alloc_chain;
+  struct kmp_bitset *mutexset = nullptr;
+  struct kmp_taskgraph_region *mutexset_parent = nullptr;
+  // Pointer to reduction input data for the region.  We only expect to see
+  // this on TASKGRAPH_REGION_PARALLEL regions.
+  kmp_taskgraph_reduce_input_data_t *reduce_input = nullptr;
+  enum kmp_taskgraph_region_type type;
+  enum kmp_taskgraph_mark mark = TASKGRAPH_UNMARKED;
+  kmp_int32 timestamp = 0;
+  kmp_int32 level = -1;
+  union {
+    struct {
+      kmp_taskgraph_node_t *node;
+    } task;
+    struct {
+      struct kmp_taskgraph_region **children;
+      kmp_int32 num_children;
+    } inner;
+  };
+  // NODE/WAIT region.
+  kmp_taskgraph_region(struct kmp_taskgraph_record *taskgraph,
+                       struct kmp_taskgraph_region **&alloc_chain,
+                       kmp_taskgraph_node_t *node,
+                       struct kmp_taskgraph_region *parent);
+  // PARALLEL, SEQUENTIAL, etc. with trailing children.
+  kmp_taskgraph_region(struct kmp_taskgraph_record *taskgraph,
+                       struct kmp_taskgraph_region **&alloc_chain,
+                       enum kmp_taskgraph_region_type type, kmp_int32 num_nodes,
+                       struct kmp_taskgraph_region *parent);
+  // Fixed size allocation for leaf nodes.
+  static void *operator new(size_t size, kmp_info_p *thread);
+  // Variable sized allocation for nodes with trailing children.
+  static void *operator new(size_t size, kmp_info_p *thread,
+                            kmp_int32 num_nodes);
+} kmp_taskgraph_region_t;
+
+typedef struct kmp_taskgraph_record {
+  std::atomic<kmp_taskgraph_status_t> status = KMP_TDG_NONE;
+  kmp_int32 gtid = 0;
+  kmp_int32 graph_id = 0;
+  // A lock that protects the record_map and num_tasks fields from being
+  // modified by multiple threads.
+  // For now, we also use this whilst the taskgraph is being replayed.
+  // This should be replaced with an invocation counter when we implement
+  // concurrent replay of the taskgraph from different threads.
+  kmp_lock_t map_lock;
+  kmp_taskgraph_node_t *record_map = nullptr;
+  kmp_int32 num_tasks = 0;
+  kmp_int32 nodes_allocated = 0;
+  kmp_taskgraph_region_t *root = nullptr;
+  kmp_taskgraph_region_t *alloc_root = nullptr;
+  kmp_taskgraph_region_dep_t *recycled_deps = nullptr;
+  kmp_int32 num_mutexes = 0;
+  struct kmp_taskgraph_exec_descr *exec_descrs = nullptr;
+  kmp_size_t num_exec_descrs = 0;
+  kmp_lock_t replay_lock;
+  // We need a taskgroup structure to keep track of recorded tasks.  This is
+  // set to TRUE if the user requested "nogroup" on the taskgraph directive
+  // (then we can avoid blocking at the end of the taskgraph region on replay,
+  // at least).
+  bool nogroup_taskgroup = false;
+  struct kmp_taskgraph_record *next = nullptr;
+} kmp_taskgraph_record_t;
+
+typedef struct kmp_taskgraph_header {
+  kmp_taskgraph_record_t *first;
+  kmp_taskgraph_record_t *expiring;
+  kmp_lock_t header_lock;
+} kmp_taskgraph_header_t;
+
+struct kmp_taskgraph_exec_descr_elem;
+
+// A node in the flattened host-replay dataflow DAG.  Each task/wait gets one
+// of these, and so does a "parallel" region.  The latter is where we locate
+// metadata for reductions.  Other region types are represented implicitly by
+// the indegree field and successor list.
+typedef struct kmp_taskgraph_exec_descr {
+  std::atomic<kmp_int32> npredecessors;
+  // Immutable in-degree; npredecessors is reset to this at the start of every
+  // replay (the descr buffer is built once and reused).
+  kmp_int32 indegree;
+  kmp_taskgraph_region_t *region;
+  struct kmp_taskgraph_exec_descr_elem *successors;
+} kmp_taskgraph_exec_descr_t;
+
+// A single element in an exec descr list.  In the exec_descr structure, these
+// are maintained in a circular list: that means that lists can be spliced
+// together in O(1) time.
+
+typedef struct kmp_taskgraph_exec_descr_elem {
+  kmp_taskgraph_exec_descr_t *exec_descr;
+  struct kmp_taskgraph_exec_descr_elem *next;
+} kmp_taskgraph_exec_descr_elem_t;
+
+// This list type is refcounted so that we don't leak memory during e.g.
+// irreducible region traversal.
+
+typedef struct kmp_taskgraph_exec_descr_list {
+  kmp_taskgraph_exec_descr_elem_t *head;
+  kmp_int32 length;
+  kmp_int32 refcount;
+} kmp_taskgraph_exec_descr_list_t;
 
 #endif
 
@@ -2739,6 +2932,9 @@ struct kmp_taskdata { /* aligned during dynamic allocation       */
   ompt_task_info_t ompt_task_info;
 #endif
 #if OMP_TASKGRAPH_EXPERIMENTAL
+  // Whether the task is within a task dependency graph.
+  struct kmp_taskgraph_record *owning_taskgraph = nullptr;
+  struct kmp_taskgraph_exec_descr *exec_descr = nullptr;
 #endif
   kmp_target_data_t td_target_data;
 }; // struct kmp_taskdata
@@ -4246,6 +4442,15 @@ KMP_EXPORT void __kmpc_omp_taskwait_deps_51(ident_t *loc_ref, kmp_int32 gtid,
 extern kmp_int32 __kmp_omp_task(kmp_int32 gtid, kmp_task_t *new_task,
                                 bool serialize_immediate);
 
+#if OMP_TASKGRAPH_EXPERIMENTAL
+extern kmp_int32 __kmp_build_taskgraph(kmp_int32 gtid,
+                                       kmp_taskdata_t *current_taskdata,
+                                       kmp_taskgraph_record_t *taskgraph);
+// True if the user asked for tracing of taskgraph structure and replay via the
+// KMP_TASKGRAPH_TRACE environment variable.
+extern bool __kmp_taskgraph_trace();
+#endif
+
 KMP_EXPORT kmp_int32 __kmpc_cancel(ident_t *loc_ref, kmp_int32 gtid,
                                    kmp_int32 cncl_kind);
 KMP_EXPORT kmp_int32 __kmpc_cancellationpoint(ident_t *loc_ref, kmp_int32 gtid,
@@ -4314,6 +4519,27 @@ KMP_EXPORT void __kmpc_init_nest_lock_with_hint(ident_t *loc, kmp_int32 gtid,
                                                 void **user_lock,
                                                 uintptr_t hint);
 #if OMP_TASKGRAPH_EXPERIMENTAL
+KMP_EXPORT void __kmpc_taskgraph(ident_t *loc_ref, kmp_int32 gtid,
+                                 std::atomic<void *> *tdg_handle,
+                                 kmp_uint32 graph_id, kmp_int32 graph_reset,
+                                 kmp_int32 nogroup, void (*entry)(void *),
+                                 void *args);
+KMP_EXPORT kmp_uint32 __kmpc_taskgraph_task(
+    ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *new_task, kmp_int32 flags,
+    size_t sizeof_kmp_task_t, void *shareds, size_t sizeof_shareds,
+    kmp_int32 ndeps, kmp_depend_info_t *dep_list);
+KMP_EXPORT kmp_uint32 __kmpc_taskgraph_taskloop(
+    ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *new_task, kmp_int32 flags,
+    size_t sizeof_kmp_task_t, void *shareds, size_t sizeof_shareds,
+    kmp_int32 if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
+    kmp_int32 nogroup, kmp_int32 sched, kmp_uint64 grainsize,
+    kmp_int32 modifier, void *task_dup);
+KMP_EXPORT void __kmpc_taskgraph_taskwait(ident_t *loc_ref, kmp_int32 gtid,
+                                          kmp_int32 ndeps,
+                                          kmp_depend_info_t *dep_list,
+                                          kmp_int32 has_no_wait);
+KMP_EXPORT void *__kmpc_taskgraph_taskred_init(kmp_int32 gtid, kmp_int32 num,
+                                               void *data);
 #endif
 /* Interface to fast scalable reduce methods routines */
 
diff --git a/openmp/runtime/src/kmp_taskdeps.cpp b/openmp/runtime/src/kmp_taskdeps.cpp
index 6884cd144f89e..fe7c630b567b0 100644
--- a/openmp/runtime/src/kmp_taskdeps.cpp
+++ b/openmp/runtime/src/kmp_taskdeps.cpp
@@ -20,6 +20,13 @@
 #include "ompt-specific.h"
 #endif
 
+#if OMP_TASKGRAPH_EXPERIMENTAL
+#include <bit>
+#include <cstdlib>
+#include <algorithm>
+#include <cinttypes>
+#endif
+
 // TODO: Improve memory allocation? keep a list of pre-allocated structures?
 // allocate in blocks? re-use list finished list entries?
 // TODO: don't use atomic ref counters for stack-allocated nodes.
@@ -33,6 +40,14 @@
 static std::atomic<kmp_int32> kmp_node_id_seed = 0;
 #endif
 
+#undef DEBUG_TASKGRAPH
+
+#ifdef DEBUG_TASKGRAPH
+#define TGDBG(ARGS...) fprintf(stderr, ARGS)
+#else
+#define TGDBG(ARGS...)
+#endif
+
 static void __kmp_init_node(kmp_depnode_t *node, bool on_stack) {
   node->dn.successors = NULL;
   node->dn.task = NULL; // will point to the right task
@@ -49,6 +64,9 @@ static void __kmp_init_node(kmp_depnode_t *node, bool on_stack) {
 #ifdef KMP_SUPPORT_GRAPH_OUTPUT
   node->dn.id = KMP_ATOMIC_INC(&kmp_node_id_seed);
 #endif
+#if OMP_TASKGRAPH_EXPERIMENTAL
+  node->dn.set_membership = nullptr;
+#endif
 #if USE_ITT_BUILD && USE_ITT_NOTIFY
   __itt_sync_create(node, "OMP task dep node", NULL, 0);
 #endif
@@ -160,7 +178,12 @@ static kmp_dephash_t *__kmp_dephash_create(kmp_info_t *thread,
 
 static kmp_dephash_entry *__kmp_dephash_find(kmp_info_t *thread,
                                              kmp_dephash_t **hash,
-                                             kmp_intptr_t addr) {
+                                             kmp_intptr_t addr
+#if OMP_TASKGRAPH_EXPERIMENTAL
+                                             ,
+                                             bool taskgraph_p
+#endif
+) {
   kmp_dephash_t *h = *hash;
   if (h->nelements != 0 && h->nconflicts / h->size >= 1) {
     *hash = __kmp_dephash_extend(thread, h);
@@ -190,7 +213,12 @@ static kmp_dephash_entry *__kmp_dephash_find(kmp_info_t *thread,
     entry->last_set = NULL;
     entry->prev_set = NULL;
     entry->last_flag = 0;
-    entry->mtx_lock = NULL;
+#if OMP_TASKGRAPH_EXPERIMENTAL
+    if (taskgraph_p)
+      entry->set_num = -1;
+    else
+#endif
+      entry->mtx_lock = NULL;
     entry->next_in_bucket = h->buckets[bucket];
     h->buckets[bucket] = entry;
     h->nelements++;
@@ -200,6 +228,7 @@ static kmp_dephash_entry *__kmp_dephash_find(kmp_info_t *thread,
   return entry;
 }
 
+template <bool refcounting>
 static kmp_depnode_list_t *__kmp_add_node(kmp_info_t *thread,
                                           kmp_depnode_list_t *list,
                                           kmp_depnode_t *node) {
@@ -213,7 +242,11 @@ static kmp_depnode_list_t *__kmp_add_node(kmp_info_t *thread,
       thread, sizeof(kmp_depnode_list_t));
 #endif
 
-  new_head->node = __kmp_node_ref(node);
+  if (refcounting) {
+    new_head->node = __kmp_node_ref(node);
+  } else {
+    new_head->node = node;
+  }
   new_head->next = list;
 
   return new_head;
@@ -275,7 +308,8 @@ __kmp_depnode_link_successor(kmp_int32 gtid, kmp_info_t *thread,
       if (dep->dn.task) {
         if (!dep->dn.successors || dep->dn.successors->node != node) {
           __kmp_track_dependence(gtid, dep, node, task);
-          dep->dn.successors = __kmp_add_node(thread, dep->dn.successors, node);
+          dep->dn.successors =
+              __kmp_add_node<true>(thread, dep->dn.successors, node);
           KA_TRACE(40, ("__kmp_process_deps: T#%d adding dependence from %p to "
                         "%p\n",
                         gtid, KMP_TASK_TO_TASKDATA(dep->dn.task),
@@ -304,7 +338,8 @@ static inline kmp_int32 __kmp_depnode_link_successor(kmp_int32 gtid,
     if (sink->dn.task) {
       if (!sink->dn.successors || sink->dn.successors->node != source) {
         __kmp_track_dependence(gtid, sink, source, task);
-        sink->dn.successors = __kmp_add_node(thread, sink->dn.successors, source);
+        sink->dn.successors =
+            __kmp_add_node<true>(thread, sink->dn.successors, source);
         KA_TRACE(40, ("__kmp_process_deps: T#%d adding dependence from %p to "
                     "%p\n",
                     gtid, KMP_TASK_TO_TASKDATA(sink->dn.task),
@@ -317,21 +352,249 @@ static inline kmp_int32 __kmp_depnode_link_successor(kmp_int32 gtid,
   return npredecessors;
 }
 
+#if OMP_TASKGRAPH_EXPERIMENTAL
+kmp_taskgraph_region_dep_t *__kmp_region_deplist_add(
+    kmp_info_t *thread, kmp_taskgraph_region_dep_t **recycled_deps,
+    kmp_taskgraph_region_t *region, kmp_taskgraph_region_dep_t *list) {
+  kmp_taskgraph_region_dep_t *head;
+  if (*recycled_deps) {
+    head = *recycled_deps;
+    *recycled_deps = (*recycled_deps)->next;
+  } else
+    head = (kmp_taskgraph_region_dep_t *)__kmp_fast_allocate(
+        thread, sizeof(kmp_taskgraph_region_dep_t));
+  head->region = region;
+  head->next = list;
+  return head;
+}
+
+kmp_taskgraph_region_t *
+__kmp_region_worklist_reverse(kmp_taskgraph_region_t *list) {
+  kmp_taskgraph_region_t *last = nullptr;
+  while (list) {
+    kmp_taskgraph_region_t *next = list->next;
+    list->next = last;
+    last = list;
+    list = next;
+  }
+  return last;
+}
+
+static kmp_depnode_t *__kmp_find_in_depnode_list(kmp_depnode_t *node,
+                                                 kmp_depnode_list_t *list) {
+  for (; list; list = list->next)
+    if (list->node == node)
+      return list->node;
+  return nullptr;
+}
+
+// A trivial fixed-size bitset implementation.
+
+typedef struct kmp_bitset {
+  kmp_uint64 *bits;
+  kmp_size_t bitsize;
+  kmp_size_t num_chunks;
+} kmp_bitset_t;
+
+static kmp_bitset_t *__kmp_bitset_alloc(kmp_info_t *thread,
+                                        kmp_size_t bitsize) {
+  kmp_size_t bytesize = (bitsize + 7) / 8;
+  kmp_size_t num_chunks =
+      (bytesize + sizeof(kmp_uint64) - 1) / sizeof(kmp_uint64);
+  kmp_bitset_t *bitset = (kmp_bitset_t *)__kmp_fast_allocate(
+      thread, sizeof(kmp_bitset_t) + sizeof(kmp_uint64) * num_chunks);
+  bitset->bits = (kmp_uint64 *)&bitset[1];
+  memset(bitset->bits, 0, sizeof(kmp_uint64) * num_chunks);
+  bitset->bitsize = bitsize;
+  bitset->num_chunks = num_chunks;
+  return bitset;
+}
+
+static void __kmp_bitset_free(kmp_info_t *thread, kmp_bitset_t *bitset) {
+  __kmp_fast_free(thread, bitset);
+}
+
+static void __kmp_bitset_set(kmp_bitset_t *bitset, kmp_size_t bitnum) {
+  kmp_size_t chunk = bitnum / (8 * sizeof(kmp_uint64));
+  if (bitnum < bitset->bitsize)
+    bitset->bits[chunk] |= (kmp_uint64)1 << (bitnum & 63);
+}
+
+static void __kmp_bitset_clearall(kmp_bitset_t *bitset) {
+  if (bitset)
+    memset(bitset->bits, 0, sizeof(kmp_int64) * bitset->num_chunks);
+}
+
+static void __kmp_bitset_setall(kmp_bitset_t *bitset) {
+  for (kmp_int32 chunk = 0; chunk < bitset->num_chunks - 1; chunk++)
+    bitset->bits[chunk] = ~(kmp_uint64)0;
+  kmp_int32 last_chunk_numbits = bitset->bitsize & 63;
+  if (last_chunk_numbits > 0) {
+    kmp_uint64 last_chunk_bits = ~((~(kmp_uint64)0) << last_chunk_numbits);
+    bitset->bits[bitset->num_chunks - 1] = last_chunk_bits;
+  }
+}
+
+static void __kmp_bitset_copy(kmp_bitset_t *dst, const kmp_bitset_t *src) {
+  assert(dst->num_chunks == src->num_chunks);
+  assert(dst->bitsize == src->bitsize);
+  memcpy(dst->bits, src->bits, sizeof(kmp_uint64) * dst->num_chunks);
+}
+
+/// Return TRUE if \c b is a subset of \c a.
+
+static bool __kmp_bitset_subset_p(const kmp_bitset_t *a,
+                                  const kmp_bitset_t *b) {
+  if (!b)
+    return true;
+  kmp_size_t chunk_max = std::max(a->num_chunks, b->num_chunks);
+  for (kmp_size_t chunk = 0; chunk < chunk_max; chunk++) {
+    kmp_uint64 a_bits = chunk < a->num_chunks ? a->bits[chunk] : 0;
+    kmp_uint64 b_bits = chunk < b->num_chunks ? b->bits[chunk] : 0;
+    if ((a_bits & b_bits) != b_bits)
+      return false;
+  }
+  return true;
+}
+
+static void __kmp_bitset_and(kmp_bitset_t *a, kmp_bitset_t *b,
+                             kmp_bitset_t *c) {
+  kmp_size_t chunk_max = std::max(b->num_chunks, c->num_chunks);
+  for (kmp_size_t chunk = 0; chunk < chunk_max; chunk++) {
+    kmp_uint64 b_bits = chunk < b->num_chunks ? b->bits[chunk] : 0;
+    kmp_uint64 c_bits = chunk < c->num_chunks ? c->bits[chunk] : 0;
+    a->bits[chunk] = b_bits & c_bits;
+  }
+}
+
+static void __kmp_bitset_and_not(kmp_bitset_t *a, kmp_bitset_t *b,
+                                 kmp_bitset_t *c) {
+  if (!c)
+    __kmp_bitset_copy(a, b);
+  else {
+    kmp_size_t chunk_max = std::max(b->num_chunks, c->num_chunks);
+    for (kmp_size_t chunk = 0; chunk < chunk_max; chunk++) {
+      kmp_uint64 b_bits = chunk < b->num_chunks ? b->bits[chunk] : 0;
+      kmp_uint64 c_bits = chunk < c->num_chunks ? c->bits[chunk] : 0;
+      a->bits[chunk] = b_bits & ~c_bits;
+    }
+  }
+}
+
+static void __kmp_bitset_or(kmp_bitset_t *a, kmp_bitset_t *b, kmp_bitset_t *c) {
+  if (!b && !c)
+    __kmp_bitset_clearall(a);
+  else if (!b)
+    __kmp_bitset_copy(a, c);
+  else if (!c)
+    __kmp_bitset_copy(a, b);
+  else {
+    kmp_size_t chunk_max = std::max(b->num_chunks, c->num_chunks);
+    for (kmp_size_t chunk = 0; chunk < chunk_max; chunk++) {
+      kmp_uint64 b_bits = chunk < b->num_chunks ? b->bits[chunk] : 0;
+      kmp_uint64 c_bits = chunk < c->num_chunks ? c->bits[chunk] : 0;
+      a->bits[chunk] = b_bits | c_bits;
+    }
+  }
+}
+
+static bool __kmp_bitset_empty_p(kmp_bitset_t *bitset) {
+  if (!bitset)
+    return true;
+  for (kmp_size_t chunk = 0; chunk < bitset->num_chunks; chunk++) {
+    if (bitset->bits[chunk] != 0)
+      return false;
+  }
+  return true;
+}
+
+/// Test two bitsets for equality.  Note that any unused bits at the end of the
+/// last chunk are kept as zero.
+
+static bool __kmp_bitset_equal(kmp_bitset_t *a, kmp_bitset_t *b) {
+  if (!b)
+    return __kmp_bitset_empty_p(a);
+  kmp_size_t chunk_max = std::max(a->num_chunks, b->num_chunks);
+  for (kmp_size_t chunk = 0; chunk < chunk_max; chunk++) {
+    kmp_uint64 a_bits = chunk < a->num_chunks ? a->bits[chunk] : 0;
+    kmp_uint64 b_bits = chunk < b->num_chunks ? b->bits[chunk] : 0;
+    if (a_bits != b_bits)
+      return false;
+  }
+  return true;
+}
+
+static bool __kmp_bitset_intersect_p(kmp_bitset_t *a, kmp_bitset_t *b) {
+  if (!a || !b)
+    return false;
+  kmp_size_t chunk_max = std::max(a->num_chunks, b->num_chunks);
+  for (kmp_size_t chunk = 0; chunk < chunk_max; chunk++) {
+    kmp_uint64 a_bits = chunk < a->num_chunks ? a->bits[chunk] : 0;
+    kmp_uint64 b_bits = chunk < b->num_chunks ? b->bits[chunk] : 0;
+    if ((a_bits & b_bits) != 0)
+      return true;
+  }
+  return false;
+}
+
+static kmp_int32 __kmp_bitset_popcount(kmp_bitset_t *bitset) {
+  if (!bitset)
+    return 0;
+  kmp_int32 accum = 0;
+  for (kmp_int32 c = 0; c < bitset->num_chunks; c++) {
+    accum += std::__popcount(bitset->bits[c]);
+  }
+  return accum;
+}
+
+static kmp_int32 __kmp_taskgraph_add_dep(kmp_info_t *thread,
+                                         kmp_depnode_t *node,
+                                         kmp_depnode_list_t *plist) {
+  kmp_int32 npredecessors = 0;
+  for (; plist; plist = plist->next) {
+    kmp_depnode_t *dep = plist->node;
+    if (!dep->dn.successors ||
+        !__kmp_find_in_depnode_list(node, dep->dn.successors)) {
+      dep->dn.successors =
+          __kmp_add_node<false>(thread, dep->dn.successors, node);
+      npredecessors++;
+    }
+  }
+  return npredecessors;
+}
+
+static kmp_int32 __kmp_taskgraph_add_dep(kmp_info_t *thread,
+                                         kmp_depnode_t *source,
+                                         kmp_depnode_t *sink) {
+  if (!sink)
+    return 0;
+  kmp_int32 npredecessors = 0;
+  if (!sink->dn.successors || sink->dn.successors->node != source) {
+    if (!__kmp_find_in_depnode_list(source, sink->dn.successors)) {
+      sink->dn.successors =
+          __kmp_add_node<false>(thread, sink->dn.successors, source);
+      npredecessors++;
+    }
+  }
+  return npredecessors;
+}
+#endif
+
+template <typename T>
 static inline kmp_int32
 __kmp_process_dep_all(kmp_int32 gtid, kmp_depnode_t *node, kmp_dephash_t *h,
                       bool dep_barrier, kmp_task_t *task) {
-  KA_TRACE(30, ("__kmp_process_dep_all: T#%d processing dep_all, "
+  KA_TRACE(30, ("__kmp_process_dep_all<%s>: T#%d processing dep_all, "
                 "dep_barrier = %d\n",
-                gtid, dep_barrier));
+                T::name, gtid, dep_barrier));
   kmp_info_t *thread = __kmp_threads[gtid];
   kmp_int32 npredecessors = 0;
 
   // process previous omp_all_memory node if any
-  npredecessors +=
-      __kmp_depnode_link_successor(gtid, thread, task, node, h->last_all);
-  __kmp_node_deref(thread, h->last_all);
+  npredecessors += T::link_successor(gtid, thread, task, node, h->last_all);
+  T::deref(thread, h->last_all);
   if (!dep_barrier) {
-    h->last_all = __kmp_node_ref(node);
+    h->last_all = T::ref(node);
   } else {
     // if this is a sync point in the serial sequence, then the previous
     // outputs are guaranteed to be completed after the execution of this
@@ -350,38 +613,37 @@ __kmp_process_dep_all(kmp_int32 gtid, kmp_depnode_t *node, kmp_dephash_t *h,
       kmp_depnode_list_t *last_set = info->last_set;
       kmp_depnode_list_t *prev_set = info->prev_set;
       if (last_set) {
-        npredecessors +=
-            __kmp_depnode_link_successor(gtid, thread, task, node, last_set);
-        __kmp_depnode_list_free(thread, last_set);
-        __kmp_depnode_list_free(thread, prev_set);
+        npredecessors += T::link_successor(gtid, thread, task, node, last_set);
+        __kmp_depnode_list_free<T::rc>(thread, last_set);
+        __kmp_depnode_list_free<T::rc>(thread, prev_set);
         info->last_set = NULL;
         info->prev_set = NULL;
         info->last_flag = 0; // no sets in this dephash entry
       } else {
-        npredecessors +=
-            __kmp_depnode_link_successor(gtid, thread, task, node, last_out);
+        npredecessors += T::link_successor(gtid, thread, task, node, last_out);
       }
-      __kmp_node_deref(thread, last_out);
+      T::deref(thread, last_out);
       if (!dep_barrier) {
-        info->last_out = __kmp_node_ref(node);
+        info->last_out = T::ref(node);
       } else {
         info->last_out = NULL;
       }
     }
   }
-  KA_TRACE(30, ("__kmp_process_dep_all: T#%d found %d predecessors\n", gtid,
-                npredecessors));
+  KA_TRACE(30, ("__kmp_process_dep_all<%s>: T#%d found %d predecessors\n",
+                T::name, gtid, npredecessors));
   return npredecessors;
 }
 
-template <bool filter>
+template <typename T>
 static inline kmp_int32
 __kmp_process_deps(kmp_int32 gtid, kmp_depnode_t *node, kmp_dephash_t **hash,
                    bool dep_barrier, kmp_int32 ndeps,
-                   kmp_depend_info_t *dep_list, kmp_task_t *task) {
-  KA_TRACE(30, ("__kmp_process_deps<%d>: T#%d processing %d dependences : "
-                "dep_barrier = %d\n",
-                filter, gtid, ndeps, dep_barrier));
+                   kmp_depend_info_t *dep_list, kmp_task_t *task,
+                   kmp_int32 &next_mutex_set, bool filter = true) {
+  KA_TRACE(30, ("__kmp_process_deps<%s>: T#%d processing %d dependences : "
+                "dep_barrier = %d, filter = %d\n",
+                T::name, gtid, ndeps, dep_barrier, filter));
 
   kmp_info_t *thread = __kmp_threads[gtid];
   kmp_int32 npredecessors = 0;
@@ -391,28 +653,31 @@ __kmp_process_deps(kmp_int32 gtid, kmp_depnode_t *node, kmp_dephash_t **hash,
     if (filter && dep->base_addr == 0)
       continue; // skip filtered entries
 
+#if OMP_TASKGRAPH_EXPERIMENTAL
+    kmp_dephash_entry_t *info =
+        __kmp_dephash_find(thread, hash, dep->base_addr, !T::rc);
+#else
     kmp_dephash_entry_t *info =
         __kmp_dephash_find(thread, hash, dep->base_addr);
+#endif
     kmp_depnode_t *last_out = info->last_out;
     kmp_depnode_list_t *last_set = info->last_set;
     kmp_depnode_list_t *prev_set = info->prev_set;
 
     if (dep->flags.out) { // out or inout --> clean lists if any
       if (last_set) {
-        npredecessors +=
-            __kmp_depnode_link_successor(gtid, thread, task, node, last_set);
-        __kmp_depnode_list_free(thread, last_set);
-        __kmp_depnode_list_free(thread, prev_set);
+        npredecessors += T::link_successor(gtid, thread, task, node, last_set);
+        __kmp_depnode_list_free<T::rc>(thread, last_set);
+        __kmp_depnode_list_free<T::rc>(thread, prev_set);
         info->last_set = NULL;
         info->prev_set = NULL;
         info->last_flag = 0; // no sets in this dephash entry
       } else {
-        npredecessors +=
-            __kmp_depnode_link_successor(gtid, thread, task, node, last_out);
+        npredecessors += T::link_successor(gtid, thread, task, node, last_out);
       }
-      __kmp_node_deref(thread, last_out);
+      T::deref(thread, last_out);
       if (!dep_barrier) {
-        info->last_out = __kmp_node_ref(node);
+        info->last_out = T::ref(node);
       } else {
         // if this is a sync point in the serial sequence, then the previous
         // outputs are guaranteed to be completed after the execution of this
@@ -423,27 +688,24 @@ __kmp_process_deps(kmp_int32 gtid, kmp_depnode_t *node, kmp_dephash_t **hash,
       if (info->last_flag == 0 || info->last_flag == dep->flag) {
         // last_set either didn't exist or of same dep kind
         // link node as successor of the last_out if any
-        npredecessors +=
-            __kmp_depnode_link_successor(gtid, thread, task, node, last_out);
+        npredecessors += T::link_successor(gtid, thread, task, node, last_out);
         // link node as successor of all nodes in the prev_set if any
-        npredecessors +=
-            __kmp_depnode_link_successor(gtid, thread, task, node, prev_set);
+        npredecessors += T::link_successor(gtid, thread, task, node, prev_set);
         if (dep_barrier) {
           // clean last_out and prev_set if any; don't touch last_set
-          __kmp_node_deref(thread, last_out);
+          T::deref(thread, last_out);
           info->last_out = NULL;
-          __kmp_depnode_list_free(thread, prev_set);
+          __kmp_depnode_list_free<T::rc>(thread, prev_set);
           info->prev_set = NULL;
         }
       } else { // last_set is of different dep kind, make it prev_set
         // link node as successor of all nodes in the last_set
-        npredecessors +=
-            __kmp_depnode_link_successor(gtid, thread, task, node, last_set);
+        npredecessors += T::link_successor(gtid, thread, task, node, last_set);
         // clean last_out if any
-        __kmp_node_deref(thread, last_out);
+        T::deref(thread, last_out);
         info->last_out = NULL;
         // clean prev_set if any
-        __kmp_depnode_list_free(thread, prev_set);
+        __kmp_depnode_list_free<T::rc>(thread, prev_set);
         if (!dep_barrier) {
           // move last_set to prev_set, new last_set will be allocated
           info->prev_set = last_set;
@@ -457,62 +719,133 @@ __kmp_process_deps(kmp_int32 gtid, kmp_depnode_t *node, kmp_dephash_t **hash,
       // 0 if last_set is empty, unchanged otherwise
       if (!dep_barrier) {
         info->last_flag = dep->flag; // store dep kind of the last_set
-        info->last_set = __kmp_add_node(thread, info->last_set, node);
+        info->last_set = __kmp_add_node<T::rc>(thread, info->last_set, node);
       }
       // check if we are processing MTX dependency
       if (dep->flag == KMP_DEP_MTX) {
-        if (info->mtx_lock == NULL) {
-          info->mtx_lock = (kmp_lock_t *)__kmp_allocate(sizeof(kmp_lock_t));
-          __kmp_init_lock(info->mtx_lock);
-        }
-        KMP_DEBUG_ASSERT(node->dn.mtx_num_locks < MAX_MTX_DEPS);
-        kmp_int32 m;
-        // Save lock in node's array
-        for (m = 0; m < MAX_MTX_DEPS; ++m) {
-          // sort pointers in decreasing order to avoid potential livelock
-          if (node->dn.mtx_locks[m] < info->mtx_lock) {
-            KMP_DEBUG_ASSERT(!node->dn.mtx_locks[node->dn.mtx_num_locks]);
-            for (int n = node->dn.mtx_num_locks; n > m; --n) {
-              // shift right all lesser non-NULL pointers
-              KMP_DEBUG_ASSERT(node->dn.mtx_locks[n - 1] != NULL);
-              node->dn.mtx_locks[n] = node->dn.mtx_locks[n - 1];
-            }
-            node->dn.mtx_locks[m] = info->mtx_lock;
-            break;
-          }
-        }
-        KMP_DEBUG_ASSERT(m < MAX_MTX_DEPS); // must break from loop
-        node->dn.mtx_num_locks++;
+        T::mutex_dep(thread, info, node, next_mutex_set);
       }
     }
   }
-  KA_TRACE(30, ("__kmp_process_deps<%d>: T#%d found %d predecessors\n", filter,
-                gtid, npredecessors));
+  KA_TRACE(30,
+           ("__kmp_process_deps<%s>: T#%d found %d predecessors (filter: %d)\n",
+            T::name, gtid, npredecessors, filter));
   return npredecessors;
 }
 
-#define NO_DEP_BARRIER (false)
-#define DEP_BARRIER (true)
+struct normal_deps {
+  static constexpr char name[] = "normal";
+  static constexpr bool rc = true;
+  static kmp_int32 link_successor(kmp_int32 gtid, kmp_info_t *thread,
+                                  kmp_task_t *task, kmp_depnode_t *source,
+                                  kmp_depnode_t *sink);
+  static kmp_int32 link_successor(kmp_int32 gtid, kmp_info_t *thread,
+                                  kmp_task_t *task, kmp_depnode_t *node,
+                                  kmp_depnode_list_t *plist);
+  static kmp_depnode_t *ref(kmp_depnode_t *node);
+  static void deref(kmp_info_t *thread, kmp_depnode_t *node);
+  static void mutex_dep(kmp_info_t *thread, kmp_dephash_entry_t *info,
+                        kmp_depnode_t *node, kmp_int32 &next_mutex_set);
+};
+
+kmp_int32 normal_deps::link_successor(kmp_int32 gtid, kmp_info_t *thread,
+                                      kmp_task_t *task, kmp_depnode_t *source,
+                                      kmp_depnode_t *sink) {
+  return __kmp_depnode_link_successor(gtid, thread, task, source, sink);
+}
 
-// returns true if the task has any outstanding dependence
-static bool __kmp_check_deps(kmp_int32 gtid, kmp_depnode_t *node,
-                             kmp_task_t *task, kmp_dephash_t **hash,
-                             bool dep_barrier, kmp_int32 ndeps,
-                             kmp_depend_info_t *dep_list,
-                             kmp_int32 ndeps_noalias,
-                             kmp_depend_info_t *noalias_dep_list) {
-  int i, n_mtxs = 0, dep_all = 0;
-#if KMP_DEBUG
-  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
+kmp_int32 normal_deps::link_successor(kmp_int32 gtid, kmp_info_t *thread,
+                                      kmp_task_t *task, kmp_depnode_t *node,
+                                      kmp_depnode_list_t *plist) {
+  return __kmp_depnode_link_successor(gtid, thread, task, node, plist);
+}
+
+kmp_depnode_t *normal_deps::ref(kmp_depnode_t *node) {
+  return __kmp_node_ref(node);
+}
+
+void normal_deps::deref(kmp_info_t *thread, kmp_depnode_t *node) {
+  __kmp_node_deref(thread, node);
+}
+
+void normal_deps::mutex_dep(kmp_info_t *thread, kmp_dephash_entry_t *info,
+                            kmp_depnode_t *node, kmp_int32 &next_mutex_set) {
+  if (info->mtx_lock == NULL) {
+    info->mtx_lock = (kmp_lock_t *)__kmp_allocate(sizeof(kmp_lock_t));
+    __kmp_init_lock(info->mtx_lock);
+  }
+  KMP_DEBUG_ASSERT(node->dn.mtx_num_locks < MAX_MTX_DEPS);
+  kmp_int32 m;
+  // Save lock in node's array
+  for (m = 0; m < MAX_MTX_DEPS; ++m) {
+    // sort pointers in decreasing order to avoid potential livelock
+    if (node->dn.mtx_locks[m] < info->mtx_lock) {
+      KMP_DEBUG_ASSERT(!node->dn.mtx_locks[node->dn.mtx_num_locks]);
+      for (int n = node->dn.mtx_num_locks; n > m; --n) {
+        // shift right all lesser non-NULL pointers
+        KMP_DEBUG_ASSERT(node->dn.mtx_locks[n - 1] != NULL);
+        node->dn.mtx_locks[n] = node->dn.mtx_locks[n - 1];
+      }
+      node->dn.mtx_locks[m] = info->mtx_lock;
+      break;
+    }
+  }
+  KMP_DEBUG_ASSERT(m < MAX_MTX_DEPS); // must break from loop
+  node->dn.mtx_num_locks++;
+}
+
+#if OMP_TASKGRAPH_EXPERIMENTAL
+struct taskgraph_deps {
+  static constexpr char name[] = "taskgraph";
+  static constexpr bool rc = false;
+  static kmp_int32 link_successor(kmp_int32 gtid, kmp_info_t *thread,
+                                  kmp_task_t *task, kmp_depnode_t *source,
+                                  kmp_depnode_t *sink);
+  static kmp_int32 link_successor(kmp_int32 gtid, kmp_info_t *thread,
+                                  kmp_task_t *task, kmp_depnode_t *node,
+                                  kmp_depnode_list_t *plist);
+  static kmp_depnode_t *ref(kmp_depnode_t *node) { return node; }
+  static void deref(kmp_info_t *thread, kmp_depnode_t *node) {}
+  static void mutex_dep(kmp_info_t *thread, kmp_dephash_entry_t *info,
+                        kmp_depnode_t *node, kmp_int32 &next_mutex_set);
+};
+
+kmp_int32 taskgraph_deps::link_successor(kmp_int32 gtid, kmp_info_t *thread,
+                                         kmp_task_t *task,
+                                         kmp_depnode_t *source,
+                                         kmp_depnode_t *sink) {
+  return __kmp_taskgraph_add_dep(thread, source, sink);
+}
+
+kmp_int32 taskgraph_deps::link_successor(kmp_int32 gtid, kmp_info_t *thread,
+                                         kmp_task_t *task, kmp_depnode_t *node,
+                                         kmp_depnode_list_t *plist) {
+  return __kmp_taskgraph_add_dep(thread, node, plist);
+}
+
+void taskgraph_deps::mutex_dep(kmp_info_t *thread, kmp_dephash_entry_t *info,
+                               kmp_depnode_t *node, kmp_int32 &next_mutex_set) {
+  if (info->set_num == -1) {
+    info->set_num = next_mutex_set++;
+  }
+  if (!node->dn.set_membership) {
+    node->dn.set_membership = __kmp_bitset_alloc(thread, 64);
+  }
+  __kmp_bitset_set(node->dn.set_membership, info->set_num);
+}
 #endif
-  KA_TRACE(20, ("__kmp_check_deps: T#%d checking dependences for task %p : %d "
-                "possibly aliased dependences, %d non-aliased dependences : "
-                "dep_barrier=%d .\n",
-                gtid, taskdata, ndeps, ndeps_noalias, dep_barrier));
+
+/// Search for aliased (same base address) dependencies in \c dep_list, and
+/// nullify duplicates.  Return TRUE if we have an 'all' dependency, FALSE
+/// otherwise.  Return number of mutex dependencies in *N_MTXS.
+static bool __kmp_filter_aliased_deps(kmp_int32 ndeps,
+                                      kmp_depend_info_t *dep_list,
+                                      kmp_task_t *task, int *n_mtxs) {
+  *n_mtxs = 0;
 
   // Filter deps in dep_list
   // TODO: Different algorithm for large dep_list ( > 10 ? )
-  for (i = 0; i < ndeps; i++) {
+  for (int i = 0; i < ndeps; i++) {
     if (dep_list[i].base_addr != 0 &&
         dep_list[i].base_addr != (kmp_intptr_t)KMP_SIZE_T_MAX) {
       KMP_DEBUG_ASSERT(
@@ -530,8 +863,8 @@ static bool __kmp_check_deps(kmp_int32 gtid, kmp_depnode_t *node,
       }
       if (dep_list[i].flag == KMP_DEP_MTX) {
         // limit number of mtx deps to MAX_MTX_DEPS per node
-        if (n_mtxs < MAX_MTX_DEPS && task != NULL) {
-          ++n_mtxs;
+        if (*n_mtxs < MAX_MTX_DEPS && task != NULL) {
+          ++(*n_mtxs);
         } else {
           dep_list[i].flag = KMP_DEP_OUT; // downgrade mutexinoutset to inout
         }
@@ -541,11 +874,2321 @@ static bool __kmp_check_deps(kmp_int32 gtid, kmp_depnode_t *node,
       // omp_all_memory dependence can be marked by compiler by either
       // (addr=0 && flag=0x80) (flag KMP_DEP_ALL), or (addr=-1).
       // omp_all_memory overrides all other dependences if any
-      dep_all = 1;
+      return true;
+    }
+  }
+  return false;
+}
+
+#if OMP_TASKGRAPH_EXPERIMENTAL
+// Round up a size to a power of two specified by val: Used to insert padding
+// between structures co-allocated using a single malloc() call
+// FIXME: We copy+pasted this, put it somewhere else instead.
+static size_t __kmp_round_up_to_val(size_t size, size_t val) {
+  if (size & (val - 1)) {
+    size &= ~(val - 1);
+    if (size <= KMP_SIZE_T_MAX - val) {
+      size += val; // Round up if there is no overflow.
+    }
+  }
+  return size;
+} // __kmp_round_up_to_val
+
+void *kmp_taskgraph_region::operator new(size_t size, kmp_info_t *thread) {
+  return __kmp_fast_allocate(thread, size);
+}
+
+void *kmp_taskgraph_region::operator new(size_t size, kmp_info_t *thread,
+                                         kmp_int32 num_nodes) {
+  size += num_nodes * sizeof(kmp_taskgraph_region_t *);
+  size = __kmp_round_up_to_val(size, sizeof(kmp_taskgraph_region_t *));
+  return __kmp_fast_allocate(thread, size);
+}
+
+kmp_taskgraph_region::kmp_taskgraph_region(
+    kmp_taskgraph_record_t *taskgraph, kmp_taskgraph_region_t **&alloc_chain,
+    kmp_taskgraph_node_t *node, kmp_taskgraph_region_t *parent) {
+  owner = taskgraph;
+  type = node ? TASKGRAPH_REGION_NODE : TASKGRAPH_REGION_WAIT;
+  task.node = node;
+  this->parent = parent;
+  *alloc_chain = this;
+  alloc_chain = &this->alloc_chain;
+}
+
+kmp_taskgraph_region::kmp_taskgraph_region(
+    kmp_taskgraph_record_t *taskgraph, kmp_taskgraph_region_t **&alloc_chain,
+    enum kmp_taskgraph_region_type type, kmp_int32 num_nodes,
+    kmp_taskgraph_region_t *parent) {
+  owner = taskgraph;
+  this->type = type;
+  inner.children = (kmp_taskgraph_region **)(this + 1); // trailing storage
+  inner.num_children = num_nodes;
+  this->parent = parent;
+  *alloc_chain = this;
+  alloc_chain = &this->alloc_chain;
+}
+
+static kmp_taskgraph_region_t *__kmp_taskgraph_region_alloc(
+    kmp_info_t *thread, kmp_taskgraph_record_t *taskgraph,
+    kmp_taskgraph_region_t **&alloc_chain, kmp_taskgraph_node_t *node,
+    kmp_taskgraph_region_t *parent) {
+  return new (thread)
+      kmp_taskgraph_region_t(taskgraph, alloc_chain, node, parent);
+}
+
+static kmp_taskgraph_region_t *__kmp_taskgraph_region_alloc(
+    kmp_info_t *thread, kmp_taskgraph_record_t *taskgraph,
+    kmp_taskgraph_region_t **&alloc_chain, enum kmp_taskgraph_region_type type,
+    kmp_int32 num_nodes, kmp_taskgraph_region_t *parent) {
+  return new (thread, num_nodes)
+      kmp_taskgraph_region_t(taskgraph, alloc_chain, type, num_nodes, parent);
+}
+
+static kmp_int32
+__kmp_taskgraph_topological_order(kmp_taskgraph_region_t *region,
+                                  kmp_taskgraph_region_t **order_out,
+                                  kmp_int32 *outidx) {
+  if (region->mark == TASKGRAPH_PERMANENT_MARK)
+    return region->level;
+
+  assert(region->mark != TASKGRAPH_TEMP_MARK);
+
+  region->mark = TASKGRAPH_TEMP_MARK;
+
+  kmp_int32 max_level = -1;
+  for (kmp_taskgraph_region_dep_t *s = region->predecessors; s; s = s->next) {
+    kmp_int32 pred_level =
+        __kmp_taskgraph_topological_order(s->region, order_out, outidx);
+    max_level = pred_level > max_level ? pred_level : max_level;
+  }
+
+  region->level = max_level + 1;
+  region->mark = TASKGRAPH_PERMANENT_MARK;
+  order_out[(*outidx)++] = region;
+
+  return region->level;
+}
+
+static void
+__kmp_taskgraph_region_chain_clear_marks(kmp_taskgraph_region_t *region) {
+  for (; region; region = region->next)
+    region->mark = TASKGRAPH_UNMARKED;
+}
+
+static void
+__kmp_taskgraph_region_chain_prune(kmp_taskgraph_region_t **region_p) {
+  kmp_taskgraph_region_t *pruned_region = nullptr, *region = *region_p;
+  kmp_taskgraph_region_t **pruned_region_p = &pruned_region;
+
+  TGDBG("pruning worklist...\n");
+
+  // NOTE: Pruning and deletion look the same here with respect to the handling
+  // of the worklist, but deleted nodes are freed from the taskgraph structure
+  // during cleanup, whereas combined nodes are retained.
+  for (; region; region = region->next) {
+    if (region->mark == TASKGRAPH_COMBINED || region->mark == TASKGRAPH_DELETED)
+      *pruned_region_p = region->next;
+    else {
+      *pruned_region_p = region;
+      pruned_region_p = &region->next;
+    }
+  }
+
+  *pruned_region_p = nullptr;
+  *region_p = pruned_region;
+}
+
+static kmp_int32 __kmp_region_deplist_len(kmp_taskgraph_region_dep_t *list) {
+  kmp_int32 len = 0;
+  for (; list; list = list->next)
+    ++len;
+  return len;
+}
+
+void __kmp_region_deplist_free(kmp_info_t *thread,
+                               kmp_taskgraph_region_dep_t *list) {
+  while (list) {
+    kmp_taskgraph_region_dep_t *next = list->next;
+    __kmp_fast_free(thread, list);
+    list = next;
+  }
+}
+
+static void __kmp_region_dep_recycle(kmp_taskgraph_region_dep_t **recycled,
+                                     kmp_taskgraph_region_dep_t *dep) {
+  dep->next = *recycled;
+  *recycled = dep;
+}
+
+void __kmp_region_deplist_recycle(kmp_taskgraph_region_dep_t **recycled,
+                                  kmp_taskgraph_region_dep_t *list) {
+  while (list) {
+    kmp_taskgraph_region_dep_t *next = list->next;
+    __kmp_region_dep_recycle(recycled, list);
+    list = next;
+  }
+}
+
+static bool __kmp_taskgraph_collapse_sequence(
+    kmp_info_t *thread, kmp_taskgraph_record_t *taskgraph,
+    kmp_taskgraph_region_t **&alloc_chain, kmp_taskgraph_region_t **region_p,
+    kmp_taskgraph_region_t *parent, kmp_int32 &stamp) {
+  kmp_taskgraph_region_t *region = *region_p;
+  kmp_taskgraph_region_t *chain_start = region;
+  kmp_taskgraph_region_t *chain_end = region;
+  kmp_int32 chain_len = 1;
+
+  if (region->type == TASKGRAPH_REGION_ENTRY)
+    return false;
+
+  while (__kmp_region_deplist_len(chain_end->successors) == 1) {
+    kmp_taskgraph_region_t *past_end = chain_end->successors->region;
+    if (__kmp_region_deplist_len(past_end->predecessors) == 1) {
+      if (past_end->type == TASKGRAPH_REGION_EXIT)
+        break;
+      else {
+        chain_end = past_end;
+        ++chain_len;
+      }
+    } else
       break;
+  }
+
+  if (chain_len <= 1)
+    return false;
+
+  auto *seq_region = __kmp_taskgraph_region_alloc(
+      thread, taskgraph, alloc_chain, TASKGRAPH_REGION_SEQUENTIAL, chain_len,
+      parent);
+  TGDBG("allocated new seq region: %p (length %d)\n", seq_region, chain_len);
+  kmp_taskgraph_region_t **worklist_p = region_p;
+  *worklist_p = seq_region;
+  seq_region->next = chain_start->next;
+  kmp_int32 level = -1;
+  for (kmp_int32 i = 0; i < chain_len; i++) {
+    seq_region->inner.children[i] = chain_start;
+    TGDBG("mark node %p as combined\n", chain_start);
+    chain_start->mark = TASKGRAPH_COMBINED;
+    chain_start->timestamp = stamp;
+    chain_start->parent = seq_region;
+    // The level of the sequence is the level of the first node.
+    if (level == -1)
+      level = chain_start->level;
+
+    if (i < chain_len - 1) {
+      chain_start = chain_start->successors->region;
+    }
+  }
+
+  seq_region->level = level;
+  seq_region->predecessors = seq_region->inner.children[0]->predecessors;
+  seq_region->successors =
+      seq_region->inner.children[chain_len - 1]->successors;
+  seq_region->inner.children[0]->predecessors = nullptr;
+  seq_region->inner.children[chain_len - 1]->successors = nullptr;
+
+  // Update predecessors to point to new seq region.
+  for (kmp_taskgraph_region_dep_t *pred = seq_region->predecessors; pred;
+       pred = pred->next) {
+    for (kmp_taskgraph_region_dep_t *succ = pred->region->successors; succ;
+         succ = succ->next) {
+      if (succ->region == seq_region->inner.children[0]) {
+        succ->region = seq_region;
+      }
+    }
+  }
+
+  // Update successors to point back to new seq region.
+  for (kmp_taskgraph_region_dep_t *succ = seq_region->successors; succ;
+       succ = succ->next) {
+    for (kmp_taskgraph_region_dep_t *pred = succ->region->predecessors; pred;
+         pred = pred->next) {
+      if (pred->region == seq_region->inner.children[chain_len - 1]) {
+        pred->region = seq_region;
+      }
+    }
+  }
+
+  return true;
+}
+
+static const char *
+__kmp_taskgraph_region_type_name(kmp_taskgraph_region_type type);
+
+static void __kmp_taskgraph_region_dfs(kmp_taskgraph_region_t *region,
+                                       kmp_taskgraph_region_t **order,
+                                       kmp_int32 &idx, bool use_preds) {
+  if (order) {
+    region->timestamp = --idx;
+    order[idx] = region;
+  }
+  region->mark = TASKGRAPH_TEMP_MARK;
+  for (kmp_taskgraph_region_dep_t *reg = use_preds ? region->predecessors
+                                                   : region->successors;
+       reg; reg = reg->next) {
+    if (reg->region->mark == TASKGRAPH_UNMARKED)
+      __kmp_taskgraph_region_dfs(reg->region, order, idx, use_preds);
+  }
+}
+
+#if defined(DEBUG_TASKGRAPH) && defined(CHECK_WORKLIST)
+
+static void __kmp_taskgraph_region_gather_deps(
+    kmp_info_t *thread, kmp_taskgraph_record_t *taskgraph,
+    kmp_taskgraph_region_t *region, kmp_taskgraph_region_dep_t **deplist,
+    bool &ok) {
+  for (kmp_taskgraph_region_dep_t *dep = *deplist; dep; dep = dep->next) {
+    if (dep->region == region)
+      return;
+  }
+
+  *deplist = __kmp_region_deplist_add(thread, &taskgraph->recycled_deps, region,
+                                      *deplist);
+
+  for (kmp_taskgraph_region_dep_t *pred = region->predecessors; pred;
+       pred = pred->next) {
+    if (pred->region->mark == TASKGRAPH_DELETED) {
+      fprintf(stderr, "*** Region %p's predecessor %p is a deleted node\n",
+              region, pred->region);
+      ok = false;
+    }
+    __kmp_taskgraph_region_gather_deps(thread, taskgraph, pred->region, deplist,
+                                       ok);
+  }
+
+  for (kmp_taskgraph_region_dep_t *succ = region->successors; succ;
+       succ = succ->next) {
+    if (succ->region->mark == TASKGRAPH_DELETED) {
+      fprintf(stderr, "*** Region %p's successor %p is a deleted node\n",
+              region, succ->region);
+      ok = false;
+    }
+    __kmp_taskgraph_region_gather_deps(thread, taskgraph, succ->region, deplist,
+                                       ok);
+  }
+}
+
+static bool __kmp_taskgraph_region_worklist_check(
+    kmp_info_t *thread, kmp_taskgraph_record_t *taskgraph,
+    kmp_taskgraph_region_t *region, const char *where) {
+  kmp_taskgraph_region_dep_t *collected_nodes = nullptr;
+  bool ok = true;
+  __kmp_taskgraph_region_gather_deps(thread, taskgraph, region,
+                                     &collected_nodes, ok);
+
+  // Check all collected nodes are in the region's worklist.
+  for (kmp_taskgraph_region_dep_t *cn = collected_nodes; cn; cn = cn->next) {
+    bool in_list = false;
+    for (kmp_taskgraph_region_t *r = region; r; r = r->next) {
+      if (r == cn->region) {
+        in_list = true;
+        break;
+      }
+    }
+    if (!in_list) {
+      fprintf(stderr,
+              "*** Region %p is in dependency graph but not worklist (%s)\n",
+              cn->region, where);
+      ok = false;
+    }
+  }
+
+  for (kmp_taskgraph_region_t *r = region; r; r = r->next) {
+    bool in_list = false;
+    for (kmp_taskgraph_region_dep_t *cn = collected_nodes; cn; cn = cn->next) {
+      if (r == cn->region) {
+        in_list = true;
+        break;
+      }
+    }
+    if (!in_list) {
+      fprintf(stderr,
+              "*** Region %p is in worklist but not dependency graph (%s)\n", r,
+              where);
+      ok = false;
+    }
+  }
+
+  __kmp_region_deplist_recycle(&taskgraph->recycled_deps, collected_nodes);
+
+  return ok;
+}
+#else
+static bool __kmp_taskgraph_region_worklist_check(
+    kmp_info_t *thread, kmp_taskgraph_record_t *taskgraph,
+    kmp_taskgraph_region_t *region, const char *where) {
+  return true;
+}
+#endif
+
+static kmp_taskgraph_region_t *__kmp_taskgraph_region_dom_intersect(
+    kmp_taskgraph_region_t **order, kmp_taskgraph_region_t **doms,
+    kmp_taskgraph_region_t *b1, kmp_taskgraph_region_t *b2) {
+  kmp_int32 finger1 = b1->timestamp;
+  kmp_int32 finger2 = b2->timestamp;
+  while (finger1 != finger2) {
+    while (finger1 < finger2)
+      finger1 = doms[finger1]->timestamp;
+    while (finger2 < finger1)
+      finger2 = doms[finger2]->timestamp;
+  }
+  return order[finger1];
+}
+
+static void __kmp_taskgraph_region_doms(kmp_taskgraph_region_t **order,
+                                        kmp_taskgraph_region_t **doms,
+                                        kmp_int32 worklist_length,
+                                        bool postdom) {
+  bool changed = true;
+  // Set doms[start_node] <- start_node
+  doms[worklist_length - 1] = order[worklist_length - 1];
+  order[worklist_length - 1]->mark = TASKGRAPH_PERMANENT_MARK;
+  while (changed) {
+    changed = false;
+    for (int n = 0; n < worklist_length - 1; n++) {
+      kmp_taskgraph_region_t *b = order[n];
+      kmp_taskgraph_region_t *new_idom = nullptr;
+      for (kmp_taskgraph_region_dep_t *pred = postdom ? b->successors
+                                                      : b->predecessors;
+           pred; pred = pred->next) {
+        if (pred->region->mark == TASKGRAPH_PERMANENT_MARK) {
+          new_idom = pred->region;
+          break;
+        }
+      }
+      for (kmp_taskgraph_region_dep_t *pred = postdom ? b->successors
+                                                      : b->predecessors;
+           pred; pred = pred->next) {
+        if (pred->region == new_idom)
+          continue;
+        if (doms[pred->region->timestamp]) {
+          new_idom = __kmp_taskgraph_region_dom_intersect(
+              order, doms, pred->region, new_idom);
+        }
+      }
+      if (doms[b->timestamp] != new_idom) {
+        doms[b->timestamp] = new_idom;
+        order[b->timestamp]->mark = TASKGRAPH_PERMANENT_MARK;
+        changed = true;
+      }
+    }
+  }
+}
+
+static bool __kmp_taskgraph_region_mutex_p(kmp_taskgraph_region_t *reg) {
+  if (reg->type == TASKGRAPH_REGION_NODE)
+    return reg->mutexset != nullptr;
+  return false;
+}
+
+// This function collapses graph regions with forms like this:
+//
+//  1.    A(pp)    2.        A               3.     A(pp)
+//       / \              /     \                 /   \
+//      B   C           B(pp)    E(pp)           B(pp)  E
+//       \ /           / \       / \            / \    /
+//       D(*)         C   D     F   G           C  D  /
+//                     \   \   /   /             \ | /
+//                      `---H(*)--'               F(*)
+//
+// We look for a node with more than one predecessor (*), where each of those
+// predecessors has a single successor and a single predecessor (pp).  We group
+// nodes by which pp (predecessor-predecessor) they have: for (1), nodes B & C
+// share a pp; for (2), C & D share a pp, and F & G share a pp; for (3), C & D
+// share a pp, and E has a separate pp.
+//
+// We choose the pp with the highest level ("furthest down the graph"), and
+// collapse the subgraph into a parallel region.
+
+static bool __kmp_taskgraph_collapse_par_exclusive(
+    kmp_info_t *thread, kmp_taskgraph_record_t *taskgraph,
+    kmp_taskgraph_region_t **&alloc_chain, kmp_taskgraph_region_t **region_p,
+    kmp_taskgraph_region_t *parent, kmp_int32 &stamp) {
+  kmp_taskgraph_region_t *region = *region_p;
+  kmp_int32 num_predecessors = __kmp_region_deplist_len(region->predecessors);
+
+  TGDBG("predecessors %d, successors %d\n",
+        __kmp_region_deplist_len(region->predecessors),
+        __kmp_region_deplist_len(region->successors));
+
+  if (num_predecessors <= 1)
+    return false;
+
+  TGDBG("found multiple predecessors, creating parallel/unordered region\n");
+  kmp_taskgraph_region_dep_t *pred_preds = nullptr;
+  kmp_int32 highest_level = -1;
+
+  for (kmp_taskgraph_region_dep_t *pred = region->predecessors; pred;
+       pred = pred->next) {
+    TGDBG("consider predecessor: %p\n", pred->region);
+    TGDBG("-- successors %d, predecessors %d\n",
+          __kmp_region_deplist_len(pred->region->successors),
+          __kmp_region_deplist_len(pred->region->predecessors));
+    if (highest_level == -1 || pred->region->level > highest_level)
+      highest_level = pred->region->level;
+    kmp_taskgraph_region_t *pred_region = pred->region;
+    if (__kmp_region_deplist_len(pred_region->successors) != 1)
+      continue;
+    if (__kmp_region_deplist_len(pred_region->predecessors) != 1)
+      continue;
+    bool in_list = false;
+    TGDBG("pp region: %p (%s)\n", pred_region->predecessors->region,
+          __kmp_taskgraph_region_type_name(
+              pred_region->predecessors->region->type));
+    kmp_taskgraph_region_t *pp_region = pred_region->predecessors->region;
+    for (kmp_taskgraph_region_dep_t *pp = pred_preds; pp; pp = pp->next) {
+      if (pp->region == pp_region) {
+        in_list = true;
+        break;
+      }
+    }
+    if (!in_list) {
+      pred_preds = __kmp_region_deplist_add(thread, &taskgraph->recycled_deps,
+                                            pp_region, pred_preds);
+      TGDBG("add %p to list: len(pred_preds)=%d\n", pp_region,
+            __kmp_region_deplist_len(pred_preds));
+    }
+  }
+
+  kmp_int32 num_pps = __kmp_region_deplist_len(pred_preds);
+  if (num_pps == 0) {
+    TGDBG("no collapsible regions, bailing out\n");
+    return false;
+  }
+  TGDBG("found %d predecessor-predecessors\n", num_pps);
+  TGDBG("highest pred level: %d\n", highest_level);
+
+  kmp_int32 pp_idx = 0;
+
+  bool changed = false;
+
+  for (kmp_taskgraph_region_dep_t *pp = pred_preds; pp; pp = pp->next) {
+    kmp_taskgraph_region_dep_t *par_succs = nullptr;
+    kmp_taskgraph_region_dep_t *par_preds = nullptr;
+    kmp_int32 preds_for_pp = 0;
+    bool any_mutex_p = false;
+    for (kmp_taskgraph_region_dep_t *pred = region->predecessors; pred;
+         pred = pred->next) {
+      kmp_taskgraph_region_t *pred_region = pred->region;
+      if (!pred_region->predecessors)
+        continue;
+      if (pred_region->level < highest_level)
+        continue;
+      if (__kmp_region_deplist_len(pred_region->predecessors) != 1 ||
+          __kmp_region_deplist_len(pred_region->successors) != 1)
+        continue;
+      TGDBG("counting pred region: %p (%s)\n", pred_region,
+            __kmp_taskgraph_region_type_name(pred_region->type));
+      if (pred_region->predecessors->region == pp->region) {
+        ++preds_for_pp;
+        if (__kmp_taskgraph_region_mutex_p(pred_region))
+          any_mutex_p = true;
+      }
+    }
+    TGDBG("found %d preds for pp region %p\n", preds_for_pp, pp->region);
+    if (preds_for_pp < 2)
+      continue;
+    kmp_taskgraph_region_type region_type =
+        any_mutex_p ? TASKGRAPH_REGION_EXCLUSIVE : TASKGRAPH_REGION_PARALLEL;
+    auto *par_region = __kmp_taskgraph_region_alloc(
+        thread, taskgraph, alloc_chain, region_type, preds_for_pp, parent);
+    changed = true;
+    TGDBG("allocated %s region: %p\n",
+          region_type == TASKGRAPH_REGION_EXCLUSIVE ? "exclusive" : "parallel",
+          par_region);
+    kmp_taskgraph_region_dep_t *pred = region->predecessors;
+    kmp_int32 level = -1;
+    bool found_reduction_data = false;
+    for (kmp_int32 i = 0; pred; pred = pred->next) {
+      kmp_taskgraph_region_t *pred_region = pred->region;
+      TGDBG("considering pred region: %p\n", pred_region);
+      if (!pred_region->predecessors) {
+        TGDBG("bailing (no predecessors)\n");
+        continue;
+      }
+      if (pred_region->predecessors->region != pp->region) {
+        TGDBG("bailing (wrong pp region)\n");
+        continue;
+      }
+      if (__kmp_region_deplist_len(pred_region->predecessors) != 1 ||
+          __kmp_region_deplist_len(pred_region->successors) != 1) {
+        TGDBG("bailing (non-unit pred/succ list length)\n");
+        continue;
+      }
+      TGDBG("process region %p (%d/%d), level %d\n", pred->region, i + 1,
+            preds_for_pp, pred_region->level);
+      par_region->inner.children[i] = pred_region;
+      pred_region->mark = TASKGRAPH_COMBINED;
+      pred_region->timestamp = stamp;
+      pred_region->parent = par_region;
+
+      // Reduction handling.  The reduction input data is now attached to one
+      // of the tasks participating in the reduction.  Move it to the enclosing
+      // parallel region instead.
+      if (pred_region->type == TASKGRAPH_REGION_NODE &&
+          pred_region->task.node->reduce_input) {
+        // We should only be doing this once per par region.
+        assert(!par_region->reduce_input);
+        par_region->reduce_input = pred_region->task.node->reduce_input;
+        pred_region->task.node->reduce_input = nullptr;
+        found_reduction_data = true;
+      }
+
+      // We expect all the predecessor regions to be at the same level.
+      if (level == -1)
+        level = pred_region->level;
+      else
+        assert(level == pred_region->level);
+      if (!par_succs) {
+        // Copy one list of predecessors/successors for the predecessor region.
+        // We know these are of length one by checks above.  We'll re-use them
+        // for the created parallel region.
+        par_preds = pred_region->predecessors;
+        par_succs = pred_region->successors;
+        pred_region->predecessors = nullptr;
+        pred_region->successors = nullptr;
+      }
+      i++;
+    }
+    par_region->level = level;
+    par_region->predecessors = par_preds;
+    par_region->successors = par_succs;
+
+    if (region->type == TASKGRAPH_REGION_WAIT && !found_reduction_data) {
+      // If we have no reduction data, we will not create a taskgroup for this
+      // parallel region at replay time, so we don't need to terminate/discard
+      // that region when we're done.  Clear the taskloop_task flag.
+      region->task.node->taskloop_task = false;
+    }
+
+    // Add the new parallel region to the worklist. FIXME: We're reprocessing
+    // the 'region' node here -- we don't need to do that if it's fully
+    // consumed.)
+    par_region->next = region->next;
+    region->next = par_region;
+  }
+
+#ifdef DEBUG_TASKGRAPH
+  TGDBG("before pred fixup:\n");
+  for (kmp_taskgraph_region_dep_t *pred = region->predecessors; pred;
+       pred = pred->next) {
+    TGDBG("region %p, pred region: %p\n", region, pred->region);
+  }
+#endif
+
+  // Now, fix up predecessor list for 'region', and successor lists for each
+  // predecessor-predecessor.
+  kmp_taskgraph_region_dep_t **dep_p = &region->predecessors;
+  while (*dep_p) {
+    kmp_taskgraph_region_dep_t *dep = *dep_p;
+    if (dep->region->mark == TASKGRAPH_COMBINED) {
+      if (!dep->region->successors) {
+        dep->region = dep->region->parent;
+        dep_p = &dep->next;
+      } else {
+        kmp_taskgraph_region_dep_t *next = dep->next;
+        __kmp_region_dep_recycle(&taskgraph->recycled_deps, dep);
+        *dep_p = next;
+      }
+    } else {
+      dep_p = &dep->next;
     }
   }
 
+#ifdef DEBUG_TASKGRAPH
+  TGDBG("after pred fixup:\n");
+  for (kmp_taskgraph_region_dep_t *pred = region->predecessors; pred;
+       pred = pred->next) {
+    TGDBG("region %p, pred region: %p\n", region, pred->region);
+  }
+#endif
+
+  for (kmp_taskgraph_region_dep_t *pp = pred_preds; pp; pp = pp->next) {
+    kmp_taskgraph_region_t *pp_region = pp->region;
+    dep_p = &pp_region->successors;
+    while (*dep_p) {
+      kmp_taskgraph_region_dep_t *dep = *dep_p;
+      if (dep->region->mark == TASKGRAPH_COMBINED) {
+        if (!dep->region->predecessors) {
+          dep->region = dep->region->parent;
+          dep_p = &dep->next;
+        } else {
+          kmp_taskgraph_region_dep_t *next = dep->next;
+          __kmp_region_dep_recycle(&taskgraph->recycled_deps, dep);
+          *dep_p = next;
+        }
+      } else {
+        dep_p = &dep->next;
+      }
+    }
+  }
+
+  return changed;
+}
+
+static void __kmp_taskgraph_region_dot(kmp_taskgraph_region_t *region,
+                                       const char *name) {
+  fprintf(stderr, "digraph %s {\n", name);
+  for (kmp_taskgraph_region_t *r = region; r; r = r->next) {
+    if (r->mark == TASKGRAPH_DELETED) {
+      fprintf(stderr, "\"%p\" [shape=box, label=\"%p(%s) (deleted)\"]\n", r, r,
+              __kmp_taskgraph_region_type_name(r->type));
+    } else if (r->level == -1) {
+      fprintf(stderr, "\"%p\" [shape=box, label=\"%p(%s) (new)\"]\n", r, r,
+              __kmp_taskgraph_region_type_name(r->type));
+    } else {
+      fprintf(stderr, "\"%p\" [shape=box, label=\"%p(%s)\"]\n", r, r,
+              __kmp_taskgraph_region_type_name(r->type));
+    }
+    for (kmp_taskgraph_region_dep_t *succ = r->successors; succ;
+         succ = succ->next) {
+      fprintf(stderr, "  \"%p\" -> \"%p\" [color=green]\n", r, succ->region);
+    }
+    for (kmp_taskgraph_region_dep_t *pred = r->predecessors; pred;
+         pred = pred->next) {
+      fprintf(stderr, "  \"%p\" -> \"%p\" [color=red, constraint=false]\n", r,
+              pred->region);
+    }
+  }
+  fprintf(stderr, "}\n");
+}
+
+/// Wrap a single-entry / single-exit (SESE) irreducible region into one
+/// TASKGRAPH_REGION_IRREDUCIBLE region.
+//
+// `entryregion` and `exitregion`  are the SESE region's boundary
+// sentinels, and `members` (length `num_children`) are its interior nodes,
+// which we know at this point to be non-series/parallel reducible.  Because
+// the region is SESE, every edge into the interior comes through the entry and
+// every edge out of it leaves through the exit, so we can collapse the
+// interior into one node without disturbing anything outside the region.
+//
+// We collect the members as the children of one IRREDUCIBLE region and
+// *retain* their child<->child (intra-kernel) predecessor/successor edges --
+// those edges are the kernel's internal DAG.  Edges to the entry/exit
+// sentinels are dropped from the children and replaced by a single
+// entry -> irreducible -> exit bracket, so the surrounding graph becomes
+// trivially series-parallel around the kernel and the build loop makes
+// progress.
+
+static kmp_taskgraph_region_t *__kmp_taskgraph_carve_irreducible(
+    kmp_info_t *thread, kmp_taskgraph_record_t *taskgraph,
+    kmp_taskgraph_region_t **&alloc_chain, kmp_taskgraph_region_t *entryregion,
+    kmp_taskgraph_region_t *exitregion, kmp_taskgraph_region_t **members,
+    kmp_int32 num_children) {
+  auto *irr = __kmp_taskgraph_region_alloc(thread, taskgraph, alloc_chain,
+                                           TASKGRAPH_REGION_IRREDUCIBLE,
+                                           num_children, nullptr);
+
+  for (kmp_int32 c = 0; c < num_children; c++) {
+    kmp_taskgraph_region_t *r = members[c];
+    irr->inner.children[c] = r;
+    r->mark = TASKGRAPH_COMBINED;
+    r->parent = irr;
+    // We've clobbered level info already: this isn't valid now, so don't
+    // pretend it is.
+    r->level = -1;
+
+    // Drop the child's edges to the entry/exit sentinels; intra-kernel edges
+    // (to other children) are kept as the irreducible region's structure.
+    kmp_taskgraph_region_dep_t **predp = &r->predecessors;
+    while (*predp) {
+      if ((*predp)->region == entryregion) {
+        kmp_taskgraph_region_dep_t *dead = *predp;
+        *predp = dead->next;
+        __kmp_region_dep_recycle(&taskgraph->recycled_deps, dead);
+      } else {
+        predp = &(*predp)->next;
+      }
+    }
+    kmp_taskgraph_region_dep_t **succp = &r->successors;
+    while (*succp) {
+      if ((*succp)->region == exitregion) {
+        kmp_taskgraph_region_dep_t *dead = *succp;
+        *succp = dead->next;
+        __kmp_region_dep_recycle(&taskgraph->recycled_deps, dead);
+      } else {
+        succp = &(*succp)->next;
+      }
+    }
+  }
+  irr->level = -1;
+
+  // Bracket the kernel: entry -> irr -> exit (collapse the fan-in/out into a
+  // single edge each way).  Because the region is SESE, every successor of the
+  // entry leads into the interior and every predecessor of the exit comes from
+  // the interior, so replacing those lists wholesale is safe.
+  __kmp_region_deplist_recycle(&taskgraph->recycled_deps,
+                               entryregion->successors);
+  entryregion->successors =
+      __kmp_region_deplist_add(thread, &taskgraph->recycled_deps, irr, nullptr);
+  __kmp_region_deplist_recycle(&taskgraph->recycled_deps,
+                               exitregion->predecessors);
+  exitregion->predecessors =
+      __kmp_region_deplist_add(thread, &taskgraph->recycled_deps, irr, nullptr);
+  irr->predecessors = __kmp_region_deplist_add(
+      thread, &taskgraph->recycled_deps, entryregion, nullptr);
+  irr->successors = __kmp_region_deplist_add(thread, &taskgraph->recycled_deps,
+                                             exitregion, nullptr);
+
+  // Splice the irreducible region into the worklist right after the entry
+  // sentinel; the now-COMBINED children are pruned by the caller's next pass.
+  irr->next = entryregion->next;
+  entryregion->next = irr;
+  return irr;
+}
+
+// Decompose the irreducible residual into its nested single-entry / single-exit
+// (SESE) regions using immediate dominators and postdominators, and carve each
+// minimal (leaf) SESE region into its own IRREDUCIBLE node.  Only the genuine
+// non-series-parallel knots become IRREDUCIBLE; the surrounding reducible /
+// reduction structure is left for the series/parallel collapse to handle on a
+// subsequent pass of the build loop.  Returns true if at least one region was
+// carved.
+//
+// On a DAG with a single source (ENTRY) and single sink (EXIT), a pair
+// (entry, exit) bounds a canonical SESE region iff exit = ipdom[entry] and
+// entry = idom[exit].  Read: every path leaving the entry reconverges at the
+// (single) exit and every path reaching the exit comes through the (single)
+// entry, so everything strictly between the entry and exit is enterable
+// only via the entry and leavable only via the exit.  These pairs form a
+// laminar (properly nested) family; a leaf is one whose interior contains no
+// other pair's entry.
+//
+// After the series/parallel collapse and the transitive-drop / twin-merge
+// passes have run to fixpoint, any surviving leaf region is genuinely
+// irreducible -- a clean series/parallel interior would already have been
+// collapsed -- so carving leaves is exactly right and preserves everything
+// reducible around the knots.
+
+static bool __kmp_taskgraph_carve_sese(kmp_info_t *thread,
+                                       kmp_taskgraph_record_t *taskgraph,
+                                       kmp_taskgraph_region_t **&alloc_chain,
+                                       kmp_taskgraph_region_t **order,
+                                       kmp_taskgraph_region_t **idom,
+                                       kmp_int32 worklist_length,
+                                       kmp_taskgraph_region_t *exitregion) {
+  kmp_int32 n = worklist_length;
+
+  // At the start, we already have immediate-dominator information.  The
+  // 'order' array points to regions in reverse postorder, and each region
+  // has a timestamp field that contains its own index in that array.
+  // The 'idom' array is indexed by reverse-postorder (i.e. the region's
+  // timestamp field), which means we should copy that timestamp field before
+  // the second (post) dominator-finding calls below.  We're not otherwise
+  // using the level field at this point.
+  for (kmp_int32 i = 0; i < n; i++) {
+    order[i]->level = order[i]->timestamp;
+  }
+
+  // Compute immediate postdominators.  Mirror the dominator setup: DFS from the
+  // EXIT sentinel following predecessors so the exit lands at index n-1, then
+  // run the same dominator solver in postdom mode (which walks successors).
+  // Reset marks first: after the dominator solve every region is PERMANENT, but
+  // region_dfs only recurses into UNMARKED neighbours.
+  kmp_taskgraph_region_t **order_pd =
+      (kmp_taskgraph_region_t **)__kmp_fast_allocate(
+          thread, n * sizeof(kmp_taskgraph_region_t *));
+  kmp_taskgraph_region_t **ipdom =
+      (kmp_taskgraph_region_t **)__kmp_fast_allocate(
+          thread, n * sizeof(kmp_taskgraph_region_t *));
+  memset(ipdom, 0, n * sizeof(kmp_taskgraph_region_t *));
+  for (kmp_int32 i = 0; i < n; i++)
+    order[i]->mark = TASKGRAPH_UNMARKED;
+  kmp_int32 cursor = n;
+  __kmp_taskgraph_region_dfs(exitregion, order_pd, cursor, /*use_preds=*/true);
+  assert(cursor == 0);
+  __kmp_taskgraph_region_doms(order_pd, ipdom, n, /*postdom=*/true);
+
+#ifdef DEBUG_TASKGRAPH
+  fprintf(stderr, "digraph {\n");
+  for (kmp_int32 i = 0; i < worklist_length; i++) {
+    kmp_taskgraph_region_t *b = order[i];
+    for (kmp_taskgraph_region_dep_t *succ = b->successors; succ;
+         succ = succ->next) {
+      fprintf(stderr, "  \"%d\" -> \"%d\"\n", b->timestamp,
+              succ->region->timestamp);
+    }
+    fprintf(stderr, "  \"%d\" -> \"%d\" [color=green, constraint=false]\n",
+            b->timestamp, idom[b->timestamp]->timestamp);
+    fprintf(stderr, "  \"%d\" -> \"%d\" [color=red, constraint=false]\n",
+            b->timestamp, ipdom[b->timestamp]->timestamp);
+  }
+  fprintf(stderr, "}\n");
+#endif
+
+  auto dominates = [&](kmp_taskgraph_region_t *a,
+                       kmp_taskgraph_region_t *b) -> bool {
+    kmp_taskgraph_region_t *cur = b;
+    while (true) {
+      // A node dominates itself.
+      if (cur == a)
+        return true;
+      kmp_taskgraph_region_t *anc = idom[cur->level];
+      // We've reached the entry node without encountering 'a':
+      // 'a' does not dominate 'b'.
+      if (anc == cur)
+        return false;
+      cur = anc;
+    }
+  };
+  auto postdominates = [&](kmp_taskgraph_region_t *a,
+                           kmp_taskgraph_region_t *b) -> bool {
+    kmp_taskgraph_region_t *cur = b;
+    while (true) {
+      // A node postdominates itself.
+      if (cur == a)
+        return true;
+      kmp_taskgraph_region_t *anc = ipdom[cur->timestamp];
+      // We've reached the exit node without encountering 'a':
+      // 'a' does not postdominate 'b'.
+      if (anc == cur)
+        return false;
+      cur = anc;
+    }
+  };
+
+  // Matched SESE pairs: matched[i] is pdom(i) if matched[i] starts the SESE
+  // region and pdom(i) ends it.
+  kmp_taskgraph_region_t **matched =
+      (kmp_taskgraph_region_t **)__kmp_fast_allocate(
+          thread, n * sizeof(kmp_taskgraph_region_t *));
+  for (kmp_int32 i = 0; i < n; i++) {
+    matched[i] = nullptr;
+    kmp_taskgraph_region_t *region = order[i];
+    kmp_taskgraph_region_t *pdom = ipdom[region->timestamp];
+    // Only true for exit region.
+    if (region == pdom)
+      continue;
+    if (idom[pdom->level] == region)
+      matched[i] = pdom;
+  }
+
+  // Carve every leaf pair with a non-empty interior.  Leaves are vertex-
+  // disjoint and each carve only rewires edges incident to its own interior,
+  // so carving them all in one pass is safe.  Membership uses the pre-carve
+  // idom/ipdom relations, which the carve does not modify.
+  bool carved = false;
+  kmp_taskgraph_region_t **members =
+      (kmp_taskgraph_region_t **)__kmp_fast_allocate(
+          thread, n * sizeof(kmp_taskgraph_region_t *));
+  for (kmp_int32 i = 0; i < n; i++) {
+    if (!matched[i])
+      continue;
+    kmp_taskgraph_region_t *entry = order[i];
+    kmp_taskgraph_region_t *exit = matched[i];
+
+    // Leaf test: no other matched pair's entry lies strictly inside
+    // (entry, exit).
+    bool is_leaf = true;
+    for (kmp_int32 k = 0; k < n && is_leaf; k++) {
+      if (k == i || !matched[k])
+        continue;
+      kmp_taskgraph_region_t *region = order[k];
+      if (region == entry || region == exit)
+        continue;
+      if (dominates(entry, region) && postdominates(exit, region))
+        is_leaf = false;
+    }
+    if (!is_leaf)
+      continue;
+
+    // Gather the interior: nodes dominated by entry and postdominated by exit,
+    // other than the boundary sentinels (and skipping anything already carved).
+    kmp_int32 num_members = 0;
+    for (kmp_int32 m = 0; m < n; m++) {
+      kmp_taskgraph_region_t *region = order[m];
+      if (region == entry || region == exit ||
+          region->mark == TASKGRAPH_COMBINED)
+        continue;
+      if (dominates(entry, region) && postdominates(exit, region)) {
+        TGDBG("carving member: %d\n", region->timestamp);
+        members[num_members++] = region;
+      }
+    }
+    if (num_members == 0)
+      continue;
+
+    TGDBG("carving %d nodes from %d to %d\n", num_members, entry->timestamp,
+          exit->timestamp);
+    __kmp_taskgraph_carve_irreducible(thread, taskgraph, alloc_chain, entry,
+                                      exit, members, num_members);
+    carved = true;
+  }
+
+  __kmp_fast_free(thread, members);
+  __kmp_fast_free(thread, matched);
+  __kmp_fast_free(thread, ipdom);
+  __kmp_fast_free(thread, order_pd);
+  return carved;
+}
+
+/// This function uses several strategies to turn an irreducible taskgraph
+/// into a reducible taskgraph.
+//
+// 1. If a node C depends on node B and also node A which dominates C,
+//    and if B also dominates C, then the dependency of C on A can be dropped.
+//    That is, we know B must execute after A, so we can say execution must
+//    proceed A->B->C, and we don't also need to specify the transitive A->C
+//    dependency directly.
+//
+//            A          A
+//           / \         |
+//          B   )   ->   B
+//           \ /         |
+//            C          C
+//
+// 2. Two nodes with the same set of predecessors and successors are turned
+//    into a parallel region.  This graph form can arise from use of
+//    "inoutset" dependencies.
+//
+//        A  B  C                               A  B  C
+//       / \/ \/ \                              |  |  |
+//      /__/\ /\__\                             |  |  |
+//      D____X____E  (A+B+C->D & A+B+C->E)  ->  par(D,E)
+//       \       /                                 |
+//        '--F--'                                  F
+//
+// 3. Otherwise the residual is genuinely non-series-parallel (no series,
+//    parallel, transitive-drop or twin merge applies).  We decompose the
+//    residual into its nested single-entry / single-exit (SESE) regions via
+//    immediate dominators/postdominators (__kmp_taskgraph_carve_sese) and
+//    carve only the minimal (leaf) irreducible regions into
+//    TASKGRAPH_REGION_IRREDUCIBLE nodes.  See the commentary on
+//    __kmp_taskgraph_carve_sese and __kmp_taskgraph_carve_irreducible.
+
+static bool __kmp_taskgraph_rewrite_irreducible(
+    kmp_info_t *thread, kmp_taskgraph_record_t *taskgraph,
+    kmp_taskgraph_region_t **&alloc_chain, kmp_taskgraph_region_t **region_p,
+    kmp_taskgraph_region_t *exitregion) {
+  kmp_taskgraph_region_t *entryregion = *region_p;
+
+  kmp_int32 worklist_length = 0;
+  for (kmp_taskgraph_region_t *r = entryregion; r; r = r->next) {
+    // Deleted regions stay deleted.  (We could actually remove these from
+    // the worklist here, I think.)
+    if (r->mark == TASKGRAPH_DELETED)
+      continue;
+    r->mark = TASKGRAPH_UNMARKED;
+    worklist_length++;
+  }
+
+#ifdef DEBUG_TASKGRAPH
+  TGDBG("worklist length: %d\n", worklist_length);
+
+  __kmp_taskgraph_region_dot(entryregion, "PredsAndSuccs");
+#endif
+
+  kmp_taskgraph_region_t **order =
+      (kmp_taskgraph_region_t **)__kmp_fast_allocate(
+          thread, worklist_length * sizeof(kmp_taskgraph_region_t *));
+  kmp_taskgraph_region_t **doms =
+      (kmp_taskgraph_region_t **)__kmp_fast_allocate(
+          thread, worklist_length * sizeof(kmp_taskgraph_region_t *));
+  memset(doms, 0, worklist_length * sizeof(kmp_taskgraph_region_t *));
+  kmp_int32 cursor = worklist_length;
+  assert(entryregion->type == TASKGRAPH_REGION_ENTRY);
+  __kmp_taskgraph_region_dfs(entryregion, order, cursor, false);
+  assert(cursor == 0);
+  __kmp_taskgraph_region_doms(order, doms, worklist_length, false);
+
+  bool dropped_preds_p = false;
+
+  for (kmp_int32 i = 0; i < worklist_length; i++) {
+    kmp_taskgraph_region_t *region = order[i];
+    if (__kmp_region_deplist_len(region->predecessors) < 2)
+      continue;
+    TGDBG("checking region %p for redundant predecessors\n", region);
+    kmp_taskgraph_region_dep_t **predp = &region->predecessors;
+    while (*predp) {
+      kmp_taskgraph_region_dep_t *pred = *predp;
+
+      bool passes_pred = false;
+      for (kmp_taskgraph_region_dep_t *rest = region->predecessors; rest;
+           rest = rest->next) {
+        if (rest->region == pred->region)
+          continue;
+        kmp_taskgraph_region_t *dom = doms[rest->region->timestamp];
+        TGDBG("pred region: %p, next: %p\n", pred->region, rest->region);
+        while (true) {
+          TGDBG("check against dom: %p\n", dom);
+          if (dom == pred->region) {
+            passes_pred = true;
+            break;
+          } else if (dom == doms[dom->timestamp]) {
+            break;
+          } else {
+            dom = doms[dom->timestamp];
+          }
+        }
+        if (passes_pred)
+          break;
+      }
+
+      if (passes_pred) {
+        // We can drop this predecessor.
+        TGDBG("dropping pred %p from region %p, dom %p\n", pred->region, region,
+              doms[pred->region->timestamp]);
+        kmp_taskgraph_region_dep_t *next = pred->next;
+        kmp_taskgraph_region_dep_t **succp = &pred->region->successors;
+        while (*succp) {
+          kmp_taskgraph_region_dep_t *succ = *succp;
+          if (succ->region == region) {
+            kmp_taskgraph_region_dep_t *nexts = succ->next;
+            __kmp_region_dep_recycle(&taskgraph->recycled_deps, succ);
+            *succp = nexts;
+          } else {
+            succp = &succ->next;
+          }
+        }
+        __kmp_region_dep_recycle(&taskgraph->recycled_deps, pred);
+        *predp = next;
+        dropped_preds_p = true;
+      } else {
+        predp = &pred->next;
+      }
+    }
+  }
+
+  if (dropped_preds_p) {
+    __kmp_fast_free(thread, order);
+    __kmp_fast_free(thread, doms);
+    return true;
+  }
+
+  kmp_bitset_t **pred_bitsets = nullptr;
+  kmp_bitset_t **succ_bitsets = nullptr;
+
+  bool regions_combined_p = false;
+
+  for (kmp_int32 i = 0; i < worklist_length; i++) {
+    kmp_taskgraph_region_t *region = order[i];
+    struct {
+      kmp_taskgraph_region_t *dom;
+      kmp_int32 count;
+    } dom_groups[worklist_length];
+    kmp_int32 num_groups = 0;
+    kmp_int32 npreds = __kmp_region_deplist_len(region->predecessors);
+    if (npreds >= 2) {
+      kmp_taskgraph_region_dep_t *pred;
+      for (pred = region->predecessors; pred; pred = pred->next) {
+        kmp_taskgraph_region_t *pred_region = pred->region;
+        kmp_taskgraph_region_t *this_dom = doms[pred_region->timestamp];
+        bool found = false;
+        for (kmp_int32 grp = 0; grp < num_groups; grp++) {
+          if (dom_groups[grp].dom == this_dom) {
+            dom_groups[grp].count++;
+            found = true;
+            break;
+          }
+        }
+        if (!found) {
+          dom_groups[num_groups].dom = this_dom;
+          dom_groups[num_groups].count = 1;
+          num_groups++;
+        }
+      }
+
+      if (num_groups == 1 && region->mark != TASKGRAPH_COMBINED) {
+        TGDBG("region %p: all predecessors have a single dominator\n", region);
+
+        if (!pred_bitsets) {
+          pred_bitsets = (kmp_bitset_t **)__kmp_fast_allocate(
+              thread, sizeof(kmp_bitset_t *) * worklist_length);
+          succ_bitsets = (kmp_bitset_t **)__kmp_fast_allocate(
+              thread, sizeof(kmp_bitset_t *) * worklist_length);
+
+          for (kmp_int32 i = 0; i < worklist_length; i++) {
+            pred_bitsets[i] = __kmp_bitset_alloc(thread, worklist_length);
+            succ_bitsets[i] = __kmp_bitset_alloc(thread, worklist_length);
+          }
+
+          for (kmp_int32 j = 0; j < worklist_length; j++) {
+            kmp_taskgraph_region_t *reg = order[j];
+
+            for (pred = reg->predecessors; pred; pred = pred->next) {
+              __kmp_bitset_set(pred_bitsets[reg->timestamp],
+                               pred->region->timestamp);
+            }
+
+            for (kmp_taskgraph_region_dep_t *succ = reg->successors; succ;
+                 succ = succ->next) {
+              __kmp_bitset_set(succ_bitsets[reg->timestamp],
+                               succ->region->timestamp);
+            }
+          }
+        }
+
+        kmp_taskgraph_region_dep_t *equal_deps_chain = nullptr;
+
+        kmp_int32 same_preds_and_succs = 1;
+        bool any_mutex_p = __kmp_taskgraph_region_mutex_p(region);
+        // FIXME: We might be able to do a bit better than this by hashing.
+        for (kmp_int32 j = i + 1; j < worklist_length; j++) {
+          if (order[j]->mark != TASKGRAPH_COMBINED &&
+              __kmp_bitset_equal(pred_bitsets[j], pred_bitsets[i]) &&
+              __kmp_bitset_equal(succ_bitsets[j], succ_bitsets[i])) {
+            TGDBG("regions %p and %p share all predecessors/successors\n",
+                  order[i], order[j]);
+            same_preds_and_succs++;
+            equal_deps_chain = __kmp_region_deplist_add(
+                thread, &taskgraph->recycled_deps, order[j], equal_deps_chain);
+            if (__kmp_taskgraph_region_mutex_p(order[j]))
+              any_mutex_p = true;
+          }
+        }
+        if (same_preds_and_succs > 1) {
+          kmp_taskgraph_region_type region_type =
+              any_mutex_p ? TASKGRAPH_REGION_EXCLUSIVE
+                          : TASKGRAPH_REGION_PARALLEL;
+          auto *par_region = __kmp_taskgraph_region_alloc(
+              thread, taskgraph, alloc_chain, region_type, same_preds_and_succs,
+              nullptr);
+          par_region->inner.children[0] = region;
+          region->mark = TASKGRAPH_COMBINED;
+          region->parent = par_region;
+          for (kmp_int32 j = 1; j < same_preds_and_succs; j++) {
+            kmp_taskgraph_region_dep_t *next = equal_deps_chain->next;
+            par_region->inner.children[j] = equal_deps_chain->region;
+            equal_deps_chain->region->mark = TASKGRAPH_COMBINED;
+            equal_deps_chain->region->parent = par_region;
+            __kmp_region_dep_recycle(&taskgraph->recycled_deps,
+                                     equal_deps_chain);
+            equal_deps_chain = next;
+          }
+          par_region->predecessors =
+              par_region->inner.children[0]->predecessors;
+          par_region->inner.children[0]->predecessors = nullptr;
+          par_region->successors = par_region->inner.children[0]->successors;
+          par_region->inner.children[0]->successors = nullptr;
+
+          // Redirect incoming deps to point to new parallel region.
+          for (pred = par_region->predecessors; pred; pred = pred->next) {
+            kmp_taskgraph_region_t *pred_region = pred->region;
+            kmp_taskgraph_region_dep_t **succp = &pred_region->successors;
+            while (*succp) {
+              kmp_taskgraph_region_dep_t *succ = *succp;
+              if (succ->region == par_region->inner.children[0]) {
+                succ->region = par_region;
+                succp = &succ->next;
+              } else {
+                bool found = false;
+                for (kmp_int32 j = 1; j < same_preds_and_succs; j++) {
+                  if (succ->region == par_region->inner.children[j]) {
+                    found = true;
+                    break;
+                  }
+                }
+                if (found) {
+                  kmp_taskgraph_region_dep_t *next = succ->next;
+                  __kmp_region_dep_recycle(&taskgraph->recycled_deps, succ);
+                  *succp = next;
+                } else {
+                  succp = &succ->next;
+                }
+              }
+            }
+          }
+
+          for (kmp_taskgraph_region_dep_t *succ = par_region->successors; succ;
+               succ = succ->next) {
+            kmp_taskgraph_region_t *succ_region = succ->region;
+            kmp_taskgraph_region_dep_t **predp = &succ_region->predecessors;
+            while (*predp) {
+              kmp_taskgraph_region_dep_t *pred = *predp;
+              if (pred->region == par_region->inner.children[0]) {
+                pred->region = par_region;
+                predp = &pred->next;
+              } else {
+                bool found = false;
+                for (kmp_int32 j = 1; j < same_preds_and_succs; j++) {
+                  if (pred->region == par_region->inner.children[j]) {
+                    found = true;
+                    break;
+                  }
+                }
+                if (found) {
+                  kmp_taskgraph_region_dep_t *next = pred->next;
+                  __kmp_region_dep_recycle(&taskgraph->recycled_deps, pred);
+                  *predp = next;
+                } else {
+                  predp = &pred->next;
+                }
+              }
+            }
+          }
+
+          par_region->next = region->next;
+          region->next = par_region;
+
+          regions_combined_p = true;
+        }
+      }
+    }
+  }
+
+  if (pred_bitsets) {
+    for (kmp_int32 j = 0; j < worklist_length; j++) {
+      __kmp_bitset_free(thread, pred_bitsets[j]);
+      __kmp_bitset_free(thread, succ_bitsets[j]);
+    }
+    __kmp_fast_free(thread, pred_bitsets);
+    __kmp_fast_free(thread, succ_bitsets);
+  }
+
+  // Case 2 made progress: let the build loop re-run the series/parallel
+  // collapse before we look at irreducibility again.
+  if (regions_combined_p) {
+    __kmp_fast_free(thread, order);
+    __kmp_fast_free(thread, doms);
+    return true;
+  }
+
+  // Case 3: neither the transitive drop nor the twin merge applied, so the
+  // residual is genuinely irreducible.  Decompose it into nested SESE regions
+  // and carve only the minimal (leaf) irreducible ones, leaving any reducible
+  // structure (and its reductions) around the knots for the series/parallel
+  // collapse to pick up on the next pass.  Each carve replaces a knot with a
+  // single IRREDUCIBLE atom (fewer live regions), so the build loop makes
+  // monotone progress and terminates.
+  bool carved = __kmp_taskgraph_carve_sese(
+      thread, taskgraph, alloc_chain, order, doms, worklist_length, exitregion);
+
+  assert(carved);
+
+  __kmp_fast_free(thread, order);
+  __kmp_fast_free(thread, doms);
+  return true;
+}
+
+/// Build a nested region structure out of a recorded taskgraph.
+//
+// The algorithm proceeds by alternating two phases until a single top-level
+// node is reached.  Briefly, and glossing over some details:
+//
+// 1. Serial-parallel decomposition.  Chains of single-successor,
+//    single-predecessor nodes are collapsed into a "sequential" region, and
+//    nodes with >1 predecessor, where each predecessor has a single
+//    predecessor and a single successor, are collapsed into "parallel" regions.
+//
+// 2. Irreducible-graph processing.  Several techniques are used to turn graphs
+//    not handled by step (1) into graphs that can be handled by that step.
+//
+// Notably, simple graphs that can be handled entirely by step (1) avoid doing
+// much of the heavier processing involved in step (2), so the common case
+// should be relatively fast.
+
+static kmp_taskgraph_region_t *__kmp_taskgraph_build_regions(
+    kmp_info_t *thread, kmp_taskgraph_record_t *taskgraph,
+    kmp_taskgraph_region_t **&alloc_chain, kmp_taskgraph_region_t *entryregion,
+    kmp_taskgraph_region_t *exitregion) {
+  bool changed;
+  kmp_int32 phase = 0;
+
+#ifdef DEBUG_TASKGRAPH
+  __kmp_taskgraph_region_dot(entryregion, "InitialPredsAndSuccs");
+#endif
+
+  __kmp_taskgraph_region_chain_clear_marks(entryregion);
+
+  while (true) {
+    do {
+      changed = false;
+      TGDBG("starting seq pass\n");
+      for (kmp_taskgraph_region_t **seq_head = &entryregion; *seq_head;
+           seq_head = &(*seq_head)->next) {
+        TGDBG("consider %s region: %p\n",
+              __kmp_taskgraph_region_type_name((*seq_head)->type), *seq_head);
+        if ((*seq_head)->mark == TASKGRAPH_COMBINED) {
+          TGDBG("already combined\n");
+          continue;
+        }
+        changed |= __kmp_taskgraph_collapse_sequence(thread, taskgraph,
+                                                     alloc_chain, seq_head,
+                                                     /*parent=*/nullptr, phase);
+        TGDBG("changed: %s\n", changed ? "true" : "false");
+      }
+      ++phase;
+      __kmp_taskgraph_region_chain_prune(&entryregion);
+      __kmp_taskgraph_region_worklist_check(thread, taskgraph, entryregion,
+                                            "after seq collapse");
+      TGDBG("starting par/unordered pass\n");
+      for (kmp_taskgraph_region_t **par_head = &entryregion; *par_head;
+           par_head = &(*par_head)->next) {
+        TGDBG("consider %s region: %p\n",
+              __kmp_taskgraph_region_type_name((*par_head)->type), *par_head);
+        if ((*par_head)->mark == TASKGRAPH_COMBINED) {
+          TGDBG("already combined\n");
+          continue;
+        }
+        changed |= __kmp_taskgraph_collapse_par_exclusive(
+            thread, taskgraph, alloc_chain, par_head, /*parent=*/nullptr,
+            phase);
+        TGDBG("changed: %s\n", changed ? "true" : "false");
+      }
+      ++phase;
+      __kmp_taskgraph_region_chain_prune(&entryregion);
+      __kmp_taskgraph_region_worklist_check(thread, taskgraph, entryregion,
+                                            "after par collapse");
+    } while (changed);
+
+    if (entryregion->type == TASKGRAPH_REGION_ENTRY) {
+      if (__kmp_region_deplist_len(entryregion->successors) == 1) {
+        kmp_taskgraph_region_t *one_region = entryregion->successors->region;
+        if (__kmp_region_deplist_len(one_region->successors) == 1) {
+          kmp_taskgraph_region_t *maybe_exit = one_region->successors->region;
+          if (maybe_exit->type == TASKGRAPH_REGION_EXIT)
+            return one_region;
+        }
+      }
+    } else {
+      fprintf(stderr, "FIXME: Expected entry region!\n");
+      return entryregion;
+    }
+
+    TGDBG("attempting to collapse irreducible regions\n");
+
+    changed |= __kmp_taskgraph_rewrite_irreducible(
+        thread, taskgraph, alloc_chain, &entryregion, exitregion);
+
+    // rewrite_irreducible always makes progress now: it either applies a safe
+    // reduction (transitive drop / twin merge) or carves a subgraph of the
+    // residual into a single IRREDUCIBLE region, so the loop is guaranteed to
+    // terminate.
+    KMP_DEBUG_ASSERT(changed);
+    if (!changed)
+      return entryregion;
+  }
+
+  return entryregion;
+}
+
+static void __kmp_taskgraph_gather_mutex_sets(kmp_info_t *thread,
+                                              kmp_taskgraph_region_t *region,
+                                              const kmp_bitset_t *held) {
+  switch (region->type) {
+  case TASKGRAPH_REGION_ENTRY:
+  case TASKGRAPH_REGION_EXIT:
+  case TASKGRAPH_REGION_WAIT:
+    return;
+  case TASKGRAPH_REGION_NODE: {
+#ifdef DEBUG_TASKGRAPH
+    if (region->mutexset && __kmp_bitset_subset_p(held, region->mutexset)) {
+      TGDBG("node is mutually exclusive with held: 0x%llx <: 0x%llx\n",
+            (unsigned long long)region->mutexset->bits[0],
+            (unsigned long long)held->bits[0]);
+    }
+#endif
+    return;
+  }
+  case TASKGRAPH_REGION_SEQUENTIAL: {
+    kmp_bitset_t *seq_held = __kmp_bitset_alloc(thread, held->bitsize);
+    __kmp_bitset_clearall(seq_held);
+    for (kmp_int32 child = 0; child < region->inner.num_children; child++) {
+      __kmp_taskgraph_gather_mutex_sets(thread, region->inner.children[child],
+                                        held);
+      if (region->inner.children[child]->mutexset)
+        __kmp_bitset_or(seq_held, seq_held,
+                        region->inner.children[child]->mutexset);
+    }
+    region->mutexset = seq_held;
+    return;
+  }
+  case TASKGRAPH_REGION_PARALLEL:
+  case TASKGRAPH_REGION_EXCLUSIVE: {
+    kmp_bitset_t *par_held = __kmp_bitset_alloc(thread, held->bitsize);
+    kmp_bitset_t *conflicts = __kmp_bitset_alloc(thread, held->bitsize);
+    while (true) {
+      __kmp_bitset_clearall(par_held);
+      for (kmp_int32 child = 0; child < region->inner.num_children; child++) {
+        __kmp_bitset_clearall(conflicts);
+        for (kmp_int32 other = 0; other < region->inner.num_children; other++) {
+          if (other != child) {
+            if (!region->inner.children[other]->mutexset)
+              __kmp_taskgraph_gather_mutex_sets(
+                  thread, region->inner.children[other], held);
+            if (region->inner.children[other]->mutexset)
+              __kmp_bitset_or(conflicts, conflicts,
+                              region->inner.children[other]->mutexset);
+          }
+        }
+        __kmp_taskgraph_gather_mutex_sets(thread, region->inner.children[child],
+                                          conflicts);
+        if (region->inner.children[child]->mutexset)
+          __kmp_bitset_or(par_held, par_held,
+                          region->inner.children[child]->mutexset);
+      }
+      if (!region->mutexset) {
+        region->mutexset = par_held;
+      } else if (__kmp_bitset_equal(region->mutexset, par_held)) {
+        TGDBG("par mutexes stabilized, exiting loop\n");
+        break;
+      } else {
+        TGDBG("par mutexes not stable, iterating\n");
+        __kmp_bitset_copy(region->mutexset, par_held);
+        __kmp_bitset_free(thread, par_held);
+      }
+    }
+    __kmp_bitset_free(thread, conflicts);
+    return;
+  }
+  }
+}
+
+static int __kmp_popcount_cmp(const void *a, const void *b) {
+  const kmp_taskgraph_region_t *reg_a = *(kmp_taskgraph_region_t **)a;
+  const kmp_taskgraph_region_t *reg_b = *(kmp_taskgraph_region_t **)b;
+  kmp_int32 popc_a = 0, popc_b = 0;
+  if (reg_a->mutexset)
+    popc_a = __kmp_bitset_popcount(reg_a->mutexset);
+  if (reg_b->mutexset)
+    popc_b = __kmp_bitset_popcount(reg_b->mutexset);
+  if (popc_a > popc_b)
+    return -1;
+  else if (popc_a < popc_b)
+    return 1;
+  return 0;
+}
+
+/// Find "mutexinoutset" regions that can be represented without explicit
+// mutexes, i.e. using "TASKGRAPH_REGION_EXCLUSIVE".
+
+static void __kmp_taskgraph_find_exclusive_regions(
+    kmp_info_t *thread, kmp_taskgraph_record_t *taskgraph,
+    kmp_taskgraph_region_t **&alloc_chain, kmp_taskgraph_region_t **region_p) {
+  kmp_taskgraph_region_t *region = *region_p;
+  switch (region->type) {
+  case TASKGRAPH_REGION_ENTRY:
+  case TASKGRAPH_REGION_EXIT:
+  case TASKGRAPH_REGION_NODE:
+  case TASKGRAPH_REGION_WAIT:
+    break;
+  case TASKGRAPH_REGION_SEQUENTIAL:
+  case TASKGRAPH_REGION_PARALLEL:
+  // A carved irreducible region is a structural container: it has no mutex set
+  // of its own, but its children may, so recurse like the other containers.
+  case TASKGRAPH_REGION_IRREDUCIBLE: {
+    for (kmp_int32 c = 0; c < region->inner.num_children; c++) {
+      __kmp_taskgraph_find_exclusive_regions(thread, taskgraph, alloc_chain,
+                                             &region->inner.children[c]);
+    }
+    break;
+  }
+  case TASKGRAPH_REGION_EXCLUSIVE: {
+    qsort(region->inner.children, region->inner.num_children,
+          sizeof(kmp_taskgraph_region_t *), __kmp_popcount_cmp);
+    for (kmp_int32 c = 0; c < region->inner.num_children; c++) {
+      TGDBG("building tree: region mutexset = 0x%llx\n",
+            (unsigned long long)region->inner.children[c]->mutexset
+                ? region->inner.children[c]->mutexset->bits[0]
+                : 0);
+      region->inner.children[c]->mark = TASKGRAPH_UNMARKED;
+    }
+    kmp_bitset_t *conflicts =
+        __kmp_bitset_alloc(thread, region->mutexset->bitsize);
+    kmp_bitset_t *subsets_cover =
+        __kmp_bitset_alloc(thread, region->mutexset->bitsize);
+    __kmp_bitset_copy(conflicts, region->mutexset);
+    bool irregular = false;
+    kmp_int32 combined_children = 0;
+    for (kmp_int32 c = 0; c < region->inner.num_children; c++) {
+      kmp_bitset_t *candidate = region->inner.children[c]->mutexset;
+      if (__kmp_bitset_empty_p(candidate))
+        continue;
+      __kmp_bitset_clearall(subsets_cover);
+      bool found_subset = false;
+      bool other_overlaps = false;
+      for (kmp_int32 d = c + 1; d < region->inner.num_children; d++) {
+        // This could test for a subset in some cases, but that adds
+        // complication for later processing.  Maybe revisit later if it
+        // seems worthwhile.
+        // E.g. if we have deps like this:
+        //
+        // #pragma omp task depend(mutexinoutset: deps[0], deps[1]) { /*a*/ }
+        // #pragma omp task depend(mutexinoutset: deps[0]) { /*b*/ }
+        // #pragma omp task depend(mutexinoutset: deps[1]) { /*c*/ }
+        //
+        // This could be represented as:
+        //
+        // exclusive {
+        //   node: a
+        //   parallel {
+        //     node: b
+        //     node: c
+        //   }
+        // }
+        //
+        // We're not doing that yet though.
+        if (__kmp_bitset_equal(candidate,
+                               region->inner.children[d]->mutexset)) {
+          found_subset = true;
+          __kmp_bitset_or(subsets_cover, subsets_cover,
+                          region->inner.children[d]->mutexset);
+        } else if (__kmp_bitset_intersect_p(
+                       candidate, region->inner.children[d]->mutexset)) {
+          other_overlaps = true;
+          break;
+        }
+      }
+      if (!found_subset || other_overlaps)
+        continue;
+      if (!__kmp_bitset_equal(subsets_cover, candidate)) {
+        TGDBG("subsets cover: 0x%llx, candidate: 0x%llx\n",
+              (unsigned long long)subsets_cover->bits[0],
+              (unsigned long long)candidate->bits[0]);
+        irregular = true;
+        break;
+      }
+      for (kmp_int32 d = c + 1; d < region->inner.num_children; d++) {
+        if (region->inner.children[d]->mutexset_parent)
+          continue;
+        // As above wrt. subsets.
+        if (__kmp_bitset_equal(candidate,
+                               region->inner.children[d]->mutexset)) {
+          TGDBG("set index %d's parent to index %d\n", d, c);
+          region->inner.children[d]->mutexset_parent =
+              region->inner.children[c];
+          combined_children++;
+          __kmp_bitset_and_not(conflicts, conflicts, candidate);
+        }
+      }
+    }
+    TGDBG("irregular: %s\n", irregular ? "true" : "false");
+    TGDBG("final conflicts: 0x%llx\n", (unsigned long long)conflicts->bits[0]);
+    __kmp_bitset_free(thread, subsets_cover);
+    region->type = TASKGRAPH_REGION_PARALLEL;
+    if (!irregular && __kmp_bitset_empty_p(conflicts)) {
+      TGDBG("transforming exclusive region %p\n", region);
+      TGDBG("orig region children: %d\n", region->inner.num_children);
+      TGDBG("combined children: %d\n", combined_children);
+      if (region->inner.num_children == combined_children + 1) {
+        region->type = TASKGRAPH_REGION_EXCLUSIVE;
+      } else {
+        kmp_int32 par_elems = region->inner.num_children - combined_children;
+        auto *new_par = __kmp_taskgraph_region_alloc(
+            thread, taskgraph, alloc_chain, TASKGRAPH_REGION_PARALLEL,
+            par_elems, nullptr);
+        for (kmp_int32 c = region->inner.num_children - 1; c >= 0; c--) {
+          kmp_taskgraph_region_t *child = region->inner.children[c];
+          // Make mutex set into a circular list.
+          if (child->mutexset_parent && child->mark != TASKGRAPH_TEMP_MARK) {
+            if (!child->mutexset_parent->mutexset_parent) {
+              // child <-> parent
+              child->mutexset_parent->mutexset_parent = child;
+              child->mutexset_parent->mark = TASKGRAPH_TEMP_MARK;
+            } else {
+              kmp_taskgraph_region_t *parent = child->mutexset_parent;
+              child->mutexset_parent = parent->mutexset_parent;
+              parent->mutexset_parent = child;
+              parent->mark = TASKGRAPH_TEMP_MARK;
+            }
+          }
+        }
+        kmp_int32 idx = 0;
+        for (kmp_int32 c = 0; c < region->inner.num_children; c++) {
+          kmp_taskgraph_region_t *child = region->inner.children[c];
+          TGDBG("process child: %p\n", child);
+          if (child->mutexset_parent && child->mark != TASKGRAPH_COMBINED) {
+            kmp_int32 elems = 0;
+            kmp_taskgraph_region_t *next = child;
+            do {
+              elems++;
+              next = next->mutexset_parent;
+            } while (next != child);
+            TGDBG("make exclusive region with %d children\n", elems);
+            auto *excl_region = __kmp_taskgraph_region_alloc(
+                thread, taskgraph, alloc_chain, TASKGRAPH_REGION_EXCLUSIVE,
+                elems, nullptr);
+            kmp_int32 excl_child = 0;
+            next = child;
+            do {
+              excl_region->inner.children[excl_child++] = next;
+              next->mark = TASKGRAPH_COMBINED;
+              next = next->mutexset_parent;
+            } while (next != child);
+            assert(excl_child == excl_region->inner.num_children);
+            new_par->inner.children[idx++] = excl_region;
+          } else if (!child->mutexset_parent) {
+            new_par->inner.children[idx++] = child;
+          }
+        }
+        TGDBG("idx=%d, supposed to be %d\n", idx, new_par->inner.num_children);
+        assert(idx == new_par->inner.num_children);
+        *region_p = new_par;
+        region->mark = TASKGRAPH_DELETED;
+      }
+    }
+    __kmp_bitset_free(thread, conflicts);
+    break;
+  }
+  default:
+    assert(false && "unreachable");
+  }
+}
+
+/// Strip mutex sets from taskgraph region, except those needed at runtime.
+
+static kmp_int32
+__kmp_taskgraph_strip_mutex_sets(kmp_info_t *thread,
+                                 kmp_taskgraph_region_t *region,
+                                 bool in_exclusive = false) {
+  kmp_int32 mutexes_needed = 0;
+  switch (region->type) {
+  case TASKGRAPH_REGION_ENTRY:
+  case TASKGRAPH_REGION_EXIT:
+  case TASKGRAPH_REGION_WAIT:
+    assert(!region->mutexset);
+    break;
+  case TASKGRAPH_REGION_NODE:
+    if (region->mutexset) {
+      if (in_exclusive) {
+        __kmp_bitset_free(thread, region->mutexset);
+        region->mutexset = nullptr;
+      } else {
+        // FIXME: This might be pessimistic -- the remaining mutex sets might
+        // have holes or duplicates.  We could compact them.
+        kmp_int32 m = region->mutexset->bitsize;
+        mutexes_needed = std::max(mutexes_needed, m);
+      }
+    }
+    break;
+  case TASKGRAPH_REGION_EXCLUSIVE: {
+    if (region->mutexset) {
+      __kmp_bitset_free(thread, region->mutexset);
+      region->mutexset = nullptr;
+    }
+    for (kmp_int32 c = 0; c < region->inner.num_children; c++) {
+      kmp_int32 m = __kmp_taskgraph_strip_mutex_sets(
+          thread, region->inner.children[c], true);
+      mutexes_needed = std::max(mutexes_needed, m);
+    }
+    break;
+  }
+  default: {
+    if (region->mutexset) {
+      __kmp_bitset_free(thread, region->mutexset);
+      region->mutexset = nullptr;
+    }
+    for (kmp_int32 c = 0; c < region->inner.num_children; c++) {
+      kmp_int32 m = __kmp_taskgraph_strip_mutex_sets(
+          thread, region->inner.children[c], in_exclusive);
+      mutexes_needed = std::max(mutexes_needed, m);
+    }
+  }
+  }
+  return mutexes_needed;
+}
+
+static void __kmp_taskgraph_exclusive_regions(
+    kmp_info_t *thread, kmp_taskgraph_record_t *taskgraph,
+    kmp_taskgraph_region_t **&alloc_chain, kmp_taskgraph_region_t **region_p,
+    kmp_int32 max_mutex) {
+  kmp_bitset_t *top = __kmp_bitset_alloc(thread, max_mutex);
+  __kmp_bitset_clearall(top);
+  __kmp_taskgraph_gather_mutex_sets(thread, *region_p, top);
+  __kmp_taskgraph_find_exclusive_regions(thread, taskgraph, alloc_chain,
+                                         region_p);
+  kmp_int32 num_mutexes = __kmp_taskgraph_strip_mutex_sets(thread, *region_p);
+  taskgraph->num_mutexes = num_mutexes;
+}
+
+static const char *
+__kmp_taskgraph_region_type_name(kmp_taskgraph_region_type type) {
+  switch (type) {
+  case TASKGRAPH_REGION_ENTRY:
+    return "entry";
+  case TASKGRAPH_REGION_EXIT:
+    return "exit";
+  case TASKGRAPH_REGION_NODE:
+    return "node";
+  case TASKGRAPH_REGION_WAIT:
+    return "wait";
+  case TASKGRAPH_REGION_PARALLEL:
+    return "parallel";
+  case TASKGRAPH_REGION_EXCLUSIVE:
+    return "exclusive";
+  case TASKGRAPH_REGION_SEQUENTIAL:
+    return "sequential";
+  case TASKGRAPH_REGION_IRREDUCIBLE:
+    return "irreducible";
+  default:
+    return "<unknown>";
+  }
+}
+
+static void __kmp_dump_taskgraph_regions(FILE *f,
+                                         kmp_taskgraph_region_t *region,
+                                         int indent = 0) {
+  switch (region->type) {
+  case TASKGRAPH_REGION_ENTRY:
+  case TASKGRAPH_REGION_EXIT:
+    fprintf(f, "%*s%s node\n", indent, "",
+            __kmp_taskgraph_region_type_name(region->type));
+    break;
+  case TASKGRAPH_REGION_NODE:
+  case TASKGRAPH_REGION_WAIT: {
+    char set_membership[40];
+    if (region->mutexset)
+      sprintf(set_membership, " [sets: 0x%llx]",
+              (unsigned long long)region->mutexset->bits[0]);
+    else
+      strcpy(set_membership, "");
+    fprintf(f, "%*s%s: %p%s\n", indent, "",
+            __kmp_taskgraph_region_type_name(region->type), region->task.node,
+            set_membership);
+    break;
+  }
+  default: {
+    char set_membership[40];
+    if (region->mutexset)
+      sprintf(set_membership, " [sets: 0x%llx]",
+              (unsigned long long)region->mutexset->bits[0]);
+    else
+      strcpy(set_membership, "");
+    fprintf(f, "%*s%s%s {\n", indent, "",
+            __kmp_taskgraph_region_type_name(region->type), set_membership);
+    for (kmp_int32 c = 0; c < region->inner.num_children; c++) {
+      __kmp_dump_taskgraph_regions(f, region->inner.children[c], indent + 2);
+    }
+    fprintf(f, "%*s}\n", indent, "");
+  }
+  }
+}
+
+#ifdef DEBUG_TASKGRAPH
+
+static kmp_taskgraph_region_dep_t *
+__kmp_dump_find_parent_regions(kmp_info *thd, kmp_taskgraph_record_t *taskgraph,
+                               kmp_taskgraph_region_t *region, int numregions,
+                               kmp_taskgraph_region_dep_t *list = nullptr) {
+  for (int r = 0; r < numregions; r++) {
+    if (!region[r].parent)
+      continue;
+    bool in_list = false;
+    for (kmp_taskgraph_region_dep_t *dep = list; dep; dep = dep->next) {
+      if (dep->region == region[r].parent) {
+        in_list = true;
+        break;
+      }
+    }
+    if (!in_list) {
+      list = __kmp_region_deplist_add(thd, &taskgraph->recycled_deps,
+                                      region[r].parent, list);
+      list = __kmp_dump_find_parent_regions(thd, taskgraph, region[r].parent, 1,
+                                            list);
+    }
+  }
+  return list;
+}
+
+static void __kmp_dump_raw_taskgraph_regions(FILE *f, kmp_info *thd,
+                                             kmp_taskgraph_record_t *taskgraph,
+                                             kmp_taskgraph_region_t *region,
+                                             int numregions, int indent = 0) {
+  kmp_taskgraph_region_dep_t *parentlist = nullptr;
+  kmp_taskgraph_region_dep_t *printedlist = nullptr;
+  for (int r = 0; r < numregions; r++) {
+    int children = 0;
+    if (region[r].type == TASKGRAPH_REGION_PARALLEL ||
+        region[r].type == TASKGRAPH_REGION_SEQUENTIAL ||
+        region[r].type == TASKGRAPH_REGION_EXCLUSIVE ||
+        region[r].type == TASKGRAPH_REGION_IRREDUCIBLE)
+      children = region[r].inner.num_children;
+    fprintf(
+        f,
+        "%*sregion %d (%p): %s%s (%d children) parent %p succs %d preds %d\n",
+        indent, "", r, &region[r],
+        __kmp_taskgraph_region_type_name(region[r].type),
+        region[r].mark == TASKGRAPH_COMBINED ? " (combined)" : "", children,
+        region[r].parent, __kmp_region_deplist_len(region[r].successors),
+        __kmp_region_deplist_len(region[r].predecessors));
+    if (children > 0) {
+      for (int c = 0; c < children; c++)
+        __kmp_dump_raw_taskgraph_regions(
+            f, thd, taskgraph, region->inner.children[c], 1, indent + 2);
+    }
+  }
+  if (indent == 0) {
+    parentlist =
+        __kmp_dump_find_parent_regions(thd, taskgraph, region, numregions);
+    fprintf(stderr, "%*sfound %d parent region(s):\n", indent, "",
+            __kmp_region_deplist_len(parentlist));
+    for (kmp_taskgraph_region_dep_t *p = parentlist; p; p = p->next) {
+      __kmp_dump_raw_taskgraph_regions(f, thd, taskgraph, p->region, 1,
+                                       indent + 2);
+    }
+    __kmp_region_deplist_recycle(&taskgraph->recycled_deps, parentlist);
+  }
+}
+#endif
+
+bool __kmp_taskgraph_trace() {
+  static int trace_enabled = -1;
+  if (trace_enabled == -1) {
+    trace_enabled = getenv("KMP_TASKGRAPH_TRACE") != nullptr;
+  }
+  return trace_enabled;
+}
+
+/// Build a nested region structure from a "raw" recorded taskgraph, and mark
+/// the taskgraph ready for replay.
+//
+// The input to this function consists of tasks with *data* dependencies
+// between them.  The output of the function is a nested tree structure: the
+// dependencies between tasks implicitly become *control* dependencies.  In
+// the common case, these ought to map straightforwardly to hardware-provided
+// execution primitives (e.g. on a GPU), or to runtime-provided primitives (for
+// the CPU).
+//
+// Here is an example taskgraph:
+//
+// #pragma omp taskgraph
+// {
+//   #pragma omp task depend(out: deps[2])
+//   { }
+//   #pragma omp task depend(out: deps[0], deps[1])
+//   { }
+//   #pragma omp task depend(inout: deps[0])
+//   { }
+//   #pragma omp task depend(inout: deps[1])
+//   { }
+//   #pragma omp task depend(inout: deps[2])
+//   { }
+//   #pragma omp task depend(in: deps[0], deps[1], deps[2])
+//   { }
+// }
+//
+// This dependency graph is "reducible", and the resulting tree looks like this:
+//
+// sequential {
+//   parallel {
+//     sequential {
+//       node: 0x588aa11021b0
+//       node: 0x588aa1102250
+//     }
+//     sequential {
+//       node: 0x588aa11021d8
+//       parallel {
+//         node: 0x588aa1102228
+//         node: 0x588aa1102200
+//       }
+//     }
+//   }
+//   node: 0x588aa1102278
+// }
+//
+// Each node represents a task, and the containing parallel and sequential
+// regions represent sub-regions that can be executed in parallel, or
+// one-at-a-time, in order.
+//
+// In some cases, the data-dependency graph may not be trivially reducible to
+// parallel and sequential regions.  In this case, several techniques are used
+// to produce a reducible graph from an irreducible graph (see
+// __kmp_taskgraph_rewrite_irreducible).
+//
+// For example in this graph:
+//
+// #pragma omp taskgraph
+// {
+//   #pragma omp task depend(out: deps[0], deps[1])
+//   { }
+//   #pragma omp task depend(out: deps[2], deps[3])
+//   { }
+//   #pragma omp task depend(inout: deps[0])
+//   { }
+//   #pragma omp task depend(inout: deps[1])
+//   { }
+//   #pragma omp task depend(inout: deps[2])
+//   { }
+//   #pragma omp task depend(inout: deps[3])
+//   { }
+//   #pragma omp task depend(in: deps[0], deps[1], deps[2], deps[3])
+//   { }
+//   #pragma omp task depend(in: deps[1], deps[2])
+//   { }
+// }
+//
+// The final two tasks overlap data dependencies in such a way that the
+// resulting dependency graph cannot be trivially decomposed to parallel and
+// sequential regions.  In this case the whole graph is reduced to an
+// irreducible region:
+//
+// irreducible {
+//   node: 0x7803e00fb8d8
+//   node: 0x7803e00fb8a8
+//   node: 0x7803e00fb848
+//   node: 0x7803e00fb938
+//   node: 0x7803e00fb998
+//   node: 0x7803e00fb968
+//   node: 0x7803e00fb908
+//   node: 0x7803e00fb878
+// }
+//
+// If *part* of the graph is reducible, it should still be handled without
+// falling back to an irreducible region.  If we chnage the input as follows:
+//
+// #pragma omp taskgraph
+// {
+//   #pragma omp task depend(out: deps[0], deps[1])
+//   { }
+//   #pragma omp task depend(out: deps[2], deps[3])
+//   { }
+//   #pragma omp task depend(inout: deps[0])
+//   { }
+//   #pragma omp task depend(inout: deps[1])
+//   { }
+//   #pragma omp task depend(inout: deps[2])
+//   { }
+//   #pragma omp task depend(inout: deps[3])
+//   { }
+//   #pragma omp task depend(in: deps[0], deps[1], deps[2], deps[3])
+//   { }
+//   #pragma omp task depend(in: deps[1], deps[2])
+//   { }
+//   int res = 0;
+//   #pragma omp taskloop reduction(+: res) num_tasks(4)
+//   {
+//     for (int i = 0; i < 16; i++)
+//       res += i;
+//   }
+// }
+//
+// We will now see:
+//
+// irreducible {
+//   node: 0x58de797a9328
+//   node: 0x58de797a92f8
+//   node: 0x58de797a9298
+//   node: 0x58de797a9388
+//   node: 0x58de797a93e8
+//   node: 0x58de797a93b8
+//   node: 0x58de797a9358
+//   node: 0x58de797a92c8
+//   sequential {
+//     parallel {
+//       node: 0x58de797a94a8
+//       node: 0x58de797a9478
+//       node: 0x58de797a9448
+//       node: 0x58de797a9418
+//     }
+//     wait: 0x58de797a94d8
+//   }
+// }
+//
+// This is particularly important for reductions, which won't work with the
+// "unstructured" irreducible region.
+//
+// The final region type is "exclusive", which arises for "mutexinoutset"
+// dependencies that are able to be abstracted away (we can't do this in all
+// cases: when we can't, we still use explicit mutexes).
+//
+// An example of this:
+//
+// #pragma omp taskgraph
+// {
+//   #pragma omp task depend(mutexinoutset: deps[0])
+//   { }
+//   #pragma omp task depend(mutexinoutset: deps[1])
+//   { }
+//   #pragma omp task depend(mutexinoutset: deps[0])
+//   { }
+//   #pragma omp task depend(mutexinoutset: deps[1])
+//   { }
+//   #pragma omp task depend(mutexinoutset: deps[0])
+//   { }
+//   #pragma omp task depend(mutexinoutset: deps[1])
+//   { }
+//   #pragma omp task depend(mutexinoutset: deps[0])
+//   { }
+//   #pragma omp task depend(mutexinoutset: deps[1])
+//   { }
+// }
+//
+// Results in this structure:
+//
+// parallel {
+//   exclusive {
+//     node: 0x5c0c5c571120
+//     node: 0x5c0c5c5710d0
+//     node: 0x5c0c5c571080
+//     node: 0x5c0c5c571030
+//   }
+//   exclusive {
+//     node: 0x5c0c5c5710f8
+//     node: 0x5c0c5c5710a8
+//     node: 0x5c0c5c571058
+//     node: 0x5c0c5c571008
+//   }
+// }
+//
+// The meaning of "exclusive" here is for each of the child regions (task
+// nodes in this case) to be executed in some unspecified order, one at a
+// time relative to the other regions in the structure.  E.g. a GPU
+// implementation could try to dynamically schedule tasks such that they fit
+// instantaneously-available execution resources.
+//
+// In cases where mutexes cannot be abstracted, each affected task node is
+// annotated with a set of mutexes that must be held while executing the task.
+// (Shown with [sets: 0xN] in dump output).
+
+kmp_int32 __kmp_build_taskgraph(kmp_int32 gtid,
+                                kmp_taskdata_t *current_taskdata,
+                                kmp_taskgraph_record_t *taskgraph) {
+  kmp_int32 numnodes = taskgraph->num_tasks;
+  kmp_int32 numregions = numnodes + 2;
+  kmp_taskgraph_node_t *nodes = taskgraph->record_map;
+  kmp_info_t *thread = __kmp_threads[gtid];
+  kmp_dephash_t *hash = __kmp_dephash_create(thread, current_taskdata);
+  bool dep_barrier = false;
+
+  // We need to take special care to align the all_depnodes array to the cache
+  // line size, because kmp_depnode_t is marked as 64-byte aligned and
+  // otherwise the compiler might generate faulting memory accesses based on
+  // that alignment assumption.
+  size_t all_depnodes_size = numregions * sizeof(kmp_depnode_t);
+  // The maximum amount of padding we need is CACHE_LINE - 1 bytes.
+  all_depnodes_size = all_depnodes_size + CACHE_LINE - 1;
+  char *all_depnodes_misaligned =
+      (char *)__kmp_thread_malloc(thread, all_depnodes_size);
+  kmp_depnode_t *all_depnodes =
+      (kmp_depnode_t *)((((intptr_t)all_depnodes_misaligned) + CACHE_LINE - 1) &
+                        ~(CACHE_LINE - 1));
+  kmp_int32 next_mutex_set = 0;
+
+  for (kmp_int32 i = 0; i < numnodes; i++) {
+    int n_mtxs = 0;
+    bool dep_all;
+
+    dep_all = __kmp_filter_aliased_deps(nodes[i].u.unresolved.ndeps,
+                                        nodes[i].u.unresolved.dep_list,
+                                        nodes[i].task, &n_mtxs);
+    kmp_depnode_t *node = &all_depnodes[i];
+    __kmp_init_node(node, /*on_stack=*/false);
+    node->dn.task = nodes[i].task;
+    dep_barrier = !nodes[i].task && nodes[i].taskloop_task;
+    if (!dep_all) {
+      __kmp_process_deps<taskgraph_deps>(
+          gtid, node, &hash, dep_barrier, nodes[i].u.unresolved.ndeps,
+          nodes[i].u.unresolved.dep_list, nodes[i].task, next_mutex_set);
+    } else {
+      __kmp_process_dep_all<taskgraph_deps>(gtid, node, hash, dep_barrier,
+                                            nodes[i].task);
+    }
+  }
+
+  kmp_taskgraph_region_t *order_out[numregions];
+  kmp_int32 outidx = 0;
+
+  kmp_taskgraph_region_t *initial_regions =
+      (kmp_taskgraph_region_t *)__kmp_fast_allocate(
+          thread, sizeof(kmp_taskgraph_region_t) * numregions);
+  memset(initial_regions, 0, sizeof(kmp_taskgraph_region_t) * numregions);
+
+  kmp_taskgraph_region_t *cfg_barrier = nullptr;
+
+  for (kmp_int32 i = 0; i < numnodes; i++) {
+    initial_regions[i].type =
+        nodes[i].task ? TASKGRAPH_REGION_NODE : TASKGRAPH_REGION_WAIT;
+    initial_regions[i].task.node = &nodes[i];
+    initial_regions[i].parent = nullptr;
+    if (i < numnodes - 1) {
+      initial_regions[i].next = &initial_regions[i + 1];
+    } else {
+      initial_regions[i].next = nullptr;
+    }
+    kmp_depnode_t *depnode = &all_depnodes[i];
+    initial_regions[i].mutexset = depnode->dn.set_membership;
+    for (kmp_depnode_list_t *succ = depnode->dn.successors; succ;
+         succ = succ->next) {
+      kmp_int32 succ_idx = succ->node - all_depnodes;
+      kmp_taskgraph_region_t *tg_succ = &initial_regions[succ_idx];
+      tg_succ->predecessors =
+          __kmp_region_deplist_add(thread, &taskgraph->recycled_deps,
+                                   &initial_regions[i], tg_succ->predecessors);
+      initial_regions[i].successors =
+          __kmp_region_deplist_add(thread, &taskgraph->recycled_deps, tg_succ,
+                                   initial_regions[i].successors);
+    }
+    // Handle control flow dependencies.  If a node (e.g. a taskloop task) has
+    // a wait after it corresponding to the end of an implicit taskgroup, join
+    // the task to the wait.  The wait then becomes a barrier; any tasks after
+    // it will depend on the barrier.
+    if (nodes[i].u.unresolved.cfg_successor != -1) {
+      kmp_int32 cfg_succ = nodes[i].u.unresolved.cfg_successor;
+      initial_regions[i].successors = __kmp_region_deplist_add(
+          thread, &taskgraph->recycled_deps, &initial_regions[cfg_succ],
+          initial_regions[i].successors);
+      initial_regions[cfg_succ].predecessors = __kmp_region_deplist_add(
+          thread, &taskgraph->recycled_deps, &initial_regions[i],
+          initial_regions[cfg_succ].predecessors);
+    }
+    if (nodes[i].taskloop_task && !nodes[i].task) {
+      cfg_barrier = &initial_regions[i];
+    } else if (cfg_barrier) {
+      cfg_barrier->successors = __kmp_region_deplist_add(
+          thread, &taskgraph->recycled_deps, &initial_regions[i],
+          cfg_barrier->successors);
+      initial_regions[i].predecessors = __kmp_region_deplist_add(
+          thread, &taskgraph->recycled_deps, cfg_barrier,
+          initial_regions[i].predecessors);
+    }
+  }
+
+  __kmp_dephash_free<false>(thread, hash);
+  __kmp_thread_free(thread, all_depnodes_misaligned);
+
+  // We're done with the "unresolved" data now.
+  for (kmp_int32 i = 0; i < numnodes; i++)
+    __kmp_thread_free(thread, nodes[i].u.unresolved.dep_list);
+
+  // Use these indices for the virtual entry and exit regions
+  kmp_int32 entryregion = numnodes, exitregion = numnodes + 1;
+
+  // Set entry/exit node types, and add to worklist
+  initial_regions[entryregion].type = TASKGRAPH_REGION_ENTRY;
+  initial_regions[entryregion].next = &initial_regions[0];
+  initial_regions[exitregion].type = TASKGRAPH_REGION_EXIT;
+  initial_regions[numnodes - 1].next = &initial_regions[exitregion];
+
+  // Join entry and exit nodes up to the graph
+  for (kmp_int32 i = 0; i < numnodes; i++) {
+    kmp_taskgraph_region_t *region = &initial_regions[i];
+    kmp_int32 npreds = __kmp_region_deplist_len(region->predecessors);
+    kmp_int32 nsuccs = __kmp_region_deplist_len(region->successors);
+    if (npreds == 0) {
+      initial_regions[entryregion].successors =
+          __kmp_region_deplist_add(thread, &taskgraph->recycled_deps, region,
+                                   initial_regions[entryregion].successors);
+      region->predecessors = __kmp_region_deplist_add(
+          thread, &taskgraph->recycled_deps, &initial_regions[entryregion],
+          region->predecessors);
+    }
+    if (nsuccs == 0) {
+      initial_regions[exitregion].predecessors =
+          __kmp_region_deplist_add(thread, &taskgraph->recycled_deps, region,
+                                   initial_regions[exitregion].predecessors);
+      region->successors = __kmp_region_deplist_add(
+          thread, &taskgraph->recycled_deps, &initial_regions[exitregion],
+          region->successors);
+    }
+    region->owner = taskgraph;
+  }
+
+  kmp_int32 max_level = -1;
+
+  for (kmp_int32 i = 0; i < numregions; i++)
+    initial_regions[i].timestamp = i;
+
+  for (kmp_int32 i = 0; i < numregions; i++) {
+    if (initial_regions[i].mark == TASKGRAPH_UNMARKED) {
+      kmp_int32 level = __kmp_taskgraph_topological_order(&initial_regions[i],
+                                                          order_out, &outidx);
+      max_level = level > max_level ? level : max_level;
+    }
+  }
+
+  assert(outidx == numregions);
+
+#ifdef DEBUG_TASKGRAPH
+  fprintf(stderr, "topological order (max level: %d):\n", max_level);
+
+  for (kmp_int32 i = 0; i < outidx; i++) {
+    fprintf(stderr, "node %d (region %p), level %d\n", order_out[i]->timestamp,
+            order_out[i], order_out[i]->level);
+  }
+#endif
+
+  kmp_taskgraph_region_t **alloc_chain = &initial_regions[0].alloc_chain;
+
+  kmp_taskgraph_region_t *root_region = __kmp_taskgraph_build_regions(
+      thread, taskgraph, alloc_chain, &initial_regions[entryregion],
+      &initial_regions[exitregion]);
+
+  __kmp_taskgraph_exclusive_regions(thread, taskgraph, alloc_chain,
+                                    &root_region, next_mutex_set);
+
+  *alloc_chain = nullptr;
+
+  taskgraph->root = root_region;
+  taskgraph->alloc_root = initial_regions;
+
+  // Free dependency lists and deleted regions.
+  kmp_taskgraph_region_t **regp = &taskgraph->alloc_root;
+  while (*regp) {
+    kmp_taskgraph_region_t *reg = *regp;
+    // The direct children of a carved IRREDUCIBLE region keep their
+    // predecessor / successor lists alive.  They are freed at record teardown
+    // (__kmp_free_taskgraph_record).
+    bool keep_edges =
+        reg->parent && reg->parent->type == TASKGRAPH_REGION_IRREDUCIBLE;
+    if (!keep_edges) {
+      __kmp_region_deplist_free(thread, reg->predecessors);
+      __kmp_region_deplist_free(thread, reg->successors);
+      reg->predecessors = nullptr;
+      reg->successors = nullptr;
+    }
+    if (reg->mark == TASKGRAPH_DELETED) {
+      kmp_taskgraph_region_t *chain_next = reg->alloc_chain;
+      TGDBG("deleted region from alloc chain: %p\n", reg);
+      __kmp_fast_free(thread, reg);
+      *regp = chain_next;
+    } else {
+      regp = &reg->alloc_chain;
+    }
+  }
+  // Free recycled dep list.  We could pass this along to the next invocation
+  // of this function instead, but we *mostly* don't do that yet
+  // (ownership/thread safety needs careful consideration if we do that).
+  // The "mostly" concerns irreducible-region edges: we recycle those
+  // opportunistically in __kmp_free_taskgraph.
+  for (kmp_taskgraph_region_dep_t *dep = taskgraph->recycled_deps; dep;) {
+    kmp_taskgraph_region_dep_t *next = dep->next;
+    TGDBG("free dep from recycled list\n");
+    __kmp_fast_free(thread, dep);
+    dep = next;
+  }
+  taskgraph->recycled_deps = nullptr;
+
+  if (__kmp_taskgraph_trace()) {
+    fprintf(stderr, "Processed taskgraph %p (graph_id %" PRIx64 "):\n",
+            taskgraph, taskgraph->graph_id);
+    __kmp_dump_taskgraph_regions(stderr, root_region);
+  }
+
+#ifdef DEBUG_TASKGRAPH
+//__kmp_dump_taskgraph_regions(stderr, root_region);
+//__kmp_dump_raw_taskgraph_regions(stderr, thread, taskgraph,
+//                                 &initial_regions[0], numregions);
+#endif
+
+  KMP_ATOMIC_ST_REL(&taskgraph->status, KMP_TDG_READY);
+
+  return 0;
+}
+#endif
+
+#define NO_DEP_BARRIER (false)
+#define DEP_BARRIER (true)
+
+// returns true if the task has any outstanding dependence
+static bool __kmp_check_deps(kmp_int32 gtid, kmp_depnode_t *node,
+                             kmp_task_t *task, kmp_dephash_t **hash,
+                             bool dep_barrier, kmp_int32 ndeps,
+                             kmp_depend_info_t *dep_list,
+                             kmp_int32 ndeps_noalias,
+                             kmp_depend_info_t *noalias_dep_list) {
+  int n_mtxs = 0, dep_all = 0;
+#if KMP_DEBUG
+  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
+#endif
+  KA_TRACE(20, ("__kmp_check_deps: T#%d checking dependences for task %p : %d "
+                "possibly aliased dependences, %d non-aliased dependences : "
+                "dep_barrier=%d .\n",
+                gtid, taskdata, ndeps, ndeps_noalias, dep_barrier));
+
+  dep_all = __kmp_filter_aliased_deps(ndeps, dep_list, task, &n_mtxs);
+
   // doesn't need to be atomic as no other thread is going to be accessing this
   // node just yet.
   // npredecessors is set -1 to ensure that none of the releasing tasks queues
@@ -555,14 +3198,17 @@ static bool __kmp_check_deps(kmp_int32 gtid, kmp_depnode_t *node,
   // used to pack all npredecessors additions into a single atomic operation at
   // the end
   int npredecessors;
+  kmp_int32 next_mutex = 0;
 
   if (!dep_all) { // regular dependences
-    npredecessors = __kmp_process_deps<true>(gtid, node, hash, dep_barrier,
-                                             ndeps, dep_list, task);
-    npredecessors += __kmp_process_deps<false>(
-        gtid, node, hash, dep_barrier, ndeps_noalias, noalias_dep_list, task);
+    npredecessors = __kmp_process_deps<normal_deps>(
+        gtid, node, hash, dep_barrier, ndeps, dep_list, task, next_mutex);
+    npredecessors += __kmp_process_deps<normal_deps>(
+        gtid, node, hash, dep_barrier, ndeps_noalias, noalias_dep_list, task,
+        next_mutex, false);
   } else { // omp_all_memory dependence
-    npredecessors = __kmp_process_dep_all(gtid, node, *hash, dep_barrier, task);
+    npredecessors = __kmp_process_dep_all<normal_deps>(gtid, node, *hash,
+                                                       dep_barrier, task);
   }
 
   node->dn.task = task;
diff --git a/openmp/runtime/src/kmp_taskdeps.h b/openmp/runtime/src/kmp_taskdeps.h
index 71e8e69d44593..e7f23745967e0 100644
--- a/openmp/runtime/src/kmp_taskdeps.h
+++ b/openmp/runtime/src/kmp_taskdeps.h
@@ -40,6 +40,7 @@ static inline void __kmp_node_deref(kmp_info_t *thread, kmp_depnode_t *node) {
   }
 }
 
+template <bool refcounting>
 static inline void __kmp_depnode_list_free(kmp_info_t *thread,
                                            kmp_depnode_list *list) {
   kmp_depnode_list *next;
@@ -47,7 +48,8 @@ static inline void __kmp_depnode_list_free(kmp_info_t *thread,
   for (; list; list = next) {
     next = list->next;
 
-    __kmp_node_deref(thread, list->node);
+    if (refcounting)
+      __kmp_node_deref(thread, list->node);
 #if USE_FAST_MEMORY
     __kmp_fast_free(thread, list);
 #else
@@ -56,6 +58,7 @@ static inline void __kmp_depnode_list_free(kmp_info_t *thread,
   }
 }
 
+template <bool refcounting>
 static inline void __kmp_dephash_free_entries(kmp_info_t *thread,
                                               kmp_dephash_t *h) {
   for (size_t i = 0; i < h->size; i++) {
@@ -63,12 +66,14 @@ static inline void __kmp_dephash_free_entries(kmp_info_t *thread,
       kmp_dephash_entry_t *next;
       for (kmp_dephash_entry_t *entry = h->buckets[i]; entry; entry = next) {
         next = entry->next_in_bucket;
-        __kmp_depnode_list_free(thread, entry->last_set);
-        __kmp_depnode_list_free(thread, entry->prev_set);
-        __kmp_node_deref(thread, entry->last_out);
-        if (entry->mtx_lock) {
-          __kmp_destroy_lock(entry->mtx_lock);
-          __kmp_free(entry->mtx_lock);
+        __kmp_depnode_list_free<refcounting>(thread, entry->last_set);
+        __kmp_depnode_list_free<refcounting>(thread, entry->prev_set);
+        if (refcounting) {
+          __kmp_node_deref(thread, entry->last_out);
+          if (entry->mtx_lock) {
+            __kmp_destroy_lock(entry->mtx_lock);
+            __kmp_free(entry->mtx_lock);
+          }
         }
 #if USE_FAST_MEMORY
         __kmp_fast_free(thread, entry);
@@ -79,12 +84,14 @@ static inline void __kmp_dephash_free_entries(kmp_info_t *thread,
       h->buckets[i] = 0;
     }
   }
-  __kmp_node_deref(thread, h->last_all);
+  if (refcounting)
+    __kmp_node_deref(thread, h->last_all);
   h->last_all = NULL;
 }
 
+template <bool refcounting>
 static inline void __kmp_dephash_free(kmp_info_t *thread, kmp_dephash_t *h) {
-  __kmp_dephash_free_entries(thread, h);
+  __kmp_dephash_free_entries<refcounting>(thread, h);
 #if USE_FAST_MEMORY
   __kmp_fast_free(thread, h);
 #else
@@ -112,7 +119,7 @@ static inline void __kmp_release_deps(kmp_int32 gtid, kmp_taskdata_t *task) {
     KA_TRACE(
         40, ("__kmp_release_deps: T#%d freeing dependencies hash of task %p.\n",
              gtid, task));
-    __kmp_dephash_free(thread, task->td_dephash);
+    __kmp_dephash_free<true>(thread, task->td_dephash);
     task->td_dephash = NULL;
   }
 
diff --git a/openmp/runtime/src/kmp_tasking.cpp b/openmp/runtime/src/kmp_tasking.cpp
index 23092927babc1..895248cb8c250 100644
--- a/openmp/runtime/src/kmp_tasking.cpp
+++ b/openmp/runtime/src/kmp_tasking.cpp
@@ -17,6 +17,8 @@
 #include "kmp_wait_release.h"
 #include "kmp_taskdeps.h"
 
+#undef DEBUG_TASKGRAPH
+
 #if OMPT_SUPPORT
 #include "ompt-specific.h"
 #endif
@@ -718,7 +720,7 @@ static void __kmp_free_task_and_ancestors(kmp_int32 gtid,
                            "dephash of implicit task %p\n",
                            gtid, taskdata));
             // cleanup dephash of finished implicit task
-            __kmp_dephash_free_entries(thread, taskdata->td_dephash);
+            __kmp_dephash_free_entries<true>(thread, taskdata->td_dephash);
           }
         }
       }
@@ -753,6 +755,12 @@ static bool __kmp_track_children_task(kmp_taskdata_t *taskdata) {
   return ret;
 }
 
+#if OMP_TASKGRAPH_EXPERIMENTAL
+static bool
+__kmp_taskgraph_exec_descr_finish(kmp_int32 gtid, kmp_info_t *thread,
+                                  kmp_taskgraph_exec_descr_t *descr);
+#endif
+
 // __kmp_task_finish: bookkeeping to do when a task finishes execution
 //
 // gtid: global thread ID for calling thread
@@ -769,6 +777,9 @@ static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task,
   kmp_info_t *thread = __kmp_threads[gtid];
   kmp_task_team_t *task_team =
       thread->th.th_task_team; // might be NULL for serial teams...
+#if OMP_TASKGRAPH_EXPERIMENTAL
+  bool is_taskgraph;
+#endif
 #if KMP_DEBUG
   kmp_int32 children = 0;
 #endif
@@ -778,6 +789,10 @@ static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task,
 
   KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT);
 
+#if OMP_TASKGRAPH_EXPERIMENTAL
+  is_taskgraph = taskdata->owning_taskgraph;
+#endif
+
   if (UNLIKELY(taskdata->td_flags.tiedness == TASK_UNTIED)) {
     // untied task needs to check the counter so that the task structure is not
     // freed prematurely
@@ -888,6 +903,19 @@ static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task,
     if (ompt)
       __ompt_task_finish(task, resumed_task, ompt_task_complete);
 #endif
+
+#if OMP_TASKGRAPH_EXPERIMENTAL
+    if (is_taskgraph) {
+      __kmp_taskgraph_exec_descr_finish(gtid, thread, taskdata->exec_descr);
+      KMP_ATOMIC_DEC(&taskdata->td_parent->td_incomplete_child_tasks);
+      if (taskdata->td_taskgroup)
+        KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count);
+      thread->th.th_current_task = resumed_task;
+      resumed_task->td_flags.executing = 1; // resume previous task
+      return;
+    }
+#endif
+
     // TODO: What would be the balance between the conditions in the function
     // and an atomic operation?
     if (__kmp_track_children_task(taskdata)) {
@@ -1106,7 +1134,7 @@ void __kmp_finish_implicit_task(kmp_info_t *thread) {
         KA_TRACE(100, ("__kmp_finish_implicit_task: T#%d cleans "
                        "dephash of implicit task %p\n",
                        thread->th.th_info.ds.ds_gtid, task));
-        __kmp_dephash_free_entries(thread, task->td_dephash);
+        __kmp_dephash_free_entries<true>(thread, task->td_dephash);
       }
     }
   }
@@ -1119,7 +1147,7 @@ void __kmp_finish_implicit_task(kmp_info_t *thread) {
 void __kmp_free_implicit_task(kmp_info_t *thread) {
   kmp_taskdata_t *task = thread->th.th_current_task;
   if (task && task->td_dephash) {
-    __kmp_dephash_free(thread, task->td_dephash);
+    __kmp_dephash_free<true>(thread, task->td_dephash);
     task->td_dephash = NULL;
   }
 }
@@ -1134,7 +1162,7 @@ static size_t __kmp_round_up_to_val(size_t size, size_t val) {
     }
   }
   return size;
-} // __kmp_round_up_to_va
+} // __kmp_round_up_to_val
 
 // __kmp_task_alloc: Allocate the taskdata and task data structures for a task
 //
@@ -1323,6 +1351,10 @@ kmp_task_t *__kmp_task_alloc(ident_t *loc_ref, kmp_int32 gtid,
   taskdata->td_flags.executing = 0;
   taskdata->td_flags.complete = 0;
   taskdata->td_flags.freed = 0;
+#if OMP_TASKGRAPH_EXPERIMENTAL
+  taskdata->owning_taskgraph = nullptr;
+  taskdata->exec_descr = nullptr;
+#endif
   KMP_ATOMIC_ST_RLX(&taskdata->td_incomplete_child_tasks, 0);
   // start at one because counts current task and children
   KMP_ATOMIC_ST_RLX(&taskdata->td_allocated_child_tasks, 1);
@@ -2079,6 +2111,493 @@ kmp_int32 __kmpc_omp_taskyield(ident_t *loc_ref, kmp_int32 gtid, int end_part) {
   return TASK_CURRENT_NOT_QUEUED;
 }
 
+#if OMP_TASKGRAPH_EXPERIMENTAL
+
+// This is copy/pasted, better make it global.
+static kmp_int32 __kmp_region_deplist_len(kmp_taskgraph_region_dep_t *list) {
+  kmp_int32 len = 0;
+  for (; list; list = list->next)
+    ++len;
+  return len;
+}
+
+// Count the number of exec_descr and successor-list structures that we need
+// to flatten REGION.  Follows the structure of __kmp_build_exec_descrs_1.
+
+kmp_int32 __kmp_count_exec_descrs_1(kmp_taskgraph_region_t *region,
+                                    kmp_int32 npreds, kmp_int32 &successors,
+                                    kmp_int32 &descrs) {
+  switch (region->type) {
+  case TASKGRAPH_REGION_NODE:
+  case TASKGRAPH_REGION_WAIT:
+  case TASKGRAPH_REGION_ENTRY:
+  case TASKGRAPH_REGION_EXIT: {
+    descrs++;
+    if (npreds > 0) {
+      successors += npreds;
+    }
+    return 1;
+  }
+  case TASKGRAPH_REGION_PARALLEL: {
+    if (npreds > 0)
+      successors += npreds;
+    descrs++;
+    npreds = 1;
+    kmp_int32 out_preds = 0;
+    for (kmp_int32 c = 0; c < region->inner.num_children; c++) {
+      kmp_int32 child_preds = __kmp_count_exec_descrs_1(
+          region->inner.children[c], npreds, successors, descrs);
+      out_preds += child_preds;
+    }
+    return out_preds;
+  }
+  case TASKGRAPH_REGION_SEQUENTIAL:
+  case TASKGRAPH_REGION_EXCLUSIVE: {
+    // In this loop, PREDS can expire per-iteration and be reinitialised.  So
+    // we shouldn't alter its refcount.
+    for (kmp_int32 c = 0; c < region->inner.num_children; c++) {
+      npreds = __kmp_count_exec_descrs_1(region->inner.children[c], npreds,
+                                         successors, descrs);
+    }
+    return npreds;
+  }
+  case TASKGRAPH_REGION_IRREDUCIBLE: {
+    kmp_int32 out_preds = 0;
+    for (kmp_int32 c = 0; c < region->inner.num_children; c++) {
+      kmp_taskgraph_region_t *child_region = region->inner.children[c];
+      kmp_int32 inner_or_outer_preds = npreds;
+      if (child_region->predecessors)
+        inner_or_outer_preds =
+            __kmp_region_deplist_len(child_region->predecessors);
+      kmp_int32 child_preds = __kmp_count_exec_descrs_1(
+          child_region, inner_or_outer_preds, successors, descrs);
+      if (!child_region->successors)
+        out_preds += child_preds;
+    }
+    return out_preds;
+  }
+  }
+}
+
+void __kmp_count_exec_descrs(kmp_taskgraph_region_t *region,
+                             kmp_int32 &successors, kmp_int32 &descrs) {
+  kmp_int32 npreds;
+  npreds = __kmp_count_exec_descrs_1(region, 0, successors, descrs);
+  // We want to reach the exit...
+  assert(npreds > 0);
+}
+
+kmp_taskgraph_exec_descr_list_t *
+__kmp_exec_descr_list_add(kmp_info_t *thread,
+                          kmp_taskgraph_exec_descr_elem_t **recycled_elems,
+                          kmp_taskgraph_exec_descr_t *exec_descr,
+                          kmp_taskgraph_exec_descr_list_t *list) {
+  kmp_taskgraph_exec_descr_elem_t *head;
+  if (*recycled_elems) {
+    head = *recycled_elems;
+    *recycled_elems = (*recycled_elems)->next;
+  } else
+    head = (kmp_taskgraph_exec_descr_elem_t *)__kmp_fast_allocate(
+        thread, sizeof(kmp_taskgraph_exec_descr_elem_t));
+  head->exec_descr = exec_descr;
+  if (!list) {
+    list = (kmp_taskgraph_exec_descr_list_t *)__kmp_fast_allocate(
+        thread, sizeof(kmp_taskgraph_exec_descr_list_t));
+    list->head = head;
+    list->refcount = 1;
+    list->length = 1;
+    head->next = head;
+  } else {
+    kmp_taskgraph_exec_descr_elem_t *list_head = list->head;
+    kmp_taskgraph_exec_descr_elem_t *list_head_next = list_head->next;
+    list_head->next = head;
+    head->next = list_head_next;
+    list->head = head;
+    list->length++;
+  }
+  return list;
+}
+
+kmp_taskgraph_exec_descr_list_t *__kmp_exec_descr_singleton_list(
+    kmp_info_t *thread, kmp_taskgraph_exec_descr_elem_t **recycled_elems,
+    kmp_taskgraph_exec_descr_t *exec_descr) {
+  return __kmp_exec_descr_list_add(thread, recycled_elems, exec_descr, nullptr);
+}
+
+static kmp_int32
+__kmp_exec_descr_list_len(kmp_taskgraph_exec_descr_list_t *list) {
+  return list ? list->length : 0;
+}
+
+void __kmp_exec_descr_free_recycled(kmp_info_t *thread,
+                                    kmp_taskgraph_exec_descr_elem_t *elems) {
+  while (elems) {
+    kmp_taskgraph_exec_descr_elem_t *next = elems->next;
+    __kmp_fast_free(thread, elems);
+    elems = next;
+  }
+}
+
+static void
+__kmp_exec_descr_elem_recycle(kmp_taskgraph_exec_descr_elem_t **recycled,
+                              kmp_taskgraph_exec_descr_elem_t *elem) {
+  elem->next = *recycled;
+  *recycled = elem;
+}
+
+void __kmp_exec_descr_list_recycle(kmp_info_t *thread,
+                                   kmp_taskgraph_exec_descr_elem_t **recycled,
+                                   kmp_taskgraph_exec_descr_list_t *list) {
+  if (!list)
+    return;
+  kmp_taskgraph_exec_descr_elem_t *head = list->head, *walk;
+  walk = head;
+  do {
+    kmp_taskgraph_exec_descr_elem_t *next = walk->next;
+    __kmp_exec_descr_elem_recycle(recycled, walk);
+    walk = next;
+  } while (walk != head);
+  __kmp_fast_free(thread, list);
+}
+
+kmp_taskgraph_exec_descr_list_t *
+__kmp_exec_descr_list_ref(kmp_taskgraph_exec_descr_list_t *list) {
+  if (list)
+    list->refcount++;
+  return list;
+}
+
+kmp_taskgraph_exec_descr_list_t *
+__kmp_exec_descr_list_deref(kmp_info_t *thread,
+                            kmp_taskgraph_exec_descr_elem_t **recycled,
+                            kmp_taskgraph_exec_descr_list_t *list) {
+  if (!list)
+    return nullptr;
+
+  list->refcount--;
+  if (list->refcount == 0) {
+    __kmp_exec_descr_list_recycle(thread, recycled, list);
+    return nullptr;
+  }
+  return list;
+}
+
+// Destructively append the elements of TO_ADD to DST_LIST.  The TO_ADD list
+// header is freed and shouldn't be used after calling this function.
+
+kmp_taskgraph_exec_descr_list_t *
+__kmp_exec_descr_splice_list(kmp_info_t *thread,
+                             kmp_taskgraph_exec_descr_list_t *dst_list,
+                             kmp_taskgraph_exec_descr_list_t *to_add) {
+  if (dst_list == nullptr && to_add == nullptr)
+    return nullptr;
+  else if (to_add == nullptr)
+    return dst_list;
+  else if (dst_list == nullptr) {
+    // We're splicing into a "new" list, so reset the refcount.
+    to_add->refcount = 1;
+    return to_add;
+  }
+
+  kmp_taskgraph_exec_descr_elem_t *dst_head = dst_list->head;
+  kmp_taskgraph_exec_descr_elem_t *dst_head_next = dst_head->next;
+  kmp_taskgraph_exec_descr_elem_t *add_head = to_add->head;
+  kmp_taskgraph_exec_descr_elem_t *add_head_next = add_head->next;
+
+  dst_list->length += to_add->length;
+  dst_head->next = add_head_next;
+  add_head->next = dst_head_next;
+
+  // Order doesn't really matter, but this makes the head pointer point to the
+  // start of the added list, so TO_ADD is neatly prepended to the dst list,
+  // rather than having (old head->appended list->remainder of old list).
+  dst_list->head = add_head;
+
+  // Free the added list header.
+  __kmp_fast_free(thread, to_add);
+
+  return dst_list;
+}
+
+// The PREDS list gets consumed ONCE by a call to this function, so if you need
+// to keep it alive, take another reference on it.
+
+kmp_taskgraph_exec_descr_list_t *__kmp_build_exec_descrs_1(
+    kmp_info_t *thread, kmp_taskgraph_exec_descr_t *exec_descr_arr,
+    kmp_int32 &idx, kmp_taskgraph_exec_descr_elem_t **recycled,
+    kmp_taskgraph_region_t *region, kmp_taskgraph_exec_descr_list_t *preds,
+    kmp_taskgraph_exec_descr_elem_t **successor_tail) {
+  auto descr_for_region =
+      [&](kmp_taskgraph_region_t *region,
+          kmp_int32 npreds) -> kmp_taskgraph_exec_descr_t * {
+    kmp_taskgraph_exec_descr_t *ed = nullptr;
+    if (region->exec_descr) {
+      assert(region->exec_descr->region == region);
+      ed = region->exec_descr;
+      if (ed->indegree == -1 && npreds > 0)
+        ed->indegree = npreds;
+    } else {
+      ed = &exec_descr_arr[idx++];
+      ed->region = region;
+      ed->indegree = npreds;
+      ed->successors = nullptr;
+      region->exec_descr = ed;
+    }
+    return ed;
+  };
+  auto fill = [&](kmp_taskgraph_exec_descr_t *exec_descr,
+                  kmp_taskgraph_exec_descr_list_t *preds_list) -> void {
+    // kmp_taskgraph_exec_descr_elem_t *successors = *successor_tail;
+    if (!preds_list)
+      return;
+    kmp_taskgraph_exec_descr_elem_t *pred = preds_list->head, *walk;
+    kmp_int32 succ_idx = 0;
+    walk = pred;
+    do {
+      kmp_taskgraph_exec_descr_t *pred_descr = walk->exec_descr;
+      // Build a simple linked list (null-terminated) of successors.  We're
+      // using a bump-allocated region because we know how many successors
+      // we need (we counted already).
+      kmp_taskgraph_exec_descr_elem_t *new_successor = *successor_tail;
+      (*successor_tail)++;
+      new_successor->exec_descr = exec_descr;
+      new_successor->next = pred_descr->successors;
+      pred_descr->successors = new_successor;
+      walk = walk->next;
+    } while (walk != pred);
+  };
+  switch (region->type) {
+  case TASKGRAPH_REGION_NODE:
+  case TASKGRAPH_REGION_WAIT:
+  case TASKGRAPH_REGION_ENTRY:
+  case TASKGRAPH_REGION_EXIT: {
+    kmp_taskgraph_exec_descr_t *exec_descr =
+        descr_for_region(region, __kmp_exec_descr_list_len(preds));
+    // Edit the taskdata for this specific instantiation.  At present the
+    // task/taskdata structures cannot be used simultaneously by different
+    // threads. We could duplicate the structures to allow simultaneous issue,
+    // but that's not done yet.  The exec_descr can already by thread-local,
+    // in principle, but for now it points to the taskgraph's single copy
+    // of each task/taskdata structure.
+    if (region->type == TASKGRAPH_REGION_NODE) {
+      kmp_task_t *task = exec_descr->region->task.node->task;
+      kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
+      taskdata->exec_descr = exec_descr;
+    }
+    fill(exec_descr, preds);
+    __kmp_exec_descr_list_deref(thread, recycled, preds);
+    return __kmp_exec_descr_singleton_list(thread, recycled, exec_descr);
+  }
+  case TASKGRAPH_REGION_PARALLEL: {
+    kmp_int32 npreds = __kmp_exec_descr_list_len(preds);
+    kmp_taskgraph_exec_descr_t *gather_descr = descr_for_region(region, npreds);
+    fill(gather_descr, preds);
+    __kmp_exec_descr_list_deref(thread, recycled, preds);
+    preds = __kmp_exec_descr_singleton_list(thread, recycled, gather_descr);
+    kmp_taskgraph_exec_descr_list_t *out_preds = nullptr;
+    for (kmp_int32 c = 0; c < region->inner.num_children; c++) {
+      kmp_taskgraph_exec_descr_list_t *child_preds;
+      // The preds are reused for each iteration, so keep them alive over this
+      // iteration of the loop.
+      __kmp_exec_descr_list_ref(preds);
+      child_preds = __kmp_build_exec_descrs_1(
+          thread, exec_descr_arr, idx, recycled, region->inner.children[c],
+          preds, successor_tail);
+      out_preds = __kmp_exec_descr_splice_list(thread, out_preds, child_preds);
+    }
+    __kmp_exec_descr_list_deref(thread, recycled, preds);
+    return out_preds;
+  }
+  case TASKGRAPH_REGION_SEQUENTIAL:
+  case TASKGRAPH_REGION_EXCLUSIVE: {
+    // In this loop, PREDS expires per-iteration and a fresh list is returned
+    // (with refcount==1).  So we don't need to make any further refcount
+    // adjustments.
+    for (kmp_int32 c = 0; c < region->inner.num_children; c++) {
+      preds = __kmp_build_exec_descrs_1(thread, exec_descr_arr, idx, recycled,
+                                        region->inner.children[c], preds,
+                                        successor_tail);
+    }
+    return preds;
+  }
+  case TASKGRAPH_REGION_IRREDUCIBLE: {
+    kmp_taskgraph_exec_descr_list_t *out_preds = nullptr;
+    for (kmp_int32 c = 0; c < region->inner.num_children; c++) {
+      kmp_taskgraph_region_t *child_region = region->inner.children[c];
+      kmp_taskgraph_exec_descr_list_t *inner_or_outer_preds = preds;
+      if (child_region->predecessors) {
+        inner_or_outer_preds = nullptr;
+        for (kmp_taskgraph_region_dep_t *p = child_region->predecessors; p;
+             p = p->next) {
+          assert(p->region);
+          kmp_taskgraph_exec_descr_t *pred_descr =
+              descr_for_region(p->region, -1);
+          inner_or_outer_preds = __kmp_exec_descr_list_add(
+              thread, recycled, pred_descr, inner_or_outer_preds);
+        }
+      } else {
+        // PREDS may be reused on successive iterations of the loop: make sure
+        // it stays alive.
+        __kmp_exec_descr_list_ref(preds);
+      }
+      // If INNER_OR_OUTER_PREDS is local, it will be consumed by the call.  If
+      // it is the incoming PREDS, we bumped the reference count above.  In
+      // either case the recursive call consumes it once.
+      kmp_taskgraph_exec_descr_list_t *child_preds = __kmp_build_exec_descrs_1(
+          thread, exec_descr_arr, idx, recycled, child_region,
+          inner_or_outer_preds, successor_tail);
+      if (!child_region->successors)
+        out_preds =
+            __kmp_exec_descr_splice_list(thread, out_preds, child_preds);
+      else
+        // This child's output preds are not a region exit and are not needed
+        // by any successor (successors wire themselves up via their own
+        // 'predecessors' lists), so drop the fresh list here to avoid leaking
+        // its header and elements.
+        __kmp_exec_descr_list_deref(thread, recycled, child_preds);
+    }
+    // We do this unconditionally even though it looks like it could depend on
+    // the flow in the loop above, but the (explicit) ref by the _list_ref call
+    // and (implicit) deref by __kmp_build_exec_descrs balance out, so it's OK.
+    // The next line dereferences PREDS for *this* function activation, i.e.
+    // for our caller.
+    __kmp_exec_descr_list_deref(thread, recycled, preds);
+    return out_preds;
+  }
+  }
+}
+
+kmp_taskgraph_exec_descr_t *
+__kmp_build_exec_descrs(kmp_info_t *thread, kmp_taskgraph_region_t *region,
+                        kmp_int32 *num_descrs) {
+  kmp_int32 ndescrs = 0, nsuccessors = 0;
+  __kmp_count_exec_descrs(region, nsuccessors, ndescrs);
+  char *arena = (char *)__kmp_thread_malloc(
+      thread, ndescrs * sizeof(kmp_taskgraph_exec_descr_t) +
+                  nsuccessors * sizeof(kmp_taskgraph_exec_descr_elem_t));
+  kmp_taskgraph_exec_descr_t *exec_descr_arr =
+      (kmp_taskgraph_exec_descr_t *)arena;
+  kmp_taskgraph_exec_descr_elem_t *successors =
+      (kmp_taskgraph_exec_descr_elem_t *)&exec_descr_arr[ndescrs];
+  kmp_taskgraph_exec_descr_elem_t *successor_tail = successors;
+  kmp_taskgraph_exec_descr_elem_t *recycled_elems = nullptr;
+  kmp_int32 idx = 0;
+  kmp_taskgraph_exec_descr_list_t *out_preds =
+      __kmp_build_exec_descrs_1(thread, exec_descr_arr, idx, &recycled_elems,
+                                region, nullptr, &successor_tail);
+  assert(idx == ndescrs);
+  assert(successor_tail - successors == nsuccessors);
+  __kmp_exec_descr_list_deref(thread, &recycled_elems, out_preds);
+  __kmp_exec_descr_free_recycled(thread, recycled_elems);
+  *num_descrs = ndescrs;
+  return exec_descr_arr;
+}
+
+/// Reset, reparent and regroup the recorded task TASK and re-invoke it.
+
+static void __kmp_omp_tg_task(kmp_int32 gtid, kmp_task_t *task,
+                              kmp_taskgroup_t *taskgroup,
+                              kmp_taskdata_t *parent_taskdata,
+                              bool serialize_immediate) {
+  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
+  taskdata->td_parent = parent_taskdata;
+
+  taskdata->td_flags.complete = 0;
+  taskdata->td_flags.started = 0;
+  taskdata->td_flags.freed = 0;
+  taskdata->td_flags.executing = 0;
+  taskdata->td_flags.task_serial =
+      (parent_taskdata->td_flags.final || taskdata->td_flags.team_serial ||
+       taskdata->td_flags.tasking_ser);
+
+  KMP_ATOMIC_ST_RLX(&taskdata->td_untied_count, 0);
+  KMP_ATOMIC_ST_RLX(&taskdata->td_incomplete_child_tasks, 0);
+  // Start at one because counter represents current task and children.
+  KMP_ATOMIC_ST_RLX(&taskdata->td_allocated_child_tasks, 1);
+
+  taskdata->td_taskgroup = taskgroup;
+  KMP_ATOMIC_INC(&taskgroup->count);
+  KMP_ATOMIC_INC(&parent_taskdata->td_incomplete_child_tasks);
+  if (parent_taskdata->td_flags.tasktype == TASK_EXPLICIT)
+    KMP_ATOMIC_INC(&parent_taskdata->td_allocated_child_tasks);
+
+  __kmp_omp_task(gtid, task, false);
+}
+
+struct kmp_taskred_input;
+template <typename T>
+void *__kmp_task_reduction_init(int gtid, int num, T *data);
+
+// Decrement DESCR's predecessor count; if it reaches zero, fire it.  TASKGROUP
+// is the taskgroup any task spawned by DESCR should join.
+static void __kmp_taskgraph_exec_descr_start(kmp_int32 gtid, kmp_info_t *thread,
+                                             kmp_taskgraph_exec_descr_t *descr,
+                                             kmp_taskgroup_t *taskgroup) {
+  kmp_int32 npredecessors = KMP_ATOMIC_DEC(&descr->npredecessors) - 1;
+  if (npredecessors > 0)
+    return;
+
+  switch (descr->region->type) {
+  case TASKGRAPH_REGION_NODE: {
+    // Spawn the task; its successors are triggered later, on completion, by
+    // __kmp_taskgraph_exec_descr_finish.
+    kmp_taskgraph_node_t *node = descr->region->task.node;
+    kmp_taskdata_t *current_taskdata = thread->th.th_current_task;
+    __kmp_omp_tg_task(gtid, node->task, taskgroup, current_taskdata, false);
+    break;
+  }
+  case TASKGRAPH_REGION_PARALLEL:
+    if (descr->region->reduce_input) {
+      // If there are reductions associated with this parallel region, we
+      // start a new taskgroup here.
+      __kmpc_taskgroup(/*loc=*/nullptr, gtid);
+      // Update variable to the newly-created taskgroup.
+      taskgroup = thread->th.th_current_task->td_taskgroup;
+      __kmp_task_reduction_init(
+          gtid, descr->region->reduce_input->reduce_num_data,
+          (struct kmp_taskred_input *)descr->region->reduce_input->reduce_data);
+    }
+    KMP_FALLTHROUGH();
+  case TASKGRAPH_REGION_WAIT:
+    for (kmp_taskgraph_exec_descr_elem_t *s = descr->successors; s; s = s->next)
+      __kmp_taskgraph_exec_descr_start(gtid, thread, s->exec_descr, taskgroup);
+    if (descr->region->type == TASKGRAPH_REGION_PARALLEL &&
+        descr->region->reduce_input)
+      __kmpc_end_taskgroup(/*loc=*/nullptr, gtid);
+    break;
+  case TASKGRAPH_REGION_IRREDUCIBLE:
+  case TASKGRAPH_REGION_EXCLUSIVE:
+  case TASKGRAPH_REGION_SEQUENTIAL:
+  case TASKGRAPH_REGION_ENTRY:
+  case TASKGRAPH_REGION_EXIT: {
+    // We don't expect to see any of these region types here.
+    abort();
+  }
+  }
+}
+
+// Called when DESCR's task completes: release its dataflow successors into the
+// same taskgroup the task ran in.  Returns true iff there were any successors.
+static bool
+__kmp_taskgraph_exec_descr_finish(kmp_int32 gtid, kmp_info_t *thread,
+                                  kmp_taskgraph_exec_descr_t *descr) {
+  KMP_DEBUG_ASSERT(descr->region->type == TASKGRAPH_REGION_NODE);
+  kmp_task_t *task = descr->region->task.node->task;
+  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
+  taskdata->td_flags.started = 0;
+  taskdata->td_flags.executing = 0;
+  taskdata->td_flags.complete = 0;
+  taskdata->td_flags.freed = 0;
+  bool any_successors = false;
+  for (kmp_taskgraph_exec_descr_elem_t *s = descr->successors; s; s = s->next) {
+    any_successors = true;
+    __kmp_taskgraph_exec_descr_start(gtid, thread, s->exec_descr,
+                                     taskdata->td_taskgroup);
+  }
+  return any_successors;
+}
+#endif
+
 // Task Reduction implementation
 //
 // Note: initial implementation didn't take into account the possibility
@@ -2266,6 +2785,48 @@ void *__kmpc_taskred_init(int gtid, int num, void *data) {
   return __kmp_task_reduction_init(gtid, num, (kmp_taskred_input_t *)data);
 }
 
+#if OMP_TASKGRAPH_EXPERIMENTAL
+static kmp_taskgraph_record_t *
+__kmp_taskgraph_or_parent_recording(kmp_taskgroup_t *taskgroup) {
+  kmp_taskgraph_record_t *rec = nullptr;
+
+  for (; taskgroup; taskgroup = taskgroup->parent) {
+    rec = KMP_ATOMIC_LD_ACQ(&taskgroup->taskgraph.recording);
+    if (rec)
+      return rec;
+  }
+
+  return nullptr;
+}
+
+void *__kmpc_taskgraph_taskred_init(kmp_int32 gtid, kmp_int32 num, void *data) {
+  kmp_info_t *thread = __kmp_threads[gtid];
+  kmp_taskgroup_t *taskgroup = thread->th.th_current_task->td_taskgroup;
+  kmp_taskgraph_record_t *rec = __kmp_taskgraph_or_parent_recording(taskgroup);
+
+  if (rec) {
+    kmp_taskgraph_status_t status = KMP_ATOMIC_LD_ACQ(&rec->status);
+    if (status == KMP_TDG_RECORDING) {
+      kmp_taskgraph_reduce_input_data_t *input_data =
+          (kmp_taskgraph_reduce_input_data_t *)__kmp_fast_allocate(
+              thread, sizeof(kmp_taskgraph_reduce_input_data_t));
+      // The compiler might build the reduction input data on the stack, so
+      // we must make a copy.
+      input_data->reduce_data =
+          __kmp_fast_allocate(thread, sizeof(kmp_taskred_input_t) * num);
+      KMP_MEMCPY(input_data->reduce_data, data,
+                 sizeof(kmp_taskred_input_t) * num);
+      ;
+      input_data->reduce_num_data = num;
+      taskgroup->taskgraph.reduce_input = input_data;
+    } else if (status == KMP_TDG_READY)
+      assert(false &&
+             "unexpected __kmpc_taskgraph_taskred_init with ready taskgraph");
+  }
+  return __kmp_task_reduction_init(gtid, num, (kmp_taskred_input_t *)data);
+}
+#endif
+
 // Copy task reduction data (except for shared pointers).
 template <typename T>
 void __kmp_task_reduction_init_copy(kmp_info_t *thr, int num, T *data,
@@ -2302,7 +2863,11 @@ void *__kmpc_task_reduction_get_th_data(int gtid, void *tskgrp, void *data) {
     return data; // nothing to do
 
   kmp_taskgroup_t *tg = (kmp_taskgroup_t *)tskgrp;
+#if OMP_TASKGRAPH_EXPERIMENTAL
+  if (tg == NULL || thread->th.th_current_task->owning_taskgraph)
+#else
   if (tg == NULL)
+#endif
     tg = thread->th.th_current_task->td_taskgroup;
   KMP_ASSERT(tg != NULL);
   kmp_taskred_data_t *arr;
@@ -2513,6 +3078,10 @@ void __kmpc_taskgroup(ident_t *loc, int gtid) {
   tg_new->reduce_data = NULL;
   tg_new->reduce_num_data = 0;
   tg_new->gomp_data = NULL;
+#if OMP_TASKGRAPH_EXPERIMENTAL
+  tg_new->taskgraph.recording = nullptr;
+  tg_new->taskgraph.reduce_input = nullptr;
+#endif
   taskdata->td_taskgroup = tg_new;
 
 #if OMPT_SUPPORT && OMPT_OPTIONAL
@@ -2668,6 +3237,13 @@ void __kmpc_end_taskgroup(ident_t *loc, int gtid) {
       __kmp_task_reduction_fini(thread, taskgroup);
     }
   }
+
+#if OMP_TASKGRAPH_EXPERIMENTAL
+  // This should have been moved to a task node within the group, else it will
+  // leak here.
+  assert(!taskgroup->taskgraph.reduce_input);
+#endif
+
   // Restore parent taskgroup for the current task
   taskdata->td_taskgroup = taskgroup->parent;
   __kmp_thread_free(thread, taskgroup);
@@ -2684,6 +3260,37 @@ void __kmpc_end_taskgroup(ident_t *loc, int gtid) {
 #endif
 }
 
+#if OMP_TASKGRAPH_EXPERIMENTAL
+static void __kmp_replay_taskgraph(kmp_int32 gtid,
+                                   kmp_taskdata_t *current_taskdata,
+                                   kmp_taskgraph_record_t *taskgraph,
+                                   kmp_intptr_t graph_id,
+                                   kmp_taskgroup_t *taskgroup) {
+  kmp_info_t *thread = __kmp_threads[gtid];
+
+  // An empty graph (no recorded task nodes) has nothing to replay.
+  // Bail out early.
+  if (taskgraph->num_tasks == 0)
+    return;
+
+  // Build the flat dataflow DAG once per taskgraph (cached on the record);
+  // re-arm the predecessor counters on every replay.
+  if (!taskgraph->exec_descrs) {
+    kmp_int32 num_descrs;
+    taskgraph->exec_descrs =
+        __kmp_build_exec_descrs(thread, taskgraph->root, &num_descrs);
+    taskgraph->num_exec_descrs = num_descrs;
+  }
+
+  kmp_taskgraph_exec_descr_t *exec_descrs = taskgraph->exec_descrs;
+  for (kmp_size_t i = 0; i < taskgraph->num_exec_descrs; i++)
+    exec_descrs[i].npredecessors = exec_descrs[i].indegree;
+
+  __kmp_taskgraph_exec_descr_start(gtid, thread, taskgraph->exec_descrs,
+                                   taskgroup);
+}
+#endif
+
 static kmp_task_t *__kmp_get_priority_task(kmp_int32 gtid,
                                            kmp_task_team_t *task_team,
                                            kmp_int32 is_constrained) {
@@ -4276,7 +4883,12 @@ void __kmp_fulfill_event(kmp_event_t *event) {
 // taskloop_recur: used only when dealing with taskgraph,
 //      indicating whether we need to update task->td_task_id
 // returns:  a pointer to the allocated kmp_task_t structure (task).
-kmp_task_t *__kmp_task_dup_alloc(kmp_info_t *thread, kmp_task_t *task_src) {
+kmp_task_t *__kmp_task_dup_alloc(kmp_info_t *thread, kmp_task_t *task_src
+#if OMP_TASKGRAPH_EXPERIMENTAL
+                                 ,
+                                 bool taskgraph
+#endif
+) {
   kmp_task_t *task;
   kmp_taskdata_t *taskdata;
   kmp_taskdata_t *taskdata_src = KMP_TASK_TO_TASKDATA(task_src);
@@ -4322,7 +4934,12 @@ kmp_task_t *__kmp_task_dup_alloc(kmp_info_t *thread, kmp_task_t *task_src) {
 
   // Only need to keep track of child task counts if team parallel and tasking
   // not serialized
+#if OMP_TASKGRAPH_EXPERIMENTAL
+  if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser ||
+        taskgraph)) {
+#else
   if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser)) {
+#endif
     KMP_ATOMIC_INC(&parent_task->td_incomplete_child_tasks);
     if (parent_task->td_taskgroup)
       KMP_ATOMIC_INC(&parent_task->td_taskgroup->count);
@@ -4455,6 +5072,47 @@ class kmp_taskloop_bounds_t {
   }
 };
 
+#if OMP_TASKGRAPH_EXPERIMENTAL
+kmp_taskgraph_node_t *
+__kmp_taskgraph_node_alloc(kmp_taskgraph_record_t *rec, kmp_task_t *task,
+                           kmp_size_t *index_p = nullptr) {
+  kmp_int32 gtid = rec->gtid;
+  kmp_info_t *thread = __kmp_threads[gtid];
+  kmp_taskgraph_node_t *new_task = nullptr;
+
+  __kmp_acquire_lock(&rec->map_lock, gtid);
+
+  if (!rec->record_map) {
+    rec->nodes_allocated = 4;
+    rec->record_map = (kmp_taskgraph_node_t *)__kmp_thread_malloc(
+        thread, rec->nodes_allocated * sizeof(kmp_taskgraph_node_t));
+  }
+
+  if (rec->num_tasks >= rec->nodes_allocated) {
+    rec->record_map = (kmp_taskgraph_node_t *)__kmp_thread_realloc(
+        thread, rec->record_map,
+        2 * rec->nodes_allocated * sizeof(kmp_taskgraph_node_t));
+    rec->nodes_allocated *= 2;
+  }
+
+  new_task = &rec->record_map[rec->num_tasks];
+  if (index_p)
+    *index_p = rec->num_tasks;
+  ++rec->num_tasks;
+
+  __kmp_release_lock(&rec->map_lock, gtid);
+
+  new_task->task = task;
+  new_task->taskloop_task = false;
+  new_task->reduce_input = nullptr;
+  new_task->u.unresolved.ndeps = 0;
+  new_task->u.unresolved.dep_list = nullptr;
+  new_task->u.unresolved.cfg_successor = -1;
+
+  return new_task;
+}
+#endif
+
 // __kmp_taskloop_linear: Start tasks of the taskloop linearly
 //
 // loc        Source location information
@@ -4471,15 +5129,21 @@ class kmp_taskloop_bounds_t {
 // tc         Iterations count
 // task_dup   Tasks duplication routine
 // codeptr_ra Return address for OMPT events
-void __kmp_taskloop_linear(ident_t *loc, int gtid, kmp_task_t *task,
-                           kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
-                           kmp_uint64 ub_glob, kmp_uint64 num_tasks,
-                           kmp_uint64 grainsize, kmp_uint64 extras,
-                           kmp_int64 last_chunk, kmp_uint64 tc,
+static void
+__kmp_taskloop_linear(ident_t *loc, int gtid, kmp_task_t *task, kmp_uint64 *lb,
+                      kmp_uint64 *ub, kmp_int64 st, kmp_int32 nogroup,
+                      kmp_uint64 ub_glob, kmp_uint64 num_tasks,
+                      kmp_uint64 grainsize, kmp_uint64 extras,
+                      kmp_int64 last_chunk, kmp_uint64 tc,
 #if OMPT_SUPPORT
-                           void *codeptr_ra,
+                      void *codeptr_ra,
+#endif
+                      void *task_dup
+#if OMP_TASKGRAPH_EXPERIMENTAL
+                      ,
+                      kmp_taskgraph_record_t *taskgraph_rec = nullptr
 #endif
-                           void *task_dup) {
+) {
   KMP_COUNT_BLOCK(OMP_TASKLOOP);
   KMP_TIME_PARTITIONED_BLOCK(OMP_taskloop_scheduling);
   p_task_dup_t ptask_dup = (p_task_dup_t)task_dup;
@@ -4492,6 +5156,9 @@ void __kmp_taskloop_linear(ident_t *loc, int gtid, kmp_task_t *task,
   kmp_taskdata_t *current_task = thread->th.th_current_task;
   kmp_task_t *next_task;
   kmp_int32 lastpriv = 0;
+#if OMP_TASKGRAPH_EXPERIMENTAL
+  kmp_int32 taskloop_prev_idx = -1, taskloop_first_idx = -1;
+#endif
 
   KMP_DEBUG_ASSERT(tc == num_tasks * grainsize +
                              (last_chunk < 0 ? last_chunk : extras));
@@ -4532,7 +5199,11 @@ void __kmp_taskloop_linear(ident_t *loc, int gtid, kmp_task_t *task,
       }
     }
 
+#if OMP_TASKGRAPH_EXPERIMENTAL
+    next_task = __kmp_task_dup_alloc(thread, task, /*taskgraph=*/taskgraph_rec);
+#else
     next_task = __kmp_task_dup_alloc(thread, task); // allocate new task
+#endif
 
     kmp_taskdata_t *next_taskdata = KMP_TASK_TO_TASKDATA(next_task);
     kmp_taskloop_bounds_t next_task_bounds =
@@ -4554,24 +5225,78 @@ void __kmp_taskloop_linear(ident_t *loc, int gtid, kmp_task_t *task,
               gtid, i, next_task, lower, upper, st,
               next_task_bounds.get_lower_offset(),
               next_task_bounds.get_upper_offset()));
+#if OMP_TASKGRAPH_EXPERIMENTAL
+    if (taskgraph_rec) {
+      kmp_size_t rec_index = -1;
+      // Record the task in the taskgraph.
+      kmp_taskgraph_node_t *node =
+          __kmp_taskgraph_node_alloc(taskgraph_rec, next_task, &rec_index);
+      kmp_taskgroup_t *taskgroup = current_task->td_taskgroup;
+      if (taskgroup->taskgraph.reduce_input) {
+        node->reduce_input = taskgroup->taskgraph.reduce_input;
+        taskgroup->taskgraph.reduce_input = nullptr;
+      }
+      node->taskloop_task = true;
+      next_taskdata->owning_taskgraph = taskgraph_rec;
+      // FIXME: These dependency fields might be back-filled by the as-yet
+      // unimplemented task_iteration subsidiary directive.  We'll need a way
+      // to locate the correct task given the value of the iteration variable,
+      // or similar.
+      node->u.unresolved.ndeps = 0;
+      node->u.unresolved.dep_list = nullptr;
+      if (nogroup)
+        taskgraph_rec->record_map[rec_index].u.unresolved.cfg_successor = -1;
+      else if (taskloop_prev_idx != -1)
+        taskgraph_rec->record_map[taskloop_prev_idx]
+            .u.unresolved.cfg_successor = rec_index;
+      if (taskloop_first_idx == -1)
+        taskloop_first_idx = rec_index;
+      taskloop_prev_idx = rec_index;
+    } else {
+#endif
 #if OMPT_SUPPORT
-    __kmp_omp_taskloop_task(NULL, gtid, next_task,
-                            codeptr_ra); // schedule new task
+      __kmp_omp_taskloop_task(NULL, gtid, next_task,
+                              codeptr_ra); // schedule new task
 #if OMPT_OPTIONAL
-    if (ompt_enabled.ompt_callback_dispatch) {
-      OMPT_GET_DISPATCH_CHUNK(next_taskdata->ompt_task_info.dispatch_chunk,
-                              lower, upper, st);
-    }
+      if (ompt_enabled.ompt_callback_dispatch) {
+        OMPT_GET_DISPATCH_CHUNK(next_taskdata->ompt_task_info.dispatch_chunk,
+                                lower, upper, st);
+      }
 #endif // OMPT_OPTIONAL
 #else
     __kmp_omp_task(gtid, next_task, true); // schedule new task
+#endif
+#if OMP_TASKGRAPH_EXPERIMENTAL
+    }
 #endif
     lower = upper + st; // adjust lower bound for the next iteration
   }
-  // free the pattern task and exit
-  __kmp_task_start(gtid, task, current_task); // make internal bookkeeping
-  // do not execute the pattern task, just do internal bookkeeping
-  __kmp_task_finish<false>(gtid, task, current_task);
+#if OMP_TASKGRAPH_EXPERIMENTAL
+  if (taskgraph_rec) {
+    if (taskloop_prev_idx != -1 && !nogroup) {
+      // Create a node to act as an "end group" marker.
+      kmp_size_t endgroup_idx = -1;
+      kmp_taskgraph_node_t *endgrpnode =
+          __kmp_taskgraph_node_alloc(taskgraph_rec, nullptr, &endgroup_idx);
+      endgrpnode->taskloop_task = true;
+      // Point all the cfg_successor indices to this node now.
+      for (kmp_int32 looptask = taskloop_first_idx; looptask != -1;) {
+        kmp_int32 next_task =
+            taskgraph_rec->record_map[looptask].u.unresolved.cfg_successor;
+        taskgraph_rec->record_map[looptask].u.unresolved.cfg_successor =
+            endgroup_idx;
+        looptask = next_task;
+      }
+    }
+  } else {
+#endif
+    // free the pattern task and exit
+    __kmp_task_start(gtid, task, current_task); // make internal bookkeeping
+    // do not execute the pattern task, just do internal bookkeeping
+    __kmp_task_finish<false>(gtid, task, current_task);
+#if OMP_TASKGRAPH_EXPERIMENTAL
+  }
+#endif
 }
 
 // Structure to keep taskloop parameters for auxiliary task
@@ -4641,8 +5366,8 @@ int __kmp_taskloop_task(int gtid, void *ptask) {
 #endif
                          task_dup);
   else
-    __kmp_taskloop_linear(NULL, gtid, task, lb, ub, st, ub_glob, num_tasks,
-                          grainsize, extras, last_chunk, tc,
+    __kmp_taskloop_linear(NULL, gtid, task, lb, ub, st, /*nogroup=*/true,
+                          ub_glob, num_tasks, grainsize, extras, last_chunk, tc,
 #if OMPT_SUPPORT
                           codeptr_ra,
 #endif
@@ -4730,7 +5455,11 @@ void __kmp_taskloop_recur(ident_t *loc, int gtid, kmp_task_t *task,
   lb1 = ub0 + st;
 
   // create pattern task for 2nd half of the loop
+#if OMP_TASKGRAPH_EXPERIMENTAL
+  next_task = __kmp_task_dup_alloc(thread, task, /*taskgraph=*/false);
+#else
   next_task = __kmp_task_dup_alloc(thread, task); // duplicate the task
+#endif
   // adjust lower bound (upper bound is not changed) for the 2nd half
   *(kmp_uint64 *)((char *)next_task + lower_offset) = lb1;
   if (ptask_dup != NULL) // construct firstprivates, etc.
@@ -4763,6 +5492,11 @@ void __kmp_taskloop_recur(ident_t *loc, int gtid, kmp_task_t *task,
   p->codeptr_ra = codeptr_ra;
 #endif
 
+#if OMP_TASKGRAPH_EXPERIMENTAL
+  kmp_taskdata_t *new_task_data = KMP_TASK_TO_TASKDATA(new_task);
+  new_task_data->owning_taskgraph = nullptr;
+#endif
+
 #if OMPT_SUPPORT
   // schedule new task with correct return address for OMPT events
   __kmp_omp_taskloop_task(NULL, gtid, new_task, codeptr_ra);
@@ -4779,8 +5513,8 @@ void __kmp_taskloop_recur(ident_t *loc, int gtid, kmp_task_t *task,
 #endif
                          task_dup);
   else
-    __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, n_tsk0,
-                          gr_size0, ext0, last_chunk0, tc0,
+    __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, /*nogroup=*/true,
+                          ub_glob, n_tsk0, gr_size0, ext0, last_chunk0, tc0,
 #if OMPT_SUPPORT
                           codeptr_ra,
 #endif
@@ -4792,14 +5526,22 @@ void __kmp_taskloop_recur(ident_t *loc, int gtid, kmp_task_t *task,
 static void __kmp_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
                            kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
                            int nogroup, int sched, kmp_uint64 grainsize,
-                           int modifier, void *task_dup) {
+                           int modifier, void *task_dup
+#if OMP_TASKGRAPH_EXPERIMENTAL
+                           ,
+                           kmp_taskgraph_record_t *taskgraph_rec = nullptr
+#endif
+) {
   kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
   KMP_DEBUG_ASSERT(task != NULL);
   if (nogroup == 0) {
 #if OMPT_SUPPORT && OMPT_OPTIONAL
     OMPT_STORE_RETURN_ADDRESS(gtid);
 #endif
-    __kmpc_taskgroup(loc, gtid);
+#if OMP_TASKGRAPH_EXPERIMENTAL
+    if (!taskgraph_rec)
+#endif
+      __kmpc_taskgroup(loc, gtid);
   }
   // =========================================================================
   // calculate loop parameters
@@ -4831,17 +5573,27 @@ static void __kmp_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
   }
   if (tc == 0) {
     KA_TRACE(20, ("__kmp_taskloop(exit): T#%d zero-trip loop\n", gtid));
-    // free the pattern task and exit
-    __kmp_task_start(gtid, task, current_task);
-    // do not execute anything for zero-trip loop
-    __kmp_task_finish<false>(gtid, task, current_task);
+#if OMP_TASKGRAPH_EXPERIMENTAL
+    if (!taskgraph_rec) {
+#endif
+      // free the pattern task and exit
+      __kmp_task_start(gtid, task, current_task);
+      // do not execute anything for zero-trip loop
+      __kmp_task_finish<false>(gtid, task, current_task);
+#if OMP_TASKGRAPH_EXPERIMENTAL
+    }
+#endif
     return;
   }
 
 #if OMPT_SUPPORT && OMPT_OPTIONAL
   ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);
   ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
+#if OMP_TASKGRAPH_EXPERIMENTAL
+  if (ompt_enabled.ompt_callback_work && !taskgraph_rec) {
+#else
   if (ompt_enabled.ompt_callback_work) {
+#endif
     ompt_callbacks.ompt_callback(ompt_callback_work)(
         ompt_work_taskloop, ompt_scope_begin, &(team_info->parallel_data),
         &(task_info->task_data), tc, OMPT_GET_RETURN_ADDRESS(0));
@@ -4898,14 +5650,31 @@ static void __kmp_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
   KMP_DEBUG_ASSERT(num_tasks > 0);
   // =========================================================================
 
-  // check if clause value first
-  // Also require GOMP_taskloop to reduce to linear (taskdata->td_flags.native)
-  if (if_val == 0) { // if(0) specified, mark task as serial
+#if OMP_TASKGRAPH_EXPERIMENTAL
+  // Handle taskgraph case first.  We just generate tasks and record them in
+  // the graph, but we do not execute them here.
+  if (taskgraph_rec) {
+    if (if_val == 0) {
+      taskdata->td_flags.task_serial = 1;
+      taskdata->td_flags.tiedness = TASK_TIED;
+    }
+    __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, nogroup, ub_glob,
+                          num_tasks, grainsize, extras, last_chunk, tc,
+#if OMPT_SUPPORT
+                          OMPT_GET_RETURN_ADDRESS(0),
+#endif
+                          task_dup, taskgraph_rec);
+    // check if clause value next
+    // Also require GOMP_taskloop to reduce to linear
+    // (taskdata->td_flags.native)
+  } else
+#endif
+      if (if_val == 0) { // if(0) specified, mark task as serial
     taskdata->td_flags.task_serial = 1;
     taskdata->td_flags.tiedness = TASK_TIED; // AC: serial task cannot be untied
     // always start serial tasks linearly
-    __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, num_tasks,
-                          grainsize, extras, last_chunk, tc,
+    __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, nogroup, ub_glob,
+                          num_tasks, grainsize, extras, last_chunk, tc,
 #if OMPT_SUPPORT
                           OMPT_GET_RETURN_ADDRESS(0),
 #endif
@@ -4928,8 +5697,8 @@ static void __kmp_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
                   "(%lld), grain %llu, extras %llu, last_chunk %lld\n",
                   gtid, tc, num_tasks, num_tasks_min, grainsize, extras,
                   last_chunk));
-    __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, num_tasks,
-                          grainsize, extras, last_chunk, tc,
+    __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, nogroup, ub_glob,
+                          num_tasks, grainsize, extras, last_chunk, tc,
 #if OMPT_SUPPORT
                           OMPT_GET_RETURN_ADDRESS(0),
 #endif
@@ -4948,7 +5717,10 @@ static void __kmp_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
 #if OMPT_SUPPORT && OMPT_OPTIONAL
     OMPT_STORE_RETURN_ADDRESS(gtid);
 #endif
-    __kmpc_end_taskgroup(loc, gtid);
+#if OMP_TASKGRAPH_EXPERIMENTAL
+    if (!taskgraph_rec)
+#endif
+      __kmpc_end_taskgroup(loc, gtid);
   }
   KA_TRACE(20, ("__kmp_taskloop(exit): T#%d\n", gtid));
 }
@@ -5048,3 +5820,310 @@ bool __kmpc_omp_has_task_team(kmp_int32 gtid) {
 
   return taskdata->td_task_team != NULL;
 }
+
+#if OMP_TASKGRAPH_EXPERIMENTAL
+
+static void __kmp_taskgraph_reset(kmp_taskgraph_record_t *rec, kmp_int32 gtid,
+                                  kmp_intptr_t graph_id) {
+  rec->status = KMP_TDG_RECORDING;
+  rec->gtid = gtid;
+  rec->graph_id = graph_id;
+  rec->record_map = nullptr;
+  rec->alloc_root = nullptr;
+  rec->recycled_deps = nullptr;
+  rec->num_tasks = 0;
+  rec->nodes_allocated = 0;
+  rec->num_mutexes = 0;
+  rec->exec_descrs = nullptr;
+  rec->num_exec_descrs = 0;
+  rec->taskgraph_args = nullptr;
+  rec->next = nullptr;
+}
+
+static kmp_taskgraph_record_t *__kmp_taskgraph_alloc(kmp_int32 gtid,
+                                                     kmp_intptr_t graph_id) {
+  kmp_info_t *thread = __kmp_threads[gtid];
+  kmp_taskgraph_record_t *new_rec =
+      (kmp_taskgraph_record_t *)__kmp_fast_allocate(
+          thread, sizeof(kmp_taskgraph_record_t));
+  new_rec->status = KMP_TDG_RECORDING;
+  new_rec->gtid = gtid;
+  new_rec->graph_id = graph_id;
+  __kmp_init_lock(&new_rec->map_lock);
+  new_rec->record_map = nullptr;
+  new_rec->alloc_root = nullptr;
+  new_rec->recycled_deps = nullptr;
+  new_rec->num_tasks = 0;
+  new_rec->nodes_allocated = 0;
+  new_rec->num_mutexes = 0;
+  new_rec->exec_descrs = nullptr;
+  new_rec->exec_descr_size = 0;
+  new_rec->next = nullptr;
+  return new_rec;
+}
+
+static kmp_taskgraph_header_t *__kmp_taskgraph_header_alloc(kmp_int32 gtid) {
+  kmp_taskgraph_header_t *new_hdr =
+      (kmp_taskgraph_header_t *)__kmp_allocate(sizeof(kmp_taskgraph_header_t));
+  new_hdr->first = nullptr;
+  new_hdr->expiring = nullptr;
+  __kmp_init_lock(&new_hdr->header_lock);
+  return new_hdr;
+}
+
+// Clone a (new) task that has had its private variables and shared variables
+// initialised already.
+static kmp_task_t *__kmp_taskgraph_clone_task(kmp_info_t *thread,
+                                              kmp_taskgraph_record_t *taskgraph,
+                                              kmp_task_t *orig,
+                                              size_t sizeof_kmp_task_t,
+                                              size_t sizeof_shareds) {
+  // FIXME: This should use a "taskdup" function like taskloops in cases where
+  // private variables are not trivially copyable.  For now, do it by plain
+  // bitwise copy.
+  // FIXME 2: It's intended that this copy be persistent, and can be
+  // re-executed on taskgraph replay.  Make sure that works (for shared
+  // variables) if stack addresses change (i.e. a task-generating function is
+  // called from different call stack depths).
+  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(orig);
+  size_t shareds_offset = sizeof(kmp_taskdata_t) + sizeof_kmp_task_t;
+  shareds_offset = __kmp_round_up_to_val(shareds_offset, sizeof(kmp_uint64));
+  kmp_taskdata_t *copy_td = (kmp_taskdata_t *)__kmp_fast_allocate(
+      thread, shareds_offset + sizeof_shareds);
+  KMP_MEMCPY(copy_td, taskdata, shareds_offset + sizeof_shareds);
+  // Tasks cloned for a taskgraph always have this field set.
+  copy_td->owning_taskgraph = taskgraph;
+  return KMP_TASKDATA_TO_TASK(copy_td);
+}
+
+// __kmpc_taskgraph: record or replay taskgraph
+// loc_ref:     Location of TDG, not used yet
+// gtid:        Global Thread ID of the encountering thread
+// tdg_handle:  Handle of taskgraph -- the address of a slot in the host
+//              program that we write the taskgraph (list) pointer back to.
+// graph_id:    Graph ID for the taskgraph.
+// graph_reset: 1 to reset taskgraph for this taskgraph/graph_id, 0 to replay
+//              (or record, initially).
+// nogroup:     1 to omit implicit taskgroup, 0 to include it.
+// entry:       Pointer to the entry function
+// args:        Pointer to the function arguments
+void __kmpc_taskgraph(ident_t *loc_ref, kmp_int32 gtid,
+                      std::atomic<void *> *tdg_handle, kmp_uint32 graph_id,
+                      kmp_int32 graph_reset, kmp_int32 nogroup,
+                      void (*entry)(void *), void *args) {
+  kmp_taskgraph_header_t *header =
+      (kmp_taskgraph_header_t *)KMP_ATOMIC_LD_ACQ(tdg_handle);
+  kmp_taskgraph_record_t *record = nullptr, **record_p = nullptr;
+  kmp_info_t *thread = __kmp_threads[gtid];
+  kmp_taskgroup_t *taskgroup;
+
+  __kmpc_taskgroup(loc_ref, gtid);
+
+  taskgroup = thread->th.th_current_task->td_taskgroup;
+
+  // FIXME: Implement graph_id and graph_reset functionality.  For graph_id, we
+  // will form a singly-linked list of task records chained through their
+  // "next" pointers (per taskgraph construct handle).  Thread safety and
+  // locking need careful consideration.  We could use a "list header" node
+  // consisting of a lock and a pointer to
+  // the list proper, perhaps.  Ideally we'd want to avoid locking/unlocking in
+  // the common case (replay).
+
+  if (!header) {
+    header = __kmp_taskgraph_header_alloc(gtid);
+    // Another thread may have allocated the header at the same time.  Grab
+    // their copy if so and forget ours.
+    kmp_taskgraph_header_t *other =
+        (kmp_taskgraph_header_t *)KMP_COMPARE_AND_STORE_RET64(tdg_handle,
+                                                              nullptr, header);
+    if (other != nullptr) {
+      __kmp_free(header);
+      header = other;
+    }
+  }
+
+  __kmp_acquire_lock(&header->header_lock, gtid);
+  record = header->first;
+  if (!record) {
+    record = __kmp_taskgraph_alloc(gtid, graph_id);
+    // We record 'nogroup' here.  We always create a group for recording the
+    // taskgraph, but we could avoid doing so for replay.  That's not done
+    // yet though.
+    record->nogroup_taskgroup = nogroup;
+    record->next = header->first;
+    header->first = record;
+    // Store our taskgraph record into the taskgraph directive's implicit
+    // taskgroup.
+    KMP_ATOMIC_ST_REL(&taskgroup->taskgraph.recording, record);
+  }
+  __kmp_release_lock(&header->header_lock, gtid);
+
+  kmp_taskgraph_status_t status = KMP_ATOMIC_LD_ACQ(&record->status);
+  if (status == KMP_TDG_RECORDING)
+    entry(args);
+  else if (status == KMP_TDG_READY) {
+    kmp_taskdata *current_taskdata = thread->th.th_current_task;
+    if (!record) {
+      if (__kmp_taskgraph_trace())
+        fprintf(stderr, "Skipping replay of empty taskgraph from task %p\n",
+                KMP_TASKDATA_TO_TASK(current_taskdata));
+      return;
+    }
+    if (__kmp_taskgraph_trace())
+      fprintf(stderr, "Replay taskgraph %p from task %p\n", record,
+              KMP_TASKDATA_TO_TASK(current_taskdata));
+    __kmp_acquire_lock(&record->map_lock, gtid);
+    __kmp_replay_taskgraph(gtid, current_taskdata, record, graph_id, taskgroup);
+    __kmpc_end_taskgroup(loc_ref, gtid);
+    __kmp_release_lock(&record->map_lock, gtid);
+    return;
+  }
+
+  __kmpc_end_taskgroup(loc_ref, gtid);
+
+  // This could perhaps be spawned as a separate task in order to avoid
+  // blocking this thread.
+  if (record->gtid == gtid) {
+    kmp_taskdata *current_taskdata = thread->th.th_current_task;
+    __kmp_build_taskgraph(gtid, current_taskdata, record);
+  }
+}
+
+kmp_uint32 __kmpc_taskgraph_task(ident_t *loc_ref, kmp_int32 gtid,
+                                 kmp_task_t *new_task, kmp_int32 flags,
+                                 size_t sizeof_kmp_task_t, void *shareds,
+                                 size_t sizeof_shareds, kmp_int32 ndeps,
+                                 kmp_depend_info_t *dep_list) {
+  kmp_info_t *thread = __kmp_threads[gtid];
+  kmp_taskgroup_t *taskgroup = thread->th.th_current_task->td_taskgroup;
+  kmp_taskgraph_record_t *rec = __kmp_taskgraph_or_parent_recording(taskgroup);
+
+  if (rec) {
+    kmp_taskgraph_status_t status = KMP_ATOMIC_LD_ACQ(&rec->status);
+    if (status == KMP_TDG_RECORDING) {
+      kmp_task_t *cloned_task = __kmp_taskgraph_clone_task(
+          thread, rec, new_task, sizeof_kmp_task_t, sizeof_shareds);
+      kmp_taskgraph_node_t *node = __kmp_taskgraph_node_alloc(rec, cloned_task);
+      if (taskgroup->taskgraph.reduce_input) {
+        node->reduce_input = taskgroup->taskgraph.reduce_input;
+        taskgroup->taskgraph.reduce_input = nullptr;
+      }
+#if defined(DEBUG_TASKGRAPH)
+      fprintf(stderr, "__kmpc_taskgraph_task: record task here!\n");
+      fprintf(stderr, "private size: %d, shared size: %d\n",
+              (int)(sizeof_kmp_task_t - sizeof(kmp_task_t)),
+              (int)sizeof_shareds);
+      fprintf(stderr, "ndeps: %d\n", (int)ndeps);
+      fprintf(stderr, "gtid: %d rec->gtid: %d\n", gtid, rec->gtid);
+      fprintf(stderr, "taskgroup: %p\n",
+              thread->th.th_current_task->td_taskgroup);
+      kmp_taskdata_t *parent = thread->th.th_current_task->td_parent;
+      while (parent) {
+        fprintf(stderr, "  parent: %p (taskgroup %p)\n", parent,
+                parent->td_taskgroup);
+        parent = parent->td_parent;
+      }
+#endif
+      node->u.unresolved.ndeps = ndeps;
+      node->u.unresolved.dep_list = (kmp_depend_info_t *)__kmp_thread_malloc(
+          thread, ndeps * sizeof(kmp_depend_info_t));
+      KMP_MEMCPY(node->u.unresolved.dep_list, dep_list,
+                 ndeps * sizeof(kmp_depend_info_t));
+    } else if (status == KMP_TDG_READY) {
+#ifdef DEBUG_TASKGRAPH
+      fprintf(stderr,
+              "non-taskgraph task entry point for task in finalized taskgraph");
+#endif
+      return 0;
+    }
+  } else {
+    kmp_taskdata_t *parent = thread->th.th_current_task->td_parent;
+    while (parent) {
+      parent = parent->td_parent;
+    }
+  }
+
+  kmp_int32 res;
+  if (ndeps == 0)
+    res = __kmpc_omp_task(loc_ref, gtid, new_task);
+  else
+    res = __kmpc_omp_task_with_deps(loc_ref, gtid, new_task, ndeps, dep_list, 0,
+                                    nullptr);
+
+  return res;
+}
+
+void __kmpc_taskgraph_taskwait(ident_t *loc_ref, kmp_int32 gtid,
+                               kmp_int32 ndeps, kmp_depend_info_t *dep_list,
+                               kmp_int32 has_no_wait) {
+  kmp_info_t *thread = __kmp_threads[gtid];
+  kmp_taskgroup_t *taskgroup = thread->th.th_current_task->td_taskgroup;
+  kmp_taskgraph_record_t *rec = __kmp_taskgraph_or_parent_recording(taskgroup);
+
+  if (rec) {
+    kmp_taskgraph_status_t status = KMP_ATOMIC_LD_ACQ(&rec->status);
+    if (status == KMP_TDG_RECORDING) {
+      kmp_taskgraph_node_t *node = __kmp_taskgraph_node_alloc(rec, nullptr);
+#ifdef DEBUG_TASKGRAPH
+      fprintf(stderr, "__kmpc_taskgraph_taskwait: record taskwait here!\n");
+      fprintf(stderr, "ndeps: %d\n", (int)ndeps);
+#endif
+      node->u.unresolved.ndeps = ndeps;
+      node->u.unresolved.dep_list = (kmp_depend_info_t *)__kmp_thread_malloc(
+          thread, ndeps * sizeof(kmp_depend_info_t));
+      KMP_MEMCPY(node->u.unresolved.dep_list, dep_list,
+                 ndeps * sizeof(kmp_depend_info_t));
+      // TODO: Record has_no_wait somewhere?
+      // if (has_no_wait)
+      //  return;
+    } else if (status == KMP_TDG_READY) {
+#ifdef DEBUG_TASKGRAPH
+      fprintf(stderr, "non-taskgraph taskwait entry point for taskwait in "
+                      "finalized taskgraph\n");
+#endif
+      return;
+    }
+  }
+
+  __kmpc_omp_taskwait_deps_51(loc_ref, gtid, ndeps, dep_list, 0, nullptr,
+                              has_no_wait);
+}
+
+kmp_uint32 __kmpc_taskgraph_taskloop(ident_t *loc_ref, kmp_int32 gtid,
+                                     kmp_task_t *new_task, kmp_int32 flags,
+                                     size_t sizeof_kmp_task_t, void *shareds,
+                                     size_t sizeof_shareds, kmp_int32 if_val,
+                                     kmp_uint64 *lb, kmp_uint64 *ub,
+                                     kmp_int64 st, kmp_int32 nogroup,
+                                     kmp_int32 sched, kmp_uint64 grainsize,
+                                     kmp_int32 modifier, void *task_dup) {
+  kmp_info_t *thread = __kmp_threads[gtid];
+  kmp_taskgroup_t *taskgroup = thread->th.th_current_task->td_taskgroup;
+  kmp_taskgraph_record_t *rec = __kmp_taskgraph_or_parent_recording(taskgroup);
+
+  if (rec) {
+    kmp_taskgraph_status_t status = KMP_ATOMIC_LD_ACQ(&rec->status);
+    if (status == KMP_TDG_RECORDING)
+      __kmp_taskloop(loc_ref, gtid, new_task, if_val, lb, ub, st, nogroup,
+                     sched, grainsize, modifier, task_dup, rec);
+    else if (status == KMP_TDG_READY) {
+#ifdef DEBUG_TASKGRAPH
+      fprintf(stderr, "non-taskgraph taskloop entry point for taskloop in "
+                      "finalized taskgraph\n");
+#endif
+      return 0;
+    }
+  }
+
+  // For 'nogroup' here, we pass TRUE because this entry point does want to
+  // know if we originally had the 'nogroup' clause or not -- but the group is
+  // created using separate API calls wrapping this one (or __kmpc_taskloop).
+  // We don't want to create another taskgroup in __kmp_taskloop here in any
+  // case.
+  __kmp_taskloop(loc_ref, gtid, new_task, if_val, lb, ub, st, /*nogroup=*/true,
+                 sched, grainsize, modifier, task_dup);
+
+  return 0;
+}
+
+#endif
diff --git a/openmp/runtime/test/tasking/omp_taskgraph_irreducible.cpp b/openmp/runtime/test/tasking/omp_taskgraph_irreducible.cpp
new file mode 100644
index 0000000000000..36d50090f1365
--- /dev/null
+++ b/openmp/runtime/test/tasking/omp_taskgraph_irreducible.cpp
@@ -0,0 +1,84 @@
+// clang-format off
+// REQUIRES: omp_taskgraph_experimental
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run
+// clang-format on
+//
+// NxN lattice "wavefront": cell (i,j) depends on (i-1,j) and (i,j-1).  This is
+// the canonical non-series-parallel (irreducible) dependence graph.  It used to
+// livelock the taskgraph region builder, which tried to make the graph
+// reducible by node-splitting -- a rewrite that does not converge on
+// lattice-like graphs.  The builder now carves the residual into a single
+// TASKGRAPH_REGION_IRREDUCIBLE region instead, which is guaranteed to
+// terminate.
+//
+// Each cell accumulates the number of monotone lattice paths reaching it
+// (val[i][j] == val[i-1][j] + val[i][j-1], with val[0][0] == 1), i.e. the
+// binomial C(i+j, i).  N is kept small enough that the values fit in long long.
+#include <cassert>
+#include <iostream>
+
+#define N 12
+#define ITERS 20
+
+static long long val[N][N];
+static long long ref[N][N];
+
+int main() {
+  // Serial reference, computed with the same recurrence to avoid any
+  // closed-form overflow surprises.
+  for (int i = 0; i < N; ++i)
+    for (int j = 0; j < N; ++j)
+      ref[i][j] =
+          (i == 0 && j == 0)
+              ? 1
+              : ((i > 0 ? ref[i - 1][j] : 0) + (j > 0 ? ref[i][j - 1] : 0));
+
+  for (int iter = 0; iter < ITERS; ++iter) {
+    for (int i = 0; i < N; ++i)
+      for (int j = 0; j < N; ++j)
+        val[i][j] = -1;
+
+#pragma omp parallel
+#pragma omp single
+    {
+#pragma omp taskgraph
+      {
+        for (int i = 0; i < N; ++i)
+          for (int j = 0; j < N; ++j) {
+            long long *me = &val[i][j];
+            long long *up = (i > 0) ? &val[i - 1][j] : nullptr;
+            long long *left = (j > 0) ? &val[i][j - 1] : nullptr;
+            if (i == 0 && j == 0) {
+#pragma omp task depend(out : me[0])
+              {
+                *me = 1;
+              }
+            } else if (i == 0) {
+#pragma omp task depend(in : left[0]) depend(out : me[0])
+              {
+                *me = *left;
+              }
+            } else if (j == 0) {
+#pragma omp task depend(in : up[0]) depend(out : me[0])
+              {
+                *me = *up;
+              }
+            } else {
+#pragma omp task depend(in : up[0], left[0]) depend(out : me[0])
+              {
+                *me = *up + *left;
+              }
+            }
+          }
+      }
+    }
+
+    for (int i = 0; i < N; ++i)
+      for (int j = 0; j < N; ++j)
+        assert(val[i][j] == ref[i][j] && "wavefront value mismatch");
+  }
+
+  std::cout << "Passed" << std::endl;
+  return 0;
+}
+// CHECK: Passed

>From fd0a272d0f7e0c9db04dc2463313a259075ef4cb Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Fri, 27 Mar 2026 09:40:01 -0500
Subject: [PATCH 03/26] [OpenMP] Add graph_id and graph_reset clause support
 (for taskgraph directive)

This patch adds support for the graph_id and graph_reset clauses for the
OpenMP 6.0 "taskgraph" directive.  Relative to previously-posted versions,
this one does not have the clauses inherit from "OMPClauseWithPreInit",
since AFAICT it shouldn't be necessary to do that.  It also avoids
casting the graph_id argument to a boolean, and allows the expression
to be omitted for the graph_reset clause (which seems of limited use,
but is allowed by the spec).

Co-Authored-By: Josep Pinot <jpinot at bsc.es>

commit-id:bef7e706

Pull Request: https://github.com/llvm/llvm-project/pull/194048
---
 clang/include/clang/AST/OpenMPClause.h        | 95 +++++++++++++++++++
 clang/include/clang/AST/RecursiveASTVisitor.h | 13 +++
 clang/include/clang/Sema/SemaOpenMP.h         |  9 ++
 clang/lib/AST/OpenMPClause.cpp                | 26 +++++
 clang/lib/AST/StmtProfile.cpp                 |  8 ++
 clang/lib/Basic/OpenMPKinds.cpp               |  4 +
 clang/lib/Parse/ParseOpenMP.cpp               | 15 +++
 clang/lib/Sema/SemaOpenMP.cpp                 | 60 ++++++++++++
 clang/lib/Sema/TreeTransform.h                | 43 +++++++++
 clang/lib/Serialization/ASTReader.cpp         | 16 ++++
 clang/lib/Serialization/ASTWriter.cpp         | 10 ++
 clang/tools/libclang/CIndex.cpp               |  6 ++
 llvm/include/llvm/Frontend/OpenMP/OMP.td      |  2 +
 13 files changed, 307 insertions(+)

diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 9e9295e1a0c54..4c3ab20c64b33 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -8632,6 +8632,101 @@ class OMPIsDevicePtrClause final
   }
 };
 
+/// This represents clause 'graph_id' in the '#pragma omp taskgraph"
+/// directives.
+///
+/// \code
+/// #pragma omp taskgraph graph_id(a)
+class OMPGraphIdClause final
+    : public OMPOneStmtClause<llvm::omp::OMPC_graph_id, OMPClause> {
+  friend class OMPClauseReader;
+
+  /// Set condition.
+  void setId(Expr *Id) { setStmt(Id); }
+
+public:
+  /// Build 'graph_id' clause with identifier value \a Id.
+  ///
+  /// \param Id Id value for the clause.
+  /// \param StartLoc Starting location of the clause.
+  /// \param LParenLoc Location of '('.
+  /// \param EndLoc Ending location of the clause.
+  OMPGraphIdClause(Expr *Id, SourceLocation StartLoc, SourceLocation LParenLoc,
+                   SourceLocation EndLoc)
+      : OMPOneStmtClause(Id, StartLoc, LParenLoc, EndLoc) {}
+
+  /// Build an empty clause.
+  OMPGraphIdClause() : OMPOneStmtClause() {}
+
+  /// Returns condition.
+  Expr *getId() const { return getStmtAs<Expr>(); }
+};
+
+// This represents clause 'graph_reset' in the '#pragma omp taskgraph"
+/// directives.
+///
+/// \code
+/// #pragma omp taskgraph graph_reset(true)
+class OMPGraphResetClause final : public OMPClause {
+  friend class OMPClauseReader;
+
+  /// Location of '('.
+  SourceLocation LParenLoc;
+
+  /// Condition of the 'graph_reset' clause.
+  Stmt *Condition = nullptr;
+
+public:
+  /// Build 'graph_reset' clause with condition \a Cond.
+  ///
+  /// \param Cond Condition of the clause.
+  /// \param StartLoc Starting location of the clause.
+  /// \param LParenLoc Location of '('.
+  /// \param EndLoc Ending location of the clause.
+  OMPGraphResetClause(Expr *Cond, SourceLocation StartLoc,
+                      SourceLocation LParenLoc, SourceLocation EndLoc)
+      : OMPClause(llvm::omp::OMPC_graph_reset, StartLoc, EndLoc),
+        LParenLoc(LParenLoc), Condition(Cond) {}
+
+  /// Build an empty clause.
+  OMPGraphResetClause()
+      : OMPClause(llvm::omp::OMPC_graph_reset, SourceLocation(),
+                  SourceLocation()) {}
+
+  /// Sets the location of '('.
+  void setLParenLoc(SourceLocation Loc) { LParenLoc = Loc; }
+
+  /// Returns the location of '('.
+  SourceLocation getLParenLoc() const { return LParenLoc; }
+
+  /// Set condition.
+  void setCondition(Expr *Cond) { Condition = Cond; }
+
+  /// Returns condition.
+  Expr *getCondition() const { return cast_or_null<Expr>(Condition); }
+
+  child_range children() {
+    if (Condition)
+      return child_range(&Condition, &Condition + 1);
+    return child_range(child_iterator(), child_iterator());
+  }
+
+  const_child_range children() const {
+    if (Condition)
+      return const_child_range(&Condition, &Condition + 1);
+    return const_child_range(const_child_iterator(), const_child_iterator());
+  }
+
+  child_range used_children();
+  const_child_range used_children() const {
+    return const_cast<OMPGraphResetClause *>(this)->used_children();
+  }
+
+  static bool classof(const OMPClause *T) {
+    return T->getClauseKind() == llvm::omp::OMPC_graph_reset;
+  }
+};
+
 /// This represents clause 'has_device_ptr' in the '#pragma omp ...'
 /// directives.
 ///
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index 6913e9b614315..097ce12038528 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -4190,6 +4190,19 @@ bool RecursiveASTVisitor<Derived>::VisitOMPIsDevicePtrClause(
   return true;
 }
 
+template <typename Derived>
+bool RecursiveASTVisitor<Derived>::VisitOMPGraphIdClause(OMPGraphIdClause *C) {
+  TRY_TO(TraverseStmt(C->getId()));
+  return true;
+}
+
+template <typename Derived>
+bool RecursiveASTVisitor<Derived>::VisitOMPGraphResetClause(
+    OMPGraphResetClause *C) {
+  TRY_TO(TraverseStmt(C->getCondition()));
+  return true;
+}
+
 template <typename Derived>
 bool RecursiveASTVisitor<Derived>::VisitOMPHasDeviceAddrClause(
     OMPHasDeviceAddrClause *C) {
diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index 361473140e236..f8ad9ebc619c1 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -952,6 +952,15 @@ class SemaOpenMP : public SemaBase {
   ActOnOpenMPOrderedClause(SourceLocation StartLoc, SourceLocation EndLoc,
                            SourceLocation LParenLoc = SourceLocation(),
                            Expr *NumForLoops = nullptr);
+  /// Called on well-formed 'graph_id' clause.
+  OMPClause *ActOnOpenMPGraphIdClause(Expr *Id, SourceLocation StartLoc,
+                                      SourceLocation LParenLoc,
+                                      SourceLocation EndLoc);
+  /// Called on well-formed 'graph_reset' clause.
+  OMPClause *ActOnOpenMPGraphResetClause(Expr *Condition,
+                                         SourceLocation StartLoc,
+                                         SourceLocation LParenLoc,
+                                         SourceLocation EndLoc);
   /// Called on well-formed 'grainsize' clause.
   OMPClause *ActOnOpenMPGrainsizeClause(OpenMPGrainsizeClauseModifier Modifier,
                                         Expr *Size, SourceLocation StartLoc,
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index cac701994def8..cc3b126f3c791 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -254,6 +254,8 @@ const OMPClauseWithPostUpdate *OMPClauseWithPostUpdate::get(const OMPClause *C)
   case OMPC_thread_limit:
   case OMPC_priority:
   case OMPC_grainsize:
+  case OMPC_graph_id:
+  case OMPC_graph_reset:
   case OMPC_nogroup:
   case OMPC_num_tasks:
   case OMPC_hint:
@@ -328,6 +330,12 @@ OMPClause::child_range OMPGrainsizeClause::used_children() {
   return child_range(&Grainsize, &Grainsize + 1);
 }
 
+OMPClause::child_range OMPGraphResetClause::used_children() {
+  if (Condition)
+    return child_range(&Condition, &Condition + 1);
+  return children();
+}
+
 OMPClause::child_range OMPNumTasksClause::used_children() {
   if (Stmt **C = getAddrOfExprAsWritten(getPreInitStmt()))
     return child_range(C, C + 1);
@@ -2418,6 +2426,24 @@ void OMPClausePrinter::VisitOMPGrainsizeClause(OMPGrainsizeClause *Node) {
   OS << ")";
 }
 
+void OMPClausePrinter::VisitOMPGraphIdClause(OMPGraphIdClause *Node) {
+  OS << "graph_id";
+  if (Expr *E = Node->getId()) {
+    OS << "(";
+    E->printPretty(OS, nullptr, Policy, 0);
+    OS << ")";
+  }
+}
+
+void OMPClausePrinter::VisitOMPGraphResetClause(OMPGraphResetClause *Node) {
+  OS << "graph_reset";
+  if (Expr *E = Node->getCondition()) {
+    OS << "(";
+    E->printPretty(OS, nullptr, Policy, 0);
+    OS << ")";
+  }
+}
+
 void OMPClausePrinter::VisitOMPNumTasksClause(OMPNumTasksClause *Node) {
   OS << "num_tasks(";
   OpenMPNumTasksClauseModifier Modifier = Node->getModifier();
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index ec4493d0afa41..80509c3f7ffd8 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -950,6 +950,14 @@ void OMPClauseProfiler::VisitOMPGrainsizeClause(const OMPGrainsizeClause *C) {
   if (C->getGrainsize())
     Profiler->VisitStmt(C->getGrainsize());
 }
+void OMPClauseProfiler::VisitOMPGraphIdClause(const OMPGraphIdClause *C) {
+  if (C->getId())
+    Profiler->VisitStmt(C->getId());
+}
+void OMPClauseProfiler::VisitOMPGraphResetClause(const OMPGraphResetClause *C) {
+  if (C->getCondition())
+    Profiler->VisitStmt(C->getCondition());
+}
 void OMPClauseProfiler::VisitOMPNumTasksClause(const OMPNumTasksClause *C) {
   VisitOMPClauseWithPreInit(C);
   if (C->getNumTasks())
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index 1a2a963243f6c..1e8d32c059282 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -330,6 +330,8 @@ unsigned clang::getOpenMPSimpleClauseType(OpenMPClauseKind Kind, StringRef Str,
   case OMPC_when:
   case OMPC_append_args:
   case OMPC_looprange:
+  case OMPC_graph_id:
+  case OMPC_graph_reset:
     break;
   default:
     break;
@@ -728,6 +730,8 @@ const char *clang::getOpenMPSimpleClauseTypeName(OpenMPClauseKind Kind,
   case OMPC_when:
   case OMPC_append_args:
   case OMPC_looprange:
+  case OMPC_graph_id:
+  case OMPC_graph_reset:
     break;
   default:
     break;
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index f466b01975ac5..5f9845145669c 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -3254,6 +3254,7 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
   case OMPC_partial:
   case OMPC_align:
   case OMPC_message:
+  case OMPC_graph_id:
   case OMPC_ompx_dyn_cgroup_mem:
   case OMPC_dyn_groupprivate:
   case OMPC_transparent:
@@ -3405,6 +3406,20 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
     else
       Clause = ParseOpenMPClause(CKind, WrongDirective);
     break;
+  case OMPC_graph_reset:
+    if (!FirstClause) {
+      Diag(Tok, diag::err_omp_more_one_clause)
+          << getOpenMPDirectiveName(DKind, OMPVersion)
+          << getOpenMPClauseName(CKind) << 0;
+      ErrorFound = true;
+    }
+
+    if (PP.LookAhead(/*N=*/0).is(tok::l_paren)) {
+      Clause = ParseOpenMPSingleExprClause(CKind, WrongDirective);
+    } else {
+      Clause = ParseOpenMPClause(CKind, WrongDirective);
+    }
+    break;
   case OMPC_self_maps:
     // OpenMP [6.0, self_maps clause]
     if (getLangOpts().OpenMP < 60) {
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 51164744b345a..a67eedfeaf0cc 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -6820,6 +6820,8 @@ StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
       case OMPC_final:
       case OMPC_priority:
       case OMPC_novariants:
+      case OMPC_graph_id:
+      case OMPC_graph_reset:
       case OMPC_nocontext:
         // Do not analyze if no parent parallel directive.
         if (isOpenMPParallelDirective(Kind))
@@ -17030,6 +17032,12 @@ OMPClause *SemaOpenMP::ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind,
   case OMPC_detach:
     Res = ActOnOpenMPDetachClause(Expr, StartLoc, LParenLoc, EndLoc);
     break;
+  case OMPC_graph_id:
+    Res = ActOnOpenMPGraphIdClause(Expr, StartLoc, LParenLoc, EndLoc);
+    break;
+  case OMPC_graph_reset:
+    Res = ActOnOpenMPGraphResetClause(Expr, StartLoc, LParenLoc, EndLoc);
+    break;
   case OMPC_novariants:
     Res = ActOnOpenMPNovariantsClause(Expr, StartLoc, LParenLoc, EndLoc);
     break;
@@ -17820,6 +17828,8 @@ OMPClause *SemaOpenMP::ActOnOpenMPSimpleClause(
   case OMPC_match:
   case OMPC_nontemporal:
   case OMPC_destroy:
+  case OMPC_graph_id:
+  case OMPC_graph_reset:
   case OMPC_novariants:
   case OMPC_nocontext:
   case OMPC_detach:
@@ -18561,6 +18571,8 @@ OMPClause *SemaOpenMP::ActOnOpenMPSingleExprWithArgClause(
   case OMPC_severity:
   case OMPC_message:
   case OMPC_destroy:
+  case OMPC_graph_id:
+  case OMPC_graph_reset:
   case OMPC_novariants:
   case OMPC_nocontext:
   case OMPC_detach:
@@ -18780,6 +18792,10 @@ OMPClause *SemaOpenMP::ActOnOpenMPClause(OpenMPClauseKind Kind,
   case OMPC_ompx_bare:
     Res = ActOnOpenMPXBareClause(StartLoc, EndLoc);
     break;
+  case OMPC_graph_reset:
+    Res = ActOnOpenMPGraphResetClause(/*Condition=*/nullptr, StartLoc,
+                                      SourceLocation(), EndLoc);
+    break;
   case OMPC_if:
   case OMPC_final:
   case OMPC_num_threads:
@@ -18834,6 +18850,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPClause(OpenMPClauseKind Kind,
   case OMPC_at:
   case OMPC_severity:
   case OMPC_message:
+  case OMPC_graph_id:
   case OMPC_novariants:
   case OMPC_nocontext:
   case OMPC_detach:
@@ -19180,6 +19197,47 @@ OMPClause *SemaOpenMP::ActOnOpenMPDestroyClause(Expr *InteropVar,
       OMPDestroyClause(InteropVar, StartLoc, LParenLoc, VarLoc, EndLoc);
 }
 
+OMPClause *SemaOpenMP::ActOnOpenMPGraphIdClause(Expr *Id,
+                                                SourceLocation StartLoc,
+                                                SourceLocation LParenLoc,
+                                                SourceLocation EndLoc) {
+  Expr *ValExpr = Id;
+
+  if (!Id->isValueDependent() && !Id->isTypeDependent() &&
+      !Id->isInstantiationDependent() &&
+      !Id->containsUnexpandedParameterPack()) {
+    ExprResult Val = PerformOpenMPImplicitIntegerConversion(LParenLoc, Id);
+    if (Val.isInvalid())
+      return nullptr;
+
+    ValExpr = Val.get();
+  }
+
+  return new (getASTContext())
+      OMPGraphIdClause(ValExpr, StartLoc, LParenLoc, EndLoc);
+}
+
+OMPClause *SemaOpenMP::ActOnOpenMPGraphResetClause(Expr *Condition,
+                                                   SourceLocation StartLoc,
+                                                   SourceLocation LParenLoc,
+                                                   SourceLocation EndLoc) {
+  Expr *ValExpr = Condition;
+  if (Condition && LParenLoc.isValid()) {
+    if (!Condition->isValueDependent() && !Condition->isTypeDependent() &&
+        !Condition->isInstantiationDependent() &&
+        !Condition->containsUnexpandedParameterPack()) {
+      ExprResult Val = SemaRef.CheckBooleanCondition(StartLoc, Condition);
+      if (Val.isInvalid())
+        return nullptr;
+
+      ValExpr = Val.get();
+    }
+  }
+
+  return new (getASTContext())
+      OMPGraphResetClause(ValExpr, StartLoc, LParenLoc, EndLoc);
+}
+
 OMPClause *SemaOpenMP::ActOnOpenMPNovariantsClause(Expr *Condition,
                                                    SourceLocation StartLoc,
                                                    SourceLocation LParenLoc,
@@ -19493,6 +19551,8 @@ OMPClause *SemaOpenMP::ActOnOpenMPVarListClause(OpenMPClauseKind Kind,
   case OMPC_severity:
   case OMPC_message:
   case OMPC_destroy:
+  case OMPC_graph_id:
+  case OMPC_graph_reset:
   case OMPC_novariants:
   case OMPC_nocontext:
   case OMPC_detach:
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 2083fcd372e81..de7435b0b1c93 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -2181,6 +2181,29 @@ class TreeTransform {
                                                         LParenLoc, EndLoc);
   }
 
+  /// Build a new OpenMP 'graph_id' clause.
+  ///
+  /// By default, performs semantic analysis to build the new OpenMP clause.
+  /// Subclasses may override this routine to provide different behavior.
+  OMPClause *RebuildOMPGraphIdClause(Expr *Condition, SourceLocation StartLoc,
+                                     SourceLocation LParenLoc,
+                                     SourceLocation EndLoc) {
+    return getSema().OpenMP().ActOnOpenMPGraphIdClause(Condition, StartLoc,
+                                                       LParenLoc, EndLoc);
+  }
+
+  /// Build a new OpenMP 'graph_reset' clause.
+  ///
+  /// By default, performs semantic analysis to build the new OpenMP clause.
+  /// Subclasses may override this routine to provide different behavior.
+  OMPClause *RebuildOMPGraphResetClause(Expr *Condition,
+                                        SourceLocation StartLoc,
+                                        SourceLocation LParenLoc,
+                                        SourceLocation EndLoc) {
+    return getSema().OpenMP().ActOnOpenMPGraphResetClause(Condition, StartLoc,
+                                                          LParenLoc, EndLoc);
+  }
+
   /// Build a new OpenMP 'grainsize' clause.
   ///
   /// By default, performs semantic analysis to build the new statement.
@@ -11881,6 +11904,26 @@ TreeTransform<Derived>::TransformOMPPriorityClause(OMPPriorityClause *C) {
       E.get(), C->getBeginLoc(), C->getLParenLoc(), C->getEndLoc());
 }
 
+template <typename Derived>
+OMPClause *
+TreeTransform<Derived>::TransformOMPGraphIdClause(OMPGraphIdClause *C) {
+  ExprResult Cond = getDerived().TransformExpr(C->getId());
+  if (Cond.isInvalid())
+    return nullptr;
+  return getDerived().RebuildOMPGraphIdClause(
+      Cond.get(), C->getBeginLoc(), C->getLParenLoc(), C->getEndLoc());
+}
+
+template <typename Derived>
+OMPClause *
+TreeTransform<Derived>::TransformOMPGraphResetClause(OMPGraphResetClause *C) {
+  ExprResult Cond = getDerived().TransformExpr(C->getCondition());
+  if (Cond.isInvalid())
+    return nullptr;
+  return getDerived().RebuildOMPGraphResetClause(
+      Cond.get(), C->getBeginLoc(), C->getLParenLoc(), C->getEndLoc());
+}
+
 template <typename Derived>
 OMPClause *
 TreeTransform<Derived>::TransformOMPGrainsizeClause(OMPGrainsizeClause *C) {
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index ffac764e46d4f..d0dea09aaf5c0 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -11729,6 +11729,12 @@ OMPClause *OMPClauseReader::readClause() {
   case llvm::omp::OMPC_grainsize:
     C = new (Context) OMPGrainsizeClause();
     break;
+  case llvm::omp::OMPC_graph_id:
+    C = new (Context) OMPGraphIdClause();
+    break;
+  case llvm::omp::OMPC_graph_reset:
+    C = new (Context) OMPGraphResetClause();
+    break;
   case llvm::omp::OMPC_num_tasks:
     C = new (Context) OMPNumTasksClause();
     break;
@@ -12657,6 +12663,16 @@ void OMPClauseReader::VisitOMPPriorityClause(OMPPriorityClause *C) {
   C->setLParenLoc(Record.readSourceLocation());
 }
 
+void OMPClauseReader::VisitOMPGraphIdClause(OMPGraphIdClause *C) {
+  C->setId(Record.readSubExpr());
+  C->setLParenLoc(Record.readSourceLocation());
+}
+
+void OMPClauseReader::VisitOMPGraphResetClause(OMPGraphResetClause *C) {
+  C->setCondition(Record.readSubExpr());
+  C->setLParenLoc(Record.readSourceLocation());
+}
+
 void OMPClauseReader::VisitOMPGrainsizeClause(OMPGrainsizeClause *C) {
   VisitOMPClauseWithPreInit(C);
   C->setModifier(Record.readEnum<OpenMPGrainsizeClauseModifier>());
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index f34b2ff182bc8..a6260ff34fa13 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -8631,6 +8631,16 @@ void OMPClauseWriter::VisitOMPPriorityClause(OMPPriorityClause *C) {
   Record.AddSourceLocation(C->getLParenLoc());
 }
 
+void OMPClauseWriter::VisitOMPGraphIdClause(OMPGraphIdClause *C) {
+  Record.AddStmt(C->getId());
+  Record.AddSourceLocation(C->getLParenLoc());
+}
+
+void OMPClauseWriter::VisitOMPGraphResetClause(OMPGraphResetClause *C) {
+  Record.AddStmt(C->getCondition());
+  Record.AddSourceLocation(C->getLParenLoc());
+}
+
 void OMPClauseWriter::VisitOMPGrainsizeClause(OMPGrainsizeClause *C) {
   VisitOMPClauseWithPreInit(C);
   Record.writeEnum(C->getModifier());
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 034f89029122c..37cfe8890e1b1 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2764,6 +2764,12 @@ void OMPClauseEnqueue::VisitOMPIsDevicePtrClause(
     const OMPIsDevicePtrClause *C) {
   VisitOMPClauseList(C);
 }
+void OMPClauseEnqueue::VisitOMPGraphIdClause(const OMPGraphIdClause *C) {
+  Visitor->AddStmt(C->getId());
+}
+void OMPClauseEnqueue::VisitOMPGraphResetClause(const OMPGraphResetClause *C) {
+  Visitor->AddStmt(C->getCondition());
+}
 void OMPClauseEnqueue::VisitOMPHasDeviceAddrClause(
     const OMPHasDeviceAddrClause *C) {
   VisitOMPClauseList(C);
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index 037a506c8b175..e405799ccad82 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -255,9 +255,11 @@ def OMPC_GrainSize : Clause<[Spelling<"grainsize">]> {
   ];
 }
 def OMPC_GraphId : Clause<[Spelling<"graph_id">]> {
+  let clangClass = "OMPGraphIdClause";
   let flangClass = "OmpGraphIdClause";
 }
 def OMPC_GraphReset : Clause<[Spelling<"graph_reset">]> {
+  let clangClass = "OMPGraphResetClause";
   let flangClass = "OmpGraphResetClause";
   let isValueOptional = true;
 }

>From 8f5db5a0132f02544239116fcb26cd58d77a3010 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Fri, 27 Mar 2026 12:54:04 -0500
Subject: [PATCH 04/26] [OpenMP] Add OpenMP 6.0 taskgraph parsing/trivial
 semantics

This patch adds OpenMP 6.0 'taskgraph' support, as a no-op: i.e. the
contents of the taskgraph region are just executed as if the directive
was omitted.

Co-Authored-By: Adrian Munera <adrian.munera at bsc.es>
Co-Authored-By: Jose M Monsalve Diaz <JoseM.MonsalveDiaz at amd.com>

Pull Request: https://github.com/llvm/llvm-project/pull/194050
---
 clang/bindings/python/clang/cindex.py         |  3 ++
 clang/include/clang-c/Index.h                 |  4 ++
 clang/include/clang/AST/RecursiveASTVisitor.h |  3 ++
 clang/include/clang/AST/StmtOpenMP.h          | 49 +++++++++++++++++++
 clang/include/clang/Basic/StmtNodes.td        |  1 +
 clang/include/clang/Sema/SemaOpenMP.h         |  4 ++
 .../include/clang/Serialization/ASTBitCodes.h |  1 +
 clang/lib/AST/StmtOpenMP.cpp                  | 15 ++++++
 clang/lib/AST/StmtPrinter.cpp                 |  5 ++
 clang/lib/AST/StmtProfile.cpp                 |  4 ++
 clang/lib/Basic/OpenMPKinds.cpp               |  4 +-
 clang/lib/CodeGen/CGOpenMPRuntime.cpp         | 40 +++++++++++++++
 clang/lib/CodeGen/CGOpenMPRuntime.h           | 13 +++++
 clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp      |  2 +
 clang/lib/CodeGen/CGStmt.cpp                  |  3 ++
 clang/lib/CodeGen/CGStmtOpenMP.cpp            | 15 ++++++
 clang/lib/CodeGen/CodeGenFunction.h           |  1 +
 clang/lib/Sema/SemaExceptionSpec.cpp          |  1 +
 clang/lib/Sema/SemaOpenMP.cpp                 | 40 +++++++++++++--
 clang/lib/Sema/TreeTransform.h                | 11 +++++
 clang/lib/Serialization/ASTReaderStmt.cpp     | 10 ++++
 clang/lib/Serialization/ASTWriterStmt.cpp     |  6 +++
 clang/lib/StaticAnalyzer/Core/ExprEngine.cpp  |  1 +
 clang/test/OpenMP/taskgraph_bad_clauses.cpp   | 19 +++++++
 .../OpenMP/unimplemented_clause_messages.cpp  |  1 -
 clang/tools/libclang/CIndex.cpp               |  2 +
 clang/tools/libclang/CXCursor.cpp             |  3 ++
 27 files changed, 255 insertions(+), 6 deletions(-)
 create mode 100644 clang/test/OpenMP/taskgraph_bad_clauses.cpp

diff --git a/clang/bindings/python/clang/cindex.py b/clang/bindings/python/clang/cindex.py
index 126c7bf44d1d1..dcff85ef4e5dd 100644
--- a/clang/bindings/python/clang/cindex.py
+++ b/clang/bindings/python/clang/cindex.py
@@ -1456,6 +1456,9 @@ def is_unexposed(self):
     # OpenMP split directive.
     OMP_SPLIT_DIRECTIVE = 312
 
+    # OpenMP taskgraph directive.
+    OMP_TASKGRAPH_DIRECTIVE = 313
+
     # OpenACC Compute Construct.
     OPEN_ACC_COMPUTE_DIRECTIVE = 320
 
diff --git a/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h
index 8427236e0b444..b97eabeedec51 100644
--- a/clang/include/clang-c/Index.h
+++ b/clang/include/clang-c/Index.h
@@ -2170,6 +2170,10 @@ enum CXCursorKind {
    */
   CXCursor_OMPSplitDirective = 312,
 
+  /** OpenMP taskgraph directive.
+   */
+  CXCursor_OMPTaskgraphDirective = 313,
+
   /** OpenACC Compute Construct.
    */
   CXCursor_OpenACCComputeConstruct = 320,
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index 097ce12038528..1d7006921f3dd 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -3301,6 +3301,9 @@ DEF_TRAVERSE_STMT(OMPBarrierDirective,
 DEF_TRAVERSE_STMT(OMPTaskwaitDirective,
                   { TRY_TO(TraverseOMPExecutableDirective(S)); })
 
+DEF_TRAVERSE_STMT(OMPTaskgraphDirective,
+                  { TRY_TO(TraverseOMPExecutableDirective(S)); })
+
 DEF_TRAVERSE_STMT(OMPTaskgroupDirective,
                   { TRY_TO(TraverseOMPExecutableDirective(S)); })
 
diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index dbc76e7df8ecd..4aa0baed7eb9e 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -2760,6 +2760,55 @@ class OMPTaskwaitDirective : public OMPExecutableDirective {
   }
 };
 
+/// This represents '#pragma omp taskgraph' directive.
+/// Available with OpenMP 6.0.
+///
+/// \code
+/// #pragma omp taskgraph
+/// \endcode
+///
+class OMPTaskgraphDirective final : public OMPExecutableDirective {
+  friend class ASTStmtReader;
+  friend class OMPExecutableDirective;
+  /// Build directive with the given start and end location.
+  ///
+  /// \param StartLoc Starting location of the directive kind.
+  /// \param EndLoc Ending location of the directive.
+  ///
+  OMPTaskgraphDirective(SourceLocation StartLoc, SourceLocation EndLoc)
+      : OMPExecutableDirective(OMPTaskgraphDirectiveClass,
+                               llvm::omp::OMPD_taskgraph, StartLoc, EndLoc) {}
+
+  /// Build an empty directive.
+  ///
+  explicit OMPTaskgraphDirective()
+      : OMPExecutableDirective(OMPTaskgraphDirectiveClass,
+                               llvm::omp::OMPD_taskgraph, SourceLocation(),
+                               SourceLocation()) {}
+
+public:
+  /// Creates directive.
+  ///
+  /// \param C AST context.
+  /// \param StartLoc Starting location of the directive kind.
+  /// \param EndLoc Ending Location of the directive.
+  ///
+  static OMPTaskgraphDirective *
+  Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation EndLoc,
+         ArrayRef<OMPClause *> Clauses, Stmt *AssociatedStmt);
+
+  /// Creates an empty directive.
+  ///
+  /// \param C AST context.
+  ///
+  static OMPTaskgraphDirective *CreateEmpty(const ASTContext &C,
+                                            unsigned NumClauses, EmptyShell);
+
+  static bool classof(const Stmt *T) {
+    return T->getStmtClass() == OMPTaskgraphDirectiveClass;
+  }
+};
+
 /// This represents '#pragma omp taskgroup' directive.
 ///
 /// \code
diff --git a/clang/include/clang/Basic/StmtNodes.td b/clang/include/clang/Basic/StmtNodes.td
index f5fa397c92ef3..1f8ab6e216a4d 100644
--- a/clang/include/clang/Basic/StmtNodes.td
+++ b/clang/include/clang/Basic/StmtNodes.td
@@ -274,6 +274,7 @@ def OMPTaskDirective : StmtNode<OMPExecutableDirective>;
 def OMPTaskyieldDirective : StmtNode<OMPExecutableDirective>;
 def OMPBarrierDirective : StmtNode<OMPExecutableDirective>;
 def OMPTaskwaitDirective : StmtNode<OMPExecutableDirective>;
+def OMPTaskgraphDirective : StmtNode<OMPExecutableDirective>;
 def OMPTaskgroupDirective : StmtNode<OMPExecutableDirective>;
 def OMPFlushDirective : StmtNode<OMPExecutableDirective>;
 def OMPDepobjDirective : StmtNode<OMPExecutableDirective>;
diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index f8ad9ebc619c1..fe5879c3d83dc 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -564,6 +564,10 @@ class SemaOpenMP : public SemaBase {
   /// Called on well-formed '\#pragma omp barrier'.
   StmtResult ActOnOpenMPBarrierDirective(SourceLocation StartLoc,
                                          SourceLocation EndLoc);
+  /// Called on well-formed '\#pragma omp taskgraph'.
+  StmtResult ActOnOpenMPTaskgraphDirective(ArrayRef<OMPClause *> Clauses,
+                                           Stmt *AStmt, SourceLocation StartLoc,
+                                           SourceLocation EndLoc);
   /// Called on well-formed '\#pragma omp taskwait'.
   StmtResult ActOnOpenMPTaskwaitDirective(ArrayRef<OMPClause *> Clauses,
                                           SourceLocation StartLoc,
diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h
index 7dee847e44410..49a43b390200b 100644
--- a/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/clang/include/clang/Serialization/ASTBitCodes.h
@@ -2001,6 +2001,7 @@ enum StmtCode {
   STMT_OMP_ERROR_DIRECTIVE,
   STMT_OMP_BARRIER_DIRECTIVE,
   STMT_OMP_TASKWAIT_DIRECTIVE,
+  STMT_OMP_TASKGRAPH_DIRECTIVE,
   STMT_OMP_FLUSH_DIRECTIVE,
   STMT_OMP_DEPOBJ_DIRECTIVE,
   STMT_OMP_SCAN_DIRECTIVE,
diff --git a/clang/lib/AST/StmtOpenMP.cpp b/clang/lib/AST/StmtOpenMP.cpp
index 9d6b315effb41..4b56c2b649b4b 100644
--- a/clang/lib/AST/StmtOpenMP.cpp
+++ b/clang/lib/AST/StmtOpenMP.cpp
@@ -966,6 +966,21 @@ OMPTaskwaitDirective *OMPTaskwaitDirective::CreateEmpty(const ASTContext &C,
   return createEmptyDirective<OMPTaskwaitDirective>(C, NumClauses);
 }
 
+OMPTaskgraphDirective *OMPTaskgraphDirective::Create(
+    const ASTContext &C, SourceLocation StartLoc, SourceLocation EndLoc,
+    ArrayRef<OMPClause *> Clauses, Stmt *AssociatedStmt) {
+  auto *Dir = createDirective<OMPTaskgraphDirective>(
+      C, Clauses, AssociatedStmt, /*NumChildren=*/0, StartLoc, EndLoc);
+  return Dir;
+}
+
+OMPTaskgraphDirective *OMPTaskgraphDirective::CreateEmpty(const ASTContext &C,
+                                                          unsigned NumClauses,
+                                                          EmptyShell) {
+  return createEmptyDirective<OMPTaskgraphDirective>(
+      C, NumClauses, /*HasAssociatedStmt=*/true);
+}
+
 OMPTaskgroupDirective *OMPTaskgroupDirective::Create(
     const ASTContext &C, SourceLocation StartLoc, SourceLocation EndLoc,
     ArrayRef<OMPClause *> Clauses, Stmt *AssociatedStmt, Expr *ReductionRef) {
diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp
index 877191d456b35..e9cc881c83661 100644
--- a/clang/lib/AST/StmtPrinter.cpp
+++ b/clang/lib/AST/StmtPrinter.cpp
@@ -942,6 +942,11 @@ void StmtPrinter::VisitOMPAssumeDirective(OMPAssumeDirective *Node) {
   PrintOMPExecutableDirective(Node);
 }
 
+void StmtPrinter::VisitOMPTaskgraphDirective(OMPTaskgraphDirective *Node) {
+  Indent() << "#pragma omp taskgraph";
+  PrintOMPExecutableDirective(Node);
+}
+
 void StmtPrinter::VisitOMPErrorDirective(OMPErrorDirective *Node) {
   Indent() << "#pragma omp error";
   PrintOMPExecutableDirective(Node);
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index 80509c3f7ffd8..493d5e2b7c4a5 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -1190,6 +1190,10 @@ void StmtProfiler::VisitOMPAssumeDirective(const OMPAssumeDirective *S) {
   VisitOMPExecutableDirective(S);
 }
 
+void StmtProfiler::VisitOMPTaskgraphDirective(const OMPTaskgraphDirective *S) {
+  VisitOMPExecutableDirective(S);
+}
+
 void StmtProfiler::VisitOMPErrorDirective(const OMPErrorDirective *S) {
   VisitOMPExecutableDirective(S);
 }
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index 1e8d32c059282..f9cf0d8f55aca 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -982,6 +982,9 @@ void clang::getOpenMPCaptureRegions(
     case OMPD_teams:
       CaptureRegions.push_back(OMPD_teams);
       break;
+    case OMPD_taskgraph:
+      CaptureRegions.push_back(OMPD_taskgraph);
+      break;
     case OMPD_taskloop:
       CaptureRegions.push_back(OMPD_taskloop);
       break;
@@ -1005,7 +1008,6 @@ void clang::getOpenMPCaptureRegions(
     case OMPD_simd:
     case OMPD_single:
     case OMPD_target_data:
-    case OMPD_taskgraph:
     case OMPD_taskgroup:
     case OMPD_stripe:
       // These directives (when standalone) use OMPD_unknown as the region,
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 51f1e8b22fcfc..f16adaa6a4255 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -60,6 +60,8 @@ class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
     ParallelOutlinedRegion,
     /// Region with outlined function for standalone 'task' directive.
     TaskOutlinedRegion,
+    /// Region with outlined function for standalone 'taskgraph' directive.
+    TaskgraphOutlinedRegion,
     /// Region for constructs that do not require function outlining,
     /// like 'for', 'sections', 'atomic' etc. directives.
     InlinedRegion,
@@ -2233,6 +2235,33 @@ void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
     Region->emitUntiedSwitch(CGF);
 }
 
+void CGOpenMPRuntime::emitTaskgraphCall(CodeGenFunction &CGF,
+                                        SourceLocation Loc,
+                                        const OMPExecutableDirective &D,
+                                        const Expr *IfCond) {
+  if (!CGF.HaveInsertPoint())
+    return;
+
+  CodeGenFunction OutlinedCGF(CGM, /*suppressNewContext=*/true);
+
+  const auto *CS = cast<CapturedStmt>(D.getAssociatedStmt());
+  LValue CapStruct = CGF.InitCapturedStruct(*CS);
+
+  llvm::Function *OutlinedFn = OutlinedCGF.GenerateCapturedStmtFunction(*CS);
+
+  llvm::Value *CapturedArgsPtr =
+      CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+          CapStruct.getPointer(OutlinedCGF), CGM.VoidPtrTy);
+
+  auto &&CodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
+    Action.Enter(CGF);
+    CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, OutlinedFn,
+                                                        CapturedArgsPtr);
+  };
+  RegionCodeGenTy RCG(CodeGen);
+  RCG(CGF);
+}
+
 void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
                                           const RegionCodeGenTy &TaskgroupOpGen,
                                           SourceLocation Loc) {
@@ -6560,6 +6589,7 @@ const Expr *CGOpenMPRuntime::getNumTeamsExprForTargetDirective(
   case OMPD_taskyield:
   case OMPD_barrier:
   case OMPD_taskwait:
+  case OMPD_taskgraph:
   case OMPD_taskgroup:
   case OMPD_atomic:
   case OMPD_flush:
@@ -10524,6 +10554,7 @@ getNestedDistributeDirective(ASTContext &Ctx, const OMPExecutableDirective &D) {
     case OMPD_taskyield:
     case OMPD_barrier:
     case OMPD_taskwait:
+    case OMPD_taskgraph:
     case OMPD_taskgroup:
     case OMPD_atomic:
     case OMPD_flush:
@@ -11289,6 +11320,7 @@ void CGOpenMPRuntime::scanForTargetRegionsFunctions(const Stmt *S,
     case OMPD_taskyield:
     case OMPD_barrier:
     case OMPD_taskwait:
+    case OMPD_taskgraph:
     case OMPD_taskgroup:
     case OMPD_atomic:
     case OMPD_flush:
@@ -11864,6 +11896,7 @@ void CGOpenMPRuntime::emitTargetDataStandAloneCall(
     case OMPD_taskyield:
     case OMPD_barrier:
     case OMPD_taskwait:
+    case OMPD_taskgraph:
     case OMPD_taskgroup:
     case OMPD_atomic:
     case OMPD_flush:
@@ -13218,6 +13251,13 @@ void CGOpenMPSIMDRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
   llvm_unreachable("Not supported in SIMD-only mode");
 }
 
+void CGOpenMPSIMDRuntime::emitTaskgraphCall(CodeGenFunction &CGF,
+                                            SourceLocation Loc,
+                                            const OMPExecutableDirective &D,
+                                            const Expr *IfCond) {
+  llvm_unreachable("Not supported in SIMD-only mode");
+}
+
 void CGOpenMPSIMDRuntime::emitTaskgroupRegion(
     CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen,
     SourceLocation Loc) {
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h
index 73bbcc05f515a..2e9c734845af8 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.h
@@ -1380,6 +1380,11 @@ class CGOpenMPRuntime {
   virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
                                 const OMPTaskDataTy &Data);
 
+  /// Emit code for 'taskgraph' directive.
+  virtual void emitTaskgraphCall(CodeGenFunction &CGF, SourceLocation Loc,
+                                 const OMPExecutableDirective &D,
+                                 const Expr *IfCond);
+
   /// Emit code for 'cancellation point' construct.
   /// \param CancelRegion Region kind for which the cancellation point must be
   /// emitted.
@@ -2209,6 +2214,14 @@ class CGOpenMPSIMDRuntime final : public CGOpenMPRuntime {
   void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
                         const OMPTaskDataTy &Data) override;
 
+  /// Emit code for 'taskgraph' directive.
+  /// \param IfCond Expression evaluated in if clause associated with the
+  // taskgraph.
+  /// \param D Directive to emit.
+  void emitTaskgraphCall(CodeGenFunction &CGF, SourceLocation Loc,
+                         const OMPExecutableDirective &D,
+                         const Expr *IfCond) override;
+
   /// Emit code for 'cancellation point' construct.
   /// \param CancelRegion Region kind for which the cancellation point must be
   /// emitted.
diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
index 8f9000660d86b..5fc876340012e 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
@@ -575,6 +575,7 @@ static bool hasNestedSPMDDirective(ASTContext &Ctx,
     case OMPD_taskyield:
     case OMPD_barrier:
     case OMPD_taskwait:
+    case OMPD_taskgraph:
     case OMPD_taskgroup:
     case OMPD_atomic:
     case OMPD_flush:
@@ -663,6 +664,7 @@ static bool supportsSPMDExecutionMode(ASTContext &Ctx,
   case OMPD_taskyield:
   case OMPD_barrier:
   case OMPD_taskwait:
+  case OMPD_taskgraph:
   case OMPD_taskgroup:
   case OMPD_atomic:
   case OMPD_flush:
diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index 73f6c3c4aff1d..c6b21488c1ae5 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -292,6 +292,9 @@ void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {
   case Stmt::OMPTaskwaitDirectiveClass:
     EmitOMPTaskwaitDirective(cast<OMPTaskwaitDirective>(*S));
     break;
+  case Stmt::OMPTaskgraphDirectiveClass:
+    EmitOMPTaskgraphDirective(cast<OMPTaskgraphDirective>(*S));
+    break;
   case Stmt::OMPTaskgroupDirectiveClass:
     EmitOMPTaskgroupDirective(cast<OMPTaskgroupDirective>(*S));
     break;
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 97c43b4c0b384..a5cb20a4bfafa 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -1699,6 +1699,7 @@ void CodeGenFunction::EmitOMPReductionClauseInit(
     case OMPD_error:
     case OMPD_barrier:
     case OMPD_taskwait:
+    case OMPD_taskgraph:
     case OMPD_taskgroup:
     case OMPD_flush:
     case OMPD_depobj:
@@ -5913,6 +5914,20 @@ void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S) {
   CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getBeginLoc(), Data);
 }
 
+void CodeGenFunction::EmitOMPTaskgraphDirective(
+    const OMPTaskgraphDirective &S) {
+  const Expr *IfCond = nullptr;
+  for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
+    if (C->getNameModifier() == OMPD_unknown ||
+        C->getNameModifier() == OMPD_taskgraph) {
+      IfCond = C->getCondition();
+      break;
+    }
+  }
+
+  CGM.getOpenMPRuntime().emitTaskgraphCall(*this, S.getBeginLoc(), S, IfCond);
+}
+
 static bool isSupportedByOpenMPIRBuilder(const OMPTaskgroupDirective &T) {
   return T.clauses().empty();
 }
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 783f97dc354eb..b054f2d3673e1 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -3967,6 +3967,7 @@ class CodeGenFunction : public CodeGenTypeCache {
   void EmitOMPErrorDirective(const OMPErrorDirective &S);
   void EmitOMPBarrierDirective(const OMPBarrierDirective &S);
   void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S);
+  void EmitOMPTaskgraphDirective(const OMPTaskgraphDirective &S);
   void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S);
   void EmitOMPFlushDirective(const OMPFlushDirective &S);
   void EmitOMPDepobjDirective(const OMPDepobjDirective &S);
diff --git a/clang/lib/Sema/SemaExceptionSpec.cpp b/clang/lib/Sema/SemaExceptionSpec.cpp
index 0ed5c35c40c1a..476334c6ca2ff 100644
--- a/clang/lib/Sema/SemaExceptionSpec.cpp
+++ b/clang/lib/Sema/SemaExceptionSpec.cpp
@@ -1529,6 +1529,7 @@ CanThrowResult Sema::canThrow(const Stmt *S) {
   case Stmt::OMPScopeDirectiveClass:
   case Stmt::OMPTaskDirectiveClass:
   case Stmt::OMPTaskgroupDirectiveClass:
+  case Stmt::OMPTaskgraphDirectiveClass:
   case Stmt::OMPTaskLoopDirectiveClass:
   case Stmt::OMPTaskLoopSimdDirectiveClass:
   case Stmt::OMPTaskwaitDirectiveClass:
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index a67eedfeaf0cc..e4e85ebd0061c 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -4526,6 +4526,14 @@ getUnknownRegionParams(Sema &SemaRef) {
   return Params;
 }
 
+static SmallVector<SemaOpenMP::CapturedParamNameType>
+getTaskgraphRegionParams(Sema &SemaRef) {
+  SmallVector<SemaOpenMP::CapturedParamNameType> Params{
+      std::make_pair(StringRef(), QualType()) // __context with shared vars
+  };
+  return Params;
+}
+
 static SmallVector<SemaOpenMP::CapturedParamNameType>
 getTaskloopRegionParams(Sema &SemaRef) {
   ASTContext &Context = SemaRef.getASTContext();
@@ -4599,6 +4607,10 @@ static void processCapturedRegions(Sema &SemaRef, OpenMPDirectiveKind DKind,
       // function directly.
       MarkAsInlined(SemaRef.getCurCapturedRegion());
       break;
+    case OMPD_taskgraph:
+      SemaRef.ActOnCapturedRegionStart(
+          Loc, CurScope, CR_OpenMP, getTaskgraphRegionParams(SemaRef), Level);
+      break;
     case OMPD_target:
       SemaRef.ActOnCapturedRegionStart(Loc, CurScope, CR_OpenMP,
                                        getTargetRegionParams(SemaRef), Level);
@@ -6565,6 +6577,12 @@ StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
            "No associated statement allowed for 'omp taskwait' directive");
     Res = ActOnOpenMPTaskwaitDirective(ClausesWithImplicit, StartLoc, EndLoc);
     break;
+  case OMPD_taskgraph:
+    assert(AStmt &&
+           "Associated statement required for 'omp taskgraph' directive");
+    Res = ActOnOpenMPTaskgraphDirective(ClausesWithImplicit, AStmt, StartLoc,
+                                        EndLoc);
+    break;
   case OMPD_taskgroup:
     Res = ActOnOpenMPTaskgroupDirective(ClausesWithImplicit, AStmt, StartLoc,
                                         EndLoc);
@@ -6778,10 +6796,6 @@ StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
   case OMPD_begin_declare_variant:
   case OMPD_end_declare_variant:
     llvm_unreachable("OpenMP Directive is not allowed");
-  case OMPD_taskgraph:
-    Diag(StartLoc, diag::err_omp_unexpected_directive)
-        << 1 << getOpenMPDirectiveName(OMPD_taskgraph);
-    return StmtError();
   case OMPD_unknown:
   default:
     llvm_unreachable("Unknown OpenMP directive");
@@ -11663,6 +11677,24 @@ SemaOpenMP::ActOnOpenMPTaskwaitDirective(ArrayRef<OMPClause *> Clauses,
                                       Clauses);
 }
 
+StmtResult
+SemaOpenMP::ActOnOpenMPTaskgraphDirective(ArrayRef<OMPClause *> Clauses,
+                                          Stmt *AStmt, SourceLocation StartLoc,
+                                          SourceLocation EndLoc) {
+  if (!getLangOpts().OpenMP || getLangOpts().OpenMP < 60) {
+    Diag(StartLoc, diag::err_omp_unexpected_directive)
+        << 1 << getOpenMPDirectiveName(OMPD_taskgraph, getLangOpts().OpenMP);
+    return StmtError();
+  }
+  if (!AStmt)
+    return StmtError();
+
+  assert(isa<CapturedStmt>(AStmt) && "Captured statement expected");
+
+  return OMPTaskgraphDirective::Create(getASTContext(), StartLoc, EndLoc,
+                                       Clauses, AStmt);
+}
+
 StmtResult
 SemaOpenMP::ActOnOpenMPTaskgroupDirective(ArrayRef<OMPClause *> Clauses,
                                           Stmt *AStmt, SourceLocation StartLoc,
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index de7435b0b1c93..0d53a1e9ca02e 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -10247,6 +10247,17 @@ TreeTransform<Derived>::TransformOMPAssumeDirective(OMPAssumeDirective *D) {
   return Res;
 }
 
+template <typename Derived>
+StmtResult TreeTransform<Derived>::TransformOMPTaskgraphDirective(
+    OMPTaskgraphDirective *D) {
+  DeclarationNameInfo DirName;
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      OMPD_taskgraph, DirName, nullptr, D->getBeginLoc());
+  StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
+  return Res;
+}
+
 template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPErrorDirective(OMPErrorDirective *D) {
diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp
index 6cde6c1816dc7..f7d4032cf6a82 100644
--- a/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -2671,6 +2671,11 @@ void ASTStmtReader::VisitOMPAssumeDirective(OMPAssumeDirective *D) {
   VisitOMPExecutableDirective(D);
 }
 
+void ASTStmtReader::VisitOMPTaskgraphDirective(OMPTaskgraphDirective *D) {
+  VisitStmt(D);
+  VisitOMPExecutableDirective(D);
+}
+
 void ASTStmtReader::VisitOMPErrorDirective(OMPErrorDirective *D) {
   VisitStmt(D);
   // The NumClauses field was read in ReadStmtFromStream.
@@ -3852,6 +3857,11 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) {
           Context, Record[ASTStmtReader::NumStmtFields], Empty);
       break;
 
+    case STMT_OMP_TASKGRAPH_DIRECTIVE:
+      S = OMPTaskgraphDirective::CreateEmpty(
+          Context, Record[ASTStmtReader::NumStmtFields], Empty);
+      break;
+
     case STMT_OMP_ERROR_DIRECTIVE:
       S = OMPErrorDirective::CreateEmpty(
           Context, Record[ASTStmtReader::NumStmtFields], Empty);
diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp
index 10443d42df8c0..1ea818390c976 100644
--- a/clang/lib/Serialization/ASTWriterStmt.cpp
+++ b/clang/lib/Serialization/ASTWriterStmt.cpp
@@ -2762,6 +2762,12 @@ void ASTStmtWriter::VisitOMPAssumeDirective(OMPAssumeDirective *D) {
   Code = serialization::STMT_OMP_ASSUME_DIRECTIVE;
 }
 
+void ASTStmtWriter::VisitOMPTaskgraphDirective(OMPTaskgraphDirective *D) {
+  VisitStmt(D);
+  VisitOMPExecutableDirective(D);
+  Code = serialization::STMT_OMP_TASKGRAPH_DIRECTIVE;
+}
+
 void ASTStmtWriter::VisitOMPErrorDirective(OMPErrorDirective *D) {
   VisitStmt(D);
   Record.push_back(D->getNumClauses());
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
index 6373e9aafe97b..cb78031e2fcde 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
@@ -1739,6 +1739,7 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
     case Stmt::OMPTaskyieldDirectiveClass:
     case Stmt::OMPBarrierDirectiveClass:
     case Stmt::OMPTaskwaitDirectiveClass:
+    case Stmt::OMPTaskgraphDirectiveClass:
     case Stmt::OMPErrorDirectiveClass:
     case Stmt::OMPTaskgroupDirectiveClass:
     case Stmt::OMPFlushDirectiveClass:
diff --git a/clang/test/OpenMP/taskgraph_bad_clauses.cpp b/clang/test/OpenMP/taskgraph_bad_clauses.cpp
new file mode 100644
index 0000000000000..16b38a35fb4bd
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_bad_clauses.cpp
@@ -0,0 +1,19 @@
+// RUN: %clang_cc1 -verify=expected,omp51 -fopenmp -fopenmp-version=51 -ferror-limit 100 -o - %s
+// RUN: %clang_cc1 -verify=expected,omp60 -fopenmp -fopenmp-version=60 -ferror-limit 100 -o - %s
+
+int main() {
+  int data[10];
+#pragma omp taskgraph map(tofrom: data[0:10]) // expected-error {{unexpected OpenMP clause 'map' in directive '#pragma omp taskgraph'}} omp51-error {{unexpected OpenMP directive '#pragma omp taskgraph'}}
+  {
+  }
+#pragma omp taskgraph depend(inout: data) // expected-error {{unexpected OpenMP clause 'depend' in directive '#pragma omp taskgraph'}} omp51-error {{unexpected OpenMP directive '#pragma omp taskgraph'}}
+  {
+  }
+#pragma omp taskgraph if(taskgraph: 1) // omp51-error {{unexpected OpenMP directive '#pragma omp taskgraph'}}
+  {
+  }
+#pragma omp taskgraph if(cancel: 1) // omp51-error {{unexpected OpenMP directive '#pragma omp taskgraph'}} expected-error {{directive name modifier 'cancel' is not allowed for '#pragma omp taskgraph'}}
+  {
+  }
+  return 0;
+}
diff --git a/clang/test/OpenMP/unimplemented_clause_messages.cpp b/clang/test/OpenMP/unimplemented_clause_messages.cpp
index 0f51c916093ff..b83b9e9195a8e 100644
--- a/clang/test/OpenMP/unimplemented_clause_messages.cpp
+++ b/clang/test/OpenMP/unimplemented_clause_messages.cpp
@@ -76,7 +76,6 @@ void processPointsInLine() {
 // Make sure test doesn't crash.
 void test_tasgraph()
 {
-  // omp60-error at +2{{unexpected OpenMP directive '#pragma omp taskgraph'}}
   // omp51-error at +1{{unexpected OpenMP directive '#pragma omp taskgraph'}}
 #pragma omp taskgraph
   for (int i = 0; i < 10; ++i)
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index 37cfe8890e1b1..a5b33a3126b7e 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -6385,6 +6385,8 @@ CXString clang_getCursorKindSpelling(enum CXCursorKind Kind) {
     return cxstring::createRef("OMPTaskwaitDirective");
   case CXCursor_OMPAssumeDirective:
     return cxstring::createRef("OMPAssumeDirective");
+  case CXCursor_OMPTaskgraphDirective:
+    return cxstring::createRef("OMPTaskgraphDirective");
   case CXCursor_OMPErrorDirective:
     return cxstring::createRef("OMPErrorDirective");
   case CXCursor_OMPTaskgroupDirective:
diff --git a/clang/tools/libclang/CXCursor.cpp b/clang/tools/libclang/CXCursor.cpp
index fb0b3c1502574..c50b24a51723d 100644
--- a/clang/tools/libclang/CXCursor.cpp
+++ b/clang/tools/libclang/CXCursor.cpp
@@ -760,6 +760,9 @@ CXCursor cxcursor::MakeCXCursor(const Stmt *S, const Decl *Parent,
   case Stmt::OMPTaskwaitDirectiveClass:
     K = CXCursor_OMPTaskwaitDirective;
     break;
+  case Stmt::OMPTaskgraphDirectiveClass:
+    K = CXCursor_OMPTaskgraphDirective;
+    break;
   case Stmt::OMPErrorDirectiveClass:
     K = CXCursor_OMPErrorDirective;
     break;

>From d6ef3083dd86561c083ac0e8450aa92aa5cb4088 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Tue, 21 Apr 2026 17:01:24 -0500
Subject: [PATCH 05/26] [OpenMP] Add graph_id/graph_reset tests

Assisted-By: Codex with gpt-5.3

Pull Request: https://github.com/llvm/llvm-project/pull/194049
---
 .../OpenMP/taskgraph_clauses_ast_print.cpp    | 94 +++++++++++++++++++
 .../OpenMP/taskgraph_clauses_messages.cpp     | 36 +++++++
 2 files changed, 130 insertions(+)
 create mode 100644 clang/test/OpenMP/taskgraph_clauses_ast_print.cpp
 create mode 100644 clang/test/OpenMP/taskgraph_clauses_messages.cpp

diff --git a/clang/test/OpenMP/taskgraph_clauses_ast_print.cpp b/clang/test/OpenMP/taskgraph_clauses_ast_print.cpp
new file mode 100644
index 0000000000000..8ac721634380a
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_clauses_ast_print.cpp
@@ -0,0 +1,94 @@
+// Check no warnings/errors
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -fsyntax-only -verify %s
+// expected-no-diagnostics
+
+// Check AST and unparsing
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -ast-dump  %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -ast-print %s | FileCheck %s --check-prefix=PRINT
+
+// Check same results after serialization round-trip
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -emit-pch -o %t %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -include-pch %t -ast-dump-all %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -include-pch %t -ast-print    %s | FileCheck %s --check-prefix=PRINT
+
+#ifndef HEADER
+#define HEADER
+
+void taskgraph_clauses() {
+  int A = 1;
+
+  // --- graph_id clause ---
+
+  // DUMP: OMPTaskgraphDirective
+  // DUMP: OMPGraph_idClause
+  // DUMP-NEXT: IntegerLiteral {{.*}} 'int' 0
+  // PRINT: #pragma omp taskgraph graph_id(0)
+  #pragma omp taskgraph graph_id(0)
+  {}
+
+  // DUMP: OMPTaskgraphDirective
+  // DUMP: OMPGraph_idClause
+  // DUMP-NEXT: IntegerLiteral {{.*}} 'int' 42
+  // PRINT: #pragma omp taskgraph graph_id(42)
+  #pragma omp taskgraph graph_id(42)
+  {}
+
+  // DUMP: OMPTaskgraphDirective
+  // DUMP: OMPGraph_idClause
+  // DUMP: BinaryOperator {{.*}} 'int' '+'
+  // DUMP-NEXT: ImplicitCastExpr {{.*}} 'int' <LValueToRValue>
+  // DUMP-NEXT: DeclRefExpr {{.*}} 'int' lvalue Var {{.*}} 'A' 'int'
+  // DUMP-NEXT: IntegerLiteral {{.*}} 'int' 10
+  // PRINT: #pragma omp taskgraph graph_id(A + 10)
+  #pragma omp taskgraph graph_id(A + 10)
+  {}
+
+  // --- graph_reset clause ---
+
+  // DUMP: OMPTaskgraphDirective
+  // DUMP: OMPGraph_resetClause
+  // DUMP-NEXT: CXXBoolLiteralExpr {{.*}} 'bool' false
+  // PRINT: #pragma omp taskgraph graph_reset(false)
+  #pragma omp taskgraph graph_reset(false)
+  {}
+
+  // DUMP: OMPTaskgraphDirective
+  // DUMP: OMPGraph_resetClause
+  // DUMP-NEXT: CXXBoolLiteralExpr {{.*}} 'bool' true
+  // PRINT: #pragma omp taskgraph graph_reset(true)
+  #pragma omp taskgraph graph_reset(true)
+  {}
+
+  // DUMP: OMPTaskgraphDirective
+  // DUMP: OMPGraph_resetClause
+  // DUMP-NEXT: BinaryOperator {{.*}} 'bool' '>'
+  // DUMP-NEXT: ImplicitCastExpr {{.*}} 'int' <LValueToRValue>
+  // DUMP-NEXT: DeclRefExpr {{.*}} 'int' lvalue Var {{.*}} 'A' 'int'
+  // DUMP-NEXT: IntegerLiteral {{.*}} 'int' 5
+  // PRINT: #pragma omp taskgraph graph_reset(A > 5)
+  #pragma omp taskgraph graph_reset(A > 5)
+  {}
+
+  // DUMP: OMPTaskgraphDirective
+  // DUMP: OMPGraph_resetClause
+  // PRINT: #pragma omp taskgraph graph_reset
+  #pragma omp taskgraph graph_reset
+  {}
+
+  // --- Combinations using both clauses ---
+
+  // DUMP: OMPTaskgraphDirective
+  // DUMP: OMPGraph_idClause
+  // DUMP: OMPGraph_resetClause
+  // PRINT: #pragma omp taskgraph graph_id(1) graph_reset(true)
+  #pragma omp taskgraph graph_id(1) graph_reset(true)
+  {}
+
+  // DUMP: OMPTaskgraphDirective
+  // DUMP: OMPGraph_resetClause
+  // DUMP: OMPGraph_idClause
+  // PRINT: #pragma omp taskgraph graph_reset(false) graph_id(2)
+  #pragma omp taskgraph graph_reset(false) graph_id(2)
+  {}
+}
+#endif
diff --git a/clang/test/OpenMP/taskgraph_clauses_messages.cpp b/clang/test/OpenMP/taskgraph_clauses_messages.cpp
new file mode 100644
index 0000000000000..0327f52475d98
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_clauses_messages.cpp
@@ -0,0 +1,36 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=51 -verify=expected,omp51 -fsyntax-only %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -verify=expected,omp60 -fsyntax-only %s
+
+// Tests that the 'graph_id' and 'graph_reset' clauses are accepted in OpenMP 6.0
+// and rejected in prior versions on the 'taskgraph' directive. Also tests
+// duplicate-clause and invalid-condition diagnostics.
+
+void foo() {}
+
+void taskgraph_clauses_messages() {
+  int A = 1;
+
+  // Basic version error tests.
+  #pragma omp taskgraph graph_id(0) // omp51-error {{unexpected OpenMP clause 'graph_id' in directive '#pragma omp taskgraph'}} omp51-error {{unexpected OpenMP directive '#pragma omp taskgraph'}}
+  {}
+
+  #pragma omp taskgraph graph_reset(true) // omp51-error {{unexpected OpenMP clause 'graph_reset' in directive '#pragma omp taskgraph'}} omp51-error {{unexpected OpenMP directive '#pragma omp taskgraph'}}
+  {}
+
+  // Same, without argument.
+  #pragma omp taskgraph graph_reset // omp51-error {{unexpected OpenMP clause 'graph_reset' in directive '#pragma omp taskgraph'}} omp51-error {{unexpected OpenMP directive '#pragma omp taskgraph'}}
+  {}
+
+  // Duplicate clause tests (OMP 6.0 only; in OMP 5.1 both are unexpected).
+  #pragma omp taskgraph graph_id(0) graph_id(1) // omp51-error {{unexpected OpenMP clause 'graph_id' in directive '#pragma omp taskgraph'}} omp51-error {{unexpected OpenMP clause 'graph_id' in directive '#pragma omp taskgraph'}} omp51-error {{unexpected OpenMP directive '#pragma omp taskgraph'}} expected-error {{directive '#pragma omp taskgraph' cannot contain more than one 'graph_id' clause}}
+  {}
+
+  #pragma omp taskgraph graph_reset(true) graph_reset(false) // omp51-error {{unexpected OpenMP clause 'graph_reset' in directive '#pragma omp taskgraph'}} omp51-error {{unexpected OpenMP clause 'graph_reset' in directive '#pragma omp taskgraph'}} omp51-error {{unexpected OpenMP directive '#pragma omp taskgraph'}} expected-error {{directive '#pragma omp taskgraph' cannot contain more than one 'graph_reset' clause}}
+  {}
+
+  #pragma omp taskgraph graph_id(foo()) // omp51-error {{unexpected OpenMP clause 'graph_id' in directive '#pragma omp taskgraph'}} omp51-error {{unexpected OpenMP directive '#pragma omp taskgraph'}} omp60-error {{expression must have integral or unscoped enumeration type, not 'void'}}
+  {}
+
+  #pragma omp taskgraph graph_reset(foo()) // omp51-error {{unexpected OpenMP clause 'graph_reset' in directive '#pragma omp taskgraph'}} omp51-error {{unexpected OpenMP directive '#pragma omp taskgraph'}} omp60-error {{value of type 'void' is not contextually convertible to 'bool'}}
+  {}
+}

>From da6db6b99efb36dd9682c95c117f125bf0f3de1d Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Tue, 5 May 2026 08:31:54 -0500
Subject: [PATCH 06/26] [OpenMP] Support 'taskgraph' semantics via new libomp
 API entry points

This patch supports the OpenMP 6.0 'taskgraph' directive via several
new API entry points for libomp.

New entry points are used for several reasons.  The first reason
is to pass extra information from the compiler relevant to the
taskgraph-recording case -- e.g. the __kmpc_taskgraph_task entry point
has extra arguments relating to shared data.  The second reason is to
reduce the potential overhead of the taskgraph implementation on the
rest of the runtime.  A third intention is to capture OpenMP semantics
at a slightly higher level: in particular when we come to add offload
target tasks to this implementation, those will also use new API entry
points to hopefully allow dependencies to be handled entirely on the GPU,
rather than by being wrapped in a host task.

Pull Request: https://github.com/llvm/llvm-project/pull/194051
---
 .../clang/Basic/DiagnosticSemaKinds.td        |   2 +
 clang/lib/CodeGen/CGOpenMPRuntime.cpp         | 718 +++++++++++++-----
 clang/lib/CodeGen/CGOpenMPRuntime.h           |   4 +-
 clang/lib/CodeGen/CodeGenFunction.h           |  15 +
 clang/lib/Sema/SemaOpenMP.cpp                 |   4 +
 .../test/OpenMP/taskgraph_taskwait_nodeps.cpp |  12 +
 .../include/llvm/Frontend/OpenMP/OMPKinds.def |  11 +
 7 files changed, 555 insertions(+), 211 deletions(-)
 create mode 100644 clang/test/OpenMP/taskgraph_taskwait_nodeps.cpp

diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 51fe39f3733b1..3b59ef75461f3 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12441,6 +12441,8 @@ def err_omp_single_copyprivate_with_nowait : Error<
   "the 'copyprivate' clause must not be used with the 'nowait' clause">;
 def err_omp_nowait_clause_without_depend: Error<
   "directive '#pragma omp taskwait' cannot use 'nowait' clause without 'depend' clause">;
+def err_omp_taskgraph_taskwait_without_depend: Error<
+  "directive '#pragma omp taskwait' within '#pragma omp taskgraph' must use 'depend' clause to be task-generating">;
 def note_omp_nowait_clause_here : Note<
   "'nowait' clause is here">;
 def err_omp_single_decl_in_declare_simd_variant : Error<
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index f16adaa6a4255..2b760da848732 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -33,6 +33,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Bitcode/BitcodeReader.h"
+#include "llvm/Frontend/OpenMP/OMP.h.inc"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DerivedTypes.h"
 #include "llvm/IR/GlobalValue.h"
@@ -348,6 +349,26 @@ class CGOpenMPTargetRegionInfo final : public CGOpenMPRegionInfo {
   StringRef HelperName;
 };
 
+/// API for captured statement code generation in OpenMP taskgraphs.
+class CGOpenMPTaskgraphRegionInfo final : public CGOpenMPRegionInfo {
+public:
+  CGOpenMPTaskgraphRegionInfo(const CapturedStmt &CS,
+                              const RegionCodeGenTy &CodeGen)
+      : CGOpenMPRegionInfo(CS, TaskgraphOutlinedRegion, CodeGen,
+                           llvm::omp::OMPD_taskgraph, false) {}
+
+  const VarDecl *getThreadIDVariable() const override { return 0; }
+
+  /// Get the name of the capture helper.
+  StringRef getHelperName() const override { return "taskgraph.omp_outlined."; }
+
+  static bool classof(const CGCapturedStmtInfo *Info) {
+    return CGOpenMPRegionInfo::classof(Info) &&
+           cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
+               TaskgraphOutlinedRegion;
+  }
+};
+
 static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) {
   llvm_unreachable("No codegen for expressions");
 }
@@ -2242,24 +2263,98 @@ void CGOpenMPRuntime::emitTaskgraphCall(CodeGenFunction &CGF,
   if (!CGF.HaveInsertPoint())
     return;
 
+  // The nogroup clause doesn't support an argument yet.  FIXME.
+  const OMPNogroupClause *NoGroupClause = D.getSingleClause<OMPNogroupClause>();
+  llvm::Value *NoGroup;
+  if (NoGroupClause) {
+    NoGroup = CGF.Builder.getInt32(1);
+  } else {
+    NoGroup = CGF.Builder.getInt32(0);
+  }
+
+  const OMPGraphResetClause *GraphResetClause =
+      D.getSingleClause<OMPGraphResetClause>();
+  llvm::Value *GraphReset;
+  if (GraphResetClause) {
+    const Expr *Cond = GraphResetClause->getCondition();
+    llvm::Value *CondVal = CGF.EvaluateExprAsBool(Cond);
+    GraphReset =
+        CGF.Builder.CreateIntCast(CondVal, CGF.IntTy, /*isSigned=*/true);
+  } else {
+    GraphReset = CGF.Builder.getInt32(0);
+  }
+
+  llvm::Value *GraphId = CGF.Builder.getInt32(0);
+  const OMPGraphIdClause *GraphIdClause = D.getSingleClause<OMPGraphIdClause>();
+  if (GraphIdClause) {
+    const auto *E = GraphIdClause->getId();
+    auto *GraphIdVal = CGF.EmitScalarExpr(E);
+    GraphId =
+        CGF.Builder.CreateIntCast(GraphIdVal, CGM.Int32Ty, /*isSigned=*/false);
+  }
+
   CodeGenFunction OutlinedCGF(CGM, /*suppressNewContext=*/true);
 
   const auto *CS = cast<CapturedStmt>(D.getAssociatedStmt());
+
+  auto BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
+    CodeGenFunction::OMPWithinTaskgraphRAII WithinTaskgraph(CGF);
+    CGF.EmitStmt(CS->getCapturedStmt());
+  };
+
   LValue CapStruct = CGF.InitCapturedStruct(*CS);
+  CGOpenMPTaskgraphRegionInfo TaskgraphRegion(*CS, BodyGen);
+  CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(OutlinedCGF,
+                                                  &TaskgraphRegion);
 
   llvm::Function *OutlinedFn = OutlinedCGF.GenerateCapturedStmtFunction(*CS);
 
-  llvm::Value *CapturedArgsPtr =
+  // Create an internal-linkage global variable to hold the taskgraph handle.
+  std::string GraphHandleName = getName({"omp", "taskgraph", "handle"});
+  auto *GraphHandle = new llvm::GlobalVariable(
+      CGM.getModule(), CGM.VoidPtrTy,
+      /*IsConstant=*/false, llvm::GlobalValue::InternalLinkage,
+      llvm::Constant::getNullValue(CGM.VoidPtrTy), GraphHandleName);
+
+  std::array<llvm::Value *, 8> Args{
+      emitUpdateLocation(CGF, Loc),
+      getThreadID(CGF, Loc),
+      GraphHandle,
+      GraphId,
+      GraphReset,
+      NoGroup,
+      CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(OutlinedFn,
+                                                      CGM.VoidPtrTy),
       CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
-          CapStruct.getPointer(OutlinedCGF), CGM.VoidPtrTy);
+          CapStruct.getPointer(OutlinedCGF), CGM.VoidPtrTy)};
 
-  auto &&CodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
-    Action.Enter(CGF);
-    CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, OutlinedFn,
-                                                        CapturedArgsPtr);
+  auto &&ThenGen = [&CGF, this, &Args](CodeGenFunction &, PrePostActionTy &) {
+    CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+                            CGM.getModule(), OMPRTL___kmpc_taskgraph),
+                        Args);
   };
-  RegionCodeGenTy RCG(CodeGen);
-  RCG(CGF);
+  auto &&ElseGen = [&CGF, this, &OutlinedFn, &CapStruct, &Loc,
+                    &OutlinedCGF](CodeGenFunction &, PrePostActionTy &) {
+    llvm::Value *CapturedArgsPtr =
+        CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+            CapStruct.getPointer(OutlinedCGF), CGM.VoidPtrTy);
+
+    auto &&CodeGen = [&](CodeGenFunction &CGF, PrePostActionTy &Action) {
+      Action.Enter(CGF);
+      CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, OutlinedFn,
+                                                          CapturedArgsPtr);
+    };
+    RegionCodeGenTy RCG(CodeGen);
+    RCG(CGF);
+  };
+
+  if (IfCond) {
+    emitIfClause(CGF, IfCond, ThenGen, ElseGen);
+  } else {
+    CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+                            CGM.getModule(), OMPRTL___kmpc_taskgraph),
+                        Args);
+  }
 }
 
 void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF,
@@ -3797,11 +3892,11 @@ static void getKmpAffinityType(ASTContext &C, QualType &KmpTaskAffinityInfoTy) {
   }
 }
 
-CGOpenMPRuntime::TaskResultTy
-CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
-                              const OMPExecutableDirective &D,
-                              llvm::Function *TaskFunction, QualType SharedsTy,
-                              Address Shareds, const OMPTaskDataTy &Data) {
+CGOpenMPRuntime::TaskResultTy CGOpenMPRuntime::emitTaskInit(
+    CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D,
+    llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds,
+    const OMPTaskDataTy &Data, bool ForTaskgraph,
+    std::array<llvm::Value *, 3> &TaskAllocArgs) {
   ASTContext &C = CGM.getContext();
   llvm::SmallVector<PrivateDataTy, 4> Privates;
   // Aggregate privates and sort them by the alignment.
@@ -3948,6 +4043,11 @@ CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
       SharedsSize, CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
           TaskEntry, KmpRoutineEntryPtrTy)};
   llvm::Value *NewTask;
+  if (ForTaskgraph) {
+    TaskAllocArgs[0] = TaskFlags;
+    TaskAllocArgs[1] = KmpTaskTWithPrivatesTySize;
+    TaskAllocArgs[2] = SharedsSize;
+  }
   if (D.hasClausesOfKind<OMPNowaitClause>()) {
     // Check if we have any device clause associated with the directive.
     const Expr *Device = nullptr;
@@ -4706,114 +4806,169 @@ void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
   if (!CGF.HaveInsertPoint())
     return;
 
-  TaskResultTy Result =
-      emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
-  llvm::Value *NewTask = Result.NewTask;
-  llvm::Function *TaskEntry = Result.TaskEntry;
-  llvm::Value *NewTaskNewTaskTTy = Result.NewTaskNewTaskTTy;
-  LValue TDBase = Result.TDBase;
-  const RecordDecl *KmpTaskTQTyRD = Result.KmpTaskTQTyRD;
-  // Process list of dependences.
-  Address DependenciesArray = Address::invalid();
-  llvm::Value *NumOfElements;
-  std::tie(NumOfElements, DependenciesArray) =
-      emitDependClause(CGF, Data.Dependences, Loc);
-
-  // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
-  // libcall.
-  // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid,
-  // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list,
-  // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence
-  // list is not empty
-  llvm::Value *ThreadID = getThreadID(CGF, Loc);
-  llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
-  llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
-  llvm::Value *DepTaskArgs[7];
-  if (!Data.Dependences.empty()) {
-    DepTaskArgs[0] = UpLoc;
-    DepTaskArgs[1] = ThreadID;
-    DepTaskArgs[2] = NewTask;
-    DepTaskArgs[3] = NumOfElements;
-    DepTaskArgs[4] = DependenciesArray.emitRawPointer(CGF);
-    DepTaskArgs[5] = CGF.Builder.getInt32(0);
-    DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
-  }
-  auto &&ThenCodeGen = [this, &Data, TDBase, KmpTaskTQTyRD, &TaskArgs,
-                        &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) {
-    if (!Data.Tied) {
-      auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
-      LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI);
-      CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal);
+  auto &&TaskgraphTaskCodeGen = [this, &Loc, &D, TaskFunction, &SharedsTy,
+                                 &Shareds, &Data](CodeGenFunction &CGF,
+                                                  PrePostActionTy &) {
+    llvm::Value *ThreadId = getThreadID(CGF, Loc);
+    llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
+    std::array<llvm::Value *, 9> TGTaskArgs;
+    std::array<llvm::Value *, 3> TaskAllocArgs;
+    TaskResultTy Result = emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy,
+                                       Shareds, Data, true, TaskAllocArgs);
+    Address DependenciesArray = Address::invalid();
+    llvm::Value *NumOfElements;
+    std::tie(NumOfElements, DependenciesArray) =
+        emitDependClause(CGF, Data.Dependences, Loc);
+    TGTaskArgs[0] = UpLoc;
+    TGTaskArgs[1] = ThreadId;
+    TGTaskArgs[2] = Result.NewTask;
+    TGTaskArgs[3] = TaskAllocArgs[0]; // TaskFlags
+    TGTaskArgs[4] = TaskAllocArgs[1]; // KmpTaskTWithPrivatesTySize
+    TGTaskArgs[5] = Shareds.emitRawPointer(CGF);
+    TGTaskArgs[6] = TaskAllocArgs[2]; // SharedsSize
+    if (auto RecType = dyn_cast<RecordType>(SharedsTy)) {
+      auto *RD = RecType->getAsRecordDecl();
+      if (RD->fields().empty()) {
+        // FIXME: The condition might not be precisely correct here.
+        TGTaskArgs[6] = CGF.Builder.getSize(0);
+      }
     }
-    if (!Data.Dependences.empty()) {
-      CGF.EmitRuntimeCall(
-          OMPBuilder.getOrCreateRuntimeFunction(
-              CGM.getModule(), OMPRTL___kmpc_omp_task_with_deps),
-          DepTaskArgs);
+    if (Data.Dependences.size() == 0) {
+      TGTaskArgs[7] = CGF.Builder.getInt32(0);
+      TGTaskArgs[8] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
     } else {
-      CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
-                              CGM.getModule(), OMPRTL___kmpc_omp_task),
-                          TaskArgs);
+      TGTaskArgs[7] = NumOfElements;
+      TGTaskArgs[8] = DependenciesArray.emitRawPointer(CGF);
     }
-    // Check if parent region is untied and build return for untied task;
-    if (auto *Region =
-            dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
-      Region->emitUntiedSwitch(CGF);
+    CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+                            CGM.getModule(), OMPRTL___kmpc_taskgraph_task),
+                        TGTaskArgs);
   };
 
-  llvm::Value *DepWaitTaskArgs[7];
-  if (!Data.Dependences.empty()) {
-    DepWaitTaskArgs[0] = UpLoc;
-    DepWaitTaskArgs[1] = ThreadID;
-    DepWaitTaskArgs[2] = NumOfElements;
-    DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);
-    DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
-    DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
-    DepWaitTaskArgs[6] =
-        llvm::ConstantInt::get(CGF.Int32Ty, Data.HasNowaitClause);
-  }
-  auto &M = CGM.getModule();
-  auto &&ElseCodeGen = [this, &M, &TaskArgs, ThreadID, NewTaskNewTaskTTy,
-                        TaskEntry, &Data, &DepWaitTaskArgs,
-                        Loc](CodeGenFunction &CGF, PrePostActionTy &) {
-    CodeGenFunction::RunCleanupsScope LocalScope(CGF);
-    // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
-    // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
-    // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info
-    // is specified.
-    if (!Data.Dependences.empty())
-      CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
-                              M, OMPRTL___kmpc_omp_taskwait_deps_51),
-                          DepWaitTaskArgs);
-    // Call proxy_task_entry(gtid, new_task);
-    auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
-                      Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
-      Action.Enter(CGF);
-      llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
-      CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
-                                                          OutlinedFnArgs);
+  auto &&NonTaskgraphTaskCodeGen = [this, &Loc, &D, TaskFunction, &SharedsTy,
+                                    &Shareds, IfCond,
+                                    &Data](CodeGenFunction &CGF,
+                                           PrePostActionTy &) {
+    std::array<llvm::Value *, 3> DummyArray;
+    TaskResultTy Result = emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy,
+                                       Shareds, Data, false, DummyArray);
+    llvm::Value *NewTask = Result.NewTask;
+    llvm::Function *TaskEntry = Result.TaskEntry;
+    llvm::Value *NewTaskNewTaskTTy = Result.NewTaskNewTaskTTy;
+    LValue TDBase = Result.TDBase;
+    const RecordDecl *KmpTaskTQTyRD = Result.KmpTaskTQTyRD;
+    // Process list of dependences.
+    Address DependenciesArray = Address::invalid();
+    llvm::Value *NumOfElements;
+    std::tie(NumOfElements, DependenciesArray) =
+        emitDependClause(CGF, Data.Dependences, Loc);
+
+    // NOTE: routine and part_id fields are initialized by
+    // __kmpc_omp_task_alloc() libcall. Build kmp_int32
+    // __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid, kmp_task_t
+    // *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
+    // ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence list is
+    // not empty
+    llvm::Value *ThreadID = getThreadID(CGF, Loc);
+    llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
+    llvm::Value *TaskArgs[] = {UpLoc, ThreadID, NewTask};
+    llvm::Value *DepTaskArgs[7];
+    if (!Data.Dependences.empty()) {
+      DepTaskArgs[0] = UpLoc;
+      DepTaskArgs[1] = ThreadID;
+      DepTaskArgs[2] = NewTask;
+      DepTaskArgs[3] = NumOfElements;
+      DepTaskArgs[4] = DependenciesArray.emitRawPointer(CGF);
+      DepTaskArgs[5] = CGF.Builder.getInt32(0);
+      DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+    }
+    auto &&ThenCodeGen = [this, &Data, TDBase, KmpTaskTQTyRD, &TaskArgs,
+                          &DepTaskArgs](CodeGenFunction &CGF,
+                                        PrePostActionTy &) {
+      if (!Data.Tied) {
+        auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
+        LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI);
+        CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal);
+      }
+      if (!Data.Dependences.empty()) {
+        CGF.EmitRuntimeCall(
+            OMPBuilder.getOrCreateRuntimeFunction(
+                CGM.getModule(), OMPRTL___kmpc_omp_task_with_deps),
+            DepTaskArgs);
+      } else {
+        CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+                                CGM.getModule(), OMPRTL___kmpc_omp_task),
+                            TaskArgs);
+      }
+      // Check if parent region is untied and build return for untied task;
+      if (auto *Region =
+              dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
+        Region->emitUntiedSwitch(CGF);
     };
 
-    // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
-    // kmp_task_t *new_task);
-    // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
-    // kmp_task_t *new_task);
-    RegionCodeGenTy RCG(CodeGen);
-    CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
-                              M, OMPRTL___kmpc_omp_task_begin_if0),
-                          TaskArgs,
-                          OMPBuilder.getOrCreateRuntimeFunction(
-                              M, OMPRTL___kmpc_omp_task_complete_if0),
-                          TaskArgs);
-    RCG.setAction(Action);
-    RCG(CGF);
+    llvm::Value *DepWaitTaskArgs[7];
+    if (!Data.Dependences.empty()) {
+      DepWaitTaskArgs[0] = UpLoc;
+      DepWaitTaskArgs[1] = ThreadID;
+      DepWaitTaskArgs[2] = NumOfElements;
+      DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);
+      DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
+      DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+      DepWaitTaskArgs[6] =
+          llvm::ConstantInt::get(CGF.Int32Ty, Data.HasNowaitClause);
+    }
+    auto &M = CGM.getModule();
+    auto &&ElseCodeGen = [this, &M, &TaskArgs, ThreadID, NewTaskNewTaskTTy,
+                          TaskEntry, &Data, &DepWaitTaskArgs,
+                          Loc](CodeGenFunction &CGF, PrePostActionTy &) {
+      CodeGenFunction::RunCleanupsScope LocalScope(CGF);
+      // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid,
+      // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
+      // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info
+      // is specified.
+      if (!Data.Dependences.empty())
+        CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+                                M, OMPRTL___kmpc_omp_taskwait_deps_51),
+                            DepWaitTaskArgs);
+      // Call proxy_task_entry(gtid, new_task);
+      auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,
+                        Loc](CodeGenFunction &CGF, PrePostActionTy &Action) {
+        Action.Enter(CGF);
+        llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
+        CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry,
+                                                            OutlinedFnArgs);
+      };
+
+      // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid,
+      // kmp_task_t *new_task);
+      // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid,
+      // kmp_task_t *new_task);
+      RegionCodeGenTy RCG(CodeGen);
+      CommonActionTy Action(OMPBuilder.getOrCreateRuntimeFunction(
+                                M, OMPRTL___kmpc_omp_task_begin_if0),
+                            TaskArgs,
+                            OMPBuilder.getOrCreateRuntimeFunction(
+                                M, OMPRTL___kmpc_omp_task_complete_if0),
+                            TaskArgs);
+      RCG.setAction(Action);
+      RCG(CGF);
+    };
+
+    if (IfCond) {
+      emitIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);
+    } else {
+      RegionCodeGenTy ThenRCG(ThenCodeGen);
+      ThenRCG(CGF);
+    }
   };
 
-  if (IfCond) {
-    emitIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen);
+  if (CGF.getOMPWithinTaskgraph()) {
+    // Lexically within taskgraph, always replayable.
+    RegionCodeGenTy TaskgraphRCG(TaskgraphTaskCodeGen);
+    TaskgraphRCG(CGF);
   } else {
-    RegionCodeGenTy ThenRCG(ThenCodeGen);
-    ThenRCG(CGF);
+    RegionCodeGenTy NonTaskgraphRCG(NonTaskgraphTaskCodeGen);
+    NonTaskgraphRCG(CGF);
   }
 }
 
@@ -4825,15 +4980,11 @@ void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
                                        const OMPTaskDataTy &Data) {
   if (!CGF.HaveInsertPoint())
     return;
-  TaskResultTy Result =
-      emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
-  // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc()
-  // libcall.
-  // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int
-  // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int
-  // sched, kmp_uint64 grainsize, void *task_dup);
-  llvm::Value *ThreadID = getThreadID(CGF, Loc);
-  llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
+
+  std::array<llvm::Value *, 3> TaskAllocArgs;
+  TaskResultTy TaskInitResult = emitTaskInit(
+      CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data, true, TaskAllocArgs);
+
   llvm::Value *IfVal;
   if (IfCond) {
     IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy,
@@ -4842,68 +4993,177 @@ void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
     IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1);
   }
 
-  LValue LBLVal = CGF.EmitLValueForField(
-      Result.TDBase,
-      *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound));
-  const auto *LBVar =
-      cast<VarDecl>(cast<DeclRefExpr>(D.getLowerBoundVariable())->getDecl());
-  CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(), LBLVal.getQuals(),
-                       /*IsInitializer=*/true);
-  LValue UBLVal = CGF.EmitLValueForField(
-      Result.TDBase,
-      *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound));
-  const auto *UBVar =
-      cast<VarDecl>(cast<DeclRefExpr>(D.getUpperBoundVariable())->getDecl());
-  CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(), UBLVal.getQuals(),
-                       /*IsInitializer=*/true);
-  LValue StLVal = CGF.EmitLValueForField(
-      Result.TDBase,
-      *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride));
-  const auto *StVar =
-      cast<VarDecl>(cast<DeclRefExpr>(D.getStrideVariable())->getDecl());
-  CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(), StLVal.getQuals(),
-                       /*IsInitializer=*/true);
-  // Store reductions address.
-  LValue RedLVal = CGF.EmitLValueForField(
-      Result.TDBase,
-      *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions));
-  if (Data.Reductions) {
-    CGF.EmitStoreOfScalar(Data.Reductions, RedLVal);
+  enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
+
+  auto &&TaskgraphTaskloopCodeGen = [this, &Loc, &D, &TaskInitResult, &Shareds,
+                                     IfVal, &Data,
+                                     &TaskAllocArgs](CodeGenFunction &CGF,
+                                                     PrePostActionTy &) {
+    llvm::Value *ThreadId = getThreadID(CGF, Loc);
+    llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
+    std::array<llvm::Value *, 16> TGTaskLoopArgs;
+
+    // This is all copy/pasted from below. Refactor!
+    LValue LBLVal = CGF.EmitLValueForField(
+        TaskInitResult.TDBase,
+        *std::next(TaskInitResult.KmpTaskTQTyRD->field_begin(),
+                   KmpTaskTLowerBound));
+    const auto *LBVar =
+        cast<VarDecl>(cast<DeclRefExpr>(D.getLowerBoundVariable())->getDecl());
+    CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(),
+                         LBLVal.getQuals(),
+                         /*IsInitializer=*/true);
+    LValue UBLVal = CGF.EmitLValueForField(
+        TaskInitResult.TDBase,
+        *std::next(TaskInitResult.KmpTaskTQTyRD->field_begin(),
+                   KmpTaskTUpperBound));
+    const auto *UBVar =
+        cast<VarDecl>(cast<DeclRefExpr>(D.getUpperBoundVariable())->getDecl());
+    CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(),
+                         UBLVal.getQuals(),
+                         /*IsInitializer=*/true);
+    LValue StLVal = CGF.EmitLValueForField(
+        TaskInitResult.TDBase,
+        *std::next(TaskInitResult.KmpTaskTQTyRD->field_begin(),
+                   KmpTaskTStride));
+    const auto *StVar =
+        cast<VarDecl>(cast<DeclRefExpr>(D.getStrideVariable())->getDecl());
+    CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(),
+                         StLVal.getQuals(), /*IsInitializer=*/true);
+    // Store reductions address.
+    LValue RedLVal = CGF.EmitLValueForField(
+        TaskInitResult.TDBase,
+        *std::next(TaskInitResult.KmpTaskTQTyRD->field_begin(),
+                   KmpTaskTReductions));
+    if (Data.Reductions) {
+      CGF.EmitStoreOfScalar(Data.Reductions, RedLVal);
+    } else {
+      CGF.EmitNullInitialization(RedLVal.getAddress(),
+                                 CGF.getContext().VoidPtrTy);
+    }
+
+    TGTaskLoopArgs[0] = UpLoc;
+    TGTaskLoopArgs[1] = ThreadId;
+    TGTaskLoopArgs[2] = TaskInitResult.NewTask;
+    TGTaskLoopArgs[3] = TaskAllocArgs[0]; // TaskFlags
+    TGTaskLoopArgs[4] = TaskAllocArgs[1]; // KmpTaskTWithPrivatesTySize
+    TGTaskLoopArgs[5] = Shareds.emitRawPointer(CGF);
+    TGTaskLoopArgs[6] = TaskAllocArgs[2]; // SharedsSize
+    TGTaskLoopArgs[7] = IfVal;
+    TGTaskLoopArgs[8] = LBLVal.getPointer(CGF);
+    TGTaskLoopArgs[9] = UBLVal.getPointer(CGF);
+    TGTaskLoopArgs[10] = CGF.EmitLoadOfScalar(StLVal, Loc);
+    TGTaskLoopArgs[11] =
+        llvm::ConstantInt::getSigned(CGF.IntTy, Data.Nogroup ? 1 : 0);
+    TGTaskLoopArgs[12] = llvm::ConstantInt::getSigned(
+        CGF.IntTy, Data.Schedule.getPointer()
+                       ? Data.Schedule.getInt() ? NumTasks : Grainsize
+                       : NoSchedule);
+    TGTaskLoopArgs[13] =
+        Data.Schedule.getPointer()
+            ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty,
+                                        /*isSigned=*/false)
+            : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0);
+    TGTaskLoopArgs[14] =
+        llvm::ConstantInt::getSigned(CGF.IntTy, Data.HasModifier ? 1 : 0);
+    TGTaskLoopArgs[15] = TaskInitResult.TaskDupFn
+                             ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+                                   TaskInitResult.TaskDupFn, CGF.VoidPtrTy)
+                             : llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+    CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+                            CGM.getModule(), OMPRTL___kmpc_taskgraph_taskloop),
+                        TGTaskLoopArgs);
+  };
+
+  auto &&NonTaskgraphTaskloopCodeGen = [this, &Loc, &D, &TaskInitResult, IfVal,
+                                        &Data](CodeGenFunction &CGF,
+                                               PrePostActionTy &) {
+    // NOTE: routine and part_id fields are initialized by
+    // __kmpc_omp_task_alloc() libcall. Call to void __kmpc_taskloop(ident_t
+    // *loc, int gtid, kmp_task_t *task, int if_val, kmp_uint64 *lb, kmp_uint64
+    // *ub, kmp_int64 st, int nogroup, int sched, kmp_uint64 grainsize, void
+    // *task_dup);
+    llvm::Value *ThreadID = getThreadID(CGF, Loc);
+    llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
+
+    LValue LBLVal = CGF.EmitLValueForField(
+        TaskInitResult.TDBase,
+        *std::next(TaskInitResult.KmpTaskTQTyRD->field_begin(),
+                   KmpTaskTLowerBound));
+    const auto *LBVar =
+        cast<VarDecl>(cast<DeclRefExpr>(D.getLowerBoundVariable())->getDecl());
+    CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(),
+                         LBLVal.getQuals(),
+                         /*IsInitializer=*/true);
+    LValue UBLVal = CGF.EmitLValueForField(
+        TaskInitResult.TDBase,
+        *std::next(TaskInitResult.KmpTaskTQTyRD->field_begin(),
+                   KmpTaskTUpperBound));
+    const auto *UBVar =
+        cast<VarDecl>(cast<DeclRefExpr>(D.getUpperBoundVariable())->getDecl());
+    CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(),
+                         UBLVal.getQuals(),
+                         /*IsInitializer=*/true);
+    LValue StLVal = CGF.EmitLValueForField(
+        TaskInitResult.TDBase,
+        *std::next(TaskInitResult.KmpTaskTQTyRD->field_begin(),
+                   KmpTaskTStride));
+    const auto *StVar =
+        cast<VarDecl>(cast<DeclRefExpr>(D.getStrideVariable())->getDecl());
+    CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(),
+                         StLVal.getQuals(),
+                         /*IsInitializer=*/true);
+    // Store reductions address.
+    LValue RedLVal = CGF.EmitLValueForField(
+        TaskInitResult.TDBase,
+        *std::next(TaskInitResult.KmpTaskTQTyRD->field_begin(),
+                   KmpTaskTReductions));
+    if (Data.Reductions) {
+      CGF.EmitStoreOfScalar(Data.Reductions, RedLVal);
+    } else {
+      CGF.EmitNullInitialization(RedLVal.getAddress(),
+                                 CGF.getContext().VoidPtrTy);
+    }
+    llvm::SmallVector<llvm::Value *, 12> TaskArgs{
+        UpLoc,
+        ThreadID,
+        TaskInitResult.NewTask,
+        IfVal,
+        LBLVal.getPointer(CGF),
+        UBLVal.getPointer(CGF),
+        CGF.EmitLoadOfScalar(StLVal, Loc),
+        llvm::ConstantInt::getSigned(
+            CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler
+        llvm::ConstantInt::getSigned(
+            CGF.IntTy, Data.Schedule.getPointer()
+                           ? Data.Schedule.getInt() ? NumTasks : Grainsize
+                           : NoSchedule),
+        Data.Schedule.getPointer()
+            ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty,
+                                        /*isSigned=*/false)
+            : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0)};
+    if (Data.HasModifier)
+      TaskArgs.push_back(llvm::ConstantInt::get(CGF.Int32Ty, 1));
+
+    TaskArgs.push_back(TaskInitResult.TaskDupFn
+                           ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+                                 TaskInitResult.TaskDupFn, CGF.VoidPtrTy)
+                           : llvm::ConstantPointerNull::get(CGF.VoidPtrTy));
+    CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+                            CGM.getModule(), Data.HasModifier
+                                                 ? OMPRTL___kmpc_taskloop_5
+                                                 : OMPRTL___kmpc_taskloop),
+                        TaskArgs);
+  };
+
+  if (CGF.getOMPWithinTaskgraph()) {
+    // Lexically within taskgraph, always replayable.
+    RegionCodeGenTy TaskgraphRCG(TaskgraphTaskloopCodeGen);
+    TaskgraphRCG(CGF);
   } else {
-    CGF.EmitNullInitialization(RedLVal.getAddress(),
-                               CGF.getContext().VoidPtrTy);
+    RegionCodeGenTy NonTaskgraphRCG(NonTaskgraphTaskloopCodeGen);
+    NonTaskgraphRCG(CGF);
   }
-  enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
-  llvm::SmallVector<llvm::Value *, 12> TaskArgs{
-      UpLoc,
-      ThreadID,
-      Result.NewTask,
-      IfVal,
-      LBLVal.getPointer(CGF),
-      UBLVal.getPointer(CGF),
-      CGF.EmitLoadOfScalar(StLVal, Loc),
-      llvm::ConstantInt::getSigned(
-          CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler
-      llvm::ConstantInt::getSigned(
-          CGF.IntTy, Data.Schedule.getPointer()
-                         ? Data.Schedule.getInt() ? NumTasks : Grainsize
-                         : NoSchedule),
-      Data.Schedule.getPointer()
-          ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty,
-                                      /*isSigned=*/false)
-          : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0)};
-  if (Data.HasModifier)
-    TaskArgs.push_back(llvm::ConstantInt::get(CGF.Int32Ty, 1));
-
-  TaskArgs.push_back(Result.TaskDupFn
-                         ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
-                               Result.TaskDupFn, CGF.VoidPtrTy)
-                         : llvm::ConstantPointerNull::get(CGF.VoidPtrTy));
-  CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
-                          CGM.getModule(), Data.HasModifier
-                                               ? OMPRTL___kmpc_taskloop_5
-                                               : OMPRTL___kmpc_taskloop),
-                      TaskArgs);
 }
 
 /// Emit reduction operation for each element of array (required for
@@ -6038,9 +6298,15 @@ llvm::Value *CGOpenMPRuntime::emitTaskReductionInit(
       llvm::ConstantInt::get(CGM.IntTy, Size, /*isSigned=*/true),
       CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TaskRedInput.getPointer(),
                                                       CGM.VoidPtrTy)};
-  return CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
-                                 CGM.getModule(), OMPRTL___kmpc_taskred_init),
-                             Args);
+  if (CGF.getOMPWithinTaskgraph())
+    return CGF.EmitRuntimeCall(
+        OMPBuilder.getOrCreateRuntimeFunction(
+            CGM.getModule(), OMPRTL___kmpc_taskgraph_taskred_init),
+        Args);
+  else
+    return CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+                                   CGM.getModule(), OMPRTL___kmpc_taskred_init),
+                               Args);
 }
 
 void CGOpenMPRuntime::emitTaskReductionFini(CodeGenFunction &CGF,
@@ -6114,36 +6380,68 @@ void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
     llvm::Value *NumOfElements;
     std::tie(NumOfElements, DependenciesArray) =
         emitDependClause(CGF, Data.Dependences, Loc);
-    if (!Data.Dependences.empty()) {
-      llvm::Value *DepWaitTaskArgs[7];
-      DepWaitTaskArgs[0] = UpLoc;
-      DepWaitTaskArgs[1] = ThreadID;
-      DepWaitTaskArgs[2] = NumOfElements;
-      DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);
-      DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
-      DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
-      DepWaitTaskArgs[6] =
-          llvm::ConstantInt::get(CGF.Int32Ty, Data.HasNowaitClause);
-
-      CodeGenFunction::RunCleanupsScope LocalScope(CGF);
 
-      // Build void __kmpc_omp_taskwait_deps_51(ident_t *, kmp_int32 gtid,
-      // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
-      // ndeps_noalias, kmp_depend_info_t *noalias_dep_list,
-      // kmp_int32 has_no_wait); if dependence info is specified.
-      CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
-                              M, OMPRTL___kmpc_omp_taskwait_deps_51),
-                          DepWaitTaskArgs);
+    auto &&TaskgraphTaskwaitCodeGen =
+        [this, UpLoc, ThreadID, NumOfElements, &DependenciesArray,
+         &Data](CodeGenFunction &CGF, PrePostActionTy &) {
+          llvm::Value *TGTaskWaitArgs[5];
+          TGTaskWaitArgs[0] = UpLoc;
+          TGTaskWaitArgs[1] = ThreadID;
+          if (Data.Dependences.empty()) {
+            TGTaskWaitArgs[2] = CGF.Builder.getInt32(0);
+            TGTaskWaitArgs[3] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+          } else {
+            TGTaskWaitArgs[2] = NumOfElements;
+            TGTaskWaitArgs[3] = DependenciesArray.emitRawPointer(CGF);
+          }
+          TGTaskWaitArgs[4] =
+              llvm::ConstantInt::get(CGF.Int32Ty, Data.HasNowaitClause);
+          CGF.EmitRuntimeCall(
+              OMPBuilder.getOrCreateRuntimeFunction(
+                  CGM.getModule(), OMPRTL___kmpc_taskgraph_taskwait),
+              TGTaskWaitArgs);
+        };
+    auto &&NonTaskgraphTaskwaitCodeGen =
+        [this, UpLoc, ThreadID, NumOfElements, &DependenciesArray, &M,
+         &Data](CodeGenFunction &CGF, PrePostActionTy &) {
+          if (!Data.Dependences.empty()) {
+            llvm::Value *DepWaitTaskArgs[7];
+            DepWaitTaskArgs[0] = UpLoc;
+            DepWaitTaskArgs[1] = ThreadID;
+            DepWaitTaskArgs[2] = NumOfElements;
+            DepWaitTaskArgs[3] = DependenciesArray.emitRawPointer(CGF);
+            DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
+            DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+            DepWaitTaskArgs[6] =
+                llvm::ConstantInt::get(CGF.Int32Ty, Data.HasNowaitClause);
+
+            CodeGenFunction::RunCleanupsScope LocalScope(CGF);
+
+            // Build void __kmpc_omp_taskwait_deps_51(ident_t *, kmp_int32 gtid,
+            // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32
+            // ndeps_noalias, kmp_depend_info_t *noalias_dep_list,
+            // kmp_int32 has_no_wait); if dependence info is specified.
+            CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+                                    M, OMPRTL___kmpc_omp_taskwait_deps_51),
+                                DepWaitTaskArgs);
+          } else {
+            // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
+            // global_tid);
+            llvm::Value *Args[] = {UpLoc, ThreadID};
+            // Ignore return result until untied tasks are supported.
+            CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+                                    M, OMPRTL___kmpc_omp_taskwait),
+                                Args);
+          }
+        };
 
+    if (CGF.getOMPWithinTaskgraph()) {
+      // Lexically within taskgraph, always replayable.
+      RegionCodeGenTy TaskgraphRCG(TaskgraphTaskwaitCodeGen);
+      TaskgraphRCG(CGF);
     } else {
-
-      // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
-      // global_tid);
-      llvm::Value *Args[] = {UpLoc, ThreadID};
-      // Ignore return result until untied tasks are supported.
-      CGF.EmitRuntimeCall(
-          OMPBuilder.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_omp_taskwait),
-          Args);
+      RegionCodeGenTy NonTaskgraphRCG(NonTaskgraphTaskwaitCodeGen);
+      NonTaskgraphRCG(CGF);
     }
   }
 
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h
index 2e9c734845af8..99c10f135f61b 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.h
@@ -582,7 +582,9 @@ class CGOpenMPRuntime {
   TaskResultTy emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc,
                             const OMPExecutableDirective &D,
                             llvm::Function *TaskFunction, QualType SharedsTy,
-                            Address Shareds, const OMPTaskDataTy &Data);
+                            Address Shareds, const OMPTaskDataTy &Data,
+                            bool ForTaskgraph,
+                            std::array<llvm::Value *, 3> &TaskAllocArgs);
 
   /// Emit update for lastprivate conditional data.
   void emitLastprivateConditionalUpdate(CodeGenFunction &CGF, LValue IVLVal,
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index b054f2d3673e1..1450f1d1948d1 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -818,6 +818,21 @@ class CodeGenFunction : public CodeGenTypeCache {
     }
   };
 
+  bool OMPWithinTaskgraph = false;
+
+  bool getOMPWithinTaskgraph() { return OMPWithinTaskgraph; }
+  void setOMPWithinTaskgraph(bool In) { OMPWithinTaskgraph = In; }
+
+  class OMPWithinTaskgraphRAII {
+    CodeGenFunction &CGF;
+
+  public:
+    OMPWithinTaskgraphRAII(CodeGenFunction &CGF_) : CGF(CGF_) {
+      CGF.setOMPWithinTaskgraph(true);
+    }
+    ~OMPWithinTaskgraphRAII() { CGF.setOMPWithinTaskgraph(false); }
+  };
+
   template <class T>
   typename DominatingValue<T>::saved_type saveValueInCond(T value) {
     return DominatingValue<T>::save(*this, value);
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index e4e85ebd0061c..e655c828f11aa 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -11672,6 +11672,10 @@ SemaOpenMP::ActOnOpenMPTaskwaitDirective(ArrayRef<OMPClause *> Clauses,
     Diag(StartLoc, diag::err_omp_nowait_clause_without_depend);
     return StmtError();
   }
+  if (DSAStack->getParentDirective() == OMPD_taskgraph && !HasDependC) {
+    Diag(StartLoc, diag::err_omp_taskgraph_taskwait_without_depend);
+    return StmtError();
+  }
 
   return OMPTaskwaitDirective::Create(getASTContext(), StartLoc, EndLoc,
                                       Clauses);
diff --git a/clang/test/OpenMP/taskgraph_taskwait_nodeps.cpp b/clang/test/OpenMP/taskgraph_taskwait_nodeps.cpp
new file mode 100644
index 0000000000000..9f088ed523f26
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_taskwait_nodeps.cpp
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -verify=expected -fsyntax-only %s
+
+int main() {
+#pragma omp taskgraph
+  {
+#pragma omp taskwait // expected-error {{directive '#pragma omp taskwait' within '#pragma omp taskgraph' must use 'depend' clause to be task-generating}}
+  }
+
+#pragma omp taskwait
+
+  return 0;
+}
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
index ea1cef97900f5..a825585900b96 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
@@ -357,6 +357,17 @@ __OMP_RTL(__kmpc_omp_task, false, Int32, IdentPtr, Int32,
           /* kmp_task_t */ VoidPtr)
 __OMP_RTL(__kmpc_end_taskgroup, false, Void, IdentPtr, Int32)
 __OMP_RTL(__kmpc_taskgroup, false, Void, IdentPtr, Int32)
+__OMP_RTL(__kmpc_taskgraph, false, Void, IdentPtr, Int32, VoidPtrPtr, Int32,
+          Int32, Int32, VoidPtr, VoidPtr)
+__OMP_RTL(__kmpc_taskgraph_task, false, Int32, IdentPtr, Int32, VoidPtr, Int32,
+          SizeTy, VoidPtr, SizeTy, Int32, VoidPtr)
+__OMP_RTL(__kmpc_taskgraph_taskloop, false, Int32, IdentPtr, Int32, VoidPtr,
+          Int32, SizeTy, VoidPtr, SizeTy, Int32, Int64Ptr, Int64Ptr, Int64,
+          Int32, Int32, Int64, Int32, VoidPtr)
+__OMP_RTL(__kmpc_taskgraph_taskwait, false, Void, IdentPtr, Int32, Int32,
+          VoidPtr, Int32)
+__OMP_RTL(__kmpc_taskgraph_taskred_init, false, /* kmp_taskgroup */ VoidPtr,
+          Int32, Int32, VoidPtr)
 __OMP_RTL(__kmpc_omp_task_begin_if0, false, Void, IdentPtr, Int32,
           /* kmp_task_t */ VoidPtr)
 __OMP_RTL(__kmpc_omp_task_complete_if0, false, Void, IdentPtr, Int32,

>From ccb8f6442db2ae2d56d6fc9de90aa2c366868bb6 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Thu, 23 Apr 2026 14:49:14 -0500
Subject: [PATCH 07/26] [OpenMP] New Clang tests for 'taskgraph' directive

This patch adds two simple tests for parsing/serialization/deserialization
and codegen for the OpenMP 'taskgraph' directive.

Pull Request: https://github.com/llvm/llvm-project/pull/194052
---
 clang/test/OpenMP/taskgraph_ast_print.cpp | 31 +++++++++++++++
 clang/test/OpenMP/taskgraph_codegen.cpp   | 47 +++++++++++++++++++++++
 2 files changed, 78 insertions(+)
 create mode 100644 clang/test/OpenMP/taskgraph_ast_print.cpp
 create mode 100644 clang/test/OpenMP/taskgraph_codegen.cpp

diff --git a/clang/test/OpenMP/taskgraph_ast_print.cpp b/clang/test/OpenMP/taskgraph_ast_print.cpp
new file mode 100644
index 0000000000000..bfbe829fc9f31
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_ast_print.cpp
@@ -0,0 +1,31 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -ast-print %s | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -x c++ -std=c++11 -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -std=c++11 -include-pch %t -verify %s -ast-print | FileCheck %s
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+int main() {
+  int x = 0, y = 0;
+
+#pragma omp taskgraph
+// CHECK: #pragma omp taskgraph
+  {
+#pragma omp task depend(in: x) depend(out: y)
+// CHECK: #pragma omp task depend(in : x) depend(out : y)
+    {
+      y = x;
+    }
+#pragma omp task depend(inout: x, y)
+// CHECK: #pragma omp task depend(inout : x,y)
+    {
+      x++;
+      y++;
+    }
+  }
+
+  return 0;
+}
+
+#endif
diff --git a/clang/test/OpenMP/taskgraph_codegen.cpp b/clang/test/OpenMP/taskgraph_codegen.cpp
new file mode 100644
index 0000000000000..3f661e6bfe3d5
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_codegen.cpp
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --functions "main" --prefix-filecheck-ir-name _
+
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+// CHECK-LABEL: @main(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[X:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[Y:%.*]] = alloca i32, align 4
+// CHECK-NEXT:    [[AGG_CAPTURED:%.*]] = alloca [[STRUCT_ANON:%.*]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB1:[0-9]+]])
+// CHECK-NEXT:    store i32 0, ptr [[RETVAL]], align 4
+// CHECK-NEXT:    store i32 0, ptr [[X]], align 4
+// CHECK-NEXT:    store i32 0, ptr [[Y]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw [[STRUCT_ANON]], ptr [[AGG_CAPTURED]], i32 0, i32 0
+// CHECK-NEXT:    store ptr [[X]], ptr [[TMP1]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw [[STRUCT_ANON]], ptr [[AGG_CAPTURED]], i32 0, i32 1
+// CHECK-NEXT:    store ptr [[Y]], ptr [[TMP2]], align 8
+// CHECK-NEXT:    call void @__kmpc_taskgraph(ptr @[[GLOB1]], i32 [[TMP0]], ptr @.omp.taskgraph.handle, i32 0, i32 0, i32 0, ptr @taskgraph.omp_outlined., ptr [[AGG_CAPTURED]])
+// CHECK-NEXT:    ret i32 0
+//
+int main() {
+  int x = 0, y = 0;
+
+#pragma omp taskgraph
+  {
+#pragma omp task depend(in: x) depend(out: y)
+    {
+      y = x;
+    }
+#pragma omp task depend(inout: x, y)
+    {
+      x++;
+      y++;
+    }
+  }
+
+  return 0;
+}
+
+#endif

>From 3761a14733b7e0763479a516f82401a34d21671b Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Mon, 27 Apr 2026 13:55:47 -0500
Subject: [PATCH 08/26] [OpenMP] Add 'replayable' clause (for 'taskgraph'
 support)

This patch adds support for the 'replayable' clause to Clang.  This is
used to mark that the directive the clause is used on participates in
taskgraph recording and replay: thus, when the clause is present, the
new API routines introduced in a previous patch are emitted, even when
the directive is not lexically nested within a taskgraph construct.

Pull Request: https://github.com/llvm/llvm-project/pull/194053
---
 clang/include/clang/AST/OpenMPClause.h        | 69 +++++++++++++++
 clang/include/clang/AST/RecursiveASTVisitor.h |  7 ++
 clang/include/clang/Sema/SemaOpenMP.h         |  6 ++
 clang/lib/AST/OpenMPClause.cpp                | 15 ++++
 clang/lib/AST/StmtProfile.cpp                 |  5 ++
 clang/lib/CodeGen/CGOpenMPRuntime.cpp         | 87 ++++++++++++-------
 clang/lib/CodeGen/CGOpenMPRuntime.h           |  9 +-
 clang/lib/CodeGen/CGStmtOpenMP.cpp            | 76 +++++++++++++---
 clang/lib/Parse/ParseOpenMP.cpp               |  1 +
 clang/lib/Sema/SemaOpenMP.cpp                 | 29 +++++++
 clang/lib/Sema/TreeTransform.h                | 25 ++++++
 clang/lib/Serialization/ASTReader.cpp         |  8 ++
 clang/lib/Serialization/ASTWriter.cpp         |  5 ++
 clang/tools/libclang/CIndex.cpp               |  4 +
 llvm/include/llvm/Frontend/OpenMP/OMP.td      |  1 +
 15 files changed, 304 insertions(+), 43 deletions(-)

diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 4c3ab20c64b33..105bef8fb1f0d 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -2040,6 +2040,75 @@ class OMPSelfMapsClause final : public OMPClause {
   }
 };
 
+/// This represents a 'replayable' clause in the '#pragma omp target',
+// '#pragma omp target enter data', '#pragma omp target exit data',
+// '#pragma omp target update', '#pragma omp task', '#pragma omp taskloop' or
+// '#pragma omp taskwait' directive.
+///
+/// \code
+/// #pragma omp task replayable(1)
+/// \endcode
+/// In this example directive '#pragma omp task' has the 'replayable' clause.
+class OMPReplayableClause final : public OMPClause {
+  friend class OMPClauseReader;
+
+  /// Location of '('.
+  SourceLocation LParenLoc;
+
+  /// Condition of the 'replayable' clause.
+  Stmt *Condition = nullptr;
+
+public:
+  /// Build 'replayable' clause.
+  ///
+  /// \param Cond Condition of the clause.
+  /// \param StartLoc Starting location of the clause.
+  /// \param LParenLoc Location of '('.
+  /// \param EndLoc Ending location of the clause.
+  OMPReplayableClause(Expr *Cond, SourceLocation StartLoc,
+                      SourceLocation LParenLoc, SourceLocation EndLoc)
+      : OMPClause(llvm::omp::OMPC_replayable, StartLoc, EndLoc),
+        LParenLoc(LParenLoc), Condition(Cond) {}
+
+  /// Build an empty clause.
+  OMPReplayableClause()
+      : OMPClause(llvm::omp::OMPC_replayable, SourceLocation(),
+                  SourceLocation()) {}
+
+  /// Sets the location of '('.
+  void setLParenLoc(SourceLocation Loc) { LParenLoc = Loc; }
+
+  /// Returns the location of '('.
+  SourceLocation getLParenLoc() const { return LParenLoc; }
+
+  /// Set condition.
+  void setCondition(Expr *Cond) { Condition = Cond; }
+
+  /// Returns condition.
+  Expr *getCondition() const { return cast_or_null<Expr>(Condition); }
+
+  child_range children() {
+    if (Condition)
+      return child_range(&Condition, &Condition + 1);
+    return child_range(child_iterator(), child_iterator());
+  }
+
+  const_child_range children() const {
+    if (Condition)
+      return const_child_range(&Condition, &Condition + 1);
+    return const_child_range(const_child_iterator(), const_child_iterator());
+  }
+
+  child_range used_children();
+  const_child_range used_children() const {
+    return const_cast<OMPReplayableClause *>(this)->used_children();
+  }
+
+  static bool classof(const OMPClause *T) {
+    return T->getClauseKind() == llvm::omp::OMPC_replayable;
+  }
+};
+
 /// This represents 'at' clause in the '#pragma omp error' directive
 ///
 /// \code
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index 1d7006921f3dd..671a52b6cbc46 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -3687,6 +3687,13 @@ bool RecursiveASTVisitor<Derived>::VisitOMPNowaitClause(OMPNowaitClause *C) {
   return true;
 }
 
+template <typename Derived>
+bool RecursiveASTVisitor<Derived>::VisitOMPReplayableClause(
+    OMPReplayableClause *C) {
+  TRY_TO(TraverseStmt(C->getCondition()));
+  return true;
+}
+
 template <typename Derived>
 bool RecursiveASTVisitor<Derived>::VisitOMPUntiedClause(OMPUntiedClause *) {
   return true;
diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index fe5879c3d83dc..451ba634cc482 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -1174,6 +1174,12 @@ class SemaOpenMP : public SemaBase {
   OMPClause *ActOnOpenMPSelfMapsClause(SourceLocation StartLoc,
                                        SourceLocation EndLoc);
 
+  /// Called on well-formed 'replayable' clause.
+  OMPClause *ActOnOpenMPReplayableClause(SourceLocation StartLoc,
+                                         SourceLocation EndLoc,
+                                         SourceLocation LParenLoc,
+                                         Expr *Condition);
+
   /// Called on well-formed 'at' clause.
   OMPClause *ActOnOpenMPAtClause(OpenMPAtClauseKind Kind,
                                  SourceLocation KindLoc,
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index cc3b126f3c791..15bb1d460be6a 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -324,6 +324,12 @@ OMPClause::child_range OMPNowaitClause::used_children() {
   return children();
 }
 
+OMPClause::child_range OMPReplayableClause::used_children() {
+  if (Condition)
+    return child_range(&Condition, &Condition + 1);
+  return children();
+}
+
 OMPClause::child_range OMPGrainsizeClause::used_children() {
   if (Stmt **C = getAddrOfExprAsWritten(getPreInitStmt()))
     return child_range(C, C + 1);
@@ -2243,6 +2249,15 @@ void OMPClausePrinter::VisitOMPNowaitClause(OMPNowaitClause *Node) {
   }
 }
 
+void OMPClausePrinter::VisitOMPReplayableClause(OMPReplayableClause *Node) {
+  OS << "replayable";
+  if (auto *Cond = Node->getCondition()) {
+    OS << "(";
+    Cond->printPretty(OS, nullptr, Policy, 0);
+    OS << ")";
+  }
+}
+
 void OMPClausePrinter::VisitOMPUntiedClause(OMPUntiedClause *) {
   OS << "untied";
 }
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index 493d5e2b7c4a5..e8bf578f85f8c 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -620,6 +620,11 @@ void OMPClauseProfiler::VisitOMPNowaitClause(const OMPNowaitClause *C) {
     Profiler->VisitStmt(C->getCondition());
 }
 
+void OMPClauseProfiler::VisitOMPReplayableClause(const OMPReplayableClause *C) {
+  if (C->getCondition())
+    Profiler->VisitStmt(C->getCondition());
+}
+
 void OMPClauseProfiler::VisitOMPUntiedClause(const OMPUntiedClause *) {}
 
 void OMPClauseProfiler::VisitOMPMergeableClause(const OMPMergeableClause *) {}
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 2b760da848732..3fe694b54a5e8 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -4797,12 +4797,10 @@ void CGOpenMPRuntime::emitUpdateDependObjectsClause(
   CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
 }
 
-void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
-                                   const OMPExecutableDirective &D,
-                                   llvm::Function *TaskFunction,
-                                   QualType SharedsTy, Address Shareds,
-                                   const Expr *IfCond,
-                                   const OMPTaskDataTy &Data) {
+void CGOpenMPRuntime::emitTaskCall(
+    CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D,
+    llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds,
+    const Expr *IfCond, const Expr *ReplayableCond, const OMPTaskDataTy &Data) {
   if (!CGF.HaveInsertPoint())
     return;
 
@@ -4967,17 +4965,23 @@ void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
     RegionCodeGenTy TaskgraphRCG(TaskgraphTaskCodeGen);
     TaskgraphRCG(CGF);
   } else {
-    RegionCodeGenTy NonTaskgraphRCG(NonTaskgraphTaskCodeGen);
-    NonTaskgraphRCG(CGF);
+    if (ReplayableCond) {
+      // We have a replayable clause.  Task is replayable if its argument is
+      // omitted or evaluates to TRUE.
+      emitIfClause(CGF, ReplayableCond, TaskgraphTaskCodeGen,
+                   NonTaskgraphTaskCodeGen);
+    } else {
+      // Not taskgraph, not replayable.
+      RegionCodeGenTy NonTaskgraphRCG(NonTaskgraphTaskCodeGen);
+      NonTaskgraphRCG(CGF);
+    }
   }
 }
 
-void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
-                                       const OMPLoopDirective &D,
-                                       llvm::Function *TaskFunction,
-                                       QualType SharedsTy, Address Shareds,
-                                       const Expr *IfCond,
-                                       const OMPTaskDataTy &Data) {
+void CGOpenMPRuntime::emitTaskLoopCall(
+    CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D,
+    llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds,
+    const Expr *IfCond, const Expr *ReplayableCond, const OMPTaskDataTy &Data) {
   if (!CGF.HaveInsertPoint())
     return;
 
@@ -5161,8 +5165,16 @@ void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
     RegionCodeGenTy TaskgraphRCG(TaskgraphTaskloopCodeGen);
     TaskgraphRCG(CGF);
   } else {
-    RegionCodeGenTy NonTaskgraphRCG(NonTaskgraphTaskloopCodeGen);
-    NonTaskgraphRCG(CGF);
+    if (ReplayableCond) {
+      // We have a replayable clause.  Taskloop is replayable if its argument
+      // is omitted or evaluates to TRUE.
+      emitIfClause(CGF, ReplayableCond, TaskgraphTaskloopCodeGen,
+                   NonTaskgraphTaskloopCodeGen);
+    } else {
+      // Not taskgraph, not replayable.
+      RegionCodeGenTy NonTaskgraphRCG(NonTaskgraphTaskloopCodeGen);
+      NonTaskgraphRCG(CGF);
+    }
   }
 }
 
@@ -6298,15 +6310,22 @@ llvm::Value *CGOpenMPRuntime::emitTaskReductionInit(
       llvm::ConstantInt::get(CGM.IntTy, Size, /*isSigned=*/true),
       CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TaskRedInput.getPointer(),
                                                       CGM.VoidPtrTy)};
-  if (CGF.getOMPWithinTaskgraph())
+  // A task/taskloop participates in taskgraph replay either when it is
+  // lexically nested inside a #pragma omp taskgraph region or when it carries
+  // a `replayable` clause (which may also fire dynamically inside a taskgraph
+  // recording).  In both cases route through the taskgraph-aware entry point
+  // so the runtime can stash the reduction input for later replay.  The
+  // taskgraph entry point degrades to the regular init when no taskgraph
+  // recording is active, so this is safe even when `replayable(false)` at
+  // runtime.
+  if (CGF.getOMPWithinTaskgraph() || Data.ReplayableCond)
     return CGF.EmitRuntimeCall(
         OMPBuilder.getOrCreateRuntimeFunction(
             CGM.getModule(), OMPRTL___kmpc_taskgraph_taskred_init),
         Args);
-  else
-    return CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
-                                   CGM.getModule(), OMPRTL___kmpc_taskred_init),
-                               Args);
+  return CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+                                 CGM.getModule(), OMPRTL___kmpc_taskred_init),
+                             Args);
 }
 
 void CGOpenMPRuntime::emitTaskReductionFini(CodeGenFunction &CGF,
@@ -6365,6 +6384,7 @@ Address CGOpenMPRuntime::getTaskReductionItem(CodeGenFunction &CGF,
 }
 
 void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
+                                       const Expr *ReplayableCond,
                                        const OMPTaskDataTy &Data) {
   if (!CGF.HaveInsertPoint())
     return;
@@ -6440,8 +6460,16 @@ void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
       RegionCodeGenTy TaskgraphRCG(TaskgraphTaskwaitCodeGen);
       TaskgraphRCG(CGF);
     } else {
-      RegionCodeGenTy NonTaskgraphRCG(NonTaskgraphTaskwaitCodeGen);
-      NonTaskgraphRCG(CGF);
+      if (ReplayableCond) {
+        // We have a replayable clause.  Taskwait is replayable if its argument
+        // is omitted or evaluates to TRUE.
+        emitIfClause(CGF, ReplayableCond, TaskgraphTaskwaitCodeGen,
+                     NonTaskgraphTaskwaitCodeGen);
+      } else {
+        // Not taskgraph, not replayable.
+        RegionCodeGenTy NonTaskgraphRCG(NonTaskgraphTaskwaitCodeGen);
+        NonTaskgraphRCG(CGF);
+      }
     }
   }
 
@@ -13669,19 +13697,17 @@ void CGOpenMPSIMDRuntime::emitFlush(CodeGenFunction &CGF,
   llvm_unreachable("Not supported in SIMD-only mode");
 }
 
-void CGOpenMPSIMDRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc,
-                                       const OMPExecutableDirective &D,
-                                       llvm::Function *TaskFunction,
-                                       QualType SharedsTy, Address Shareds,
-                                       const Expr *IfCond,
-                                       const OMPTaskDataTy &Data) {
+void CGOpenMPSIMDRuntime::emitTaskCall(
+    CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D,
+    llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds,
+    const Expr *IfCond, const Expr *ReplayableCond, const OMPTaskDataTy &Data) {
   llvm_unreachable("Not supported in SIMD-only mode");
 }
 
 void CGOpenMPSIMDRuntime::emitTaskLoopCall(
     CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D,
     llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds,
-    const Expr *IfCond, const OMPTaskDataTy &Data) {
+    const Expr *IfCond, const Expr *ReplayableCond, const OMPTaskDataTy &Data) {
   llvm_unreachable("Not supported in SIMD-only mode");
 }
 
@@ -13722,6 +13748,7 @@ Address CGOpenMPSIMDRuntime::getTaskReductionItem(CodeGenFunction &CGF,
 
 void CGOpenMPSIMDRuntime::emitTaskwaitCall(CodeGenFunction &CGF,
                                            SourceLocation Loc,
+                                           const Expr *ReplayableCond,
                                            const OMPTaskDataTy &Data) {
   llvm_unreachable("Not supported in SIMD-only mode");
 }
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h
index 99c10f135f61b..821c1ca9bcffd 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.h
@@ -123,6 +123,7 @@ struct OMPTaskDataTy final {
   bool IsWorksharingReduction = false;
   bool HasNowaitClause = false;
   bool HasModifier = false;
+  const Expr *ReplayableCond = nullptr;
 };
 
 /// Class intended to support codegen of all kind of the reduction clauses.
@@ -1177,6 +1178,7 @@ class CGOpenMPRuntime {
                             const OMPExecutableDirective &D,
                             llvm::Function *TaskFunction, QualType SharedsTy,
                             Address Shareds, const Expr *IfCond,
+                            const Expr *ReplayableCond,
                             const OMPTaskDataTy &Data);
 
   /// Emit task region for the taskloop directive. The taskloop region is
@@ -1212,7 +1214,8 @@ class CGOpenMPRuntime {
                                 const OMPLoopDirective &D,
                                 llvm::Function *TaskFunction,
                                 QualType SharedsTy, Address Shareds,
-                                const Expr *IfCond, const OMPTaskDataTy &Data);
+                                const Expr *IfCond, const Expr *ReplayableCond,
+                                const OMPTaskDataTy &Data);
 
   /// Emit code for the directive that does not require outlining.
   ///
@@ -1380,6 +1383,7 @@ class CGOpenMPRuntime {
 
   /// Emit code for 'taskwait' directive.
   virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
+                                const Expr *ReplayableCond,
                                 const OMPTaskDataTy &Data);
 
   /// Emit code for 'taskgraph' directive.
@@ -2060,6 +2064,7 @@ class CGOpenMPSIMDRuntime final : public CGOpenMPRuntime {
                     const OMPExecutableDirective &D,
                     llvm::Function *TaskFunction, QualType SharedsTy,
                     Address Shareds, const Expr *IfCond,
+                    const Expr *ReplayableCond,
                     const OMPTaskDataTy &Data) override;
 
   /// Emit task region for the taskloop directive. The taskloop region is
@@ -2094,6 +2099,7 @@ class CGOpenMPSIMDRuntime final : public CGOpenMPRuntime {
   void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc,
                         const OMPLoopDirective &D, llvm::Function *TaskFunction,
                         QualType SharedsTy, Address Shareds, const Expr *IfCond,
+                        const Expr *ReplayableCond,
                         const OMPTaskDataTy &Data) override;
 
   /// Emit a code for reduction clause. Next code should be emitted for
@@ -2214,6 +2220,7 @@ class CGOpenMPSIMDRuntime final : public CGOpenMPRuntime {
 
   /// Emit code for 'taskwait' directive.
   void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc,
+                        const Expr *ReplayableCond,
                         const OMPTaskDataTy &Data) override;
 
   /// Emit code for 'taskgraph' directive.
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index a5cb20a4bfafa..9c261225137c1 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -5762,8 +5762,19 @@ void CodeGenFunction::EmitOMPTargetTaskBasedDirective(
   IntegerLiteral IfCond(getContext(), TrueOrFalse,
                         getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
                         SourceLocation());
+  const Expr *ReplayableCond = nullptr;
+  if (auto *RC = S.getSingleClause<OMPReplayableClause>()) {
+    ReplayableCond = RC->getCondition();
+    if (!ReplayableCond) {
+      ReplayableCond = IntegerLiteral::Create(
+          getContext(), llvm::APInt(32, 1),
+          getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
+          SourceLocation());
+    }
+  }
   CGM.getOpenMPRuntime().emitTaskCall(*this, S.getBeginLoc(), S, OutlinedFn,
-                                      SharedsTy, CapturedStruct, &IfCond, Data);
+                                      SharedsTy, CapturedStruct, &IfCond,
+                                      ReplayableCond, Data);
 }
 
 void CodeGenFunction::processInReduction(const OMPExecutableDirective &S,
@@ -5872,15 +5883,29 @@ void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) {
   OMPTaskDataTy Data;
   // Check if we should emit tied or untied task.
   Data.Tied = !S.getSingleClause<OMPUntiedClause>();
+  const Expr *ReplayableCond = nullptr;
+  if (auto *RC = S.getSingleClause<OMPReplayableClause>()) {
+    ReplayableCond = RC->getCondition();
+    if (!ReplayableCond) {
+      ReplayableCond = IntegerLiteral::Create(
+          getContext(), llvm::APInt(32, 1),
+          getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
+          SourceLocation());
+    }
+  }
+  // Propagate the replayable signal into Data so that reduction init can
+  // route to the taskgraph-aware runtime entry point when the task may
+  // participate in taskgraph replay.
+  Data.ReplayableCond = ReplayableCond;
   auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
     CGF.EmitStmt(CS->getCapturedStmt());
   };
-  auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
-                    IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
-                            const OMPTaskDataTy &Data) {
+  auto &&TaskGen = [&S, SharedsTy, CapturedStruct, IfCond, ReplayableCond](
+                       CodeGenFunction &CGF, llvm::Function *OutlinedFn,
+                       const OMPTaskDataTy &Data) {
     CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn,
                                             SharedsTy, CapturedStruct, IfCond,
-                                            Data);
+                                            ReplayableCond, Data);
   };
   auto LPCRegion =
       CGOpenMPRuntime::LastprivateConditionalRAII::disable(*this, S);
@@ -5911,7 +5936,18 @@ void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S) {
   // Build list of dependences
   buildDependences(S, Data);
   Data.HasNowaitClause = S.hasClausesOfKind<OMPNowaitClause>();
-  CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getBeginLoc(), Data);
+  const Expr *ReplayableCond = nullptr;
+  if (auto *RC = S.getSingleClause<OMPReplayableClause>()) {
+    ReplayableCond = RC->getCondition();
+    if (!ReplayableCond) {
+      ReplayableCond = IntegerLiteral::Create(
+          getContext(), llvm::APInt(32, 1),
+          getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
+          SourceLocation());
+    }
+  }
+  CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getBeginLoc(),
+                                          ReplayableCond, Data);
 }
 
 void CodeGenFunction::EmitOMPTaskgraphDirective(
@@ -8312,11 +8348,26 @@ void CodeGenFunction::EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S) {
     }
   }
 
+  const Expr *ReplayableCond = nullptr;
+  if (auto *RC = S.getSingleClause<OMPReplayableClause>()) {
+    ReplayableCond = RC->getCondition();
+    if (!ReplayableCond) {
+      ReplayableCond = IntegerLiteral::Create(
+          getContext(), llvm::APInt(32, 1),
+          getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
+          SourceLocation());
+    }
+  }
+
   OMPTaskDataTy Data;
   // Check if taskloop must be emitted without taskgroup.
   Data.Nogroup = S.getSingleClause<OMPNogroupClause>();
   // TODO: Check if we should emit tied or untied task.
   Data.Tied = true;
+  // Propagate the replayable signal into Data so that reduction init can
+  // route to the taskgraph-aware runtime entry point when the taskloop may
+  // participate in taskgraph replay.
+  Data.ReplayableCond = ReplayableCond;
   // Set scheduling for taskloop
   if (const auto *Clause = S.getSingleClause<OMPGrainsizeClause>()) {
     // grainsize clause
@@ -8431,15 +8482,16 @@ void CodeGenFunction::EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S) {
                                (*LIP)->getType(), S.getBeginLoc()));
     });
   };
-  auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
-                    IfCond](CodeGenFunction &CGF, llvm::Function *OutlinedFn,
-                            const OMPTaskDataTy &Data) {
+  auto &&TaskGen = [&S, SharedsTy, CapturedStruct, IfCond, ReplayableCond](
+                       CodeGenFunction &CGF, llvm::Function *OutlinedFn,
+                       const OMPTaskDataTy &Data) {
     auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
+                      ReplayableCond,
                       &Data](CodeGenFunction &CGF, PrePostActionTy &) {
       OMPLoopScope PreInitScope(CGF, S);
-      CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S,
-                                                  OutlinedFn, SharedsTy,
-                                                  CapturedStruct, IfCond, Data);
+      CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(
+          CGF, S.getBeginLoc(), S, OutlinedFn, SharedsTy, CapturedStruct,
+          IfCond, ReplayableCond, Data);
     };
     CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
                                                     CodeGen);
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 5f9845145669c..d79a1737fcbb2 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -3407,6 +3407,7 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
       Clause = ParseOpenMPClause(CKind, WrongDirective);
     break;
   case OMPC_graph_reset:
+  case OMPC_replayable:
     if (!FirstClause) {
       Diag(Tok, diag::err_omp_more_one_clause)
           << getOpenMPDirectiveName(DKind, OMPVersion)
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index e655c828f11aa..c31db03c62c3f 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -6837,6 +6837,7 @@ StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
       case OMPC_graph_id:
       case OMPC_graph_reset:
       case OMPC_nocontext:
+      case OMPC_replayable:
         // Do not analyze if no parent parallel directive.
         if (isOpenMPParallelDirective(Kind))
           break;
@@ -17074,6 +17075,9 @@ OMPClause *SemaOpenMP::ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind,
   case OMPC_graph_reset:
     Res = ActOnOpenMPGraphResetClause(Expr, StartLoc, LParenLoc, EndLoc);
     break;
+  case OMPC_replayable:
+    Res = ActOnOpenMPReplayableClause(StartLoc, EndLoc, LParenLoc, Expr);
+    break;
   case OMPC_novariants:
     Res = ActOnOpenMPNovariantsClause(Expr, StartLoc, LParenLoc, EndLoc);
     break;
@@ -18814,6 +18818,11 @@ OMPClause *SemaOpenMP::ActOnOpenMPClause(OpenMPClauseKind Kind,
   case OMPC_self_maps:
     Res = ActOnOpenMPSelfMapsClause(StartLoc, EndLoc);
     break;
+  case OMPC_replayable:
+    Res = ActOnOpenMPReplayableClause(StartLoc, EndLoc,
+                                      /*LParenLoc=*/SourceLocation(),
+                                      /*Condition=*/nullptr);
+    break;
   case OMPC_destroy:
     Res = ActOnOpenMPDestroyClause(/*InteropVar=*/nullptr, StartLoc,
                                    /*LParenLoc=*/SourceLocation(),
@@ -19051,6 +19060,26 @@ OMPClause *SemaOpenMP::ActOnOpenMPSelfMapsClause(SourceLocation StartLoc,
   return new (getASTContext()) OMPSelfMapsClause(StartLoc, EndLoc);
 }
 
+OMPClause *SemaOpenMP::ActOnOpenMPReplayableClause(SourceLocation StartLoc,
+                                                   SourceLocation EndLoc,
+                                                   SourceLocation LParenLoc,
+                                                   Expr *Condition) {
+  Expr *ValExpr = Condition;
+  if (Condition && LParenLoc.isValid()) {
+    if (!Condition->isValueDependent() && !Condition->isTypeDependent() &&
+        !Condition->isInstantiationDependent() &&
+        !Condition->containsUnexpandedParameterPack()) {
+      ExprResult Val = SemaRef.CheckBooleanCondition(StartLoc, Condition);
+      if (Val.isInvalid())
+        return nullptr;
+
+      ValExpr = Val.get();
+    }
+  }
+  return new (getASTContext())
+      OMPReplayableClause(ValExpr, StartLoc, LParenLoc, EndLoc);
+}
+
 StmtResult
 SemaOpenMP::ActOnOpenMPInteropDirective(ArrayRef<OMPClause *> Clauses,
                                         SourceLocation StartLoc,
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 0d53a1e9ca02e..9efbdf6ad9684 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -1906,6 +1906,18 @@ class TreeTransform {
                                                       LParenLoc, Condition);
   }
 
+  /// Build a new OpenMP 'replayable' clause.
+  ///
+  /// By default, performs semantic analysis to build the new OpenMP clause.
+  /// Subclasses may override this routine to provide different behavior.
+  OMPClause *RebuildOMPReplayableClause(Expr *Condition,
+                                        SourceLocation StartLoc,
+                                        SourceLocation LParenLoc,
+                                        SourceLocation EndLoc) {
+    return getSema().OpenMP().ActOnOpenMPReplayableClause(StartLoc, EndLoc,
+                                                          LParenLoc, Condition);
+  }
+
   /// Build a new OpenMP 'private' clause.
   ///
   /// By default, performs semantic analysis to build the new OpenMP clause.
@@ -11101,6 +11113,19 @@ TreeTransform<Derived>::TransformOMPNowaitClause(OMPNowaitClause *C) {
                                              C->getLParenLoc(), C->getEndLoc());
 }
 
+template <typename Derived>
+OMPClause *
+TreeTransform<Derived>::TransformOMPReplayableClause(OMPReplayableClause *C) {
+  ExprResult Cond;
+  if (auto *Condition = C->getCondition()) {
+    Cond = getDerived().TransformExpr(Condition);
+    if (Cond.isInvalid())
+      return nullptr;
+  }
+  return getDerived().RebuildOMPReplayableClause(
+      Cond.get(), C->getBeginLoc(), C->getLParenLoc(), C->getEndLoc());
+}
+
 template <typename Derived>
 OMPClause *
 TreeTransform<Derived>::TransformOMPUntiedClause(OMPUntiedClause *C) {
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index d0dea09aaf5c0..43dbd98851181 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -11735,6 +11735,9 @@ OMPClause *OMPClauseReader::readClause() {
   case llvm::omp::OMPC_graph_reset:
     C = new (Context) OMPGraphResetClause();
     break;
+  case llvm::omp::OMPC_replayable:
+    C = new (Context) OMPReplayableClause();
+    break;
   case llvm::omp::OMPC_num_tasks:
     C = new (Context) OMPNumTasksClause();
     break;
@@ -12043,6 +12046,11 @@ void OMPClauseReader::VisitOMPNowaitClause(OMPNowaitClause *C) {
   C->setLParenLoc(Record.readSourceLocation());
 }
 
+void OMPClauseReader::VisitOMPReplayableClause(OMPReplayableClause *C) {
+  C->setCondition(Record.readSubExpr());
+  C->setLParenLoc(Record.readSourceLocation());
+}
+
 void OMPClauseReader::VisitOMPUntiedClause(OMPUntiedClause *) {}
 
 void OMPClauseReader::VisitOMPMergeableClause(OMPMergeableClause *) {}
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index a6260ff34fa13..140e1f8366dc1 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -8199,6 +8199,11 @@ void OMPClauseWriter::VisitOMPNowaitClause(OMPNowaitClause *C) {
   Record.AddSourceLocation(C->getLParenLoc());
 }
 
+void OMPClauseWriter::VisitOMPReplayableClause(OMPReplayableClause *C) {
+  Record.AddStmt(C->getCondition());
+  Record.AddSourceLocation(C->getLParenLoc());
+}
+
 void OMPClauseWriter::VisitOMPUntiedClause(OMPUntiedClause *) {}
 
 void OMPClauseWriter::VisitOMPMergeableClause(OMPMergeableClause *) {}
diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index a5b33a3126b7e..93861ab98bbb9 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2413,6 +2413,10 @@ void OMPClauseEnqueue::VisitOMPNowaitClause(const OMPNowaitClause *C) {
   Visitor->AddStmt(C->getCondition());
 }
 
+void OMPClauseEnqueue::VisitOMPReplayableClause(const OMPReplayableClause *C) {
+  Visitor->AddStmt(C->getCondition());
+}
+
 void OMPClauseEnqueue::VisitOMPUntiedClause(const OMPUntiedClause *) {}
 
 void OMPClauseEnqueue::VisitOMPMergeableClause(const OMPMergeableClause *) {}
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index e405799ccad82..91aa99f4470c5 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -507,6 +507,7 @@ def OMPC_Release : Clause<[Spelling<"release">]> {
   let clangClass = "OMPReleaseClause";
 }
 def OMPC_Replayable : Clause<[Spelling<"replayable">]> {
+  let clangClass = "OMPReplayableClause";
   let flangClass = "OmpReplayableClause";
   let isValueOptional = true;
 }

>From 2ac5e2449721f3ed0eab3629d401e3b91991c72f Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Mon, 20 Apr 2026 06:05:36 -0500
Subject: [PATCH 09/26] [OpenMP] Add tests for 'replayable' clause

Parsing and diagnostics tests for the replayable clause added by
the previous patch, plus two libomp runtime tests
(taskgraph_replayable_nonlexical_reduction_minimal_taskloop.cpp
and taskgraph_replayable_nonlexical_recursive_reduction.cpp) for
reductions on non-lexically-nested replayable taskloops, which
exercise the Clang dispatch through __kmpc_taskgraph_taskred_init
that stashes the reduction input for later replay.

Assisted-By: Codex with gpt-5.3
Assisted-By: Claude Opus 4.7

commit-id:6a552a7c

Pull Request: https://github.com/llvm/llvm-project/pull/194054
---
 clang/test/OpenMP/replayable_ast_print.cpp    | 233 ++++++++++++++++++
 clang/test/OpenMP/replayable_messages.cpp     |  51 ++++
 ...layable_nonlexical_recursive_reduction.cpp | 147 +++++++++++
 ..._nonlexical_reduction_minimal_taskloop.cpp | 105 ++++++++
 4 files changed, 536 insertions(+)
 create mode 100644 clang/test/OpenMP/replayable_ast_print.cpp
 create mode 100644 clang/test/OpenMP/replayable_messages.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_recursive_reduction.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_reduction_minimal_taskloop.cpp

diff --git a/clang/test/OpenMP/replayable_ast_print.cpp b/clang/test/OpenMP/replayable_ast_print.cpp
new file mode 100644
index 0000000000000..cea97b8b1cd73
--- /dev/null
+++ b/clang/test/OpenMP/replayable_ast_print.cpp
@@ -0,0 +1,233 @@
+// Check no warnings/errors
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -fsyntax-only -verify %s
+// expected-no-diagnostics
+
+// Check AST and unparsing
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -ast-dump  %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -ast-print %s | FileCheck %s --check-prefix=PRINT
+
+// Check same results after serialization round-trip
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -emit-pch -o %t %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -include-pch %t -ast-dump-all %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -include-pch %t -ast-print    %s | FileCheck %s --check-prefix=PRINT
+
+#ifndef HEADER
+#define HEADER
+
+void replayable_clauses() {
+  int A = 1;
+  int B[10];
+
+  // --- omp task ---
+
+  // DUMP: OMPTaskDirective
+  // DUMP-NEXT: OMPReplayableClause
+  // PRINT: #pragma omp task replayable
+  #pragma omp task replayable
+  {}
+
+  // DUMP: OMPTaskDirective
+  // DUMP-NEXT: OMPReplayableClause
+  // DUMP-NEXT: XXBoolLiteralExpr {{.*}} 'bool' false
+  // PRINT: #pragma omp task replayable(false)
+  #pragma omp task replayable(false)
+  {}
+
+  // DUMP: OMPTaskDirective
+  // DUMP-NEXT: OMPReplayableClause
+  // DUMP-NEXT: XXBoolLiteralExpr {{.*}} 'bool' true
+  // PRINT: #pragma omp task replayable(true)
+  #pragma omp task replayable(true)
+  {}
+
+  // DUMP: OMPTaskDirective
+  // DUMP-NEXT: OMPReplayableClause
+  // DUMP-NEXT: BinaryOperator {{.*}} 'bool' '>'
+  // DUMP-NEXT: ImplicitCastExpr {{.*}} 'int' <LValueToRValue>
+  // DUMP-NEXT: DeclRefExpr {{.*}} 'int' lvalue Var {{.*}} 'A' 'int'
+  // DUMP-NEXT: IntegerLiteral {{.*}} 'int' 5
+  // PRINT: #pragma omp task replayable(A > 5)
+  #pragma omp task replayable(A > 5)
+  {}
+
+  // --- omp taskloop ---
+
+  // DUMP: OMPTaskLoopDirective
+  // DUMP-NEXT: OMPReplayableClause
+  // PRINT: #pragma omp taskloop replayable
+  #pragma omp taskloop replayable
+  for (int i = 0; i < 10; ++i)
+    {}
+
+  // DUMP: OMPTaskLoopDirective
+  // DUMP-NEXT: OMPReplayableClause
+  // DUMP-NEXT: XXBoolLiteralExpr {{.*}} 'bool' false
+  // PRINT: #pragma omp taskloop replayable(false)
+  #pragma omp taskloop replayable(false)
+  for (int i = 0; i < 10; ++i)
+    {}
+
+  // DUMP: OMPTaskLoopDirective
+  // DUMP-NEXT: OMPReplayableClause
+  // DUMP-NEXT: XXBoolLiteralExpr {{.*}} 'bool' true
+  // PRINT: #pragma omp taskloop replayable(true)
+  #pragma omp taskloop replayable(true)
+  for (int i = 0; i < 10; ++i)
+    {}
+
+  // DUMP: OMPTaskLoopDirective
+  // DUMP-NEXT: OMPReplayableClause
+  // DUMP-NEXT: BinaryOperator {{.*}} 'bool' '>'
+  // DUMP-NEXT: ImplicitCastExpr {{.*}} 'int' <LValueToRValue>
+  // DUMP-NEXT: DeclRefExpr {{.*}} 'int' lvalue Var {{.*}} 'A' 'int'
+  // DUMP-NEXT: IntegerLiteral {{.*}} 'int' 5
+  // PRINT: #pragma omp taskloop replayable(A > 5)
+  #pragma omp taskloop replayable(A > 5)
+  for (int i = 0; i < 10; ++i)
+    {}
+
+  // --- omp taskwait ---
+
+  // DUMP: OMPTaskwaitDirective
+  // DUMP-NEXT: OMPReplayableClause
+  // PRINT: #pragma omp taskwait replayable
+  #pragma omp taskwait replayable
+
+  // DUMP: OMPTaskwaitDirective
+  // DUMP-NEXT: OMPReplayableClause
+  // DUMP-NEXT: XXBoolLiteralExpr {{.*}} 'bool' false
+  // PRINT: #pragma omp taskwait replayable(false)
+  #pragma omp taskwait replayable(false)
+
+  // DUMP: OMPTaskwaitDirective
+  // DUMP-NEXT: OMPReplayableClause
+  // DUMP-NEXT: XXBoolLiteralExpr {{.*}} 'bool' true
+  // PRINT: #pragma omp taskwait replayable(true)
+  #pragma omp taskwait replayable(true)
+
+  // DUMP: OMPTaskwaitDirective
+  // DUMP-NEXT: OMPReplayableClause
+  // DUMP-NEXT: BinaryOperator {{.*}} 'bool' '>'
+  // DUMP-NEXT: ImplicitCastExpr {{.*}} 'int' <LValueToRValue>
+  // DUMP-NEXT: DeclRefExpr {{.*}} 'int' lvalue Var {{.*}} 'A' 'int'
+  // DUMP-NEXT: IntegerLiteral {{.*}} 'int' 5
+  // PRINT: #pragma omp taskwait replayable(A > 5)
+  #pragma omp taskwait replayable(A > 5)
+
+  // --- omp target ---
+
+  // DUMP: OMPTargetDirective
+  // DUMP-NEXT: OMPReplayableClause
+  // PRINT: #pragma omp target replayable
+  #pragma omp target replayable
+  {}
+
+  // DUMP: OMPTargetDirective
+  // DUMP-NEXT: OMPReplayableClause
+  // DUMP-NEXT: XXBoolLiteralExpr {{.*}} 'bool' false
+  // PRINT: #pragma omp target replayable(false)
+  #pragma omp target replayable(false)
+  {}
+
+  // DUMP: OMPTargetDirective
+  // DUMP-NEXT: OMPReplayableClause
+  // DUMP-NEXT: XXBoolLiteralExpr {{.*}} 'bool' true
+  // PRINT: #pragma omp target replayable(true)
+  #pragma omp target replayable(true)
+  {}
+
+  // DUMP: OMPTargetDirective
+  // DUMP-NEXT: OMPReplayableClause
+  // DUMP-NEXT: BinaryOperator {{.*}} 'bool' '>'
+  // DUMP-NEXT: ImplicitCastExpr {{.*}} 'int' <LValueToRValue>
+  // DUMP-NEXT: DeclRefExpr {{.*}} 'int' lvalue Var {{.*}} 'A' 'int'
+  // DUMP-NEXT: IntegerLiteral {{.*}} 'int' 5
+  // PRINT: #pragma omp target replayable(A > 5)
+  #pragma omp target replayable(A > 5)
+  {}
+
+  // --- omp target enter data ---
+
+  // DUMP: OMPTargetEnterDataDirective
+  // DUMP: OMPReplayableClause
+  // PRINT: #pragma omp target enter data map(to: A) replayable
+  #pragma omp target enter data map(to: A) replayable
+
+  // DUMP: OMPTargetEnterDataDirective
+  // DUMP: OMPReplayableClause
+  // DUMP-NEXT: XXBoolLiteralExpr {{.*}} 'bool' false
+  // PRINT: #pragma omp target enter data map(to: A) replayable(false)
+  #pragma omp target enter data map(to: A) replayable(false)
+
+  // DUMP: OMPTargetEnterDataDirective
+  // DUMP: OMPReplayableClause
+  // DUMP-NEXT: XXBoolLiteralExpr {{.*}} 'bool' true
+  // PRINT: #pragma omp target enter data map(to: A) replayable(true)
+  #pragma omp target enter data map(to: A) replayable(true)
+
+  // DUMP: OMPTargetEnterDataDirective
+  // DUMP: OMPReplayableClause
+  // DUMP-NEXT: BinaryOperator {{.*}} 'bool' '>'
+  // DUMP-NEXT: ImplicitCastExpr {{.*}} 'int' <LValueToRValue>
+  // DUMP-NEXT: DeclRefExpr {{.*}} 'int' lvalue Var {{.*}} 'A' 'int'
+  // DUMP-NEXT: IntegerLiteral {{.*}} 'int' 5
+  // PRINT: #pragma omp target enter data map(to: A) replayable(A > 5)
+  #pragma omp target enter data map(to: A) replayable(A > 5)
+
+  // --- omp target exit data ---
+
+  // DUMP: OMPTargetExitDataDirective
+  // DUMP: OMPReplayableClause
+  // PRINT: #pragma omp target exit data map(from: A) replayable
+  #pragma omp target exit data map(from: A) replayable
+
+  // DUMP: OMPTargetExitDataDirective
+  // DUMP: OMPReplayableClause
+  // DUMP-NEXT: XXBoolLiteralExpr {{.*}} 'bool' false
+  // PRINT: #pragma omp target exit data map(from: A) replayable(false)
+  #pragma omp target exit data map(from: A) replayable(false)
+
+  // DUMP: OMPTargetExitDataDirective
+  // DUMP: OMPReplayableClause
+  // DUMP-NEXT: XXBoolLiteralExpr {{.*}} 'bool' true
+  // PRINT: #pragma omp target exit data map(from: A) replayable(true)
+  #pragma omp target exit data map(from: A) replayable(true)
+
+  // DUMP: OMPTargetExitDataDirective
+  // DUMP: OMPReplayableClause
+  // DUMP-NEXT: BinaryOperator {{.*}} 'bool' '>'
+  // DUMP-NEXT: ImplicitCastExpr {{.*}} 'int' <LValueToRValue>
+  // DUMP-NEXT: DeclRefExpr {{.*}} 'int' lvalue Var {{.*}} 'A' 'int'
+  // DUMP-NEXT: IntegerLiteral {{.*}} 'int' 5
+  // PRINT: #pragma omp target exit data map(from: A) replayable(A > 5)
+  #pragma omp target exit data map(from: A) replayable(A > 5)
+
+  // --- omp target update ---
+
+  // DUMP: OMPTargetUpdateDirective
+  // DUMP: OMPReplayableClause
+  // PRINT: #pragma omp target update to(A) replayable
+  #pragma omp target update to(A) replayable
+
+  // DUMP: OMPTargetUpdateDirective
+  // DUMP: OMPReplayableClause
+  // DUMP-NEXT: XXBoolLiteralExpr {{.*}} 'bool' false
+  // PRINT: #pragma omp target update to(A) replayable(false)
+  #pragma omp target update to(A) replayable(false)
+
+  // DUMP: OMPTargetUpdateDirective
+  // DUMP: OMPReplayableClause
+  // DUMP-NEXT: XXBoolLiteralExpr {{.*}} 'bool' true
+  // PRINT: #pragma omp target update to(A) replayable(true)
+  #pragma omp target update to(A) replayable(true)
+
+  // DUMP: OMPTargetUpdateDirective
+  // DUMP: OMPReplayableClause
+  // DUMP-NEXT: BinaryOperator {{.*}} 'bool' '>'
+  // DUMP-NEXT: ImplicitCastExpr {{.*}} 'int' <LValueToRValue>
+  // DUMP-NEXT: DeclRefExpr {{.*}} 'int' lvalue Var {{.*}} 'A' 'int'
+  // DUMP-NEXT: IntegerLiteral {{.*}} 'int' 5
+  // PRINT: #pragma omp target update to(A) replayable(A > 5)
+  #pragma omp target update to(A) replayable(A > 5)
+}
+#endif
diff --git a/clang/test/OpenMP/replayable_messages.cpp b/clang/test/OpenMP/replayable_messages.cpp
new file mode 100644
index 0000000000000..10172f50162ad
--- /dev/null
+++ b/clang/test/OpenMP/replayable_messages.cpp
@@ -0,0 +1,51 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=51 -verify=expected,omp51 -fsyntax-only %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -verify=expected,omp60 -fsyntax-only %s
+
+// Tests that the 'replayable' clause is accepted in OpenMP 6.0 and rejected in
+// prior versions on all seven supported directives. Also tests duplicate-clause
+// and invalid-condition diagnostics.
+
+void foo() {}
+
+void replayable_messages() {
+  int A = 1;
+
+  #pragma omp task replayable // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp task'}}
+  {}
+
+  #pragma omp taskloop replayable // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp taskloop'}}
+  for (int i = 0; i < 10; ++i)
+    {}
+
+  #pragma omp taskwait replayable // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp taskwait'}}
+
+  #pragma omp target replayable // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp target'}}
+  {}
+
+  #pragma omp target enter data map(to: A) replayable // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp target enter data'}}
+
+  #pragma omp target exit data map(from: A) replayable // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp target exit data'}}
+
+  #pragma omp target update to(A) replayable // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp target update'}}
+
+  #pragma omp task replayable replayable // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp task'}} omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp task'}} expected-error {{directive '#pragma omp task' cannot contain more than one 'replayable' clause}}
+  {}
+
+  #pragma omp task replayable(foo()) // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp task'}} omp60-error {{value of type 'void' is not contextually convertible to 'bool'}}
+  {}
+
+  #pragma omp taskloop replayable(A > 0) // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp taskloop'}}
+  for (int i = 0; i < 10; ++i)
+    {}
+
+  #pragma omp taskwait replayable(A > 0) // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp taskwait'}}
+
+  #pragma omp target replayable(A > 0) // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp target'}}
+  {}
+
+  #pragma omp target enter data map(to: A) replayable(A > 0) // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp target enter data'}}
+
+  #pragma omp target exit data map(from: A) replayable(A > 0) // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp target exit data'}}
+
+  #pragma omp target update to(A) replayable(A > 0) // omp51-error {{unexpected OpenMP clause 'replayable' in directive '#pragma omp target update'}}
+}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_recursive_reduction.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_recursive_reduction.cpp
new file mode 100644
index 0000000000000..c349bd2f34146
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_recursive_reduction.cpp
@@ -0,0 +1,147 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// Exercises reduction(+: ...) on dynamically (non-lexically) nested
+// replayable constructs reached through a recursive driver:
+//
+//   - emit_reduction_taskloop():  replayable taskloop performing a reduction
+//                                 into a file-scope accumulator (Sum).  The
+//                                 'seed' parameter is captured as
+//                                 firstprivate(saved:) so the snapshot
+//                                 taken at recording is reused unchanged
+//                                 on every replay (OpenMP 6.0 [7.5.4] /
+//                                 [14.3]).
+//   - emit_publish_task():        replayable task that reads Sum and stores
+//                                 it into a distinct slot of a file-scope
+//                                 Snapshots[] array; the slot index is
+//                                 carried into the task as a
+//                                 firstprivate(saved:) capture so that
+//                                 every recursion-level publish lands in
+//                                 its own slot and no two publishes race
+//                                 for the same destination
+//   - driver():                   recursive function that calls
+//                                 emit_reduction_taskloop() on descent and
+//                                 emit_publish_task() on ascent, so on
+//                                 replay these constructs are dispatched
+//                                 from differing recursive stack frames
+//   - run_taskgraph():            wraps the recursive driver() in
+//                                 #pragma omp taskgraph and returns Sum
+//                                 (the cumulative reduction, which is the
+//                                 deterministic test signal; Snapshots[]
+//                                 is written purely as a side-effect to
+//                                 exercise the replayable publish tasks)
+//
+// Per the saved-snapshot semantics, every replay reproduces the recording
+// run's reduction value, independent of the seed argument passed to the
+// replay invocation.  The test therefore (i) verifies that the recording
+// matches expected_result(Seeds[0]) and (ii) verifies that every replay
+// matches the recording.  Historically this exercise hit two consecutive
+// crashes on replay: first a relocate-side #302 because the (static)
+// reduction target's shareds slot couldn't be re-projected from a
+// non-OpenMP outer scope, then a taskred-lookup assertion because the
+// recording-time taskgroup_t holding the reduction state had been torn
+// down.  Both gaps are now fixed in the compiler and runtime.
+
+#include <cstdio>
+
+static constexpr int MaxDepth = 4;
+
+static volatile int Sum = 0;
+static volatile int Snapshots[MaxDepth] = {0, 0, 0, 0};
+
+__attribute__((noinline)) static void emit_reduction_taskloop(int seed) {
+#pragma omp taskloop replayable num_tasks(8) reduction(+ : Sum)                \
+    firstprivate(saved : seed)
+  for (int i = 0; i < 16; ++i)
+    Sum += seed + i;
+}
+
+__attribute__((noinline)) static void emit_publish_task(int slot) {
+#pragma omp task replayable firstprivate(saved : slot)
+  {
+    Snapshots[slot] = Sum;
+  }
+}
+
+__attribute__((noinline)) static void driver(int depth, int seed) {
+  emit_reduction_taskloop(seed + depth);
+  if (depth == 0) {
+    emit_publish_task(depth);
+    return;
+  }
+  driver(depth - 1, seed);
+  emit_publish_task(depth);
+}
+
+__attribute__((noinline)) static int run_taskgraph(int seed) {
+  Sum = 0;
+  for (int i = 0; i < MaxDepth; ++i)
+    Snapshots[i] = 0;
+
+#pragma omp taskgraph graph_id(917)
+  {
+    driver(MaxDepth - 1, seed);
+  }
+
+  // Sum is the deterministic test signal: after the taskgraph's implicit
+  // taskwait it holds the cumulative reduction across all MaxDepth
+  // taskloops.  The per-slot Snapshots[] entries are written by the
+  // replayable publish tasks but race with the reduction taskloops, so
+  // their individual values are non-deterministic and are intentionally
+  // not compared here.  The array exists only so that each replayable
+  // publish task targets a distinct destination instead of clobbering a
+  // shared scalar.
+  return Sum;
+}
+
+__attribute__((noinline)) static int expected_result(int seed) {
+  int sum = 0;
+  for (int d = MaxDepth - 1; d >= 0; --d)
+    for (int i = 0; i < 16; ++i)
+      sum += seed + d + i;
+  return sum;
+}
+
+int main() {
+  constexpr int NumRuns = 4;
+  constexpr int Seeds[NumRuns] = {1, 5, 11, 23};
+
+  int recorded = -1;
+  bool failed = false;
+
+#pragma omp parallel num_threads(4)
+  {
+#pragma omp single
+    {
+      recorded = run_taskgraph(Seeds[0]);
+      const int exp0 = expected_result(Seeds[0]);
+      if (recorded != exp0) {
+        std::fprintf(stderr, "FAIL initial record got=%d expected=%d\n",
+                     recorded, exp0);
+        failed = true;
+      }
+
+      // Saved-snapshot semantics: every replay should reproduce the
+      // recording's reduction value, regardless of the live argument.
+      for (int i = 1; i < NumRuns; ++i) {
+        const int replayed = run_taskgraph(Seeds[i]);
+        if (replayed != recorded) {
+          std::fprintf(stderr, "FAIL replay %d seed=%d got=%d recorded=%d\n", i,
+                       Seeds[i], replayed, recorded);
+          failed = true;
+        }
+      }
+    }
+  }
+
+  if (failed)
+    return 1;
+
+  std::fprintf(stderr, "PASS non-lexical recursive reduction result=%d\n",
+               recorded);
+  return 0;
+}
+
+// CHECK: PASS non-lexical recursive reduction result=
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_reduction_minimal_taskloop.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_reduction_minimal_taskloop.cpp
new file mode 100644
index 0000000000000..17b36f16a02ef
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_reduction_minimal_taskloop.cpp
@@ -0,0 +1,105 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// Minimal end-to-end exercise of reduction(+: ...) on a replayable taskloop
+// that is dynamically (non-lexically) nested inside a taskgraph: the
+// taskloop directive lives in a helper called from inside the taskgraph
+// region, not in the taskgraph's lexical body.
+//
+//   - Sum is a file-scope static, so the reduction's accumulator address is
+//     stable across runs and the taskgraph relocate helper has no shareds
+//     slot it must refresh (the static capture is link-time-fixed).
+//   - 'seed' is captured into the taskloop's per-task '.kmp_privates.t'
+//     snapshot as firstprivate(saved: ...); per OpenMP 6.0 [7.5.4] / [14.3]
+//     the snapshot is taken at recording time and reused unchanged on every
+//     replay.
+//   - The recording run therefore produces the expected reduction for
+//     Seeds[0], and every subsequent replay (each invoked with a different
+//     seed) produces exactly the recorded value because the saved snapshot
+//     of 'seed' is what the body sees.
+//
+// Historically this test crashed on replay with OMP: Error #302 because
+// the taskgraph relocate helper refused to handle the (static) reduction
+// capture, and even after that was fixed the reduction body still
+// asserted because the recording-time taskgroup_t had been torn down and
+// the taskred state was unreachable.  Both gaps are addressed: the
+// relocate helper now treats captures of static-storage variables as
+// no-op-safe, and the runtime now stashes reduction-init input into the
+// surrounding taskgraph at recording so the replay machinery can re-create
+// the taskred state on every replay.
+
+#include <cstdio>
+
+static volatile int Sum = 0;
+
+__attribute__((noinline)) static void emit_reduction_taskloop(int seed) {
+#pragma omp taskloop replayable num_tasks(8) reduction(+ : Sum)                \
+    firstprivate(saved : seed)
+  for (int i = 0; i < 16; ++i)
+    Sum += seed + i;
+}
+
+__attribute__((noinline)) static int run_taskgraph(int seed) {
+  Sum = 0;
+
+#pragma omp taskgraph graph_id(921)
+  {
+    emit_reduction_taskloop(seed);
+  }
+
+  return Sum;
+}
+
+__attribute__((noinline)) static int expected_result(int seed) {
+  int sum = 0;
+  for (int i = 0; i < 16; ++i)
+    sum += seed + i;
+  return sum;
+}
+
+int main() {
+  constexpr int NumRuns = 4;
+  constexpr int Seeds[NumRuns] = {1, 5, 11, 23};
+
+  int recorded = -1;
+  bool failed = false;
+
+#pragma omp parallel num_threads(4)
+  {
+#pragma omp single
+    {
+      recorded = run_taskgraph(Seeds[0]);
+      const int exp0 = expected_result(Seeds[0]);
+      if (recorded != exp0) {
+        std::fprintf(stderr, "FAIL initial record got=%d expected=%d\n",
+                     recorded, exp0);
+        failed = true;
+      }
+
+      // 'seed' is firstprivate(saved:), so every replay's body sees the
+      // snapshot taken at recording time.  The reduction therefore yields
+      // the same value as the recording regardless of the live argument
+      // passed on the replay.
+      for (int i = 1; i < NumRuns; ++i) {
+        const int replayed = run_taskgraph(Seeds[i]);
+        if (replayed != recorded) {
+          std::fprintf(stderr, "FAIL replay %d seed=%d got=%d recorded=%d\n", i,
+                       Seeds[i], replayed, recorded);
+          failed = true;
+        }
+      }
+    }
+  }
+
+  if (failed)
+    return 1;
+
+  std::fprintf(stderr,
+               "PASS non-lexical replayable taskloop reduction result=%d\n",
+               recorded);
+  return 0;
+}
+
+// CHECK: PASS non-lexical replayable taskloop reduction result=

>From 2cba7b3c230c856ad9ea0e7f0256bb8692df2ee5 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Thu, 23 Apr 2026 14:47:24 -0500
Subject: [PATCH 10/26] [OpenMP] OpenMP 6.0 "taskgraph" support, additional
 tests from previous implementation

This patch contains new tests from previous iterations of taskgraph
support, with some changes to adjust to the new implementation.

Co-Authored-By: Josep Pinot <josep at bsc.es>

Pull Request: https://github.com/llvm/llvm-project/pull/194055
---
 openmp/runtime/test/tasking/omp_taskgraph.cpp | 37 ++++++++++
 .../test/tasking/omp_taskgraph_deps.cpp       | 54 +++++++++++++++
 .../test/tasking/omp_taskgraph_multiTDGs.cpp  | 68 +++++++++++++++++++
 .../test/tasking/omp_taskgraph_taskloop.cpp   | 41 +++++++++++
 4 files changed, 200 insertions(+)
 create mode 100644 openmp/runtime/test/tasking/omp_taskgraph.cpp
 create mode 100644 openmp/runtime/test/tasking/omp_taskgraph_deps.cpp
 create mode 100644 openmp/runtime/test/tasking/omp_taskgraph_multiTDGs.cpp
 create mode 100644 openmp/runtime/test/tasking/omp_taskgraph_taskloop.cpp

diff --git a/openmp/runtime/test/tasking/omp_taskgraph.cpp b/openmp/runtime/test/tasking/omp_taskgraph.cpp
new file mode 100644
index 0000000000000..67567a9b31870
--- /dev/null
+++ b/openmp/runtime/test/tasking/omp_taskgraph.cpp
@@ -0,0 +1,37 @@
+// clang-format off
+// REQUIRES: omp_taskgraph_experimental
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run
+// clang-format on
+#include <iostream>
+#include <cassert>
+#define NT 100
+
+// Compiler-generated code (emulation)
+typedef struct ident {
+  void *dummy;
+} ident_t;
+
+void func(int *num_exec) { (*num_exec)++; }
+
+int main() {
+  int num_exec = 0;
+  int num_tasks = 0;
+  int x = 0;
+#pragma omp parallel
+#pragma omp single
+  for (int iter = 0; iter < NT; ++iter) {
+#pragma omp taskgraph
+    {
+      num_tasks++;
+#pragma omp task
+      func(&num_exec);
+    }
+  }
+
+  assert(num_tasks == 1);
+  assert(num_exec == NT);
+
+  std::cout << "Passed" << std::endl;
+  return 0;
+}
+// CHECK: Passed
diff --git a/openmp/runtime/test/tasking/omp_taskgraph_deps.cpp b/openmp/runtime/test/tasking/omp_taskgraph_deps.cpp
new file mode 100644
index 0000000000000..e83e0c2f2ce62
--- /dev/null
+++ b/openmp/runtime/test/tasking/omp_taskgraph_deps.cpp
@@ -0,0 +1,54 @@
+// clang-format off
+// REQUIRES: omp_taskgraph_experimental
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run
+// clang-format on
+#include <iostream>
+#include <cassert>
+#define NT 100
+#define MULTIPLIER 100
+#define DECREMENT 5
+
+int val;
+// Compiler-generated code (emulation)
+typedef struct ident {
+  void *dummy;
+} ident_t;
+
+void sub() {
+#pragma omp atomic
+  val -= DECREMENT;
+}
+
+void add() {
+#pragma omp atomic
+  val += DECREMENT;
+}
+
+void mult() {
+  // no atomicity needed, can only be executed by 1 thread
+  // and no concurrency with other tasks possible
+  val *= MULTIPLIER;
+}
+
+int main() {
+  val = 0;
+  int *x, *y;
+#pragma omp parallel
+#pragma omp single
+  for (int iter = 0; iter < NT; ++iter) {
+#pragma omp taskgraph
+    {
+#pragma omp task depend(out : y)
+      add();
+#pragma omp task depend(out : x)
+      sub();
+#pragma omp task depend(in : x, y)
+      mult();
+    }
+  }
+  assert(val == 0);
+
+  std::cout << "Passed" << std::endl;
+  return 0;
+}
+// CHECK: Passed
diff --git a/openmp/runtime/test/tasking/omp_taskgraph_multiTDGs.cpp b/openmp/runtime/test/tasking/omp_taskgraph_multiTDGs.cpp
new file mode 100644
index 0000000000000..eedd4e642c956
--- /dev/null
+++ b/openmp/runtime/test/tasking/omp_taskgraph_multiTDGs.cpp
@@ -0,0 +1,68 @@
+// clang-format off
+// REQUIRES: omp_taskgraph_experimental
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run
+// clang-format on
+#include <iostream>
+#include <cassert>
+#define NT 20
+#define MULTIPLIER 100
+#define DECREMENT 5
+
+// Compiler-generated code (emulation)
+typedef struct ident {
+  void *dummy;
+} ident_t;
+
+int val;
+
+void sub() {
+#pragma omp atomic
+  val -= DECREMENT;
+}
+
+void add() {
+#pragma omp atomic
+  val += DECREMENT;
+}
+
+void mult() {
+  // no atomicity needed, can only be executed by 1 thread
+  // and no concurrency with other tasks possible
+  val *= MULTIPLIER;
+}
+
+int main() {
+  int num_tasks = 0;
+  int *x, *y;
+#pragma omp parallel
+#pragma omp single
+  for (int iter = 0; iter < NT; ++iter) {
+#pragma omp taskgraph
+    {
+      num_tasks++;
+#pragma omp task depend(out : y)
+      add();
+#pragma omp task depend(out : x)
+      sub();
+#pragma omp task depend(in : x, y)
+      mult();
+    }
+#pragma omp taskgraph
+    {
+      num_tasks++;
+#pragma omp task depend(out : y)
+      add();
+#pragma omp task depend(out : x)
+      sub();
+#pragma omp task depend(in : x, y)
+      mult();
+    }
+  }
+
+  assert(num_tasks == 2);
+  assert(val == 0);
+
+  std::cout << "Passed" << std::endl;
+  return 0;
+}
+// CHECK: Passed
diff --git a/openmp/runtime/test/tasking/omp_taskgraph_taskloop.cpp b/openmp/runtime/test/tasking/omp_taskgraph_taskloop.cpp
new file mode 100644
index 0000000000000..f4fe290a90f0f
--- /dev/null
+++ b/openmp/runtime/test/tasking/omp_taskgraph_taskloop.cpp
@@ -0,0 +1,41 @@
+// clang-format off
+// REQUIRES: omp_taskgraph_experimental
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run
+// clang-format on
+#include <iostream>
+#include <cassert>
+
+#define NT 20
+#define N 128 * 128
+
+typedef struct ident {
+  void *dummy;
+} ident_t;
+
+int main() {
+  int num_tasks = 0;
+
+  int array[N];
+  for (int i = 0; i < N; ++i)
+    array[i] = 1;
+
+  long sum = 0;
+#pragma omp parallel
+#pragma omp single
+  for (int iter = 0; iter < NT; ++iter) {
+#pragma omp taskgraph
+    {
+      num_tasks++;
+#pragma omp taskloop reduction(+ : sum) num_tasks(4096)
+      for (int i = 0; i < N; ++i) {
+        sum += array[i];
+      }
+    }
+  }
+  assert(sum == N * NT);
+  assert(num_tasks == 1);
+
+  std::cout << "Passed" << std::endl;
+  return 0;
+}
+// CHECK: Passed

>From 4819fd6bff79c703256bb54cf5cc31426feb2fda Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Wed, 25 Mar 2026 16:57:25 -0500
Subject: [PATCH 11/26] [OpenMP] OpenMP 6.0 "taskgraph" support, add new tests

These are new tests intended to cover the functionality present in the
new taskgraph implementation.

commit-id:4eb2eb71

Pull Request: https://github.com/llvm/llvm-project/pull/194057
---
 .../test/taskgraph/taskgraph_deps_1.cpp       |  53 +++++++++
 .../test/taskgraph/taskgraph_deps_10.cpp      |  50 +++++++++
 .../test/taskgraph/taskgraph_deps_11.cpp      |  60 ++++++++++
 .../test/taskgraph/taskgraph_deps_12.cpp      |  55 ++++++++++
 .../test/taskgraph/taskgraph_deps_13.cpp      |  45 ++++++++
 .../test/taskgraph/taskgraph_deps_14.cpp      |  48 ++++++++
 .../test/taskgraph/taskgraph_deps_15.cpp      |  75 +++++++++++++
 .../test/taskgraph/taskgraph_deps_16.cpp      |  55 ++++++++++
 .../test/taskgraph/taskgraph_deps_17.cpp      |  68 ++++++++++++
 .../test/taskgraph/taskgraph_deps_18.cpp      |  46 ++++++++
 .../test/taskgraph/taskgraph_deps_19.cpp      |  51 +++++++++
 .../test/taskgraph/taskgraph_deps_2.cpp       |  58 ++++++++++
 .../test/taskgraph/taskgraph_deps_20.cpp      |  51 +++++++++
 .../test/taskgraph/taskgraph_deps_21.cpp      |  52 +++++++++
 .../test/taskgraph/taskgraph_deps_22.cpp      |  70 ++++++++++++
 .../test/taskgraph/taskgraph_deps_23.cpp      | 103 ++++++++++++++++++
 .../test/taskgraph/taskgraph_deps_24.cpp      |  80 ++++++++++++++
 .../test/taskgraph/taskgraph_deps_25.cpp      |  89 +++++++++++++++
 .../test/taskgraph/taskgraph_deps_26.cpp      |  61 +++++++++++
 .../test/taskgraph/taskgraph_deps_27.cpp      |  63 +++++++++++
 .../test/taskgraph/taskgraph_deps_3.cpp       |  62 +++++++++++
 .../test/taskgraph/taskgraph_deps_4.cpp       |  59 ++++++++++
 .../test/taskgraph/taskgraph_deps_5.cpp       |  63 +++++++++++
 .../test/taskgraph/taskgraph_deps_6.cpp       |  59 ++++++++++
 .../test/taskgraph/taskgraph_deps_7.cpp       |  59 ++++++++++
 .../test/taskgraph/taskgraph_deps_8.cpp       |  39 +++++++
 .../test/taskgraph/taskgraph_deps_9.cpp       |  47 ++++++++
 27 files changed, 1621 insertions(+)
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_1.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_10.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_11.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_12.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_13.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_14.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_15.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_16.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_17.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_18.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_19.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_2.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_20.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_21.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_22.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_23.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_24.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_25.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_26.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_27.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_3.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_4.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_5.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_6.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_7.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_8.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_9.cpp

diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_1.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_1.cpp
new file mode 100644
index 0000000000000..6c9241206bc60
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_1.cpp
@@ -0,0 +1,53 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+  int deps[3];
+  #pragma omp parallel
+  {
+    #pragma omp single
+    {
+      for (int i = 0; i < 2; i++)
+      {
+        #pragma omp taskgraph
+        {
+          #pragma omp task depend(out: deps[2])
+          { }
+          #pragma omp task depend(out: deps[0], deps[1])
+          { }
+          #pragma omp task depend(inout: deps[0])
+          { }
+          #pragma omp task depend(inout: deps[1])
+          { }
+          #pragma omp task depend(inout: deps[2])
+          { }
+          #pragma omp task depend(in: deps[0], deps[1], deps[2])
+          { }
+        }
+      }
+    }
+  }
+  return 0;
+}
+
+// CHECK:      Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: sequential {
+// CHECK-NEXT:   parallel {
+// CHECK-NEXT:     sequential {
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:     }
+// CHECK-NEXT:     sequential {
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:       parallel {
+// CHECK-NEXT:         node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:         node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:       }
+// CHECK-NEXT:     }
+// CHECK-NEXT:    }
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_10.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_10.cpp
new file mode 100644
index 0000000000000..ff0a46b835706
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_10.cpp
@@ -0,0 +1,50 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+  int deps[5];
+  #pragma omp parallel
+  {
+    #pragma omp single
+    {
+      for (int i = 0; i < 2; i++)
+      {
+        #pragma omp taskgraph
+        {
+          #pragma omp task depend(out: deps[0], deps[1])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[0], deps[1], deps[4])
+          { }
+          #pragma omp task depend(in: deps[0], deps[1])
+          { }
+
+          #pragma omp task depend(out: deps[2], deps[3])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[2], deps[3], deps[4])
+          { }
+          #pragma omp task depend(in: deps[2], deps[3])
+          { }
+        }
+      }
+    }
+  }
+  return 0;
+}
+
+// CHECK:      Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: parallel {
+// CHECK-NEXT:   sequential {
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}} [sets: 0x1c]
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   }
+// CHECK-NEXT:   sequential {
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}} [sets: 0x7]
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   }
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_11.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_11.cpp
new file mode 100644
index 0000000000000..308232707b9af
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_11.cpp
@@ -0,0 +1,60 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+  int deps[4];
+  #pragma omp parallel
+  {
+    #pragma omp single
+    {
+      for (int i = 0; i < 2; i++)
+      {
+        #pragma omp taskgraph
+        {
+          #pragma omp task depend(out: deps[0], deps[1])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[0], deps[1])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[0], deps[1])
+          { }
+          #pragma omp task depend(in: deps[0], deps[1])
+          { }
+
+          #pragma omp task depend(out: deps[2], deps[3])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[2], deps[3])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[2], deps[3])
+          { }
+          #pragma omp task depend(in: deps[2], deps[3])
+          { }
+        }
+      }
+    }
+  }
+  return 0;
+}
+
+// CHECK:      Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: parallel {
+// CHECK-NEXT:   sequential {
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:     exclusive {
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:     }
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   }
+// CHECK-NEXT:   sequential {
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:     exclusive {
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:     }
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   }
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_12.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_12.cpp
new file mode 100644
index 0000000000000..5a4fc884ff995
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_12.cpp
@@ -0,0 +1,55 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+  int deps[2];
+  #pragma omp parallel
+  {
+    #pragma omp single
+    {
+      for (int i = 0; i < 2; i++)
+      {
+        #pragma omp taskgraph
+        {
+          #pragma omp task depend(mutexinoutset: deps[0])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[1])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[0])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[1])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[0])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[1])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[0])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[1])
+          { }
+        }
+      }
+    }
+  }
+  return 0;
+}
+
+// CHECK:      Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: parallel {
+// CHECK-NEXT:   exclusive {
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   }
+// CHECK-NEXT:   exclusive {
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   }
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_13.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_13.cpp
new file mode 100644
index 0000000000000..6247283d0cfd6
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_13.cpp
@@ -0,0 +1,45 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+  int deps[4];
+  #pragma omp parallel
+  {
+    #pragma omp single
+    {
+      for (int i = 0; i < 2; i++)
+      {
+        #pragma omp taskgraph
+        {
+          #pragma omp task depend(mutexinoutset: deps[0])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[1])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[0], deps[1])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[2])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[3])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[2], deps[3])
+          { }
+        }
+      }
+    }
+  }
+  return 0;
+}
+
+// CHECK:      Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: parallel {
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0xc]
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0x3]
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0x8]
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0x4]
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0x2]
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0x1]
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_14.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_14.cpp
new file mode 100644
index 0000000000000..ea7671f334557
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_14.cpp
@@ -0,0 +1,48 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+  int deps[4];
+  #pragma omp parallel
+  {
+    #pragma omp single
+    {
+      for (int i = 0; i < 2; i++)
+      {
+        #pragma omp taskgraph
+        {
+          #pragma omp task depend(mutexinoutset: deps[0])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[0], deps[1])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[0], deps[1], deps[2])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[0], deps[1], deps[2], deps[3])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[1], deps[2], deps[3])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[2], deps[3])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[3])
+          { }
+        }
+      }
+    }
+  }
+  return 0;
+}
+
+// CHECK:      Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: parallel {
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0xf]
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0xe]
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0x7]
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0xc]
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0x3]
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0x8]
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0x1]
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_15.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_15.cpp
new file mode 100644
index 0000000000000..f8659f9d12cb8
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_15.cpp
@@ -0,0 +1,75 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+  int deps[4];
+  #pragma omp parallel
+  {
+    #pragma omp single
+    {
+      for (int i = 0; i < 2; i++)
+      {
+        #pragma omp taskgraph
+        {
+          #pragma omp task
+          { }
+          #pragma omp task depend(mutexinoutset: deps[0])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[1])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[1], deps[0])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[2])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[2], deps[0])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[2], deps[1])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[2], deps[1], deps[0])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[3])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[3], deps[0])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[3], deps[1])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[3], deps[1], deps[0])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[3], deps[2])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[3], deps[2], deps[0])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[3], deps[2], deps[1])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[3], deps[2], deps[1], deps[0])
+          { }
+        }
+      }
+    }
+  }
+  return 0;
+}
+
+// CHECK:      Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: parallel {
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0xf]
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0xe]
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0xd]
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0xb]
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0x7]
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0xc]
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0xa]
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0x9]
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0x6]
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0x5]
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0x3]
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0x8]
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0x4]
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0x2]
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0x1]
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_16.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_16.cpp
new file mode 100644
index 0000000000000..471e19fab88bf
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_16.cpp
@@ -0,0 +1,55 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+  int deps[8];
+  #pragma omp parallel
+  {
+    #pragma omp single
+    {
+      for (int i = 0; i < 2; i++)
+      {
+        #pragma omp taskgraph
+        {
+          #pragma omp task depend(inout: deps[0])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[4], deps[7])
+          { }
+          #pragma omp task depend(inout: deps[1])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[4], deps[7])
+          { }
+          #pragma omp task depend(inout: deps[2])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[5], deps[6])
+          { }
+          #pragma omp task depend(inout: deps[3])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[5], deps[6])
+          { }
+        }
+      }
+    }
+  }
+  return 0;
+}
+
+// CHECK:      Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: parallel {
+// CHECK-NEXT:   exclusive {
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   }
+// CHECK-NEXT:   exclusive {
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   }
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_17.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_17.cpp
new file mode 100644
index 0000000000000..fda89fd39a9a0
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_17.cpp
@@ -0,0 +1,68 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+#include <cstdio>
+
+int main()
+{
+  int deps[4];
+  #pragma omp parallel
+  {
+    #pragma omp single
+    {
+      for (int i = 0; i < 2; i++)
+      {
+        #pragma omp taskgraph
+        {
+          #pragma omp task depend(out: deps[0], deps[1])
+          {
+            fprintf(stderr, "task 0\n");
+          }
+          #pragma omp task depend(out: deps[2], deps[3])
+          {
+            fprintf(stderr, "task 1\n");
+          }
+          #pragma omp task depend(inout: deps[0])
+          {
+            fprintf(stderr, "task 2\n");
+          }
+          #pragma omp task depend(inout: deps[1])
+          {
+            fprintf(stderr, "task 3\n");
+          }
+          #pragma omp task depend(inout: deps[2])
+          {
+            fprintf(stderr, "task 4\n");
+          }
+          #pragma omp task depend(inout: deps[3])
+          {
+            fprintf(stderr, "task 5\n");
+          }
+          #pragma omp task depend(in: deps[0], deps[1], deps[2], deps[3])
+          {
+            fprintf(stderr, "task 6\n");
+          }
+        }
+      }
+    }
+  }
+  return 0;
+}
+
+// CHECK-DAG: task 0
+// CHECK-DAG: task 2
+// CHECK-DAG: task 3
+// CHECK-DAG: task 1
+// CHECK-DAG: task 4
+// CHECK-DAG: task 5
+// CHECK: task 6
+
+// CHECK-DAG: task 0
+// CHECK-DAG: task 2
+// CHECK-DAG: task 3
+// CHECK-DAG: task 1
+// CHECK-DAG: task 4
+// CHECK-DAG: task 5
+// CHECK: task 6
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_18.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_18.cpp
new file mode 100644
index 0000000000000..a04e405de6c70
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_18.cpp
@@ -0,0 +1,46 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+  int deps[2];
+  #pragma omp parallel
+  {
+    #pragma omp single
+    {
+      for (int i = 0; i < 2; i++)
+      {
+        #pragma omp taskgraph
+        {
+          #pragma omp task depend(out: deps[0], deps[1])
+          { }
+          #pragma omp taskloop num_tasks(strict: 2)
+          {
+            for (int j = 0; j < 20; j++) { }
+          }
+          #pragma omp task depend(in: deps[0], deps[1])
+          { }
+        }
+      }
+    }
+  }
+  return 0;
+}
+
+// CHECK:      Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: sequential {
+// CHECK-NEXT:   parallel {
+// CHECK-NEXT:     sequential {
+// CHECK-NEXT:       parallel {
+// CHECK-NEXT:         node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:         node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:       }
+// CHECK-NEXT:       wait: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:     }
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   }
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_19.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_19.cpp
new file mode 100644
index 0000000000000..b18f8c844e8e1
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_19.cpp
@@ -0,0 +1,51 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+#include <cstdio>
+
+int main()
+{
+  int deps[3];
+  #pragma omp parallel
+  {
+    #pragma omp single
+    {
+      for (int i = 0; i < 10; i++)
+      {
+        #pragma omp taskgraph
+        {
+          #pragma omp task depend(out: deps[0])
+          {
+            fprintf(stderr, "task 0\n");
+          }
+          #pragma omp task depend(out: deps[1])
+          {
+            fprintf(stderr, "task 1\n");
+          }
+          #pragma omp task depend(out: deps[2])
+          {
+            fprintf(stderr, "task 2\n");
+          }
+          #pragma omp taskwait depend(in: deps[0], deps[1], deps[2])
+          #pragma omp task depend(in: deps[0], deps[1], deps[2])
+          {
+            fprintf(stderr, "task 3\n");
+          }
+        }
+      }
+    }
+  }
+  return 0;
+}
+
+// CHECK-DAG: task 0
+// CHECK-DAG: task 1
+// CHECK-DAG: task 2
+// CHECK: task 3
+
+// CHECK-DAG: task 0
+// CHECK-DAG: task 1
+// CHECK-DAG: task 2
+// CHECK: task 3
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_2.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_2.cpp
new file mode 100644
index 0000000000000..7b897bc92bb62
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_2.cpp
@@ -0,0 +1,58 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+  int deps[4];
+  #pragma omp parallel
+  {
+    #pragma omp single
+    {
+      for (int i = 0; i < 2; i++)
+      {
+        #pragma omp taskgraph
+        {
+          #pragma omp task depend(out: deps[0], deps[1])
+          { }
+          #pragma omp task depend(out: deps[2], deps[3])
+          { }
+          #pragma omp task depend(inout: deps[0])
+          { }
+          #pragma omp task depend(inout: deps[1])
+          { }
+          #pragma omp task depend(inout: deps[2])
+          { }
+          #pragma omp task depend(inout: deps[3])
+          { }
+          #pragma omp task depend(in: deps[0], deps[1], deps[2], deps[3])
+          { }
+        }
+      }
+    }
+  }
+  return 0;
+}
+
+// CHECK:      Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: sequential {
+// CHECK-NEXT:   parallel {
+// CHECK-NEXT:     sequential {
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:       parallel {
+// CHECK-NEXT:         node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:         node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:       }
+// CHECK-NEXT:     }
+// CHECK-NEXT:     sequential {
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:       parallel {
+// CHECK-NEXT:         node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:         node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:       }
+// CHECK-NEXT:     }
+// CHECK-NEXT:   }
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_20.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_20.cpp
new file mode 100644
index 0000000000000..549b6d6138a52
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_20.cpp
@@ -0,0 +1,51 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+#include <cstdio>
+
+int main()
+{
+  int deps[3];
+  #pragma omp parallel
+  {
+    #pragma omp single
+    {
+      for (int i = 0; i < 10; i++)
+      {
+        #pragma omp taskgraph
+        {
+          #pragma omp task depend(out: deps[0])
+          {
+            fprintf(stderr, "task 0\n");
+          }
+          #pragma omp task depend(out: deps[1])
+          {
+            fprintf(stderr, "task 1\n");
+          }
+          #pragma omp task depend(out: deps[2])
+          {
+            fprintf(stderr, "task 2\n");
+          }
+          #pragma omp taskwait depend(inoutset: deps[0], deps[1])
+          #pragma omp task depend(in: deps[0], deps[1], deps[2])
+          {
+            fprintf(stderr, "task 3\n");
+          }
+        }
+      }
+    }
+  }
+  return 0;
+}
+
+// CHECK-DAG: task 0
+// CHECK-DAG: task 1
+// CHECK-DAG: task 2
+// CHECK: task 3
+
+// CHECK-DAG: task 0
+// CHECK-DAG: task 1
+// CHECK-DAG: task 2
+// CHECK: task 3
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_21.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_21.cpp
new file mode 100644
index 0000000000000..0b86e677a13eb
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_21.cpp
@@ -0,0 +1,52 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+#include <cstdio>
+
+int main()
+{
+  int arr[100];
+
+  int res = 0;
+  for (int i = 0; i < 100; i++) {
+    arr[i] = i;
+    res += i;
+  }
+  printf("base result: %d\n", res);
+
+  #pragma omp parallel
+  {
+    #pragma omp single
+    {
+      for (int i = 0; i < 10; i++)
+      {
+        int res = 0;
+        #pragma omp taskgraph
+        {
+          #pragma omp taskloop reduction(+: res) num_tasks(10)
+          {
+            for (int j = 0; j < 100; j++) {
+              res += arr[j];
+            }
+          }
+        }
+        printf("reduction result: %d\n", res);
+      }
+    }
+  }
+  return 0;
+}
+
+// CHECK:      base result: 4950
+// CHECK-NEXT: reduction result: 4950
+// CHECK-NEXT: reduction result: 4950
+// CHECK-NEXT: reduction result: 4950
+// CHECK-NEXT: reduction result: 4950
+// CHECK-NEXT: reduction result: 4950
+// CHECK-NEXT: reduction result: 4950
+// CHECK-NEXT: reduction result: 4950
+// CHECK-NEXT: reduction result: 4950
+// CHECK-NEXT: reduction result: 4950
+// CHECK-NEXT: reduction result: 4950
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_22.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_22.cpp
new file mode 100644
index 0000000000000..66b1ec8b8093c
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_22.cpp
@@ -0,0 +1,70 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+#include <cstdio>
+
+void foo() {
+#pragma omp task replayable(1)
+  {
+    fprintf(stderr, "task outside lexical taskgraph\n");
+  }
+}
+
+int main()
+{
+  int arr[100];
+
+  int res = 0;
+  for (int i = 0; i < 100; i++) {
+    arr[i] = i;
+    res += i;
+  }
+  fprintf(stderr, "base result: %d\n", res);
+
+  #pragma omp parallel
+  {
+    #pragma omp single
+    {
+      for (int i = 0; i < 10; i++)
+      {
+        int res = 0;
+        #pragma omp taskgraph
+        {
+          #pragma omp taskloop reduction(+: res) num_tasks(10)
+          {
+            for (int j = 0; j < 100; j++) {
+              res += arr[j];
+            }
+          }
+          foo();
+        }
+        fprintf(stderr, "reduction result: %d\n", res);
+      }
+    }
+  }
+  return 0;
+}
+
+// CHECK: base result: 4950
+// CHECK-DAG: task outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: task outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: task outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: task outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: task outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: task outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: task outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: task outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: task outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: task outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_23.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_23.cpp
new file mode 100644
index 0000000000000..d5ed9cba60cda
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_23.cpp
@@ -0,0 +1,103 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+#include <cstdio>
+
+void foo() {
+  fprintf(stderr, "called function foo\n");
+#pragma omp taskloop replayable num_tasks(4)
+  {
+    for (int i = 0; i < 4; i++)
+      fprintf(stderr, "taskloop iter %d outside lexical taskgraph\n", i);
+  }
+}
+
+int main()
+{
+  int arr[100];
+
+  int res = 0;
+  for (int i = 0; i < 100; i++) {
+    arr[i] = i;
+    res += i;
+  }
+  fprintf(stderr, "base result: %d\n", res);
+
+  #pragma omp parallel
+  {
+    #pragma omp single
+    {
+      for (int i = 0; i < 10; i++)
+      {
+        int res = 0;
+        #pragma omp taskgraph
+        {
+          #pragma omp taskloop reduction(+: res) num_tasks(10)
+          {
+            for (int j = 0; j < 100; j++) {
+              res += arr[j];
+            }
+          }
+          foo();
+        }
+        fprintf(stderr, "reduction result: %d\n", res);
+      }
+    }
+  }
+  return 0;
+}
+
+// CHECK: base result: 4950
+// CHECK-NEXT: called function foo
+// CHECK-DAG: taskloop iter 0 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 1 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 2 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 3 outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: taskloop iter 0 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 1 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 2 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 3 outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: taskloop iter 0 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 1 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 2 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 3 outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: taskloop iter 0 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 1 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 2 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 3 outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: taskloop iter 0 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 1 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 2 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 3 outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: taskloop iter 0 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 1 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 2 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 3 outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: taskloop iter 0 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 1 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 2 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 3 outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: taskloop iter 0 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 1 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 2 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 3 outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: taskloop iter 0 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 1 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 2 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 3 outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
+// CHECK-DAG: taskloop iter 0 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 1 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 2 outside lexical taskgraph
+// CHECK-DAG: taskloop iter 3 outside lexical taskgraph
+// CHECK-DAG: reduction result: 4950
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_24.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_24.cpp
new file mode 100644
index 0000000000000..cb6878c82fe2f
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_24.cpp
@@ -0,0 +1,80 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+#include <cassert>
+
+int global_dep;
+
+void foo() {
+#pragma omp taskwait replayable(1) depend(in: global_dep)
+}
+
+int main()
+{
+  int arr[100];
+
+  int res = 0;
+  for (int i = 0; i < 100; i++) {
+    arr[i] = i;
+    res += i;
+  }
+
+  assert(res == 4950);
+
+  #pragma omp parallel
+  {
+    #pragma omp single
+    {
+      for (int i = 0; i < 10; i++)
+      {
+        int res = 0;
+        #pragma omp taskgraph
+        {
+          #pragma omp taskloop reduction(+: res) num_tasks(10)
+          {
+            for (int j = 0; j < 100; j++) {
+              res += arr[j];
+            }
+          }
+          #pragma omp task depend(out: global_dep)
+          { }
+          foo();
+        }
+        assert(res == 4950);
+      }
+    }
+  }
+  return 0;
+}
+
+// CHECK:      Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: sequential {
+// CHECK-NEXT:   sequential {
+// CHECK-NEXT:     parallel {
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:     }
+// CHECK-NEXT:     wait: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   }
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   wait: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_25.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_25.cpp
new file mode 100644
index 0000000000000..893f51bc1e0fb
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_25.cpp
@@ -0,0 +1,89 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+#include <cstdio>
+
+int global_dep;
+
+void foo() {
+  fprintf(stderr, "called function foo\n");
+#pragma omp task replayable(1) depend(in: global_dep)
+  {
+    fprintf(stderr, "out-of-line task created from within taskloop\n");
+  }
+}
+
+int main()
+{
+  int arr[100];
+
+  int res = 0;
+  for (int i = 0; i < 4; i++) {
+    arr[i] = i;
+    res += i;
+  }
+  fprintf(stderr, "base result: %d\n", res);
+
+  #pragma omp parallel
+  {
+    #pragma omp single
+    {
+      for (int i = 0; i < 4; i++)
+      {
+        int res = 0;
+        #pragma omp taskgraph
+        {
+          #pragma omp taskloop reduction(+: res) num_tasks(4)
+          {
+            for (int j = 0; j < 4; j++) {
+              res += arr[j];
+              foo();
+            }
+          }
+        }
+        fprintf(stderr, "reduction result: %d\n", res);
+      }
+    }
+  }
+  return 0;
+}
+
+// CHECK: base result: 6
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: reduction result: 6
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: reduction result: 6
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: reduction result: 6
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: called function foo
+// CHECK-DAG: out-of-line task created from within taskloop
+// CHECK-DAG: reduction result: 6
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_26.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_26.cpp
new file mode 100644
index 0000000000000..d98b4b60c485e
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_26.cpp
@@ -0,0 +1,61 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+#include <cstdio>
+
+int main()
+{
+  int arr[100];
+  int arr2[100];
+
+  int res = 0, res2 = 0;
+  for (int i = 0; i < 10; i++) {
+    arr[i] = i;
+    arr2[i] = 3 + i * 2;
+    res += i;
+    res2 += 3 + i * 2;
+  }
+  fprintf(stderr, "base results: %d, %d\n", res, res2);
+
+  #pragma omp parallel
+  {
+    #pragma omp single
+    {
+      for (int i = 0; i < 10; i++)
+      {
+        int res = 0, res2 = 0;
+        #pragma omp taskgraph
+        {
+          #pragma omp taskloop reduction(+: res) num_tasks(10)
+          {
+            for (int j = 0; j < 10; j++) {
+              res += arr[j];
+            }
+          }
+          #pragma omp taskloop reduction(+: res2) num_tasks(10)
+          {
+            for (int j = 0; j < 10; j++) {
+              res2 += arr2[j];
+            }
+          }
+        }
+        fprintf(stderr, "reduction results: %d, %d\n", res, res2);
+      }
+    }
+  }
+  return 0;
+}
+
+// CHECK: base results: 45, 120
+// CHECK-NEXT: reduction results: 45, 120
+// CHECK-NEXT: reduction results: 45, 120
+// CHECK-NEXT: reduction results: 45, 120
+// CHECK-NEXT: reduction results: 45, 120
+// CHECK-NEXT: reduction results: 45, 120
+// CHECK-NEXT: reduction results: 45, 120
+// CHECK-NEXT: reduction results: 45, 120
+// CHECK-NEXT: reduction results: 45, 120
+// CHECK-NEXT: reduction results: 45, 120
+// CHECK-NEXT: reduction results: 45, 120
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_27.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_27.cpp
new file mode 100644
index 0000000000000..fd2323c21fe5f
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_27.cpp
@@ -0,0 +1,63 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+#include <cstdio>
+
+int main()
+{
+  int arr[100];
+  int arr2[100];
+
+  int res = 0, res2 = 0;
+  for (int i = 0; i < 10; i++) {
+    arr[i] = i;
+    res += i;
+  }
+  for (int i = 0; i < 10; i++) {
+    arr2[i] = 3 + i * 2;
+    res2 += res * (3 + i * 2);
+  }
+  fprintf(stderr, "base results: %d, %d\n", res, res2);
+
+  #pragma omp parallel
+  {
+    #pragma omp single
+    {
+      for (int i = 0; i < 10; i++)
+      {
+        int res = 0, res2 = 0;
+        #pragma omp taskgraph
+        {
+          #pragma omp taskloop reduction(+: res) num_tasks(10)
+          {
+            for (int j = 0; j < 10; j++) {
+              res += arr[j];
+            }
+          }
+          #pragma omp taskloop reduction(+: res2) num_tasks(10)
+          {
+            for (int j = 0; j < 10; j++) {
+              res2 += res * arr2[j];
+            }
+          }
+        }
+        fprintf(stderr, "reduction results: %d, %d\n", res, res2);
+      }
+    }
+  }
+  return 0;
+}
+
+// CHECK: base results: 45, 5400
+// CHECK-NEXT: reduction results: 45, 5400
+// CHECK-NEXT: reduction results: 45, 5400
+// CHECK-NEXT: reduction results: 45, 5400
+// CHECK-NEXT: reduction results: 45, 5400
+// CHECK-NEXT: reduction results: 45, 5400
+// CHECK-NEXT: reduction results: 45, 5400
+// CHECK-NEXT: reduction results: 45, 5400
+// CHECK-NEXT: reduction results: 45, 5400
+// CHECK-NEXT: reduction results: 45, 5400
+// CHECK-NEXT: reduction results: 45, 5400
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_3.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_3.cpp
new file mode 100644
index 0000000000000..196a705e516f0
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_3.cpp
@@ -0,0 +1,62 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+  int deps[6];
+  #pragma omp parallel
+  {
+    #pragma omp single
+    {
+      for (int i = 0; i < 2; i++)
+      {
+        #pragma omp taskgraph
+        {
+          #pragma omp task depend(out: deps[0], deps[1])
+          { }
+          #pragma omp task depend(out: deps[2], deps[3])
+          { }
+          #pragma omp task depend(inout: deps[0])
+          { }
+          #pragma omp task depend(inout: deps[1])
+          { }
+          #pragma omp task depend(inout: deps[2])
+          { }
+          #pragma omp task depend(inout: deps[3])
+          { }
+          #pragma omp task depend(in: deps[0], deps[1], deps[2], deps[3])
+          { }
+          #pragma omp task depend(in: deps[1], deps[2]) depend(out: deps[5])
+          { }
+          #pragma omp task depend(in: deps[5])
+          { }
+        }
+      }
+    }
+  }
+  return 0;
+}
+
+// This dependence graph is irreducible (it has no series-parallel
+// decomposition), so the residual tangle is carved into a single
+// TASKGRAPH_REGION_IRREDUCIBLE container whose children carry explicit intra
+// edges.  The two tasks that form a clean producer/consumer chain on deps[5]
+// (the depend(out: deps[5]) and depend(in: deps[5]) tasks) still reduce to a
+// nested sequential.
+// CHECK:      Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: irreducible {
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   sequential {
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   }
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_4.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_4.cpp
new file mode 100644
index 0000000000000..d86633ae51e25
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_4.cpp
@@ -0,0 +1,59 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+  int deps[4];
+  #pragma omp parallel
+  {
+    #pragma omp single
+    {
+      for (int i = 0; i < 2; i++)
+      {
+        #pragma omp taskgraph
+        {
+          #pragma omp task depend(out: deps[0], deps[1])
+          { }
+          #pragma omp task depend(out: deps[2], deps[3])
+          { }
+          #pragma omp task depend(inout: deps[0])
+          { }
+          #pragma omp task depend(inout: deps[1])
+          { }
+          #pragma omp task depend(inout: deps[2])
+          { }
+          #pragma omp task depend(inout: deps[3])
+          { }
+          #pragma omp task depend(in: deps[0], deps[2], deps[3])
+          { }
+          #pragma omp task depend(in: deps[0], deps[1])
+          { }
+          #pragma omp task depend(in: deps[3])
+          { }
+        }
+      }
+    }
+  }
+  return 0;
+}
+
+// This dependence graph is irreducible (it has no series-parallel
+// decomposition), so the whole residual tangle is carved into a single
+// TASKGRAPH_REGION_IRREDUCIBLE container whose children (the nine task arms)
+// carry explicit intra edges rather than being nested into parallel/sequential
+// regions.
+// CHECK:      Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: irreducible {
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_5.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_5.cpp
new file mode 100644
index 0000000000000..18bc54a164b4c
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_5.cpp
@@ -0,0 +1,63 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+  int deps[4];
+  #pragma omp parallel
+  {
+    #pragma omp single
+    {
+      for (int i = 0; i < 2; i++)
+      {
+        #pragma omp taskgraph
+        {
+          #pragma omp task depend(out: deps[0], deps[2])
+          { }
+          #pragma omp task depend(out: deps[1], deps[3])
+          { }
+          #pragma omp task depend(inoutset: deps[0], deps[1])
+          { }
+          #pragma omp task depend(inoutset: deps[0], deps[1])
+          { }
+          #pragma omp task depend(inoutset: deps[2], deps[3])
+          { }
+          #pragma omp task depend(inoutset: deps[2], deps[3])
+          { }
+          #pragma omp task depend(in: deps[0], deps[1])
+          { }
+          #pragma omp task depend(in: deps[2], deps[3])
+          { }
+        }
+      }
+    }
+  }
+  return 0;
+}
+
+// CHECK:      Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: sequential {
+// CHECK-NEXT:   parallel {
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   }
+// CHECK-NEXT:   parallel {
+// CHECK-NEXT:     sequential {
+// CHECK-NEXT:       parallel {
+// CHECK-NEXT:         node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:         node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:       }
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:     }
+// CHECK-NEXT:     sequential {
+// CHECK-NEXT:       parallel {
+// CHECK-NEXT:         node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:         node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:       }
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:     }
+// CHECK-NEXT:   }
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_6.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_6.cpp
new file mode 100644
index 0000000000000..c7190eae0d489
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_6.cpp
@@ -0,0 +1,59 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+  int deps[4];
+  #pragma omp parallel
+  {
+    #pragma omp single
+    {
+      for (int i = 0; i < 2; i++)
+      {
+        #pragma omp taskgraph
+        {
+          #pragma omp task depend(out: deps[1], deps[3])
+          { }
+          #pragma omp task depend(out: deps[0], deps[2])
+          { }
+          #pragma omp task depend(inoutset: deps[0], deps[1])
+          { }
+          #pragma omp task depend(inoutset: deps[0], deps[1])
+          { }
+          #pragma omp task depend(inoutset: deps[2], deps[3])
+          { }
+          #pragma omp task depend(inoutset: deps[2], deps[3])
+          { }
+          #pragma omp task depend(in: deps[0], deps[2])
+          { }
+          #pragma omp task depend(in: deps[0], deps[2])
+          { }
+        }
+      }
+    }
+  }
+  return 0;
+}
+
+// CHECK:      Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: sequential {
+// CHECK-NEXT:   parallel {
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   }
+// CHECK-NEXT:   sequential {
+// CHECK-NEXT:     parallel {
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:     }
+// CHECK-NEXT:     parallel {
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:     }
+// CHECK-NEXT:   }
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_7.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_7.cpp
new file mode 100644
index 0000000000000..38f60b16314d6
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_7.cpp
@@ -0,0 +1,59 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+  int deps[4];
+  #pragma omp parallel
+  {
+    #pragma omp single
+    {
+      for (int i = 0; i < 2; i++)
+      {
+        #pragma omp taskgraph
+        {
+          #pragma omp task depend(out: deps[0], deps[1])
+          { }
+          #pragma omp task depend(out: deps[2], deps[3])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[0], deps[1])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[0], deps[1])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[2], deps[3])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[2], deps[3])
+          { }
+          #pragma omp task depend(in: deps[0], deps[1])
+          { }
+          #pragma omp task depend(in: deps[2], deps[3])
+          { }
+        }
+      }
+    }
+  }
+  return 0;
+}
+
+// CHECK:      Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: parallel {
+// CHECK-NEXT:   sequential {
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:     exclusive {
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:     }
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   }
+// CHECK-NEXT:   sequential {
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:     exclusive {
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:       node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:     }
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   }
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_8.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_8.cpp
new file mode 100644
index 0000000000000..c723212290a16
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_8.cpp
@@ -0,0 +1,39 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+  int deps[2];
+  #pragma omp parallel
+  {
+    #pragma omp single
+    {
+      for (int i = 0; i < 2; i++)
+      {
+        #pragma omp taskgraph
+        {
+          #pragma omp task depend(out: deps[0], deps[1])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[0], deps[1])
+          { }
+          #pragma omp task depend(in: deps[0], deps[1])
+          { }
+        }
+      }
+    }
+  }
+  return 0;
+}
+
+// FIXME: This isn't perfect -- we don't really need to keep the mutexes in
+// this case.
+
+// CHECK:      Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: sequential {
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0x3]
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}
diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_9.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_9.cpp
new file mode 100644
index 0000000000000..1c9e99337452b
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_9.cpp
@@ -0,0 +1,47 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env KMP_TASKGRAPH_TRACE=1 %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+int main()
+{
+  int deps[3];
+  #pragma omp parallel
+  {
+    #pragma omp single
+    {
+      for (int i = 0; i < 2; i++)
+      {
+        #pragma omp taskgraph
+        {
+          #pragma omp task depend(out: deps[0], deps[1])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[0], deps[1])
+          { }
+          #pragma omp task depend(in: deps[1]) depend(out: deps[2])
+          { }
+          #pragma omp task depend(mutexinoutset: deps[0], deps[1])
+          { }
+          #pragma omp task depend(in: deps[0], deps[1], deps[2])
+          { }
+        }
+      }
+    }
+  }
+  return 0;
+}
+
+// FIXME: This isn't perfect -- we don't really need to keep the mutexes in
+// this case.
+
+// CHECK:      Processed taskgraph 0x[[#%x,GRAPHPTR:]] (graph_id 0):
+// CHECK-NEXT: sequential {
+// CHECK-NEXT:   sequential {
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}} [sets: 0x3]
+// CHECK-NEXT:     node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT:   }
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}} [sets: 0x3]
+// CHECK-NEXT:   node: 0x{{[[:xdigit:]]+}}
+// CHECK-NEXT: }
+// CHECK-NEXT: Replay taskgraph 0x[[#GRAPHPTR]] from task 0x{{[[:xdigit:]]+}}

>From 1cc168d661b4ea53a7c3f3e40103a6f7d2b9efee Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Thu, 2 Jul 2026 11:02:04 -0500
Subject: [PATCH 12/26] [OpenMP] Runtime support for graph_id and graph_reset
 clauses

This patch adds the runtime support for graph_id and graph_reset
functionality, including freeing of old taskgraph records.

Lazy expiration of taskgraphs is partially implemented here, but won't be
wired up fully until concurrent/nowait taskgraph execution is implemented.

Pull Request: https://github.com/llvm/llvm-project/pull/216325
---
 openmp/runtime/src/kmp.h           |   4 +
 openmp/runtime/src/kmp_tasking.cpp | 144 ++++++++++++++++++++++++-----
 2 files changed, 126 insertions(+), 22 deletions(-)

diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index c2fdab3de64bf..dcf598434dd73 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -4446,6 +4446,10 @@ extern kmp_int32 __kmp_omp_task(kmp_int32 gtid, kmp_task_t *new_task,
 extern kmp_int32 __kmp_build_taskgraph(kmp_int32 gtid,
                                        kmp_taskdata_t *current_taskdata,
                                        kmp_taskgraph_record_t *taskgraph);
+extern void __kmp_region_deplist_free(kmp_info_t *thread,
+                                      kmp_taskgraph_region_dep_t *list);
+extern void __kmp_region_deplist_recycle(kmp_taskgraph_region_dep_t **recycled,
+                                         kmp_taskgraph_region_dep_t *list);
 // True if the user asked for tracing of taskgraph structure and replay via the
 // KMP_TASKGRAPH_TRACE environment variable.
 extern bool __kmp_taskgraph_trace();
diff --git a/openmp/runtime/src/kmp_tasking.cpp b/openmp/runtime/src/kmp_tasking.cpp
index 895248cb8c250..8b1396390ff8c 100644
--- a/openmp/runtime/src/kmp_tasking.cpp
+++ b/openmp/runtime/src/kmp_tasking.cpp
@@ -5846,22 +5846,87 @@ static kmp_taskgraph_record_t *__kmp_taskgraph_alloc(kmp_int32 gtid,
   kmp_taskgraph_record_t *new_rec =
       (kmp_taskgraph_record_t *)__kmp_fast_allocate(
           thread, sizeof(kmp_taskgraph_record_t));
-  new_rec->status = KMP_TDG_RECORDING;
-  new_rec->gtid = gtid;
-  new_rec->graph_id = graph_id;
   __kmp_init_lock(&new_rec->map_lock);
-  new_rec->record_map = nullptr;
-  new_rec->alloc_root = nullptr;
-  new_rec->recycled_deps = nullptr;
-  new_rec->num_tasks = 0;
-  new_rec->nodes_allocated = 0;
-  new_rec->num_mutexes = 0;
-  new_rec->exec_descrs = nullptr;
-  new_rec->exec_descr_size = 0;
-  new_rec->next = nullptr;
+  __kmp_taskgraph_reset(new_rec, gtid, graph_id);
   return new_rec;
 }
 
+static void
+__kmp_taskgraph_free_region_metadata(kmp_info_t *thread,
+                                     kmp_taskgraph_region_t *region) {
+  if (region->reduce_input) {
+    __kmp_fast_free(thread, region->reduce_input->reduce_data);
+    __kmp_fast_free(thread, region->reduce_input);
+  }
+  if (region->mutexset) {
+    __kmp_fast_free(thread, region->mutexset);
+  }
+  switch (region->type) {
+  case TASKGRAPH_REGION_ENTRY:
+  case TASKGRAPH_REGION_EXIT:
+  case TASKGRAPH_REGION_WAIT:
+  case TASKGRAPH_REGION_NODE:
+    break;
+  case TASKGRAPH_REGION_PARALLEL:
+  case TASKGRAPH_REGION_SEQUENTIAL:
+  case TASKGRAPH_REGION_EXCLUSIVE:
+  case TASKGRAPH_REGION_IRREDUCIBLE: {
+    for (int k = 0; k < region->inner.num_children; k++) {
+      __kmp_taskgraph_free_region_metadata(thread, region->inner.children[k]);
+    }
+    break;
+  }
+  default:
+    assert(false && "unreachable");
+  }
+}
+
+static void __kmp_taskgraph_free(kmp_int32 gtid, kmp_taskgraph_record_t *rec,
+                                 bool keep_rec = false) {
+  kmp_info_t *thread = __kmp_threads[gtid];
+
+  __kmp_taskgraph_free_region_metadata(thread, rec->root);
+
+  for (size_t task = 0; task < rec->num_tasks; task++) {
+    kmp_taskdata *taskdata = KMP_TASK_TO_TASKDATA(rec->record_map[task].task);
+    // Setting this here keeps an assertion in __kmp_free_task happy: the
+    // clone may never have been replayed, in which case 'complete' will be
+    // zero here, as initialized.
+    taskdata->td_flags.complete = 1;
+    // We never decrement td_allocated_child_tasks for taskgraph tasks.  This
+    // keeps another assertion happy in __kmp_free_task.
+    KMP_ATOMIC_ST_RLX(&taskdata->td_allocated_child_tasks, 0);
+    __kmp_free_task(gtid, taskdata, thread);
+  }
+  __kmp_thread_free(thread, rec->record_map);
+
+  kmp_taskgraph_region_t *region = rec->alloc_root;
+  while (region) {
+    kmp_taskgraph_region_t *next_region = region->alloc_chain;
+    // Children of a carved IRREDUCIBLE region retain their intra-kernel edge
+    // lists past build (the exec-DAG builder reads them on first replay); for
+    // all other regions these lists were already freed in
+    // __kmp_build_taskgraph.
+    if (keep_rec) {
+      __kmp_region_deplist_recycle(&rec->recycled_deps, region->predecessors);
+      __kmp_region_deplist_recycle(&rec->recycled_deps, region->successors);
+    } else {
+      __kmp_region_deplist_free(thread, region->predecessors);
+      __kmp_region_deplist_free(thread, region->successors);
+    }
+    __kmp_fast_free(thread, region);
+    region = next_region;
+  }
+
+  if (rec->exec_descrs)
+    __kmp_thread_free(thread, rec->exec_descrs);
+
+  if (!keep_rec) {
+    __kmp_destroy_lock(&rec->map_lock);
+    __kmp_fast_free(thread, rec);
+  }
+}
+
 static kmp_taskgraph_header_t *__kmp_taskgraph_header_alloc(kmp_int32 gtid) {
   kmp_taskgraph_header_t *new_hdr =
       (kmp_taskgraph_header_t *)__kmp_allocate(sizeof(kmp_taskgraph_header_t));
@@ -5893,9 +5958,36 @@ static kmp_task_t *__kmp_taskgraph_clone_task(kmp_info_t *thread,
   KMP_MEMCPY(copy_td, taskdata, shareds_offset + sizeof_shareds);
   // Tasks cloned for a taskgraph always have this field set.
   copy_td->owning_taskgraph = taskgraph;
+  KMP_ATOMIC_ST_RLX(&copy_td->td_incomplete_child_tasks, 0);
   return KMP_TASKDATA_TO_TASK(copy_td);
 }
 
+// Go through list of taskgraph records and free any that are no longer being
+// executed.  For now, this just frees everything in the list immediately: when
+// multiple concurrent playbacks are implemented, it should only free records
+// with zero usage counts.
+// When available, a kmp_taskgraph_record_t structure is returned for reuse by
+// a subsequent taskgraph recording.
+
+static kmp_taskgraph_record_t *
+__kmp_expire_taskgraph_records(kmp_int32 gtid,
+                               kmp_taskgraph_record_t **expiring_p) {
+  kmp_taskgraph_record_t *record = nullptr;
+
+  while (*expiring_p) {
+    kmp_taskgraph_record_t *expiring = *expiring_p;
+    if (!record) {
+      record = expiring;
+      __kmp_taskgraph_free(gtid, record, /*keep_rec=*/true);
+    } else {
+      __kmp_taskgraph_free(gtid, record, /*keep_rec=*/false);
+    }
+    *expiring_p = expiring->next;
+  }
+
+  return record;
+}
+
 // __kmpc_taskgraph: record or replay taskgraph
 // loc_ref:     Location of TDG, not used yet
 // gtid:        Global Thread ID of the encountering thread
@@ -5921,14 +6013,6 @@ void __kmpc_taskgraph(ident_t *loc_ref, kmp_int32 gtid,
 
   taskgroup = thread->th.th_current_task->td_taskgroup;
 
-  // FIXME: Implement graph_id and graph_reset functionality.  For graph_id, we
-  // will form a singly-linked list of task records chained through their
-  // "next" pointers (per taskgraph construct handle).  Thread safety and
-  // locking need careful consideration.  We could use a "list header" node
-  // consisting of a lock and a pointer to
-  // the list proper, perhaps.  Ideally we'd want to avoid locking/unlocking in
-  // the common case (replay).
-
   if (!header) {
     header = __kmp_taskgraph_header_alloc(gtid);
     // Another thread may have allocated the header at the same time.  Grab
@@ -5943,9 +6027,25 @@ void __kmpc_taskgraph(ident_t *loc_ref, kmp_int32 gtid,
   }
 
   __kmp_acquire_lock(&header->header_lock, gtid);
-  record = header->first;
+  for (record_p = &header->first; *record_p; record_p = &((*record_p)->next))
+    if ((*record_p)->graph_id == graph_id)
+      break;
+  record = *record_p;
+  kmp_taskgraph_record_t *reuse_record = nullptr;
+  if (record && graph_reset) {
+    // Move the existing record to the header's expiring list
+    *record_p = record->next;
+    record->next = header->expiring;
+    header->expiring = record;
+    reuse_record = __kmp_expire_taskgraph_records(gtid, &header->expiring);
+    record = nullptr;
+  }
   if (!record) {
-    record = __kmp_taskgraph_alloc(gtid, graph_id);
+    if (reuse_record) {
+      record = reuse_record;
+      __kmp_taskgraph_reset(record, gtid, graph_id);
+    } else
+      record = __kmp_taskgraph_alloc(gtid, graph_id);
     // We record 'nogroup' here.  We always create a group for recording the
     // taskgraph, but we could avoid doing so for replay.  That's not done
     // yet though.

>From 6a5d207c059b08751b3283549fc58690ed5f8698 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Thu, 30 Apr 2026 06:34:33 -0500
Subject: [PATCH 13/26] [OpenMP] New runtime tests for graph_id/graph_reset
 clauses (taskgraph)

This patch contains several new tests for graph_id/graph_reset functionality.

Assisted-By: Codex/gpt-5.3 and Claude Opus 4.7

Pull Request: https://github.com/llvm/llvm-project/pull/195075
---
 ...taskgraph_deps_irreducible_graph_reset.cpp | 50 ++++++++++++++++
 .../test/taskgraph/taskgraph_graph_id.cpp     | 56 ++++++++++++++++++
 .../taskgraph_graph_id_and_reset.cpp          | 57 +++++++++++++++++++
 .../test/taskgraph/taskgraph_graph_reset.cpp  | 57 +++++++++++++++++++
 4 files changed, 220 insertions(+)
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deps_irreducible_graph_reset.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_graph_id.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_graph_id_and_reset.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_graph_reset.cpp

diff --git a/openmp/runtime/test/taskgraph/taskgraph_deps_irreducible_graph_reset.cpp b/openmp/runtime/test/taskgraph/taskgraph_deps_irreducible_graph_reset.cpp
new file mode 100644
index 0000000000000..0e94a1644cd99
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deps_irreducible_graph_reset.cpp
@@ -0,0 +1,50 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && %libomp-run 2>&1 | FileCheck %s
+
+// REQUIRES: omp_taskgraph_experimental
+
+#include <atomic>
+#include <cstdio>
+
+// This builds a non-series-parallel dependency shape:
+// A->B, A->C, B->D, C->D, C->E.
+
+int main() {
+  int deps[4] = {0, 0, 0, 0};
+
+  for (int iter = 0; iter < 1000; ++iter) {
+    std::atomic<int> sum{0};
+
+#pragma omp parallel num_threads(4)
+    {
+#pragma omp single
+      {
+#pragma omp taskgraph graph_id(123) graph_reset(1)
+        {
+#pragma omp task depend(out : deps[0], deps[1])
+          { sum.fetch_add(1, std::memory_order_relaxed); } // A
+#pragma omp task depend(inout : deps[0])
+          { sum.fetch_add(4, std::memory_order_relaxed); } // B
+#pragma omp task depend(inout : deps[1])
+          { sum.fetch_add(8, std::memory_order_relaxed); } // C
+#pragma omp task depend(in : deps[0], deps[1], deps[2], deps[3])
+          { sum.fetch_add(64, std::memory_order_relaxed); } // D
+#pragma omp task depend(in : deps[1], deps[2])
+          { sum.fetch_add(128, std::memory_order_relaxed); } // E
+        }
+      }
+    }
+
+    const int actual = sum.load(std::memory_order_relaxed);
+    if (actual != 205) {
+      std::fprintf(stderr, "FAIL iter=%d expected=205 actual=%d\n", iter,
+                   actual);
+      return 1;
+    }
+  }
+
+  std::fprintf(stderr, "PASS\n");
+  return 0;
+}
+
+// CHECK: PASS
diff --git a/openmp/runtime/test/taskgraph/taskgraph_graph_id.cpp b/openmp/runtime/test/taskgraph/taskgraph_graph_id.cpp
new file mode 100644
index 0000000000000..f536c2a03c9b1
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_graph_id.cpp
@@ -0,0 +1,56 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <atomic>
+#include <cstdio>
+
+int main() {
+  constexpr int NumIters = 12;
+  constexpr int WorkA = 16;
+  constexpr int WorkB = 24;
+
+  std::atomic<int> total{0};
+
+#pragma omp parallel num_threads(4)
+  {
+#pragma omp single
+    {
+      for (int iter = 0; iter < NumIters; ++iter) {
+        const int gid = iter & 1;
+
+#pragma omp taskgraph graph_id(gid)
+        {
+          if (gid == 0) {
+            for (int i = 0; i < WorkA; ++i) {
+#pragma omp task
+              total.fetch_add(1, std::memory_order_relaxed);
+            }
+          } else {
+            for (int i = 0; i < WorkB; ++i) {
+#pragma omp task
+              total.fetch_add(1, std::memory_order_relaxed);
+            }
+          }
+        }
+      }
+    }
+  }
+
+  const int NumEven = (NumIters + 1) / 2;
+  const int NumOdd = NumIters / 2;
+  const int Expected = NumEven * WorkA + NumOdd * WorkB;
+  const int Actual = total.load(std::memory_order_relaxed);
+
+  if (Actual != Expected) {
+    std::fprintf(stderr, "FAIL graph_id total=%d expected=%d\n", Actual,
+                 Expected);
+    return 1;
+  }
+
+  std::fprintf(stderr, "PASS graph_id total=%d\n", Actual);
+  return 0;
+}
+
+// CHECK: PASS graph_id total=
diff --git a/openmp/runtime/test/taskgraph/taskgraph_graph_id_and_reset.cpp b/openmp/runtime/test/taskgraph/taskgraph_graph_id_and_reset.cpp
new file mode 100644
index 0000000000000..cadc527102ee7
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_graph_id_and_reset.cpp
@@ -0,0 +1,57 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <atomic>
+#include <cstdio>
+
+// Exercise graph_id and graph_reset together on a single taskgraph construct.
+//
+// graph_id is per-taskgraph: the runtime keeps one record per (construct,
+// graph_id) pair.  Here we feed the same construct two different graph_ids
+// alternately (0, 1, 0, 1, ...) so two records coexist for the same
+// directive.  graph_reset is then driven by a separate condition: every
+// fourth visit, we ask the runtime to re-record the graph_id we are about
+// to use, forcing it through the expiry-and-re-record path while the other
+// graph_id's record stays intact and continues to be replayed.
+
+int main() {
+  constexpr int NumIters = 40;
+  constexpr int TasksPerVisit = 6;
+
+  std::atomic<int> total{0};
+
+#pragma omp parallel num_threads(4)
+  {
+#pragma omp single
+    {
+      for (int iter = 0; iter < NumIters; ++iter) {
+        const int gid = iter & 1; // alternate two records
+        const bool reset = (iter % 4) == 3; // periodically force re-record
+
+#pragma omp taskgraph graph_id(gid) graph_reset(reset)
+        {
+          for (int i = 0; i < TasksPerVisit; ++i) {
+#pragma omp task
+            total.fetch_add(1, std::memory_order_relaxed);
+          }
+        }
+      }
+    }
+  }
+
+  const int Expected = NumIters * TasksPerVisit;
+  const int Actual = total.load(std::memory_order_relaxed);
+
+  if (Actual != Expected) {
+    std::fprintf(stderr, "FAIL graph_id+reset total=%d expected=%d\n", Actual,
+                 Expected);
+    return 1;
+  }
+
+  std::fprintf(stderr, "PASS graph_id+reset total=%d\n", Actual);
+  return 0;
+}
+
+// CHECK: PASS graph_id+reset total=
diff --git a/openmp/runtime/test/taskgraph/taskgraph_graph_reset.cpp b/openmp/runtime/test/taskgraph/taskgraph_graph_reset.cpp
new file mode 100644
index 0000000000000..d41592f0ac243
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_graph_reset.cpp
@@ -0,0 +1,57 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <atomic>
+#include <cstdio>
+
+int main() {
+  constexpr int NumIters = 10;
+  constexpr int GraphId = 7;
+
+  std::atomic<int> total{0};
+
+#pragma omp parallel num_threads(4)
+  {
+#pragma omp single
+    {
+      for (int iter = 0; iter < NumIters; ++iter) {
+        const bool odd = (iter & 1) != 0;
+
+        // Alternate the taskgraph shape every iteration. graph_reset(1)
+        // requires the runtime to re-record rather than replay stale shape.
+#pragma omp taskgraph graph_id(GraphId) graph_reset(1)
+        {
+          if (odd) {
+            for (int i = 0; i < 40; ++i) {
+#pragma omp task
+              total.fetch_add(2, std::memory_order_relaxed);
+            }
+          } else {
+            for (int i = 0; i < 10; ++i) {
+#pragma omp task
+              total.fetch_add(1, std::memory_order_relaxed);
+            }
+          }
+        }
+      }
+    }
+  }
+
+  const int NumOdd = NumIters / 2;
+  const int NumEven = NumIters - NumOdd;
+  const int Expected = NumOdd * 40 * 2 + NumEven * 10;
+  const int Actual = total.load(std::memory_order_relaxed);
+
+  if (Actual != Expected) {
+    std::fprintf(stderr, "FAIL graph_reset total=%d expected=%d\n", Actual,
+                 Expected);
+    return 1;
+  }
+
+  std::fprintf(stderr, "PASS graph_reset total=%d\n", Actual);
+  return 0;
+}
+
+// CHECK: PASS graph_reset total=

>From 21d33b58ecc0678cecd58973602050bfe888ded3 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Thu, 30 Apr 2026 07:02:52 -0500
Subject: [PATCH 14/26] [OpenMP] Add sanity check for concurrent taskgraph
 reset

This patch adds a sanity check for concurrent taskgraph re-recording: the
case where two threads might simultaneously hit a taskgraph region with
the same graph_id and a non-zero graph_reset clause.  If the situation
is encountered, a fatal runtime error is raised.

Pull Request: https://github.com/llvm/llvm-project/pull/195076
---
 openmp/runtime/src/i18n/en_US.txt             |  1 +
 openmp/runtime/src/kmp_tasking.cpp            | 17 +++++++
 ...taskgraph_graph_id_concurrent_rerecord.cpp | 50 +++++++++++++++++++
 3 files changed, 68 insertions(+)
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_graph_id_concurrent_rerecord.cpp

diff --git a/openmp/runtime/src/i18n/en_US.txt b/openmp/runtime/src/i18n/en_US.txt
index b3699df6db65d..46f5e794e5a27 100644
--- a/openmp/runtime/src/i18n/en_US.txt
+++ b/openmp/runtime/src/i18n/en_US.txt
@@ -484,6 +484,7 @@ AffIgnoringNonHybrid         "%1$s ignored: This machine is not a hybrid archite
 AffIgnoringNotAvailable      "%1$s ignored: %2$s is not available. Using \"%3$s\" instead."
 SetNumTeamsInParOrTeamsRegion "%1$s() called inside a parallel or teams region; call ignored."
 SetTeamsThreadLimitInParOrTeamsRegion "%1$s() called inside a parallel or teams region; call ignored."
+OmpTaskgraphConcurrentRecord "%1$s: multiple threads attempting to re-record taskgraph concurrently for graph_id %2$d"
 
 # --------------------------------------------------------------------------------------------------
 -*- HINTS -*-
diff --git a/openmp/runtime/src/kmp_tasking.cpp b/openmp/runtime/src/kmp_tasking.cpp
index 8b1396390ff8c..3b1b6d1fe5998 100644
--- a/openmp/runtime/src/kmp_tasking.cpp
+++ b/openmp/runtime/src/kmp_tasking.cpp
@@ -6033,6 +6033,23 @@ void __kmpc_taskgraph(ident_t *loc_ref, kmp_int32 gtid,
   record = *record_p;
   kmp_taskgraph_record_t *reuse_record = nullptr;
   if (record && graph_reset) {
+    kmp_taskgraph_status_t old_status = KMP_ATOMIC_LD_ACQ(&record->status);
+    // Sanity check: if the graph is not ready, it means another thread is
+    // already performing a record operation for this taskgraph/graph_id.
+    // That's likely a bug, so fail with an error.
+    if (old_status != KMP_TDG_READY) {
+      char *src_loc;
+      if (loc_ref && loc_ref->psource) {
+        kmp_str_loc_t str_loc = __kmp_str_loc_init(loc_ref->psource, false);
+        src_loc = __kmp_str_format("%s:%d:%d", str_loc.file, str_loc.line,
+                                   str_loc.col);
+        __kmp_str_loc_free(&str_loc);
+      } else {
+        src_loc = __kmp_str_format("unknown");
+      }
+      KMP_FATAL(OmpTaskgraphConcurrentRecord, src_loc, graph_id);
+      __kmp_str_free(&src_loc);
+    }
     // Move the existing record to the header's expiring list
     *record_p = record->next;
     record->next = header->expiring;
diff --git a/openmp/runtime/test/taskgraph/taskgraph_graph_id_concurrent_rerecord.cpp b/openmp/runtime/test/taskgraph/taskgraph_graph_id_concurrent_rerecord.cpp
new file mode 100644
index 0000000000000..385e1f7ed795d
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_graph_id_concurrent_rerecord.cpp
@@ -0,0 +1,50 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_DYNAMIC=FALSE %not --crash %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <atomic>
+#include <cstdio>
+#include <omp.h>
+
+static std::atomic<int> first_thread_inside{0};
+static std::atomic<int> release_first_thread{0};
+
+// Deterministically force two threads to hit the same lexical taskgraph
+// construct with the same graph_id while the first thread is still recording
+// it.  That guarantees the second thread sees an existing record in
+// KMP_TDG_RECORDING state and trips the runtime sanity check.
+static void enter_same_taskgraph(int tid) {
+#pragma omp taskgraph graph_id(17) graph_reset(1)
+  {
+    if (tid == 0) {
+      first_thread_inside.store(1, std::memory_order_release);
+      while (release_first_thread.load(std::memory_order_acquire) == 0) {
+      }
+    }
+  }
+}
+
+int main() {
+#pragma omp parallel num_threads(2)
+  {
+    const int tid = omp_get_thread_num();
+
+    if (tid == 1) {
+      while (first_thread_inside.load(std::memory_order_acquire) == 0) {
+      }
+    }
+
+    enter_same_taskgraph(tid);
+
+    if (tid == 1)
+      release_first_thread.store(1, std::memory_order_release);
+  }
+
+  std::fprintf(stderr, "UNEXPECTED SUCCESS\n");
+  return 0;
+}
+
+// CHECK: OMP: Error #{{[0-9]+}}: {{.*}}multiple threads attempting to re-record
+// CHECK-SAME: taskgraph concurrently for graph_id 17
+// CHECK-NOT: UNEXPECTED SUCCESS

>From a479822b7675d5e3fd3154711107e980e7593e97 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Tue, 26 May 2026 14:59:11 -0500
Subject: [PATCH 15/26] [OpenMP] Widen taskgraph graph_id to uintptr_t

The 'graph_id' clause introduced in OpenMP 6.0 lets the user attach an
identifying value to a taskgraph instance, and a potentially-useful
idiom is to pass a pointer expression so that each instance can be
specialized for the pointed-to data.  The corresponding runtime entry
point __kmpc_taskgraph and the kmp_taskgraph_record_t::graph_id field
were declared as 32-bit types, which silently truncates such pointer
values on LP64 targets.

Widen the field and the runtime ABI to uintptr_t, lower 'graph_id'
to CGM.IntPtrTy in emitTaskgraphCall, and mark the OMPRTL
descriptor as SizeTy so the call is typed consistently on 32- and
64-bit hosts.  Refresh the auto-generated CHECK line in
taskgraph_codegen.cpp accordingly.

Assisted-By: Claude Opus 4.7

Pull Request: https://github.com/llvm/llvm-project/pull/200403
---
 clang/lib/CodeGen/CGOpenMPRuntime.cpp          | 4 ++--
 clang/test/OpenMP/taskgraph_codegen.cpp        | 2 +-
 llvm/include/llvm/Frontend/OpenMP/OMPKinds.def | 2 +-
 openmp/runtime/src/kmp.h                       | 4 ++--
 openmp/runtime/src/kmp_tasking.cpp             | 2 +-
 5 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 3fe694b54a5e8..2d5bf9b20802b 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -2284,13 +2284,13 @@ void CGOpenMPRuntime::emitTaskgraphCall(CodeGenFunction &CGF,
     GraphReset = CGF.Builder.getInt32(0);
   }
 
-  llvm::Value *GraphId = CGF.Builder.getInt32(0);
+  llvm::Value *GraphId = llvm::ConstantInt::get(CGM.IntPtrTy, 0);
   const OMPGraphIdClause *GraphIdClause = D.getSingleClause<OMPGraphIdClause>();
   if (GraphIdClause) {
     const auto *E = GraphIdClause->getId();
     auto *GraphIdVal = CGF.EmitScalarExpr(E);
     GraphId =
-        CGF.Builder.CreateIntCast(GraphIdVal, CGM.Int32Ty, /*isSigned=*/false);
+        CGF.Builder.CreateIntCast(GraphIdVal, CGM.IntPtrTy, /*isSigned=*/false);
   }
 
   CodeGenFunction OutlinedCGF(CGM, /*suppressNewContext=*/true);
diff --git a/clang/test/OpenMP/taskgraph_codegen.cpp b/clang/test/OpenMP/taskgraph_codegen.cpp
index 3f661e6bfe3d5..e6e718c2cd6d5 100644
--- a/clang/test/OpenMP/taskgraph_codegen.cpp
+++ b/clang/test/OpenMP/taskgraph_codegen.cpp
@@ -22,7 +22,7 @@
 // CHECK-NEXT:    store ptr [[X]], ptr [[TMP1]], align 8
 // CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds nuw [[STRUCT_ANON]], ptr [[AGG_CAPTURED]], i32 0, i32 1
 // CHECK-NEXT:    store ptr [[Y]], ptr [[TMP2]], align 8
-// CHECK-NEXT:    call void @__kmpc_taskgraph(ptr @[[GLOB1]], i32 [[TMP0]], ptr @.omp.taskgraph.handle, i32 0, i32 0, i32 0, ptr @taskgraph.omp_outlined., ptr [[AGG_CAPTURED]])
+// CHECK-NEXT:    call void @__kmpc_taskgraph(ptr @[[GLOB1]], i32 [[TMP0]], ptr @.omp.taskgraph.handle, i64 0, i32 0, i32 0, ptr @taskgraph.omp_outlined., ptr [[AGG_CAPTURED]])
 // CHECK-NEXT:    ret i32 0
 //
 int main() {
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
index a825585900b96..9f2c331a092e7 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
@@ -357,7 +357,7 @@ __OMP_RTL(__kmpc_omp_task, false, Int32, IdentPtr, Int32,
           /* kmp_task_t */ VoidPtr)
 __OMP_RTL(__kmpc_end_taskgroup, false, Void, IdentPtr, Int32)
 __OMP_RTL(__kmpc_taskgroup, false, Void, IdentPtr, Int32)
-__OMP_RTL(__kmpc_taskgraph, false, Void, IdentPtr, Int32, VoidPtrPtr, Int32,
+__OMP_RTL(__kmpc_taskgraph, false, Void, IdentPtr, Int32, VoidPtrPtr, SizeTy,
           Int32, Int32, VoidPtr, VoidPtr)
 __OMP_RTL(__kmpc_taskgraph_task, false, Int32, IdentPtr, Int32, VoidPtr, Int32,
           SizeTy, VoidPtr, SizeTy, Int32, VoidPtr)
diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index dcf598434dd73..f0366e36af899 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -2756,7 +2756,7 @@ typedef struct kmp_taskgraph_region {
 typedef struct kmp_taskgraph_record {
   std::atomic<kmp_taskgraph_status_t> status = KMP_TDG_NONE;
   kmp_int32 gtid = 0;
-  kmp_int32 graph_id = 0;
+  uintptr_t graph_id = 0;
   // A lock that protects the record_map and num_tasks fields from being
   // modified by multiple threads.
   // For now, we also use this whilst the taskgraph is being replayed.
@@ -4525,7 +4525,7 @@ KMP_EXPORT void __kmpc_init_nest_lock_with_hint(ident_t *loc, kmp_int32 gtid,
 #if OMP_TASKGRAPH_EXPERIMENTAL
 KMP_EXPORT void __kmpc_taskgraph(ident_t *loc_ref, kmp_int32 gtid,
                                  std::atomic<void *> *tdg_handle,
-                                 kmp_uint32 graph_id, kmp_int32 graph_reset,
+                                 uintptr_t graph_id, kmp_int32 graph_reset,
                                  kmp_int32 nogroup, void (*entry)(void *),
                                  void *args);
 KMP_EXPORT kmp_uint32 __kmpc_taskgraph_task(
diff --git a/openmp/runtime/src/kmp_tasking.cpp b/openmp/runtime/src/kmp_tasking.cpp
index 3b1b6d1fe5998..2a04d9bb58812 100644
--- a/openmp/runtime/src/kmp_tasking.cpp
+++ b/openmp/runtime/src/kmp_tasking.cpp
@@ -6000,7 +6000,7 @@ __kmp_expire_taskgraph_records(kmp_int32 gtid,
 // entry:       Pointer to the entry function
 // args:        Pointer to the function arguments
 void __kmpc_taskgraph(ident_t *loc_ref, kmp_int32 gtid,
-                      std::atomic<void *> *tdg_handle, kmp_uint32 graph_id,
+                      std::atomic<void *> *tdg_handle, uintptr_t graph_id,
                       kmp_int32 graph_reset, kmp_int32 nogroup,
                       void (*entry)(void *), void *args) {
   kmp_taskgraph_header_t *header =

>From 8fb99c8c80297caa43587d56ae50118a6a9626dd Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Tue, 26 May 2026 15:11:50 -0500
Subject: [PATCH 16/26] [OpenMP] Relocate task shared-by-ref captures across
 taskgraph replay

OpenMP 6.0 lets a taskgraph region be recorded once and replayed many
times.  Each replay creates a fresh instance of the 'args' pointer
block passed to __kmpc_taskgraph (and may execute at a different stack
location, or even on a different stack), so by-reference captures inside a
recorded task must be re-pointed at the live host objects of the current
invocation; otherwise the recorded tasks would dereference stale memory
from the stack frame of the initial call to __kmpc_taskgraph.

This patch introduces the small infrastructure to do that and wires
it up for the explicit 'task' construct.  A subsequent patch
extends the same scheme to 'taskloop'.

On the compiler side (CGOpenMPRuntime.cpp), a new helper
emitTaskRelocationFunction emits a per-task thunk:

  void __omp_taskgraph_relocate.NN(kmp_task_t *task,
                                   void *outer_captures);

The thunk walks the task's captures and overwrites each entry of
task->shareds with the address of the corresponding field projected from
the freshly reconstructed outer pointer block.  Two classes of capture do
not need updating and are treated as no-ops by the thunk: captures that
correspond to a firstprivate list item (the body reads from the per-task
'.kmp_privates.t' snapshot, populated when the task is allocated and
-- for non-trivial types -- reset on each replay by the clone helper
introduced later), and captures of variables with static storage duration
(their address is link-time fixed).  Reductions of a local-stack variable
are intentionally not in this set: the taskred state is keyed on the
recording-time taskgroup hierarchy and is not yet usable on replay,
so we prefer to preserve today's relocate-returns-null / runtime-aborts
behaviour for that case so the limitation surfaces as a diagnostic.

emitTaskCall now emits such a thunk for each taskgraph-recorded task
and passes it as the new trailing argument of __kmpc_taskgraph_task.
The redundant 'shareds' parameter is dropped, since relocation now
provides the supported mechanism for refreshing that pointer.

On the runtime side (kmp.h, kmp_tasking.cpp, OMPKinds.def),
introduce a new typedef kmp_task_relocate_t and store the callback
on each recorded task in kmp_taskgraph_node_t::relocate, together
with the outer-record pointer captured at __kmpc_taskgraph entry in
kmp_taskgraph_record_t::taskgraph_args.  __kmp_omp_tg_task invokes
the callback on replay, and aborts with a new fatal diagnostic
(OmpTaskgraphBadCapture, i18n/en_US.txt) when a recorded task has a
non-null shareds payload but no relocation callback.  There is also a
fix for a pre-existing bug in __kmp_taskgraph_clone_task -- the cloned
task's shareds pointer was left referring to the original's payload --
which becomes observable as soon as the relocation thunk writes through
that pointer.

New libomp tests cover lexical and non-lexical shared captures,
pointer captures, non-trivial types, recursive recordings,
stack-depth differences across replays, and the saved/expired-
graph cases.

Assisted-By: Claude Opus 4.7

Pull Request: https://github.com/llvm/llvm-project/pull/200404
---
 clang/lib/CodeGen/CGOpenMPRuntime.cpp         | 184 +++++++++++++++++-
 .../include/llvm/Frontend/OpenMP/OMPKinds.def |   2 +-
 openmp/runtime/src/i18n/en_US.txt             |   1 +
 openmp/runtime/src/kmp.h                      |   7 +-
 openmp/runtime/src/kmp_tasking.cpp            |  36 +++-
 .../taskgraph_firstprivate_stack_depth.cpp    | 112 +++++++++++
 ...eplayable_lexical_shared_mixed_capture.cpp |  46 +++++
 ...layable_lexical_shared_nontrivial_type.cpp |  60 ++++++
 ...xical_shared_nontrivial_type_recursive.cpp |  88 +++++++++
 ...raph_replayable_lexical_shared_pointer.cpp |  44 +++++
 ...xical_shared_pointer_recursive_frameid.cpp |  76 ++++++++
 ...ph_replayable_lexical_shared_recursive.cpp |  46 +++++
 ...kgraph_replayable_lexical_shared_works.cpp |  42 ++++
 ...h_replayable_nonlexical_shared_fails_1.cpp |  52 +++++
 ...h_replayable_nonlexical_shared_fails_2.cpp |  70 +++++++
 ...taskgraph_replayable_saved_stack_depth.cpp | 119 +++++++++++
 .../taskgraph_shared_stack_depth.cpp          |  94 +++++++++
 17 files changed, 1060 insertions(+), 19 deletions(-)
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_firstprivate_stack_depth.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_mixed_capture.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type_recursive.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer_recursive_frameid.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_recursive.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_works.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_fails_1.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_fails_2.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_saved_stack_depth.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_shared_stack_depth.cpp

diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 2d5bf9b20802b..6f3031161bfff 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -2256,6 +2256,169 @@ void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF,
     Region->emitUntiedSwitch(CGF);
 }
 
+/// Emit a helper with the runtime relocation signature (kmp_task_relocate_t):
+///   void relocate(kmp_task_t *task, void *outer_captures);
+///
+/// On taskgraph replay the runtime invokes this helper to refresh the task's
+/// shared-pointer table. Each capture (a shared-by-ref variable or \c this)
+/// that the task body actually dereferences at execution time is
+/// re-projected from the freshly reconstructed outer record passed as
+/// \p outer_captures and stored back into \c task->shareds.
+///
+/// Captures that the body cannot observe a changed address for across
+/// replays are skipped here:
+///
+///   * captures of a variable that appears as a firstprivate list item
+///     -- the body sources the value from the per-task '.kmp_privates.t'
+///     snapshot rather than from the shareds slot, so the (potentially
+///     stale) original address in the shareds entry is harmless;
+///
+///   * captures of a variable with static (global / namespace-scope /
+///     static-local / static-data-member) storage duration -- the
+///     captured pointer is the variable's link-time-fixed address, which
+///     is identical at recording and on every replay, so no re-projection
+///     is meaningful.
+///
+/// The relocate helper is therefore only ever called upon to refresh
+/// shareds slots that the body genuinely depends on at execution time
+/// (shared-by-ref to a local variable, captured \c this on a heap or
+/// stack object, etc.).  When every capture falls in one of the
+/// skip-eligible categories the helper is emitted as a (still non-null)
+/// no-op: today's runtime only inspects null-vs-non-null, and a non-null
+/// no-op is the right signal that there is nothing the body actually
+/// needs the shareds table refreshed for.
+///
+/// Reduction captures of a local-stack variable still keep the existing
+/// null-relocate-and-abort behaviour: the taskred runtime state is keyed
+/// off the recording-time taskgroup hierarchy and is not currently usable
+/// on replay, so it is preferable to fail loudly (#302) than to silently
+/// misbehave.  Reduction captures of a static-storage variable do not run
+/// into this hazard at the relocate layer -- the captured pointer is
+/// stable -- and are no-op-skipped via the static-storage rule above;
+/// whether the reduction body itself then succeeds on replay is a
+/// separate concern.
+///
+/// Returns null only when at least one capture is genuinely shared (none
+/// of the skip-eligible categories apply) AND cannot be resolved in
+/// \p OuterCSI; in that case the caller passes a null relocation function
+/// to the runtime and the runtime fails fast at replay.
+static llvm::Function *
+emitTaskRelocationFunction(CodeGenModule &CGM, SourceLocation Loc,
+                           const CapturedStmt &CS,
+                           const CodeGenFunction::CGCapturedStmtInfo *OuterCSI,
+                           const OMPTaskDataTy &Data) {
+  ASTContext &C = CGM.getContext();
+
+  // Variables that don't need their shareds slot refreshed across replays
+  // because the body sources them from the per-task '.kmp_privates.t'
+  // snapshot.  Today this is the set of firstprivate list items (snapshot
+  // is taken at task allocation and reused unchanged by every replay).
+  llvm::SmallPtrSet<const VarDecl *, 8> NoRelocateFirstprivateVars;
+  for (const Expr *E : Data.FirstprivateVars) {
+    if (!E)
+      continue;
+    if (const auto *DRE = dyn_cast<DeclRefExpr>(E->IgnoreParenImpCasts()))
+      if (const auto *VD = dyn_cast<VarDecl>(DRE->getDecl()))
+        NoRelocateFirstprivateVars.insert(VD->getCanonicalDecl());
+  }
+
+  // A capture is "no-op-safe" with respect to taskgraph replay when
+  // refreshing its shareds slot is provably unnecessary - either because
+  // the body never reads from that slot (firstprivate) or because the
+  // captured pointer is a link-time-fixed address and is therefore
+  // identical at every replay (static storage duration).
+  auto IsNoOpRelocate = [&](const CapturedStmt::Capture &Cap) {
+    if (Cap.capturesThis() || !Cap.capturesVariable())
+      return false;
+    const VarDecl *VD = Cap.getCapturedVar();
+    if (VD->hasGlobalStorage())
+      return true;
+    return NoRelocateFirstprivateVars.contains(VD->getCanonicalDecl());
+  };
+
+  auto LookupOuterField =
+      [&](const CapturedStmt::Capture &Cap) -> const FieldDecl * {
+    if (!OuterCSI)
+      return nullptr;
+    return Cap.capturesThis() ? OuterCSI->getThisFieldDecl()
+                              : OuterCSI->lookup(Cap.getCapturedVar());
+  };
+
+  // Bail out before emitting any IR if a genuinely-shared capture cannot
+  // be resolved in the containing context.  No-op-safe captures (see the
+  // function-level comment) don't participate in this preflight; they
+  // simply cause the helper to skip their slot below.
+  if (llvm::any_of(CS.captures(), [&](const CapturedStmt::Capture &Cap) {
+        assert((Cap.capturesThis() || Cap.capturesVariable()) &&
+               "OpenMP task capture must be shared-by-ref or 'this'");
+        return !IsNoOpRelocate(Cap) && !LookupOuterField(Cap);
+      }))
+    return nullptr;
+
+  // void relocate(void *task, void *outer_captures)
+  auto *TaskArg =
+      ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
+                                C.VoidPtrTy, ImplicitParamKind::Other);
+  auto *OuterArg =
+      ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr,
+                                C.VoidPtrTy, ImplicitParamKind::Other);
+  FunctionArgList Args{TaskArg, OuterArg};
+  const CGFunctionInfo &FnInfo =
+      CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
+
+  std::string Name =
+      CGM.getOpenMPRuntime().getName({"omp", "taskgraph", "relocate", ""});
+  auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(FnInfo),
+                                    llvm::GlobalValue::InternalLinkage, Name,
+                                    &CGM.getModule());
+  CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
+  if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
+    Fn->addFnAttr("sample-profile-suffix-elision-policy", "selected");
+  Fn->setDoesNotRecurse();
+
+  CodeGenFunction CGF(CGM);
+  CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
+
+  CGBuilderTy &Bld = CGF.Builder;
+  CharUnits PtrAlign = CGF.getPointerAlign();
+
+  // Base of the reconstructed outer record for this replay.
+  llvm::Value *OuterRaw = Bld.CreateLoad(CGF.GetAddrOfLocalVar(OuterArg));
+
+  // kmp_task_t::shareds is the first field of the runtime task descriptor;
+  // load it to obtain the void* shared table that we will refresh in place.
+  // The table holds one void* per by-ref capture.
+  llvm::Value *TaskRaw = Bld.CreateLoad(CGF.GetAddrOfLocalVar(TaskArg));
+  llvm::Value *SharedRaw =
+      Bld.CreateLoad(Address(TaskRaw, CGF.VoidPtrTy, PtrAlign));
+  Address SharedTable(SharedRaw, CGF.VoidPtrTy, PtrAlign);
+
+  unsigned Index = 0;
+  for (const CapturedStmt::Capture &Cap : CS.captures()) {
+    // Always advance the slot index so that we stay aligned with the
+    // shareds-table layout established at task allocation.
+    unsigned ThisIndex = Index++;
+    if (IsNoOpRelocate(Cap))
+      continue;
+    // Project the capture's referent from the freshly reconstructed outer
+    // record. EmitLValueForField auto-loads the outer reference field, so
+    // the resulting pointer is the live referent address (not the slot).
+    const FieldDecl *OuterField = LookupOuterField(Cap);
+    assert(OuterField && "preflight should have rejected this capture");
+    QualType OuterTy =
+        C.getCanonicalTagType(cast<RecordDecl>(OuterField->getDeclContext()));
+    LValue OuterBase = CGF.MakeAddrLValue(
+        Address(OuterRaw, CGF.ConvertTypeForMem(OuterTy), PtrAlign), OuterTy);
+    llvm::Value *Mapped =
+        CGF.EmitLValueForField(OuterBase, OuterField).getPointer(CGF);
+    Mapped = Bld.CreatePointerBitCastOrAddrSpaceCast(Mapped, CGM.VoidPtrTy);
+    Bld.CreateStore(Mapped, Bld.CreateConstGEP(SharedTable, ThisIndex));
+  }
+
+  CGF.FinishFunction();
+  return Fn;
+}
+
 void CGOpenMPRuntime::emitTaskgraphCall(CodeGenFunction &CGF,
                                         SourceLocation Loc,
                                         const OMPExecutableDirective &D,
@@ -4822,22 +4985,27 @@ void CGOpenMPRuntime::emitTaskCall(
     TGTaskArgs[2] = Result.NewTask;
     TGTaskArgs[3] = TaskAllocArgs[0]; // TaskFlags
     TGTaskArgs[4] = TaskAllocArgs[1]; // KmpTaskTWithPrivatesTySize
-    TGTaskArgs[5] = Shareds.emitRawPointer(CGF);
-    TGTaskArgs[6] = TaskAllocArgs[2]; // SharedsSize
+    TGTaskArgs[5] = TaskAllocArgs[2]; // SharedsSize
     if (auto RecType = dyn_cast<RecordType>(SharedsTy)) {
       auto *RD = RecType->getAsRecordDecl();
       if (RD->fields().empty()) {
         // FIXME: The condition might not be precisely correct here.
-        TGTaskArgs[6] = CGF.Builder.getSize(0);
+        TGTaskArgs[5] = CGF.Builder.getSize(0);
       }
     }
     if (Data.Dependences.size() == 0) {
-      TGTaskArgs[7] = CGF.Builder.getInt32(0);
-      TGTaskArgs[8] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+      TGTaskArgs[6] = CGF.Builder.getInt32(0);
+      TGTaskArgs[7] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
     } else {
-      TGTaskArgs[7] = NumOfElements;
-      TGTaskArgs[8] = DependenciesArray.emitRawPointer(CGF);
-    }
+      TGTaskArgs[6] = NumOfElements;
+      TGTaskArgs[7] = DependenciesArray.emitRawPointer(CGF);
+    }
+    const auto *CS = cast<CapturedStmt>(D.getAssociatedStmt());
+    llvm::Function *RelocFn =
+        emitTaskRelocationFunction(CGM, Loc, *CS, CGF.CapturedStmtInfo, Data);
+    TGTaskArgs[8] = RelocFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+                                  RelocFn, CGM.VoidPtrTy)
+                            : llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
     CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
                             CGM.getModule(), OMPRTL___kmpc_taskgraph_task),
                         TGTaskArgs);
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
index 9f2c331a092e7..9b951d7b480b7 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
@@ -360,7 +360,7 @@ __OMP_RTL(__kmpc_taskgroup, false, Void, IdentPtr, Int32)
 __OMP_RTL(__kmpc_taskgraph, false, Void, IdentPtr, Int32, VoidPtrPtr, SizeTy,
           Int32, Int32, VoidPtr, VoidPtr)
 __OMP_RTL(__kmpc_taskgraph_task, false, Int32, IdentPtr, Int32, VoidPtr, Int32,
-          SizeTy, VoidPtr, SizeTy, Int32, VoidPtr)
+          SizeTy, SizeTy, Int32, VoidPtr, VoidPtr)
 __OMP_RTL(__kmpc_taskgraph_taskloop, false, Int32, IdentPtr, Int32, VoidPtr,
           Int32, SizeTy, VoidPtr, SizeTy, Int32, Int64Ptr, Int64Ptr, Int64,
           Int32, Int32, Int64, Int32, VoidPtr)
diff --git a/openmp/runtime/src/i18n/en_US.txt b/openmp/runtime/src/i18n/en_US.txt
index 46f5e794e5a27..5fc34123f4776 100644
--- a/openmp/runtime/src/i18n/en_US.txt
+++ b/openmp/runtime/src/i18n/en_US.txt
@@ -484,6 +484,7 @@ AffIgnoringNonHybrid         "%1$s ignored: This machine is not a hybrid archite
 AffIgnoringNotAvailable      "%1$s ignored: %2$s is not available. Using \"%3$s\" instead."
 SetNumTeamsInParOrTeamsRegion "%1$s() called inside a parallel or teams region; call ignored."
 SetTeamsThreadLimitInParOrTeamsRegion "%1$s() called inside a parallel or teams region; call ignored."
+OmpTaskgraphBadCapture       "Cannot locate captured shared variable reference for taskgraph replay"
 OmpTaskgraphConcurrentRecord "%1$s: multiple threads attempting to re-record taskgraph concurrently for graph_id %2$d"
 
 # --------------------------------------------------------------------------------------------------
diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index f0366e36af899..5ad6b88db6d35 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -2469,6 +2469,7 @@ extern kmp_uint64 __kmp_taskloop_min_tasks;
 /*!
  */
 typedef kmp_int32 (*kmp_routine_entry_t)(kmp_int32, void *);
+typedef void (*kmp_task_relocate_t)(struct kmp_task *, void *);
 
 typedef union kmp_cmplrdata {
   kmp_int32 priority; /**< priority specified by user for the task */
@@ -2690,6 +2691,7 @@ enum kmp_taskgraph_region_type {
 typedef struct kmp_taskgraph_node {
   kmp_task_t *task;
   bool taskloop_task;
+  kmp_task_relocate_t relocate;
   kmp_taskgraph_reduce_input_data_t *reduce_input;
   union {
     // Valid when KMP_TDG_RECORDING in parent taskgraph record.
@@ -2773,6 +2775,7 @@ typedef struct kmp_taskgraph_record {
   struct kmp_taskgraph_exec_descr *exec_descrs = nullptr;
   kmp_size_t num_exec_descrs = 0;
   kmp_lock_t replay_lock;
+  void *taskgraph_args = nullptr;
   // We need a taskgroup structure to keep track of recorded tasks.  This is
   // set to TRUE if the user requested "nogroup" on the taskgraph directive
   // (then we can avoid blocking at the end of the taskgraph region on replay,
@@ -4530,8 +4533,8 @@ KMP_EXPORT void __kmpc_taskgraph(ident_t *loc_ref, kmp_int32 gtid,
                                  void *args);
 KMP_EXPORT kmp_uint32 __kmpc_taskgraph_task(
     ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *new_task, kmp_int32 flags,
-    size_t sizeof_kmp_task_t, void *shareds, size_t sizeof_shareds,
-    kmp_int32 ndeps, kmp_depend_info_t *dep_list);
+    size_t sizeof_kmp_task_t, size_t sizeof_shareds, kmp_int32 ndeps,
+    kmp_depend_info_t *dep_list, kmp_task_relocate_t reloc);
 KMP_EXPORT kmp_uint32 __kmpc_taskgraph_taskloop(
     ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *new_task, kmp_int32 flags,
     size_t sizeof_kmp_task_t, void *shareds, size_t sizeof_shareds,
diff --git a/openmp/runtime/src/kmp_tasking.cpp b/openmp/runtime/src/kmp_tasking.cpp
index 2a04d9bb58812..b500652f5f10f 100644
--- a/openmp/runtime/src/kmp_tasking.cpp
+++ b/openmp/runtime/src/kmp_tasking.cpp
@@ -2495,10 +2495,11 @@ __kmp_build_exec_descrs(kmp_info_t *thread, kmp_taskgraph_region_t *region,
 
 /// Reset, reparent and regroup the recorded task TASK and re-invoke it.
 
-static void __kmp_omp_tg_task(kmp_int32 gtid, kmp_task_t *task,
+static void __kmp_omp_tg_task(kmp_int32 gtid, kmp_taskgraph_node_t *node,
                               kmp_taskgroup_t *taskgroup,
                               kmp_taskdata_t *parent_taskdata,
                               bool serialize_immediate) {
+  kmp_task_t *task = node->task;
   kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
   taskdata->td_parent = parent_taskdata;
 
@@ -2521,6 +2522,18 @@ static void __kmp_omp_tg_task(kmp_int32 gtid, kmp_task_t *task,
   if (parent_taskdata->td_flags.tasktype == TASK_EXPLICIT)
     KMP_ATOMIC_INC(&parent_taskdata->td_allocated_child_tasks);
 
+  if (node->relocate) {
+    // Call the task's relocation function with the incoming args from the
+    // owning taskgraph.  This rewrites capture-by-reference variables to point
+    // to the correct location on the replayed taskgraph's stack (which may not
+    // be the same as the location from the initial recorded taskgraph).
+    node->relocate(task, taskdata->owning_taskgraph->taskgraph_args);
+  } else if (task->shareds != NULL) {
+    // A missing relocation callback is only fatal when there is a non-empty
+    // shareds payload that may contain by-reference captures needing remap.
+    KMP_FATAL(OmpTaskgraphBadCapture);
+  }
+
   __kmp_omp_task(gtid, task, false);
 }
 
@@ -2543,7 +2556,7 @@ static void __kmp_taskgraph_exec_descr_start(kmp_int32 gtid, kmp_info_t *thread,
     // __kmp_taskgraph_exec_descr_finish.
     kmp_taskgraph_node_t *node = descr->region->task.node;
     kmp_taskdata_t *current_taskdata = thread->th.th_current_task;
-    __kmp_omp_tg_task(gtid, node->task, taskgroup, current_taskdata, false);
+    __kmp_omp_tg_task(gtid, node, taskgroup, current_taskdata, false);
     break;
   }
   case TASKGRAPH_REGION_PARALLEL:
@@ -5104,6 +5117,7 @@ __kmp_taskgraph_node_alloc(kmp_taskgraph_record_t *rec, kmp_task_t *task,
 
   new_task->task = task;
   new_task->taskloop_task = false;
+  new_task->relocate = nullptr;
   new_task->reduce_input = nullptr;
   new_task->u.unresolved.ndeps = 0;
   new_task->u.unresolved.dep_list = nullptr;
@@ -5946,10 +5960,6 @@ static kmp_task_t *__kmp_taskgraph_clone_task(kmp_info_t *thread,
   // FIXME: This should use a "taskdup" function like taskloops in cases where
   // private variables are not trivially copyable.  For now, do it by plain
   // bitwise copy.
-  // FIXME 2: It's intended that this copy be persistent, and can be
-  // re-executed on taskgraph replay.  Make sure that works (for shared
-  // variables) if stack addresses change (i.e. a task-generating function is
-  // called from different call stack depths).
   kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(orig);
   size_t shareds_offset = sizeof(kmp_taskdata_t) + sizeof_kmp_task_t;
   shareds_offset = __kmp_round_up_to_val(shareds_offset, sizeof(kmp_uint64));
@@ -5958,6 +5968,11 @@ static kmp_task_t *__kmp_taskgraph_clone_task(kmp_info_t *thread,
   KMP_MEMCPY(copy_td, taskdata, shareds_offset + sizeof_shareds);
   // Tasks cloned for a taskgraph always have this field set.
   copy_td->owning_taskgraph = taskgraph;
+  kmp_task_t *copy_task = KMP_TASKDATA_TO_TASK(copy_td);
+  if (orig->shareds) {
+    // New task's shared data has now moved.  Update the pointer.
+    copy_task->shareds = (void *)((char *)copy_td + shareds_offset);
+  }
   KMP_ATOMIC_ST_RLX(&copy_td->td_incomplete_child_tasks, 0);
   return KMP_TASKDATA_TO_TASK(copy_td);
 }
@@ -6073,6 +6088,9 @@ void __kmpc_taskgraph(ident_t *loc_ref, kmp_int32 gtid,
     // taskgroup.
     KMP_ATOMIC_ST_REL(&taskgroup->taskgraph.recording, record);
   }
+  // Keep the current taskgraph invocation's outlined-entry args for
+  // replay-time relocation of by-reference captures.
+  record->taskgraph_args = args;
   __kmp_release_lock(&header->header_lock, gtid);
 
   kmp_taskgraph_status_t status = KMP_ATOMIC_LD_ACQ(&record->status);
@@ -6108,9 +6126,10 @@ void __kmpc_taskgraph(ident_t *loc_ref, kmp_int32 gtid,
 
 kmp_uint32 __kmpc_taskgraph_task(ident_t *loc_ref, kmp_int32 gtid,
                                  kmp_task_t *new_task, kmp_int32 flags,
-                                 size_t sizeof_kmp_task_t, void *shareds,
+                                 size_t sizeof_kmp_task_t,
                                  size_t sizeof_shareds, kmp_int32 ndeps,
-                                 kmp_depend_info_t *dep_list) {
+                                 kmp_depend_info_t *dep_list,
+                                 kmp_task_relocate_t relocate) {
   kmp_info_t *thread = __kmp_threads[gtid];
   kmp_taskgroup_t *taskgroup = thread->th.th_current_task->td_taskgroup;
   kmp_taskgraph_record_t *rec = __kmp_taskgraph_or_parent_recording(taskgroup);
@@ -6146,6 +6165,7 @@ kmp_uint32 __kmpc_taskgraph_task(ident_t *loc_ref, kmp_int32 gtid,
           thread, ndeps * sizeof(kmp_depend_info_t));
       KMP_MEMCPY(node->u.unresolved.dep_list, dep_list,
                  ndeps * sizeof(kmp_depend_info_t));
+      node->relocate = relocate;
     } else if (status == KMP_TDG_READY) {
 #ifdef DEBUG_TASKGRAPH
       fprintf(stderr,
diff --git a/openmp/runtime/test/taskgraph/taskgraph_firstprivate_stack_depth.cpp b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_stack_depth.cpp
new file mode 100644
index 0000000000000..e3f8976bc1ae0
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_stack_depth.cpp
@@ -0,0 +1,112 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+static volatile int StackSink = 0;
+// Keep the observable result in stable storage so this test isolates
+// firstprivate replay across different stack depths. There is a separate bug
+// test for replayed tasks writing through a stack-local shared pointer cached
+// at record time.
+static volatile int ReplayResult = -1;
+
+struct Payload {
+  int values[6];
+  int bias;
+};
+
+__attribute__((noinline)) static int evaluate_payload(const Payload &payload,
+                                                      int seed) {
+  return seed * payload.values[0] - payload.values[1] +
+         payload.values[2] * payload.values[3] - payload.bias +
+         payload.values[4] - payload.values[5];
+}
+
+__attribute__((noinline)) static void clobber_stack(int base) {
+  volatile int scratch[4096];
+
+  for (int i = 0; i < 4096; ++i)
+    scratch[i] = base + i;
+
+  StackSink += scratch[base & 63];
+}
+
+__attribute__((noinline)) static int run_taskgraph(int seed) {
+  Payload payload{
+      {seed + 1, seed + 3, seed + 5, seed + 7, seed + 11, seed + 13},
+      seed * 17 + 19};
+  ReplayResult = -1;
+
+#pragma omp taskgraph graph_id(91)
+  {
+#pragma omp task firstprivate(payload, seed) shared(ReplayResult)
+    {
+      ReplayResult = evaluate_payload(payload, seed);
+    }
+  }
+
+  return ReplayResult;
+}
+
+__attribute__((noinline)) static int call_with_depth(int seed, int depth) {
+  volatile int padding[128];
+
+  for (int i = 0; i < 128; ++i)
+    padding[i] = seed + depth + i;
+
+  StackSink += padding[(seed + depth) & 127];
+
+  if (depth == 0)
+    return run_taskgraph(seed);
+  return call_with_depth(seed, depth - 1);
+}
+
+__attribute__((noinline)) static int expected_result(int seed) {
+  Payload payload{
+      {seed + 1, seed + 3, seed + 5, seed + 7, seed + 11, seed + 13},
+      seed * 17 + 19};
+  return evaluate_payload(payload, seed);
+}
+
+int main() {
+  constexpr int NumCalls = 4;
+  constexpr int Seeds[NumCalls] = {3, 17, 29, 41};
+  constexpr int Depths[NumCalls] = {0, 3, 1, 5};
+
+  int recorded = -1;
+  bool failed = false;
+
+#pragma omp parallel num_threads(4)
+  {
+#pragma omp single
+    {
+      recorded = call_with_depth(Seeds[0], Depths[0]);
+      if (recorded != expected_result(Seeds[0])) {
+        std::fprintf(stderr, "FAIL initial record got=%d expected=%d\n",
+                     recorded, expected_result(Seeds[0]));
+        failed = true;
+      }
+
+      for (int i = 1; i < NumCalls; ++i) {
+        clobber_stack(Seeds[i] * 1000);
+        const int replayed = call_with_depth(Seeds[i], Depths[i]);
+        if (replayed != recorded) {
+          std::fprintf(stderr,
+                       "FAIL replay %d depth=%d seed=%d got=%d expected=%d\n",
+                       i, Depths[i], Seeds[i], replayed, recorded);
+          failed = true;
+        }
+      }
+    }
+  }
+
+  if (failed)
+    return 1;
+
+  std::fprintf(stderr, "PASS firstprivate stack result=%d\n", recorded);
+  return 0;
+}
+
+// CHECK: PASS firstprivate stack result=
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_mixed_capture.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_mixed_capture.cpp
new file mode 100644
index 0000000000000..0ea8e4bdadbb3
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_mixed_capture.cpp
@@ -0,0 +1,46 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+__attribute__((noinline)) static int run_taskgraph_mixed_capture(int seed) {
+  int x = seed;
+  int y = seed * 2;
+  int out = -1;
+  int fp = 7;
+
+#pragma omp taskgraph graph_id(401)
+  {
+#pragma omp task replayable(1) shared(x, y, out) firstprivate(fp)              \
+    depend(inout : x, y)
+    {
+      x += fp;
+      y += x;
+      out = y + fp;
+    }
+  }
+
+  return out;
+}
+
+int main() {
+  const int first = run_taskgraph_mixed_capture(1);
+  const int second = run_taskgraph_mixed_capture(100);
+
+  if (first != 17 || second != 314) {
+    std::fprintf(stderr,
+                 "FAIL lexical mixed capture replay first=%d second=%d "
+                 "expected=17/314\n",
+                 first, second);
+    return 1;
+  }
+
+  std::fprintf(stderr, "PASS lexical mixed capture replay first=%d second=%d\n",
+               first, second);
+  return 0;
+}
+
+// CHECK: PASS lexical mixed capture replay first=17 second=314
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type.cpp
new file mode 100644
index 0000000000000..f3a50a27176d4
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type.cpp
@@ -0,0 +1,60 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+struct Tracker {
+  static int Ctors;
+  static int Dtors;
+
+  int Value;
+
+  explicit Tracker(int V) : Value(V) { ++Ctors; }
+  ~Tracker() { ++Dtors; }
+
+  void bump(int Delta) { Value += Delta; }
+};
+
+int Tracker::Ctors = 0;
+int Tracker::Dtors = 0;
+
+__attribute__((noinline)) static int run_taskgraph_nontrivial(int seed) {
+  Tracker Obj(seed);
+  int out = -1;
+
+#pragma omp taskgraph graph_id(403)
+  {
+#pragma omp task replayable(1) shared(Obj, out)
+    {
+      Obj.bump(11);
+      out = Obj.Value;
+    }
+  }
+
+  return out;
+}
+
+int main() {
+  const int first = run_taskgraph_nontrivial(1);
+  const int second = run_taskgraph_nontrivial(100);
+
+  if (first != 12 || second != 111 || Tracker::Ctors < 2 ||
+      Tracker::Dtors < 2 || Tracker::Ctors != Tracker::Dtors) {
+    std::fprintf(
+        stderr,
+        "FAIL lexical nontrivial replay first=%d second=%d ctors=%d dtors=%d\n",
+        first, second, Tracker::Ctors, Tracker::Dtors);
+    return 1;
+  }
+
+  std::fprintf(
+      stderr,
+      "PASS lexical nontrivial replay first=%d second=%d ctors=%d dtors=%d\n",
+      first, second, Tracker::Ctors, Tracker::Dtors);
+  return 0;
+}
+
+// CHECK: PASS lexical nontrivial replay first=12 second=111
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type_recursive.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type_recursive.cpp
new file mode 100644
index 0000000000000..6012b6194e56b
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type_recursive.cpp
@@ -0,0 +1,88 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+struct Tracker {
+  static int Ctors;
+  static int Dtors;
+
+  int Value;
+
+  explicit Tracker(int V) : Value(V) { ++Ctors; }
+  ~Tracker() { ++Dtors; }
+
+  void bump(int Delta) { Value += Delta; }
+};
+
+int Tracker::Ctors = 0;
+int Tracker::Dtors = 0;
+
+__attribute__((noinline)) static int expected_recursive(int depth, int seed,
+                                                        int run_tag) {
+  int local = seed + (depth + 1) * 5 + run_tag;
+  if (depth == 0)
+    return local;
+  return local + expected_recursive(depth - 1, seed + 9, run_tag);
+}
+
+__attribute__((noinline)) static int
+run_recursive_nontrivial(int depth, int seed, int run_tag) {
+  Tracker Obj(seed);
+  int out = -1;
+
+  int gid = 500 + depth;
+#pragma omp taskgraph graph_id(gid)
+  {
+#pragma omp task replayable(1) shared(Obj, out, depth, run_tag)
+    {
+      Obj.bump((depth + 1) * 5 + run_tag);
+      out = Obj.Value;
+    }
+  }
+
+  if (depth == 0)
+    return out;
+  return out + run_recursive_nontrivial(depth - 1, seed + 9, run_tag);
+}
+
+int main() {
+  const int depth = 3;
+  int total_actual = 0;
+  int total_expected = 0;
+
+  for (int run = 0; run < 3; ++run) {
+    const int seed = 100 * run + 1;
+    const int actual = run_recursive_nontrivial(depth, seed, run);
+    const int expected = expected_recursive(depth, seed, run);
+
+    if (actual != expected) {
+      std::fprintf(stderr,
+                   "FAIL recursive nontrivial run=%d actual=%d expected=%d\n",
+                   run, actual, expected);
+      return 1;
+    }
+
+    total_actual += actual;
+    total_expected += expected;
+  }
+
+  if (Tracker::Ctors != Tracker::Dtors || Tracker::Ctors < 12) {
+    std::fprintf(stderr,
+                 "FAIL recursive nontrivial lifetime ctors=%d dtors=%d "
+                 "total=%d expected=%d\n",
+                 Tracker::Ctors, Tracker::Dtors, total_actual, total_expected);
+    return 1;
+  }
+
+  std::fprintf(
+      stderr,
+      "PASS recursive nontrivial total=%d expected=%d ctors=%d dtors=%d\n",
+      total_actual, total_expected, Tracker::Ctors, Tracker::Dtors);
+  return 0;
+}
+
+// CHECK: PASS recursive nontrivial total=
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer.cpp
new file mode 100644
index 0000000000000..7690aff37853c
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer.cpp
@@ -0,0 +1,44 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+__attribute__((noinline)) static int run_taskgraph_pointer_shared(int seed) {
+  int value = seed;
+  int *ptr = &value;
+  int out = -1;
+
+#pragma omp taskgraph graph_id(402)
+  {
+#pragma omp task replayable(1) shared(ptr, out) depend(inout : value)
+    {
+      *ptr += 3;
+      out = *ptr;
+    }
+  }
+
+  return out;
+}
+
+int main() {
+  const int first = run_taskgraph_pointer_shared(1);
+  const int second = run_taskgraph_pointer_shared(100);
+
+  if (first != 4 || second != 103) {
+    std::fprintf(stderr,
+                 "FAIL lexical pointer shared replay first=%d second=%d "
+                 "expected=4/103\n",
+                 first, second);
+    return 1;
+  }
+
+  std::fprintf(stderr,
+               "PASS lexical pointer shared replay first=%d second=%d\n", first,
+               second);
+  return 0;
+}
+
+// CHECK: PASS lexical pointer shared replay first=4 second=103
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer_recursive_frameid.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer_recursive_frameid.cpp
new file mode 100644
index 0000000000000..142a5941f97ae
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer_recursive_frameid.cpp
@@ -0,0 +1,76 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdint>
+#include <cstdio>
+
+__attribute__((noinline)) static int expected_recursive(int depth, int seed,
+                                                        int run_tag) {
+  int value = seed;
+  value += (depth + 1) * 3 + run_tag;
+  if (depth == 0)
+    return value;
+  return value + expected_recursive(depth - 1, seed + 7, run_tag);
+}
+
+__attribute__((noinline)) static int run_recursive_frameid(int depth, int seed,
+                                                           int run_tag) {
+  int value = seed;
+  int *ptr = &value;
+  int *&ptr_ref = ptr;
+  int out = -1;
+
+  // Typically, if captured pointers refer to locations on the stack, that
+  // would not be safe for taskgraph record/replay because we in general we
+  // cannot rewrite such pointers to point to the current (live) stack frame.
+  //
+  // This is one possible way around that though: we keep a taskgraph record
+  // per stack-depth, each of which may refer to the local stack frame.
+  //
+  // I probably wouldn't recommend use of this technique in production code.
+  uintptr_t frame_gid = reinterpret_cast<uintptr_t>(__builtin_frame_address(0));
+
+#pragma omp taskgraph graph_id(frame_gid)
+  {
+#pragma omp task shared(ptr_ref, out, depth, run_tag) depend(inout : value)
+    {
+      *ptr_ref += (depth + 1) * 3 + run_tag;
+      out = *ptr_ref;
+    }
+  }
+
+  if (depth == 0)
+    return out;
+  return out + run_recursive_frameid(depth - 1, seed + 7, run_tag);
+}
+
+int main() {
+  const int depth = 3;
+  int actual_sum = 0;
+  int expected_sum = 0;
+
+  for (int run = 0; run < 3; ++run) {
+    int seed = 100 * run + 1;
+    int actual = run_recursive_frameid(depth, seed, run);
+    int expected = expected_recursive(depth, seed, run);
+    if (actual != expected) {
+      std::fprintf(
+          stderr,
+          "FAIL recursive pointer frameid run=%d actual=%d expected=%d\n", run,
+          actual, expected);
+      return 1;
+    }
+    actual_sum += actual;
+    expected_sum += expected;
+  }
+
+  std::fprintf(stderr,
+               "PASS recursive pointer frameid runs=3 total=%d expected=%d\n",
+               actual_sum, expected_sum);
+  return 0;
+}
+
+// CHECK: PASS recursive pointer frameid runs=3 total=
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_recursive.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_recursive.cpp
new file mode 100644
index 0000000000000..4d0d7b5613108
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_recursive.cpp
@@ -0,0 +1,46 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+__attribute__((noinline)) static int run_taskgraph_recursive(int depth,
+                                                             int seed) {
+  int x = seed;
+  int out = -1;
+
+#pragma omp taskgraph graph_id(450)
+  {
+#pragma omp task replayable(1) shared(x, out, depth) depend(inout : x)
+    {
+      x += depth + 1;
+      out = x;
+    }
+  }
+
+  if (depth == 0)
+    return out;
+
+  return out + run_taskgraph_recursive(depth - 1, seed + 10);
+}
+
+int main() {
+  const int first = run_taskgraph_recursive(3, 1);
+  const int second = run_taskgraph_recursive(3, 100);
+
+  if (first != 74 || second != 470) {
+    std::fprintf(
+        stderr,
+        "FAIL lexical recursive replay first=%d second=%d expected=74/470\n",
+        first, second);
+    return 1;
+  }
+
+  std::fprintf(stderr, "PASS lexical recursive replay first=%d second=%d\n",
+               first, second);
+  return 0;
+}
+
+// CHECK: PASS lexical recursive replay first=74 second=470
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_works.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_works.cpp
new file mode 100644
index 0000000000000..82be9c82d3e7c
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_works.cpp
@@ -0,0 +1,42 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+__attribute__((noinline)) static int run_taskgraph_lexical(int seed) {
+  int x = seed;
+  int out = -1;
+
+#pragma omp taskgraph graph_id(311)
+  {
+#pragma omp task replayable(1) shared(x, out) depend(inout : x)
+    {
+      x += 5;
+      out = x;
+    }
+  }
+
+  return out;
+}
+
+int main() {
+  const int first = run_taskgraph_lexical(1);
+  const int second = run_taskgraph_lexical(100);
+
+  if (first != 6 || second != 105) {
+    std::fprintf(
+        stderr,
+        "FAIL lexical shared replay first=%d second=%d expected=6/105\n", first,
+        second);
+    return 1;
+  }
+
+  std::fprintf(stderr, "PASS lexical shared replay first=%d second=%d\n", first,
+               second);
+  return 0;
+}
+
+// CHECK: PASS lexical shared replay first=6 second=105
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_fails_1.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_fails_1.cpp
new file mode 100644
index 0000000000000..e5483ae5e06bf
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_fails_1.cpp
@@ -0,0 +1,52 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %not --crash %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+// This seems like it could work in principle, but in general we don't know
+// where the targets of the referenced variables are when the task is replayed.
+__attribute__((noinline)) static void emit_nonlexical_task(int &x, int &out) {
+#pragma omp task replayable(1) shared(x, out) depend(inout : x)
+  {
+    x += 5;
+    out = x;
+  }
+}
+
+__attribute__((noinline)) static int run_taskgraph_nonlexical(int seed) {
+  int x = seed;
+  int out = -1;
+
+#pragma omp taskgraph graph_id(312)
+  {
+    emit_nonlexical_task(x, out);
+  }
+
+  return out;
+}
+
+int main() {
+  const int recorded = run_taskgraph_nonlexical(1);
+  const int replayed = run_taskgraph_nonlexical(100);
+
+  // The "non-lexical" replayable task is emitted in a helper function outside
+  // the taskgraph lexical scope.  We expect this to raise a runtime error.
+  if (recorded == replayed) {
+    std::fprintf(
+        stderr,
+        "UNEXPECTED SUCCESS nonlexical replay recorded=%d replayed=%d\n",
+        recorded, replayed);
+    return 0;
+  }
+
+  std::fprintf(stderr,
+               "EXPECTED FAILURE nonlexical replay recorded=%d replayed=%d\n",
+               recorded, replayed);
+  return 1;
+}
+
+// CHECK: OMP: Error #{{[0-9]+}}: Cannot locate captured shared variable reference for
+// taskgraph replay
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_fails_2.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_fails_2.cpp
new file mode 100644
index 0000000000000..ee593b28e3c8e
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_fails_2.cpp
@@ -0,0 +1,70 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %not --crash %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+__attribute__((noinline)) static int emit_nonlexical_task(int seed) {
+  int x = seed;
+  int out = -1;
+
+// This is syntactically valid, but a taskgraph replay that includes this
+// task cannot possibly succeed, because the stack frame containing 'x' and
+// 'out' doesn't exist at replay time.  We can raise a runtime error in that
+// case.
+// This isn't a compile error because the code is still valid if no taskgraph
+// record/replay is in progress.
+#pragma omp task replayable(1) shared(x, out) depend(inout : x)
+  {
+    x += 5;
+    out = x;
+  }
+
+  return out;
+}
+
+__attribute__((noinline)) static int run_taskgraph_nonlexical(int seed) {
+  int out;
+
+#pragma omp taskgraph graph_id(312)
+  {
+#pragma omp task shared(out)
+    {
+      out = emit_nonlexical_task(seed);
+    }
+  }
+
+  return out;
+}
+
+int main() {
+  int out = emit_nonlexical_task(50);
+  if (out != 55) {
+    std::fprintf(stderr,
+                 "UNEXPECTED FAILURE: task outside taskgraph returned %d\n",
+                 out);
+  }
+
+  const int recorded = run_taskgraph_nonlexical(1);
+  const int replayed = run_taskgraph_nonlexical(100);
+
+  // The non-lexical replayable task is emitted in a helper function outside
+  // the taskgraph lexical scope.
+  if (recorded == replayed) {
+    std::fprintf(
+        stderr,
+        "UNEXPECTED SUCCESS nonlexical replay recorded=%d replayed=%d\n",
+        recorded, replayed);
+    return 0;
+  }
+
+  std::fprintf(stderr,
+               "EXPECTED FAILURE nonlexical replay recorded=%d replayed=%d\n",
+               recorded, replayed);
+  return 1;
+}
+
+// CHECK: OMP: Error #{{[0-9]+}}: Cannot locate captured shared variable reference for
+// taskgraph replay
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_saved_stack_depth.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_saved_stack_depth.cpp
new file mode 100644
index 0000000000000..922cb85a53eec
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_saved_stack_depth.cpp
@@ -0,0 +1,119 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// XFAIL: *
+// clang-format on
+
+#include <cstdio>
+
+static volatile int StackSink = 0;
+static volatile int ReplayResult = -1;
+
+struct Payload {
+  int values[6];
+  int bias;
+};
+
+__attribute__((noinline)) static int evaluate_payload(const Payload &payload,
+                                                      int seed) {
+  return seed * payload.values[0] - payload.values[1] +
+         payload.values[2] * payload.values[3] - payload.bias +
+         payload.values[4] - payload.values[5];
+}
+
+__attribute__((noinline)) static void clobber_stack(int base) {
+  volatile int scratch[4096];
+
+  for (int i = 0; i < 4096; ++i)
+    scratch[i] = base + i;
+
+  StackSink += scratch[base & 63];
+}
+
+// Intended future usage of firstprivate(saved: ...): this replayable task is
+// not lexically nested within the taskgraph directive. It is created from a
+// helper function called inside the taskgraph region, and needs closure-like
+// capture of that helper's stack locals.
+__attribute__((noinline)) static void emit_replayable_task(int seed) {
+  Payload payload{
+      {seed + 1, seed + 3, seed + 5, seed + 7, seed + 11, seed + 13},
+      seed * 17 + 19};
+
+#pragma omp task replayable(1) firstprivate(saved : payload, seed)             \
+    shared(ReplayResult)
+  {
+    ReplayResult = evaluate_payload(payload, seed);
+  }
+}
+
+__attribute__((noinline)) static int run_taskgraph(int seed) {
+  ReplayResult = -1;
+
+#pragma omp taskgraph graph_id(93)
+  {
+    emit_replayable_task(seed);
+  }
+
+  return ReplayResult;
+}
+
+__attribute__((noinline)) static int call_with_depth(int seed, int depth) {
+  volatile int padding[128];
+
+  for (int i = 0; i < 128; ++i)
+    padding[i] = seed + depth + i;
+
+  StackSink += padding[(seed + depth) & 127];
+
+  if (depth == 0)
+    return run_taskgraph(seed);
+  return call_with_depth(seed, depth - 1);
+}
+
+__attribute__((noinline)) static int expected_result(int seed) {
+  Payload payload{
+      {seed + 1, seed + 3, seed + 5, seed + 7, seed + 11, seed + 13},
+      seed * 17 + 19};
+  return evaluate_payload(payload, seed);
+}
+
+int main() {
+  constexpr int NumCalls = 4;
+  constexpr int Seeds[NumCalls] = {3, 17, 29, 41};
+  constexpr int Depths[NumCalls] = {0, 3, 1, 5};
+
+  int recorded = -1;
+  bool failed = false;
+
+#pragma omp parallel num_threads(4)
+  {
+#pragma omp single
+    {
+      recorded = call_with_depth(Seeds[0], Depths[0]);
+      if (recorded != expected_result(Seeds[0])) {
+        std::fprintf(stderr, "FAIL initial record got=%d expected=%d\n",
+                     recorded, expected_result(Seeds[0]));
+        failed = true;
+      }
+
+      for (int i = 1; i < NumCalls; ++i) {
+        clobber_stack(Seeds[i] * 1000);
+        const int replayed = call_with_depth(Seeds[i], Depths[i]);
+        if (replayed != recorded) {
+          std::fprintf(stderr,
+                       "FAIL replay %d depth=%d seed=%d got=%d expected=%d\n",
+                       i, Depths[i], Seeds[i], replayed, recorded);
+          failed = true;
+        }
+      }
+    }
+  }
+
+  if (failed)
+    return 1;
+
+  std::fprintf(stderr, "PASS replayable saved stack result=%d\n", recorded);
+  return 0;
+}
+
+// CHECK: PASS replayable saved stack result=
diff --git a/openmp/runtime/test/taskgraph/taskgraph_shared_stack_depth.cpp b/openmp/runtime/test/taskgraph/taskgraph_shared_stack_depth.cpp
new file mode 100644
index 0000000000000..ad579e8ed1b3d
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_shared_stack_depth.cpp
@@ -0,0 +1,94 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// XFAIL: *
+// clang-format on
+
+#include <cstdio>
+
+static volatile int StackSink = 0;
+
+struct Payload {
+  int values[6];
+  int bias;
+};
+
+__attribute__((noinline)) static int evaluate_payload(const Payload &payload,
+                                                      int seed) {
+  return seed * payload.values[0] - payload.values[1] +
+         payload.values[2] * payload.values[3] - payload.bias +
+         payload.values[4] - payload.values[5];
+}
+
+__attribute__((noinline)) static void clobber_stack(int base) {
+  volatile int scratch[4096];
+
+  for (int i = 0; i < 4096; ++i)
+    scratch[i] = base + i;
+
+  StackSink += scratch[base & 63];
+}
+
+__attribute__((noinline)) static int run_taskgraph(int seed) {
+  Payload payload{
+      {seed + 1, seed + 3, seed + 5, seed + 7, seed + 11, seed + 13},
+      seed * 17 + 19};
+  int result = -1;
+
+#pragma omp taskgraph graph_id(92)
+  {
+#pragma omp task firstprivate(payload, seed) shared(result)
+    {
+      result = evaluate_payload(payload, seed);
+    }
+  }
+
+  return result;
+}
+
+__attribute__((noinline)) static int call_with_depth(int seed, int depth) {
+  volatile int padding[128];
+
+  for (int i = 0; i < 128; ++i)
+    padding[i] = seed + depth + i;
+
+  StackSink += padding[(seed + depth) & 127];
+
+  if (depth == 0)
+    return run_taskgraph(seed);
+  return call_with_depth(seed, depth - 1);
+}
+
+__attribute__((noinline)) static int expected_result(int seed) {
+  Payload payload{
+      {seed + 1, seed + 3, seed + 5, seed + 7, seed + 11, seed + 13},
+      seed * 17 + 19};
+  return evaluate_payload(payload, seed);
+}
+
+int main() {
+  constexpr int RecordSeed = 3;
+  constexpr int ReplaySeed = 17;
+
+  const int recorded = call_with_depth(RecordSeed, 0);
+  if (recorded != expected_result(RecordSeed)) {
+    std::fprintf(stderr, "FAIL initial record got=%d expected=%d\n", recorded,
+                 expected_result(RecordSeed));
+    return 1;
+  }
+
+  clobber_stack(ReplaySeed * 1000);
+  const int replayed = call_with_depth(ReplaySeed, 3);
+  if (replayed != recorded) {
+    std::fprintf(
+        stderr, "BUG shared stack replay depth=%d seed=%d got=%d expected=%d\n",
+        3, ReplaySeed, replayed, recorded);
+    return 1;
+  }
+
+  std::fprintf(stderr, "PASS shared stack replay=%d\n", replayed);
+  return 0;
+}
+
+// CHECK: PASS shared stack replay=14

>From 4984eeb1c13e7cb37d8f5138813dfae5efd2fdb7 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Tue, 26 May 2026 15:20:12 -0500
Subject: [PATCH 17/26] [OpenMP] Extend taskgraph shared-data relocation to
 taskloop

Build on the per-task relocation infrastructure introduced for
'omp task' in the previous patch and apply the same scheme to
taskloop tasks recorded inside a taskgraph.  Without this,
by-reference captures inside a recorded taskloop iteration still
point at the original recording's stack on each replay.

On the compiler side (CGOpenMPRuntime.cpp), emit a relocation
thunk for the taskloop's captured statement via the shared
emitTaskRelocationFunction helper and pass it as the new trailing
argument of __kmpc_taskgraph_taskloop.  The now-unused 'shareds'
and 'sizeof_shareds' parameters are dropped from the call and from
TGTaskLoopArgs.

On the runtime side, update __kmpc_taskgraph_taskloop to match
the new ABI (drop the dead 'shareds' / 'sizeof_shareds'
parameters, add the trailing kmp_task_relocate_t parameter) and
plumb the callback through the taskgraph variant of
__kmp_taskloop and __kmp_taskloop_linear so every recorded
subtask gets node->relocate = reloc, mirroring the explicit-task
path.  Non-taskgraph callers pass a default nullptr and are
unaffected.

Add taskloop counterparts of the omp-task runtime tests added in
the previous patch.

Assisted-By: Claude Opus 4.7

Pull Request: https://github.com/llvm/llvm-project/pull/200405
---
 clang/lib/CodeGen/CGOpenMPRuntime.cpp         |  30 ++--
 .../include/llvm/Frontend/OpenMP/OMPKinds.def |   4 +-
 openmp/runtime/src/kmp.h                      |   3 +-
 openmp/runtime/src/kmp_tasking.cpp            |  24 ++--
 ..._lexical_shared_mixed_capture_taskloop.cpp |  45 ++++++
 ...red_nontrivial_type_recursive_taskloop.cpp |  89 ++++++++++++
 ...exical_shared_nontrivial_type_taskloop.cpp |  59 ++++++++
 ...red_pointer_recursive_frameid_taskloop.cpp |  95 +++++++++++++
 ...ayable_lexical_shared_pointer_taskloop.cpp |  44 ++++++
 ...able_lexical_shared_recursive_taskloop.cpp |  72 ++++++++++
 ...playable_lexical_shared_works_taskloop.cpp |  42 ++++++
 ...ble_nonlexical_shared_fails_1_taskloop.cpp |  51 +++++++
 ...ble_nonlexical_shared_fails_2_taskloop.cpp |  62 +++++++++
 ..._replayable_saved_stack_depth_taskloop.cpp | 130 ++++++++++++++++++
 14 files changed, 721 insertions(+), 29 deletions(-)
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_mixed_capture_taskloop.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type_recursive_taskloop.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type_taskloop.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer_recursive_frameid_taskloop.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer_taskloop.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_recursive_taskloop.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_works_taskloop.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_fails_1_taskloop.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_fails_2_taskloop.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_replayable_saved_stack_depth_taskloop.cpp

diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 6f3031161bfff..8bb2c5c8d7897 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -5173,7 +5173,7 @@ void CGOpenMPRuntime::emitTaskLoopCall(
                                                      PrePostActionTy &) {
     llvm::Value *ThreadId = getThreadID(CGF, Loc);
     llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
-    std::array<llvm::Value *, 16> TGTaskLoopArgs;
+    std::array<llvm::Value *, 14> TGTaskLoopArgs;
 
     // This is all copy/pasted from below. Refactor!
     LValue LBLVal = CGF.EmitLValueForField(
@@ -5218,30 +5218,34 @@ void CGOpenMPRuntime::emitTaskLoopCall(
     TGTaskLoopArgs[1] = ThreadId;
     TGTaskLoopArgs[2] = TaskInitResult.NewTask;
     TGTaskLoopArgs[3] = TaskAllocArgs[0]; // TaskFlags
-    TGTaskLoopArgs[4] = TaskAllocArgs[1]; // KmpTaskTWithPrivatesTySize
-    TGTaskLoopArgs[5] = Shareds.emitRawPointer(CGF);
-    TGTaskLoopArgs[6] = TaskAllocArgs[2]; // SharedsSize
-    TGTaskLoopArgs[7] = IfVal;
-    TGTaskLoopArgs[8] = LBLVal.getPointer(CGF);
-    TGTaskLoopArgs[9] = UBLVal.getPointer(CGF);
-    TGTaskLoopArgs[10] = CGF.EmitLoadOfScalar(StLVal, Loc);
-    TGTaskLoopArgs[11] =
+    TGTaskLoopArgs[4] = IfVal;
+    TGTaskLoopArgs[5] = LBLVal.getPointer(CGF);
+    TGTaskLoopArgs[6] = UBLVal.getPointer(CGF);
+    TGTaskLoopArgs[7] = CGF.EmitLoadOfScalar(StLVal, Loc);
+    TGTaskLoopArgs[8] =
         llvm::ConstantInt::getSigned(CGF.IntTy, Data.Nogroup ? 1 : 0);
-    TGTaskLoopArgs[12] = llvm::ConstantInt::getSigned(
+    TGTaskLoopArgs[9] = llvm::ConstantInt::getSigned(
         CGF.IntTy, Data.Schedule.getPointer()
                        ? Data.Schedule.getInt() ? NumTasks : Grainsize
                        : NoSchedule);
-    TGTaskLoopArgs[13] =
+    TGTaskLoopArgs[10] =
         Data.Schedule.getPointer()
             ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty,
                                         /*isSigned=*/false)
             : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0);
-    TGTaskLoopArgs[14] =
+    TGTaskLoopArgs[11] =
         llvm::ConstantInt::getSigned(CGF.IntTy, Data.HasModifier ? 1 : 0);
-    TGTaskLoopArgs[15] = TaskInitResult.TaskDupFn
+    TGTaskLoopArgs[12] = TaskInitResult.TaskDupFn
                              ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
                                    TaskInitResult.TaskDupFn, CGF.VoidPtrTy)
                              : llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+    const auto *CS = cast<CapturedStmt>(D.getAssociatedStmt());
+    llvm::Function *RelocFn =
+        emitTaskRelocationFunction(CGM, Loc, *CS, CGF.CapturedStmtInfo, Data);
+    TGTaskLoopArgs[13] =
+        RelocFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(RelocFn,
+                                                                  CGM.VoidPtrTy)
+                : llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
     CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
                             CGM.getModule(), OMPRTL___kmpc_taskgraph_taskloop),
                         TGTaskLoopArgs);
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
index 9b951d7b480b7..6ff0235c04546 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
@@ -362,8 +362,8 @@ __OMP_RTL(__kmpc_taskgraph, false, Void, IdentPtr, Int32, VoidPtrPtr, SizeTy,
 __OMP_RTL(__kmpc_taskgraph_task, false, Int32, IdentPtr, Int32, VoidPtr, Int32,
           SizeTy, SizeTy, Int32, VoidPtr, VoidPtr)
 __OMP_RTL(__kmpc_taskgraph_taskloop, false, Int32, IdentPtr, Int32, VoidPtr,
-          Int32, SizeTy, VoidPtr, SizeTy, Int32, Int64Ptr, Int64Ptr, Int64,
-          Int32, Int32, Int64, Int32, VoidPtr)
+          Int32, Int32, Int64Ptr, Int64Ptr, Int64,
+          Int32, Int32, Int64, Int32, VoidPtr, VoidPtr)
 __OMP_RTL(__kmpc_taskgraph_taskwait, false, Void, IdentPtr, Int32, Int32,
           VoidPtr, Int32)
 __OMP_RTL(__kmpc_taskgraph_taskred_init, false, /* kmp_taskgroup */ VoidPtr,
diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index 5ad6b88db6d35..4d81d0afcfebe 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -4537,10 +4537,9 @@ KMP_EXPORT kmp_uint32 __kmpc_taskgraph_task(
     kmp_depend_info_t *dep_list, kmp_task_relocate_t reloc);
 KMP_EXPORT kmp_uint32 __kmpc_taskgraph_taskloop(
     ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *new_task, kmp_int32 flags,
-    size_t sizeof_kmp_task_t, void *shareds, size_t sizeof_shareds,
     kmp_int32 if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
     kmp_int32 nogroup, kmp_int32 sched, kmp_uint64 grainsize,
-    kmp_int32 modifier, void *task_dup);
+    kmp_int32 modifier, void *task_dup, kmp_task_relocate_t reloc);
 KMP_EXPORT void __kmpc_taskgraph_taskwait(ident_t *loc_ref, kmp_int32 gtid,
                                           kmp_int32 ndeps,
                                           kmp_depend_info_t *dep_list,
diff --git a/openmp/runtime/src/kmp_tasking.cpp b/openmp/runtime/src/kmp_tasking.cpp
index b500652f5f10f..2b02dd98b4640 100644
--- a/openmp/runtime/src/kmp_tasking.cpp
+++ b/openmp/runtime/src/kmp_tasking.cpp
@@ -5155,7 +5155,8 @@ __kmp_taskloop_linear(ident_t *loc, int gtid, kmp_task_t *task, kmp_uint64 *lb,
                       void *task_dup
 #if OMP_TASKGRAPH_EXPERIMENTAL
                       ,
-                      kmp_taskgraph_record_t *taskgraph_rec = nullptr
+                      kmp_taskgraph_record_t *taskgraph_rec = nullptr,
+                      kmp_task_relocate_t relocate = nullptr
 #endif
 ) {
   KMP_COUNT_BLOCK(OMP_TASKLOOP);
@@ -5251,6 +5252,7 @@ __kmp_taskloop_linear(ident_t *loc, int gtid, kmp_task_t *task, kmp_uint64 *lb,
         taskgroup->taskgraph.reduce_input = nullptr;
       }
       node->taskloop_task = true;
+      node->relocate = relocate;
       next_taskdata->owning_taskgraph = taskgraph_rec;
       // FIXME: These dependency fields might be back-filled by the as-yet
       // unimplemented task_iteration subsidiary directive.  We'll need a way
@@ -5543,7 +5545,8 @@ static void __kmp_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
                            int modifier, void *task_dup
 #if OMP_TASKGRAPH_EXPERIMENTAL
                            ,
-                           kmp_taskgraph_record_t *taskgraph_rec = nullptr
+                           kmp_taskgraph_record_t *taskgraph_rec = nullptr,
+                           kmp_task_relocate_t relocate = nullptr
 #endif
 ) {
   kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
@@ -5677,7 +5680,7 @@ static void __kmp_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
 #if OMPT_SUPPORT
                           OMPT_GET_RETURN_ADDRESS(0),
 #endif
-                          task_dup, taskgraph_rec);
+                          task_dup, taskgraph_rec, relocate);
     // check if clause value next
     // Also require GOMP_taskloop to reduce to linear
     // (taskdata->td_flags.native)
@@ -6226,14 +6229,11 @@ void __kmpc_taskgraph_taskwait(ident_t *loc_ref, kmp_int32 gtid,
                               has_no_wait);
 }
 
-kmp_uint32 __kmpc_taskgraph_taskloop(ident_t *loc_ref, kmp_int32 gtid,
-                                     kmp_task_t *new_task, kmp_int32 flags,
-                                     size_t sizeof_kmp_task_t, void *shareds,
-                                     size_t sizeof_shareds, kmp_int32 if_val,
-                                     kmp_uint64 *lb, kmp_uint64 *ub,
-                                     kmp_int64 st, kmp_int32 nogroup,
-                                     kmp_int32 sched, kmp_uint64 grainsize,
-                                     kmp_int32 modifier, void *task_dup) {
+kmp_uint32 __kmpc_taskgraph_taskloop(
+    ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *new_task, kmp_int32 flags,
+    kmp_int32 if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
+    kmp_int32 nogroup, kmp_int32 sched, kmp_uint64 grainsize,
+    kmp_int32 modifier, void *task_dup, kmp_task_relocate_t relocate) {
   kmp_info_t *thread = __kmp_threads[gtid];
   kmp_taskgroup_t *taskgroup = thread->th.th_current_task->td_taskgroup;
   kmp_taskgraph_record_t *rec = __kmp_taskgraph_or_parent_recording(taskgroup);
@@ -6242,7 +6242,7 @@ kmp_uint32 __kmpc_taskgraph_taskloop(ident_t *loc_ref, kmp_int32 gtid,
     kmp_taskgraph_status_t status = KMP_ATOMIC_LD_ACQ(&rec->status);
     if (status == KMP_TDG_RECORDING)
       __kmp_taskloop(loc_ref, gtid, new_task, if_val, lb, ub, st, nogroup,
-                     sched, grainsize, modifier, task_dup, rec);
+                     sched, grainsize, modifier, task_dup, rec, relocate);
     else if (status == KMP_TDG_READY) {
 #ifdef DEBUG_TASKGRAPH
       fprintf(stderr, "non-taskgraph taskloop entry point for taskloop in "
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_mixed_capture_taskloop.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_mixed_capture_taskloop.cpp
new file mode 100644
index 0000000000000..147e768dcc85c
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_mixed_capture_taskloop.cpp
@@ -0,0 +1,45 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+__attribute__((noinline)) static int run_taskgraph_mixed_capture(int seed) {
+  int x = seed;
+  int y = seed * 2;
+  int fp = 7;
+  int res = 0;
+
+#pragma omp taskgraph graph_id(612)
+  {
+#pragma omp taskloop replayable num_tasks(8) shared(x, y) firstprivate(fp)     \
+    reduction(+ : res)
+    for (int i = 0; i < 16; ++i) {
+      res += x + y + fp + i;
+    }
+  }
+
+  return res;
+}
+
+int main() {
+  const int first = run_taskgraph_mixed_capture(1);
+  const int second = run_taskgraph_mixed_capture(100);
+
+  if (first != 280 || second != 5032) {
+    std::fprintf(stderr,
+                 "FAIL lexical mixed capture taskloop replay first=%d "
+                 "second=%d expected=280/5032\n",
+                 first, second);
+    return 1;
+  }
+
+  std::fprintf(
+      stderr, "PASS lexical mixed capture taskloop replay first=%d second=%d\n",
+      first, second);
+  return 0;
+}
+
+// CHECK: PASS lexical mixed capture taskloop replay first=280 second=5032
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type_recursive_taskloop.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type_recursive_taskloop.cpp
new file mode 100644
index 0000000000000..ab23adf6ce477
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type_recursive_taskloop.cpp
@@ -0,0 +1,89 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+struct Tracker {
+  static int Ctors;
+  static int Dtors;
+
+  int Value;
+
+  explicit Tracker(int V) : Value(V) { ++Ctors; }
+  ~Tracker() { ++Dtors; }
+
+  void bump(int Delta) { Value += Delta; }
+};
+
+int Tracker::Ctors = 0;
+int Tracker::Dtors = 0;
+
+__attribute__((noinline)) static int expected_recursive(int depth, int seed,
+                                                        int run_tag) {
+  int local = 16 * (seed + (depth + 1) * 5 + run_tag) + 120;
+  if (depth == 0)
+    return local;
+  return local + expected_recursive(depth - 1, seed + 9, run_tag);
+}
+
+__attribute__((noinline)) static int
+run_recursive_nontrivial(int depth, int seed, int run_tag) {
+  Tracker Obj(seed);
+  int res = 0;
+
+  int gid = 620 + depth;
+#pragma omp taskgraph graph_id(gid)
+  {
+#pragma omp taskloop replayable num_tasks(8) shared(Obj, depth, run_tag)       \
+    reduction(+ : res)
+    for (int i = 0; i < 16; ++i) {
+      res += Obj.Value + (depth + 1) * 5 + run_tag + i;
+    }
+  }
+
+  if (depth == 0)
+    return res;
+  return res + run_recursive_nontrivial(depth - 1, seed + 9, run_tag);
+}
+
+int main() {
+  const int depth = 3;
+  int total_actual = 0;
+  int total_expected = 0;
+
+  for (int run = 0; run < 3; ++run) {
+    const int seed = 100 * run + 1;
+    const int actual = run_recursive_nontrivial(depth, seed, run);
+    const int expected = expected_recursive(depth, seed, run);
+
+    if (actual != expected) {
+      std::fprintf(
+          stderr,
+          "FAIL recursive nontrivial taskloop run=%d actual=%d expected=%d\n",
+          run, actual, expected);
+      return 1;
+    }
+
+    total_actual += actual;
+    total_expected += expected;
+  }
+
+  if (Tracker::Ctors != Tracker::Dtors || Tracker::Ctors < 12) {
+    std::fprintf(stderr,
+                 "FAIL recursive nontrivial taskloop lifetime ctors=%d "
+                 "dtors=%d total=%d expected=%d\n",
+                 Tracker::Ctors, Tracker::Dtors, total_actual, total_expected);
+    return 1;
+  }
+
+  std::fprintf(stderr,
+               "PASS recursive nontrivial taskloop total=%d expected=%d "
+               "ctors=%d dtors=%d\n",
+               total_actual, total_expected, Tracker::Ctors, Tracker::Dtors);
+  return 0;
+}
+
+// CHECK: PASS recursive nontrivial taskloop total=
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type_taskloop.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type_taskloop.cpp
new file mode 100644
index 0000000000000..9d4d3dd15b7d2
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_nontrivial_type_taskloop.cpp
@@ -0,0 +1,59 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+struct Tracker {
+  static int Ctors;
+  static int Dtors;
+
+  int Value;
+
+  explicit Tracker(int V) : Value(V) { ++Ctors; }
+  ~Tracker() { ++Dtors; }
+
+  void bump(int Delta) { Value += Delta; }
+};
+
+int Tracker::Ctors = 0;
+int Tracker::Dtors = 0;
+
+__attribute__((noinline)) static int run_taskgraph_nontrivial(int seed) {
+  Tracker Obj(seed);
+  int res = 0;
+
+#pragma omp taskgraph graph_id(614)
+  {
+#pragma omp taskloop replayable num_tasks(8) shared(Obj) reduction(+ : res)
+    for (int i = 0; i < 16; ++i) {
+      res += Obj.Value + i;
+    }
+  }
+
+  return res;
+}
+
+int main() {
+  const int first = run_taskgraph_nontrivial(1);
+  const int second = run_taskgraph_nontrivial(100);
+
+  if (first != 136 || second != 1720 || Tracker::Ctors < 2 ||
+      Tracker::Dtors < 2 || Tracker::Ctors != Tracker::Dtors) {
+    std::fprintf(stderr,
+                 "FAIL lexical nontrivial taskloop replay first=%d second=%d "
+                 "ctors=%d dtors=%d\n",
+                 first, second, Tracker::Ctors, Tracker::Dtors);
+    return 1;
+  }
+
+  std::fprintf(stderr,
+               "PASS lexical nontrivial taskloop replay first=%d second=%d "
+               "ctors=%d dtors=%d\n",
+               first, second, Tracker::Ctors, Tracker::Dtors);
+  return 0;
+}
+
+// CHECK: PASS lexical nontrivial taskloop replay first=136 second=1720
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer_recursive_frameid_taskloop.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer_recursive_frameid_taskloop.cpp
new file mode 100644
index 0000000000000..31db8cac9b28f
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer_recursive_frameid_taskloop.cpp
@@ -0,0 +1,95 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+#include <cstdint>
+
+__attribute__((noinline)) static int expected_recursive(int depth, int seed,
+                                                        int run_tag) {
+  // Mirror run_recursive_frameid: the taskloop adds, over i in [0, 16),
+  // delta = (depth + 1) * 3 + run_tag + i both to 'value' (through the shared
+  // pointer) and to the reduction 'sum_delta'.
+  int sum_delta = 16 * ((depth + 1) * 3 + run_tag) + 120;
+  int value = seed + sum_delta;
+  int local = value * 17 + sum_delta;
+  if (depth == 0)
+    return local;
+  return local + expected_recursive(depth - 1, seed + 7, run_tag);
+}
+
+__attribute__((noinline)) static int run_recursive_frameid(int depth, int seed,
+                                                           int run_tag) {
+  int value = seed;
+  int *ptr = &value;
+  int *&ptr_ref = ptr;
+  int sum_delta = 0;
+  uintptr_t frame_gid = reinterpret_cast<uintptr_t>(__builtin_frame_address(0));
+
+  // Typically, if captured pointers refer to locations on the stack, that
+  // would not be safe for taskgraph record/replay because we in general we
+  // cannot rewrite such pointers to point to the current (live) stack frame.
+  //
+  // This is one possible way around that though: we keep a taskgraph record
+  // per stack-depth, each of which may refer to the local stack frame.
+  //
+  // I probably wouldn't recommend use of this technique in production code.
+#pragma omp taskgraph graph_id(frame_gid)
+  {
+#pragma omp taskloop replayable num_tasks(8) shared(ptr_ref, depth, run_tag)   \
+    reduction(+ : sum_delta)
+    for (int i = 0; i < 16; ++i) {
+      int delta = (depth + 1) * 3 + run_tag + i;
+      __atomic_fetch_add(ptr_ref, delta, __ATOMIC_RELAXED);
+      sum_delta += delta;
+    }
+  }
+
+  int local = value * 17 + sum_delta;
+
+  if (depth == 0)
+    return local;
+  return local + run_recursive_frameid(depth - 1, seed + 7, run_tag);
+}
+
+int main() {
+  const int depth = 3;
+  int actual_sum = 0;
+  int expected_sum = 0;
+  int failures = 0;
+
+#pragma omp parallel
+  {
+#pragma omp single
+    {
+      for (int run = 0; run < 3; ++run) {
+        int seed = 100 * run + 1;
+        int actual = run_recursive_frameid(depth, seed, run);
+        int expected = expected_recursive(depth, seed, run);
+        if (actual != expected) {
+          std::fprintf(
+              stderr,
+              "FAIL recursive pointer taskloop replay run=%d actual=%d "
+              "expected=%d\n",
+              run, actual, expected);
+          ++failures;
+        }
+        actual_sum += actual;
+        expected_sum += expected;
+      }
+    }
+  }
+
+  if (failures != 0)
+    return 1;
+
+  std::fprintf(stderr,
+               "PASS recursive pointer taskloop replay runs=3 total=%d "
+               "expected=%d\n",
+               actual_sum, expected_sum);
+  return 0;
+}
+
+// CHECK: PASS recursive pointer taskloop replay
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer_taskloop.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer_taskloop.cpp
new file mode 100644
index 0000000000000..ac5471505e539
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_pointer_taskloop.cpp
@@ -0,0 +1,44 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+__attribute__((noinline)) static int run_taskgraph_pointer_shared(int seed) {
+  int value = seed;
+  int *ptr = &value;
+  int res = 0;
+
+#pragma omp taskgraph graph_id(613)
+  {
+#pragma omp taskloop replayable num_tasks(8) shared(ptr) reduction(+ : res)
+    for (int i = 0; i < 16; ++i) {
+      res += *ptr + i;
+    }
+  }
+
+  return res;
+}
+
+int main() {
+  const int first = run_taskgraph_pointer_shared(1);
+  const int second = run_taskgraph_pointer_shared(100);
+
+  if (first != 136 || second != 1720) {
+    std::fprintf(stderr,
+                 "FAIL lexical pointer shared taskloop replay first=%d "
+                 "second=%d expected=136/1720\n",
+                 first, second);
+    return 1;
+  }
+
+  std::fprintf(
+      stderr,
+      "PASS lexical pointer shared taskloop replay first=%d second=%d\n", first,
+      second);
+  return 0;
+}
+
+// CHECK: PASS lexical pointer shared taskloop replay first=136 second=1720
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_recursive_taskloop.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_recursive_taskloop.cpp
new file mode 100644
index 0000000000000..8d3df091be741
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_recursive_taskloop.cpp
@@ -0,0 +1,72 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+__attribute__((noinline)) static int expected_recursive(int depth, int seed) {
+  int x = seed;
+  int sum_delta = 0;
+
+  for (int i = 0; i < 16; ++i) {
+    sum_delta += depth + i + 1;
+  }
+
+  x += sum_delta;
+  int local = x * 17 + sum_delta;
+
+  if (depth == 0)
+    return local;
+
+  return local + expected_recursive(depth - 1, seed + 10);
+}
+
+__attribute__((noinline)) static int run_taskgraph_recursive(int depth,
+                                                             int seed) {
+  int x = seed;
+  int *ptr = &x;
+  int sum_delta = 0;
+  int gid = 615;
+
+#pragma omp taskgraph graph_id(gid)
+  {
+#pragma omp taskloop replayable num_tasks(8) shared(ptr, depth)                \
+    reduction(+ : sum_delta)
+    for (int i = 0; i < 16; ++i) {
+      int delta = depth + i + 1;
+      __atomic_fetch_add(ptr, delta, __ATOMIC_RELAXED);
+      sum_delta += delta;
+    }
+  }
+
+  int local = x * 17 + sum_delta;
+
+  if (depth == 0)
+    return local;
+
+  return local + run_taskgraph_recursive(depth - 1, seed + 10);
+}
+
+int main() {
+  const int first = run_taskgraph_recursive(3, 1);
+  const int second = run_taskgraph_recursive(3, 100);
+  const int expected_first = expected_recursive(3, 1);
+  const int expected_second = expected_recursive(3, 100);
+
+  if (first == expected_first && second == expected_second) {
+    std::fprintf(stderr,
+                 "PASS lexical recursive taskloop replay first=%d second=%d\n",
+                 first, second);
+    return 0;
+  }
+
+  std::fprintf(stderr,
+               "FAIL lexical recursive taskloop replay first=%d second=%d "
+               "expected=%d/%d\n",
+               first, second, expected_first, expected_second);
+  return 1;
+}
+
+// CHECK: PASS lexical recursive taskloop replay
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_works_taskloop.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_works_taskloop.cpp
new file mode 100644
index 0000000000000..beaaedfea77f3
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_lexical_shared_works_taskloop.cpp
@@ -0,0 +1,42 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+__attribute__((noinline)) static int run_taskgraph_lexical(int seed) {
+  int x = seed;
+  int res = 0;
+
+#pragma omp taskgraph graph_id(611)
+  {
+#pragma omp taskloop replayable num_tasks(8) shared(x) reduction(+ : res)
+    for (int i = 0; i < 16; ++i) {
+      res += x + i;
+    }
+  }
+
+  return res;
+}
+
+int main() {
+  const int first = run_taskgraph_lexical(1);
+  const int second = run_taskgraph_lexical(100);
+
+  if (first != 136 || second != 1720) {
+    std::fprintf(stderr,
+                 "FAIL lexical shared taskloop replay first=%d second=%d "
+                 "expected=136/1720\n",
+                 first, second);
+    return 1;
+  }
+
+  std::fprintf(stderr,
+               "PASS lexical shared taskloop replay first=%d second=%d\n",
+               first, second);
+  return 0;
+}
+
+// CHECK: PASS lexical shared taskloop replay first=136 second=1720
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_fails_1_taskloop.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_fails_1_taskloop.cpp
new file mode 100644
index 0000000000000..0728d56ef15d8
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_fails_1_taskloop.cpp
@@ -0,0 +1,51 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %not --crash %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+__attribute__((noinline)) static int emit_nonlexical_taskloop(int &x) {
+  int sum = 0;
+#pragma omp taskloop replayable num_tasks(8) shared(x) reduction(+ : sum)
+  for (int i = 0; i < 16; ++i) {
+    sum += x + i;
+  }
+  x += 5;
+  return sum;
+}
+
+__attribute__((noinline)) static int run_taskgraph_nonlexical(int seed) {
+  int x = seed;
+  int out = -1;
+
+#pragma omp taskgraph graph_id(631)
+  {
+    out = emit_nonlexical_taskloop(x);
+  }
+
+  return out;
+}
+
+int main() {
+  const int recorded = run_taskgraph_nonlexical(1);
+  const int replayed = run_taskgraph_nonlexical(100);
+
+  if (recorded == replayed) {
+    std::fprintf(stderr,
+                 "UNEXPECTED SUCCESS nonlexical taskloop replay recorded=%d "
+                 "replayed=%d\n",
+                 recorded, replayed);
+    return 0;
+  }
+
+  std::fprintf(
+      stderr,
+      "EXPECTED FAILURE nonlexical taskloop replay recorded=%d replayed=%d\n",
+      recorded, replayed);
+  return 1;
+}
+
+// CHECK: OMP: Error #{{[0-9]+}}: Cannot locate captured shared variable reference for
+// taskgraph replay
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_fails_2_taskloop.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_fails_2_taskloop.cpp
new file mode 100644
index 0000000000000..ca5c5ea0daf7d
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_nonlexical_shared_fails_2_taskloop.cpp
@@ -0,0 +1,62 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %not --crash %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+__attribute__((noinline)) static int emit_nonlexical_taskloop(int seed) {
+  int x = seed;
+  int sum = 0;
+
+#pragma omp taskloop replayable num_tasks(8) shared(x) reduction(+ : sum)
+  for (int i = 0; i < 16; ++i) {
+    sum += x + i;
+  }
+
+  return sum;
+}
+
+__attribute__((noinline)) static int run_taskgraph_nonlexical(int seed) {
+  int out;
+
+#pragma omp taskgraph graph_id(632)
+  {
+#pragma omp task shared(out)
+    {
+      out = emit_nonlexical_taskloop(seed);
+    }
+  }
+
+  return out;
+}
+
+int main() {
+  int out = emit_nonlexical_taskloop(50);
+  if (out != 920) {
+    std::fprintf(stderr,
+                 "UNEXPECTED FAILURE: taskloop outside taskgraph returned %d\n",
+                 out);
+  }
+
+  const int recorded = run_taskgraph_nonlexical(1);
+  const int replayed = run_taskgraph_nonlexical(100);
+
+  if (recorded == replayed) {
+    std::fprintf(stderr,
+                 "UNEXPECTED SUCCESS nonlexical taskloop replay recorded=%d "
+                 "replayed=%d\n",
+                 recorded, replayed);
+    return 0;
+  }
+
+  std::fprintf(
+      stderr,
+      "EXPECTED FAILURE nonlexical taskloop replay recorded=%d replayed=%d\n",
+      recorded, replayed);
+  return 1;
+}
+
+// CHECK: OMP: Error #{{[0-9]+}}: Cannot locate captured shared variable reference for
+// taskgraph replay
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_saved_stack_depth_taskloop.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_saved_stack_depth_taskloop.cpp
new file mode 100644
index 0000000000000..5a5b8352f4925
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_saved_stack_depth_taskloop.cpp
@@ -0,0 +1,130 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+#include <cstdio>
+
+static volatile int StackSink = 0;
+// Aggregated across the taskloop body via #pragma omp atomic. Lives in
+// static storage so the address never moves across replays (this test is
+// about firstprivate(saved:), not reductions: reductions on replayable
+// taskgraphs are tracked separately).
+static volatile int Aggregate = 0;
+
+struct Payload {
+  int values[6];
+  int bias;
+};
+
+__attribute__((noinline)) static int evaluate_payload(const Payload &payload,
+                                                      int seed) {
+  return seed * payload.values[0] - payload.values[1] +
+         payload.values[2] * payload.values[3] - payload.bias +
+         payload.values[4] - payload.values[5];
+}
+
+__attribute__((noinline)) static void clobber_stack(int base) {
+  volatile int scratch[4096];
+
+  for (int i = 0; i < 4096; ++i)
+    scratch[i] = base + i;
+
+  StackSink += scratch[base & 63];
+}
+
+// The taskloop directive lives in a helper function called from inside the
+// taskgraph region, so the loop's firstprivate(saved:) captures originate
+// in a non-lexical surrounding scope. The taskloop relocation helper has
+// nothing to refresh at replay since the saved snapshots are sourced from
+// '.kmp_privates.t', not the shareds slots.
+__attribute__((noinline)) static void emit_replayable_taskloop(int seed) {
+  Payload payload{
+      {seed + 1, seed + 3, seed + 5, seed + 7, seed + 11, seed + 13},
+      seed * 17 + 19};
+
+#pragma omp taskloop replayable num_tasks(8) firstprivate(saved : payload, seed)
+  for (int i = 0; i < 16; ++i) {
+    int contribution = evaluate_payload(payload, seed + i);
+#pragma omp atomic
+    Aggregate += contribution;
+  }
+}
+
+__attribute__((noinline)) static int run_taskgraph(int seed) {
+  Aggregate = 0;
+
+#pragma omp taskgraph graph_id(633)
+  {
+    emit_replayable_taskloop(seed);
+  }
+
+  return Aggregate;
+}
+
+__attribute__((noinline)) static int call_with_depth(int seed, int depth) {
+  volatile int padding[128];
+
+  for (int i = 0; i < 128; ++i)
+    padding[i] = seed + depth + i;
+
+  StackSink += padding[(seed + depth) & 127];
+
+  if (depth == 0)
+    return run_taskgraph(seed);
+  return call_with_depth(seed, depth - 1);
+}
+
+__attribute__((noinline)) static int expected_result(int seed) {
+  Payload payload{
+      {seed + 1, seed + 3, seed + 5, seed + 7, seed + 11, seed + 13},
+      seed * 17 + 19};
+  int sum = 0;
+  for (int i = 0; i < 16; ++i)
+    sum += evaluate_payload(payload, seed + i);
+  return sum;
+}
+
+int main() {
+  constexpr int NumCalls = 4;
+  constexpr int Seeds[NumCalls] = {3, 17, 29, 41};
+  constexpr int Depths[NumCalls] = {0, 3, 1, 5};
+
+  int recorded = -1;
+  bool failed = false;
+
+#pragma omp parallel num_threads(4)
+  {
+#pragma omp single
+    {
+      recorded = call_with_depth(Seeds[0], Depths[0]);
+      if (recorded != expected_result(Seeds[0])) {
+        std::fprintf(stderr,
+                     "FAIL initial taskloop record got=%d expected=%d\n",
+                     recorded, expected_result(Seeds[0]));
+        failed = true;
+      }
+
+      for (int i = 1; i < NumCalls; ++i) {
+        clobber_stack(Seeds[i] * 1000);
+        const int replayed = call_with_depth(Seeds[i], Depths[i]);
+        if (replayed != recorded) {
+          std::fprintf(
+              stderr,
+              "FAIL taskloop replay %d depth=%d seed=%d got=%d expected=%d\n",
+              i, Depths[i], Seeds[i], replayed, recorded);
+          failed = true;
+        }
+      }
+    }
+  }
+
+  if (failed)
+    return 1;
+
+  std::fprintf(stderr, "PASS replayable taskloop saved stack result=%d\n",
+               recorded);
+  return 0;
+}
+
+// CHECK: PASS replayable taskloop saved stack result=

>From 9288915c47220cdd5d5aceef316078f320a99e66 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Wed, 27 May 2026 07:51:15 -0500
Subject: [PATCH 18/26] [Clang][OpenMP] Parsing, serialisation and
 deserialisation for the 'saved' firstprivate modifier

OpenMP 6.0 adds a 'saved' modifier to the data-environment
attribute clauses.  On a 'firstprivate' clause it specifies that
the list item's value is captured when the surrounding construct
is recorded into a taskgraph, and that captured value is reused
(rather than re-evaluating the initialiser from the live original)
on every replay.  See OpenMP 6.0 [7.5.4], [6.4.7] and [14.3].

This patch implements the front-end plumbing for the modifier:
parsing, AST representation and AST printing, profile, template
instantiation and PCH serialisation, modelled directly on the
existing 'lastprivate(conditional: ...)' machinery.  No new
semantic restriction is added here (e.g. requiring a
taskgraph-nested or replayable task context), and codegen and the
runtime continue to treat 'saved' as a no-op for now -- those are
follow-up patches.

Specifically:

  - OPENMP_FIRSTPRIVATE_KIND(saved) is added in OpenMPKinds.def,
    with a matching OpenMPFirstprivateModifier enum and the
    appropriate parse/print helpers in OpenMPKinds.cpp.
  - OMPFirstprivateClause gains FPKind / FPKindLoc / ColonLoc
    fields and an updated Create() signature; OMPClausePrinter
    emits 'firstprivate(saved: ...)' when the modifier is set.
  - ParseOpenMP recognises '[modifier:]' inside a firstprivate
    clause when OpenMP >= 6.0.
  - SemaOpenMP::ActOnOpenMPFirstprivateClause now takes the
    modifier triple and diagnoses an unrecognised modifier via
    the existing err_omp_unexpected_clause_value path; the
    dispatcher in ActOnOpenMPVarListClause and the
    implicit-firstprivate emitter in ActOnOpenMPRegionEnd are
    updated accordingly.
  - TreeTransform forwards the modifier through template
    instantiation; ASTReader / ASTWriter round-trip the new
    fields.

A new lit test taskgraph_firstprivate_saved_ast_print.cpp covers
ast-print and a PCH emit -> include -> ast-print round-trip on
'omp task' and 'omp taskloop' uses of firstprivate(saved: ...).

Assisted-By: Claude Opus 4.7

Pull Request: https://github.com/llvm/llvm-project/pull/200406
---
 clang/include/clang/AST/OpenMPClause.h        | 38 +++++++++++++++--
 clang/include/clang/Basic/OpenMPKinds.def     |  7 ++++
 clang/include/clang/Basic/OpenMPKinds.h       |  7 ++++
 clang/include/clang/Sema/SemaOpenMP.h         |  8 ++--
 clang/lib/AST/OpenMPClause.cpp                | 22 ++++++----
 clang/lib/Basic/OpenMPKinds.cpp               | 17 +++++++-
 clang/lib/Parse/ParseOpenMP.cpp               | 13 ++++++
 clang/lib/Sema/SemaOpenMP.cpp                 | 30 ++++++++++----
 clang/lib/Sema/TreeTransform.h                | 10 +++--
 clang/lib/Serialization/ASTReader.cpp         |  3 ++
 clang/lib/Serialization/ASTWriter.cpp         |  3 ++
 ...taskgraph_firstprivate_saved_ast_print.cpp | 41 +++++++++++++++++++
 12 files changed, 172 insertions(+), 27 deletions(-)
 create mode 100644 clang/test/OpenMP/taskgraph_firstprivate_saved_ast_print.cpp

diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 105bef8fb1f0d..5cb07798ad549 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -3698,17 +3698,31 @@ class OMPFirstprivateClause final
   friend OMPVarListClause;
   friend TrailingObjects;
 
+  /// Optional firstprivate modifier (e.g. 'saved'), if specified by the user.
+  OpenMPFirstprivateModifier FPKind = OMPC_FIRSTPRIVATE_unknown;
+  /// Optional location of the firstprivate modifier, if specified.
+  SourceLocation FPKindLoc;
+  /// Optional location of the ':' separating the modifier from the list.
+  SourceLocation ColonLoc;
+
   /// Build clause with number of variables \a N.
   ///
   /// \param StartLoc Starting location of the clause.
   /// \param LParenLoc Location of '('.
   /// \param EndLoc Ending location of the clause.
+  /// \param FPKind Firstprivate modifier (e.g. 'saved').
+  /// \param FPKindLoc Location of the firstprivate modifier, if any.
+  /// \param ColonLoc Location of the ':' symbol, if a modifier is used.
   /// \param N Number of the variables in the clause.
   OMPFirstprivateClause(SourceLocation StartLoc, SourceLocation LParenLoc,
-                        SourceLocation EndLoc, unsigned N)
+                        SourceLocation EndLoc,
+                        OpenMPFirstprivateModifier FPKind,
+                        SourceLocation FPKindLoc, SourceLocation ColonLoc,
+                        unsigned N)
       : OMPVarListClause<OMPFirstprivateClause>(llvm::omp::OMPC_firstprivate,
                                                 StartLoc, LParenLoc, EndLoc, N),
-        OMPClauseWithPreInit(this) {}
+        OMPClauseWithPreInit(this), FPKind(FPKind), FPKindLoc(FPKindLoc),
+        ColonLoc(ColonLoc) {}
 
   /// Build an empty clause.
   ///
@@ -3719,6 +3733,13 @@ class OMPFirstprivateClause final
             SourceLocation(), N),
         OMPClauseWithPreInit(this) {}
 
+  /// Sets the firstprivate modifier kind.
+  void setKind(OpenMPFirstprivateModifier Kind) { FPKind = Kind; }
+  /// Sets the location of the firstprivate modifier.
+  void setKindLoc(SourceLocation Loc) { FPKindLoc = Loc; }
+  /// Sets the location of the ':' separator.
+  void setColonLoc(SourceLocation Loc) { ColonLoc = Loc; }
+
   /// Sets the list of references to private copies with initializers for
   /// new private variables.
   /// \param VL List of references.
@@ -3759,12 +3780,16 @@ class OMPFirstprivateClause final
   /// \param InitVL List of references to auto generated variables used for
   /// initialization of a single array element. Used if firstprivate variable is
   /// of array type.
+  /// \param FPKind Firstprivate modifier, e.g. 'saved'.
+  /// \param FPKindLoc Location of the firstprivate modifier, if any.
+  /// \param ColonLoc Location of the ':' symbol, if a modifier is used.
   /// \param PreInit Statement that must be executed before entering the OpenMP
   /// region with this clause.
   static OMPFirstprivateClause *
   Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc,
          SourceLocation EndLoc, ArrayRef<Expr *> VL, ArrayRef<Expr *> PrivateVL,
-         ArrayRef<Expr *> InitVL, Stmt *PreInit);
+         ArrayRef<Expr *> InitVL, OpenMPFirstprivateModifier FPKind,
+         SourceLocation FPKindLoc, SourceLocation ColonLoc, Stmt *PreInit);
 
   /// Creates an empty clause with the place for \a N variables.
   ///
@@ -3772,6 +3797,13 @@ class OMPFirstprivateClause final
   /// \param N The number of variables.
   static OMPFirstprivateClause *CreateEmpty(const ASTContext &C, unsigned N);
 
+  /// Firstprivate modifier (e.g. 'saved' if specified, otherwise 'unknown').
+  OpenMPFirstprivateModifier getKind() const { return FPKind; }
+  /// Returns the location of the firstprivate modifier, if any.
+  SourceLocation getKindLoc() const { return FPKindLoc; }
+  /// Returns the location of the ':' symbol, if any.
+  SourceLocation getColonLoc() const { return ColonLoc; }
+
   using private_copies_iterator = MutableArrayRef<Expr *>::iterator;
   using private_copies_const_iterator = ArrayRef<const Expr *>::iterator;
   using private_copies_range = llvm::iterator_range<private_copies_iterator>;
diff --git a/clang/include/clang/Basic/OpenMPKinds.def b/clang/include/clang/Basic/OpenMPKinds.def
index 5c396a5d2d4b3..dc0ce134ccacf 100644
--- a/clang/include/clang/Basic/OpenMPKinds.def
+++ b/clang/include/clang/Basic/OpenMPKinds.def
@@ -59,6 +59,9 @@
 #ifndef OPENMP_DEVICE_TYPE_KIND
 #define OPENMP_DEVICE_TYPE_KIND(Name)
 #endif
+#ifndef OPENMP_FIRSTPRIVATE_KIND
+#define OPENMP_FIRSTPRIVATE_KIND(Name)
+#endif
 #ifndef OPENMP_LASTPRIVATE_KIND
 #define OPENMP_LASTPRIVATE_KIND(Name)
 #endif
@@ -230,6 +233,9 @@ OPENMP_DEVICE_TYPE_KIND(host)
 OPENMP_DEVICE_TYPE_KIND(nohost)
 OPENMP_DEVICE_TYPE_KIND(any)
 
+// Type of the 'firstprivate' clause.
+OPENMP_FIRSTPRIVATE_KIND(saved)
+
 // Type of the 'lastprivate' clause.
 OPENMP_LASTPRIVATE_KIND(conditional)
 
@@ -319,6 +325,7 @@ OPENMP_USE_DEVICE_PTR_FALLBACK_MODIFIER(fb_preserve)
 #undef OPENMP_ORIGINAL_SHARING_MODIFIER
 #undef OPENMP_ORDER_KIND
 #undef OPENMP_ORDER_MODIFIER
+#undef OPENMP_FIRSTPRIVATE_KIND
 #undef OPENMP_LASTPRIVATE_KIND
 #undef OPENMP_DEVICE_TYPE_KIND
 #undef OPENMP_LINEAR_KIND
diff --git a/clang/include/clang/Basic/OpenMPKinds.h b/clang/include/clang/Basic/OpenMPKinds.h
index 36c388668a455..8b8495c143009 100644
--- a/clang/include/clang/Basic/OpenMPKinds.h
+++ b/clang/include/clang/Basic/OpenMPKinds.h
@@ -161,6 +161,13 @@ enum OpenMPDeviceType {
   OMPC_DEVICE_TYPE_unknown
 };
 
+/// OpenMP 'firstprivate' clause modifier.
+enum OpenMPFirstprivateModifier {
+#define OPENMP_FIRSTPRIVATE_KIND(Name) OMPC_FIRSTPRIVATE_##Name,
+#include "clang/Basic/OpenMPKinds.def"
+  OMPC_FIRSTPRIVATE_unknown,
+};
+
 /// OpenMP 'lastprivate' clause modifier.
 enum OpenMPLastprivateModifier {
 #define OPENMP_LASTPRIVATE_KIND(Name) OMPC_LASTPRIVATE_##Name,
diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index 451ba634cc482..507e3eb147f63 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -1276,10 +1276,10 @@ class SemaOpenMP : public SemaBase {
                                       SourceLocation LParenLoc,
                                       SourceLocation EndLoc);
   /// Called on well-formed 'firstprivate' clause.
-  OMPClause *ActOnOpenMPFirstprivateClause(ArrayRef<Expr *> VarList,
-                                           SourceLocation StartLoc,
-                                           SourceLocation LParenLoc,
-                                           SourceLocation EndLoc);
+  OMPClause *ActOnOpenMPFirstprivateClause(
+      ArrayRef<Expr *> VarList, OpenMPFirstprivateModifier FPKind,
+      SourceLocation FPKindLoc, SourceLocation ColonLoc,
+      SourceLocation StartLoc, SourceLocation LParenLoc, SourceLocation EndLoc);
   /// Called on well-formed 'lastprivate' clause.
   OMPClause *ActOnOpenMPLastprivateClause(
       ArrayRef<Expr *> VarList, OpenMPLastprivateModifier LPKind,
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index 15bb1d460be6a..de2ff950ceb72 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -483,14 +483,14 @@ void OMPFirstprivateClause::setInits(ArrayRef<Expr *> VL) {
   llvm::copy(VL, getPrivateCopies().end());
 }
 
-OMPFirstprivateClause *
-OMPFirstprivateClause::Create(const ASTContext &C, SourceLocation StartLoc,
-                              SourceLocation LParenLoc, SourceLocation EndLoc,
-                              ArrayRef<Expr *> VL, ArrayRef<Expr *> PrivateVL,
-                              ArrayRef<Expr *> InitVL, Stmt *PreInit) {
+OMPFirstprivateClause *OMPFirstprivateClause::Create(
+    const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc,
+    SourceLocation EndLoc, ArrayRef<Expr *> VL, ArrayRef<Expr *> PrivateVL,
+    ArrayRef<Expr *> InitVL, OpenMPFirstprivateModifier FPKind,
+    SourceLocation FPKindLoc, SourceLocation ColonLoc, Stmt *PreInit) {
   void *Mem = C.Allocate(totalSizeToAlloc<Expr *>(3 * VL.size()));
-  OMPFirstprivateClause *Clause =
-      new (Mem) OMPFirstprivateClause(StartLoc, LParenLoc, EndLoc, VL.size());
+  OMPFirstprivateClause *Clause = new (Mem) OMPFirstprivateClause(
+      StartLoc, LParenLoc, EndLoc, FPKind, FPKindLoc, ColonLoc, VL.size());
   Clause->setVarRefs(VL);
   Clause->setPrivateCopies(PrivateVL);
   Clause->setInits(InitVL);
@@ -2634,7 +2634,13 @@ void OMPClausePrinter::VisitOMPPrivateClause(OMPPrivateClause *Node) {
 void OMPClausePrinter::VisitOMPFirstprivateClause(OMPFirstprivateClause *Node) {
   if (!Node->varlist_empty()) {
     OS << "firstprivate";
-    VisitOMPClauseList(Node, '(');
+    OpenMPFirstprivateModifier FPKind = Node->getKind();
+    if (FPKind != OMPC_FIRSTPRIVATE_unknown) {
+      OS << "("
+         << getOpenMPSimpleClauseTypeName(OMPC_firstprivate, Node->getKind())
+         << ":";
+    }
+    VisitOMPClauseList(Node, FPKind == OMPC_FIRSTPRIVATE_unknown ? '(' : ' ');
     OS << ")";
   }
 }
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index f9cf0d8f55aca..7db404f576dc3 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -149,6 +149,11 @@ unsigned clang::getOpenMPSimpleClauseType(OpenMPClauseKind Kind, StringRef Str,
 #define OPENMP_SEVERITY_KIND(Name) .Case(#Name, OMPC_SEVERITY_##Name)
 #include "clang/Basic/OpenMPKinds.def"
         .Default(OMPC_SEVERITY_unknown);
+  case OMPC_firstprivate:
+    return llvm::StringSwitch<OpenMPFirstprivateModifier>(Str)
+#define OPENMP_FIRSTPRIVATE_KIND(Name) .Case(#Name, OMPC_FIRSTPRIVATE_##Name)
+#include "clang/Basic/OpenMPKinds.def"
+        .Default(OMPC_FIRSTPRIVATE_unknown);
   case OMPC_lastprivate:
     return llvm::StringSwitch<OpenMPLastprivateModifier>(Str)
 #define OPENMP_LASTPRIVATE_KIND(Name) .Case(#Name, OMPC_LASTPRIVATE_##Name)
@@ -281,7 +286,6 @@ unsigned clang::getOpenMPSimpleClauseType(OpenMPClauseKind Kind, StringRef Str,
   case OMPC_allocator:
   case OMPC_collapse:
   case OMPC_private:
-  case OMPC_firstprivate:
   case OMPC_shared:
   case OMPC_task_reduction:
   case OMPC_in_reduction:
@@ -495,6 +499,16 @@ const char *clang::getOpenMPSimpleClauseTypeName(OpenMPClauseKind Kind,
 #include "clang/Basic/OpenMPKinds.def"
     }
     llvm_unreachable("Invalid OpenMP 'severity' clause type");
+  case OMPC_firstprivate:
+    switch (Type) {
+    case OMPC_FIRSTPRIVATE_unknown:
+      return "unknown";
+#define OPENMP_FIRSTPRIVATE_KIND(Name)                                         \
+  case OMPC_FIRSTPRIVATE_##Name:                                               \
+    return #Name;
+#include "clang/Basic/OpenMPKinds.def"
+    }
+    llvm_unreachable("Invalid OpenMP 'firstprivate' clause type");
   case OMPC_lastprivate:
     switch (Type) {
     case OMPC_LASTPRIVATE_unknown:
@@ -681,7 +695,6 @@ const char *clang::getOpenMPSimpleClauseTypeName(OpenMPClauseKind Kind,
   case OMPC_allocator:
   case OMPC_collapse:
   case OMPC_private:
-  case OMPC_firstprivate:
   case OMPC_shared:
   case OMPC_task_reduction:
   case OMPC_in_reduction:
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index d79a1737fcbb2..dc2f5dc777c29 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -5095,6 +5095,19 @@ bool Parser::ParseOpenMPVarList(OpenMPDirectiveKind DKind,
             << "linear-modifier(list)" << getOpenMPClauseName(Kind)
             << "linear(list: [linear-modifier,] step(step-size))";
     }
+  } else if (Kind == OMPC_firstprivate) {
+    // Try to parse modifier if any.
+    Data.ExtraModifier = OMPC_FIRSTPRIVATE_unknown;
+    // The 'saved' modifier is OpenMP 6.0+ only.
+    if (getLangOpts().OpenMP >= 60 && Tok.is(tok::identifier) &&
+        PP.LookAhead(0).is(tok::colon)) {
+      Data.ExtraModifier =
+          getOpenMPSimpleClauseType(Kind, PP.getSpelling(Tok), getLangOpts());
+      Data.ExtraModifierLoc = Tok.getLocation();
+      ConsumeToken();
+      assert(Tok.is(tok::colon) && "Expected colon.");
+      Data.ColonLoc = ConsumeToken();
+    }
   } else if (Kind == OMPC_lastprivate) {
     // Try to parse modifier if any.
     Data.ExtraModifier = OMPC_LASTPRIVATE_unknown;
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index c31db03c62c3f..eb4004922ee28 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -6378,8 +6378,10 @@ StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
     }
     if (!ImpInfo.Firstprivates.empty()) {
       if (OMPClause *Implicit = ActOnOpenMPFirstprivateClause(
-              ImpInfo.Firstprivates.getArrayRef(), SourceLocation(),
-              SourceLocation(), SourceLocation())) {
+              ImpInfo.Firstprivates.getArrayRef(), OMPC_FIRSTPRIVATE_unknown,
+              /*FPKindLoc=*/SourceLocation(), /*ColonLoc=*/SourceLocation(),
+              /*StartLoc=*/SourceLocation(), /*LParenLoc=*/SourceLocation(),
+              /*EndLoc=*/SourceLocation())) {
         ClausesWithImplicit.push_back(Implicit);
         ErrorFound = cast<OMPFirstprivateClause>(Implicit)->varlist_size() !=
                      ImpInfo.Firstprivates.size();
@@ -19407,7 +19409,11 @@ OMPClause *SemaOpenMP::ActOnOpenMPVarListClause(OpenMPClauseKind Kind,
     Res = ActOnOpenMPPrivateClause(VarList, StartLoc, LParenLoc, EndLoc);
     break;
   case OMPC_firstprivate:
-    Res = ActOnOpenMPFirstprivateClause(VarList, StartLoc, LParenLoc, EndLoc);
+    assert(0 <= ExtraModifier && ExtraModifier <= OMPC_FIRSTPRIVATE_unknown &&
+           "Unexpected firstprivate modifier.");
+    Res = ActOnOpenMPFirstprivateClause(
+        VarList, static_cast<OpenMPFirstprivateModifier>(ExtraModifier),
+        ExtraModifierLoc, ColonLoc, StartLoc, LParenLoc, EndLoc);
     break;
   case OMPC_lastprivate:
     assert(0 <= ExtraModifier && ExtraModifier <= OMPC_LASTPRIVATE_unknown &&
@@ -19800,10 +19806,19 @@ OMPClause *SemaOpenMP::ActOnOpenMPPrivateClause(ArrayRef<Expr *> VarList,
                                   Vars, PrivateCopies);
 }
 
-OMPClause *SemaOpenMP::ActOnOpenMPFirstprivateClause(ArrayRef<Expr *> VarList,
-                                                     SourceLocation StartLoc,
-                                                     SourceLocation LParenLoc,
-                                                     SourceLocation EndLoc) {
+OMPClause *SemaOpenMP::ActOnOpenMPFirstprivateClause(
+    ArrayRef<Expr *> VarList, OpenMPFirstprivateModifier FPKind,
+    SourceLocation FPKindLoc, SourceLocation ColonLoc, SourceLocation StartLoc,
+    SourceLocation LParenLoc, SourceLocation EndLoc) {
+  if (FPKind == OMPC_FIRSTPRIVATE_unknown && FPKindLoc.isValid()) {
+    assert(ColonLoc.isValid() && "Colon location must be valid.");
+    Diag(FPKindLoc, diag::err_omp_unexpected_clause_value)
+        << getListOfPossibleValues(OMPC_firstprivate, /*First=*/0,
+                                   /*Last=*/OMPC_FIRSTPRIVATE_unknown)
+        << getOpenMPClauseNameForDiag(OMPC_firstprivate);
+    return nullptr;
+  }
+
   SmallVector<Expr *, 8> Vars;
   SmallVector<Expr *, 8> PrivateCopies;
   SmallVector<Expr *, 8> Inits;
@@ -20089,6 +20104,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPFirstprivateClause(ArrayRef<Expr *> VarList,
 
   return OMPFirstprivateClause::Create(
       getASTContext(), StartLoc, LParenLoc, EndLoc, Vars, PrivateCopies, Inits,
+      FPKind, FPKindLoc, ColonLoc,
       buildPreInits(getASTContext(), ExprCaptures));
 }
 
diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 9efbdf6ad9684..80d227d678cb1 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -1935,11 +1935,14 @@ class TreeTransform {
   /// By default, performs semantic analysis to build the new OpenMP clause.
   /// Subclasses may override this routine to provide different behavior.
   OMPClause *RebuildOMPFirstprivateClause(ArrayRef<Expr *> VarList,
+                                          OpenMPFirstprivateModifier FPKind,
+                                          SourceLocation FPKindLoc,
+                                          SourceLocation ColonLoc,
                                           SourceLocation StartLoc,
                                           SourceLocation LParenLoc,
                                           SourceLocation EndLoc) {
-    return getSema().OpenMP().ActOnOpenMPFirstprivateClause(VarList, StartLoc,
-                                                            LParenLoc, EndLoc);
+    return getSema().OpenMP().ActOnOpenMPFirstprivateClause(
+        VarList, FPKind, FPKindLoc, ColonLoc, StartLoc, LParenLoc, EndLoc);
   }
 
   /// Build a new OpenMP 'lastprivate' clause.
@@ -11474,7 +11477,8 @@ OMPClause *TreeTransform<Derived>::TransformOMPFirstprivateClause(
     Vars.push_back(EVar.get());
   }
   return getDerived().RebuildOMPFirstprivateClause(
-      Vars, C->getBeginLoc(), C->getLParenLoc(), C->getEndLoc());
+      Vars, C->getKind(), C->getKindLoc(), C->getColonLoc(), C->getBeginLoc(),
+      C->getLParenLoc(), C->getEndLoc());
 }
 
 template <typename Derived>
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 43dbd98851181..695df2bf40ea8 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -12244,6 +12244,9 @@ void OMPClauseReader::VisitOMPPrivateClause(OMPPrivateClause *C) {
 void OMPClauseReader::VisitOMPFirstprivateClause(OMPFirstprivateClause *C) {
   VisitOMPClauseWithPreInit(C);
   C->setLParenLoc(Record.readSourceLocation());
+  C->setKind(Record.readEnum<OpenMPFirstprivateModifier>());
+  C->setKindLoc(Record.readSourceLocation());
+  C->setColonLoc(Record.readSourceLocation());
   unsigned NumVars = C->varlist_size();
   SmallVector<Expr *, 16> Vars;
   Vars.reserve(NumVars);
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 140e1f8366dc1..3d11752a6f631 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -8352,6 +8352,9 @@ void OMPClauseWriter::VisitOMPFirstprivateClause(OMPFirstprivateClause *C) {
   Record.push_back(C->varlist_size());
   VisitOMPClauseWithPreInit(C);
   Record.AddSourceLocation(C->getLParenLoc());
+  Record.writeEnum(C->getKind());
+  Record.AddSourceLocation(C->getKindLoc());
+  Record.AddSourceLocation(C->getColonLoc());
   for (auto *VE : C->varlist()) {
     Record.AddStmt(VE);
   }
diff --git a/clang/test/OpenMP/taskgraph_firstprivate_saved_ast_print.cpp b/clang/test/OpenMP/taskgraph_firstprivate_saved_ast_print.cpp
new file mode 100644
index 0000000000000..faf6b7a2936ae
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_firstprivate_saved_ast_print.cpp
@@ -0,0 +1,41 @@
+// Check no warnings/errors
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -fsyntax-only -verify %s
+// expected-no-diagnostics
+
+// Check unparsing
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -ast-print %s | FileCheck %s
+
+// Check same results after serialization round-trip
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -emit-pch -o %t %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -include-pch %t -ast-print %s | FileCheck %s
+
+#ifndef HEADER
+#define HEADER
+
+void firstprivate_saved() {
+  int a = 1;
+  int b = 2;
+  int c = 3;
+
+  // CHECK: #pragma omp task firstprivate(a)
+  #pragma omp task firstprivate(a)
+  { (void)a; }
+
+  // CHECK: #pragma omp task firstprivate(saved: a)
+  #pragma omp task firstprivate(saved: a)
+  { (void)a; }
+
+  // CHECK: #pragma omp task firstprivate(saved: a,b,c)
+  #pragma omp task firstprivate(saved: a, b, c)
+  { (void)a; (void)b; (void)c; }
+
+  // CHECK: #pragma omp task firstprivate(saved: a) shared(b)
+  #pragma omp task firstprivate(saved: a) shared(b)
+  { (void)a; (void)b; }
+
+  // CHECK: #pragma omp taskloop firstprivate(saved: a)
+  #pragma omp taskloop firstprivate(saved: a)
+  for (int i = 0; i < 4; ++i) (void)a;
+}
+
+#endif

>From 0317524850aaceea2b558d94e6e7b177ad049a6e Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Wed, 27 May 2026 08:03:40 -0500
Subject: [PATCH 19/26] [Clang][OpenMP] Restrict 'saved' firstprivate modifier
 to admissible directives

Per OpenMP 6.0 [7.2], the 'saved' modifier on a data-environment
attribute clause has effect only when that clause appears on a
replayable construct.  Per [14.3] and [14.6], the only directives
that admit the 'replayable' clause are: 'target',
'target enter data', 'target exit data', 'target update', 'task',
'taskloop' and 'taskwait'.  Of those seven, only 'target', 'task'
and 'taskloop' also admit a 'firstprivate' clause; the remaining
four cannot syntactically carry 'firstprivate' and need no extra
sema work.  A directive outside the admitted set can never make
'firstprivate(saved: ...)' meaningful.

We do not try to flag every dead-modifier case statically:
'saved' on a directive in the admitted set but without
'replayable' (and without lexical nesting in a taskgraph) is
well-formed per [7.2] and silently has no effect.  Detecting that
would require inspecting the full clause list, and is invalidated
as soon as the user adds the clause or moves the construct inside
a taskgraph.

What we can and do diagnose is the case where the host directive
itself cannot ever be a replayable construct: 'saved' on a
'parallel', 'for', 'sections', 'single', 'distribute', 'teams',
etc. firstprivate clause has no path to replay semantics at all.
Reject that in ActOnOpenMPFirstprivateClause when CurDir is
neither an OpenMP tasking directive
(isOpenMPTaskingDirective, which covers OMPD_task and every
OMPD_*taskloop* variant) nor an OpenMP target execution directive
(isOpenMPTargetExecutionDirective, which covers OMPD_target and
every combined directive that has OMPD_target as a leaf
construct), with a new dedicated diagnostic
err_omp_firstprivate_saved_wrong_directive that points at the
modifier location and lists the three admissible directives.

The existing ast-print test is extended to cover these cases, and other
new tests have been added.

Assisted-By: Claude Opus 4.7

Pull Request: https://github.com/llvm/llvm-project/pull/200407
---
 .../clang/Basic/DiagnosticSemaKinds.td        |   3 +
 clang/lib/Sema/SemaOpenMP.cpp                 |  14 +++
 ...taskgraph_firstprivate_saved_ast_print.cpp |  52 ++++++--
 .../taskgraph_firstprivate_saved_messages.cpp | 115 ++++++++++++++++++
 4 files changed, 177 insertions(+), 7 deletions(-)
 create mode 100644 clang/test/OpenMP/taskgraph_firstprivate_saved_messages.cpp

diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 3b59ef75461f3..c4334a5e0aa3d 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12741,6 +12741,9 @@ def err_omp_one_defaultmap_each_category: Error<
 def err_omp_lastprivate_conditional_non_scalar : Error<
   "expected list item of scalar type in 'lastprivate' clause with 'conditional' modifier"
   >;
+def err_omp_firstprivate_saved_wrong_directive : Error<
+  "'saved' modifier on 'firstprivate' clause is only allowed on a 'task', "
+  "'taskloop', or 'target' construct">;
 def err_omp_flush_order_clause_and_list : Error<
   "'flush' directive with memory order clause '%0' cannot have the list">;
 def note_omp_flush_order_clause_here : Note<
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index eb4004922ee28..f0e4788a602be 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -19819,6 +19819,20 @@ OMPClause *SemaOpenMP::ActOnOpenMPFirstprivateClause(
     return nullptr;
   }
 
+  // OpenMP 6.0 [7.2]: the 'saved' modifier has effect only on a clause
+  // that appears on a replayable construct.  Per [14.6] the directives
+  // that admit a 'replayable' clause and that also admit 'firstprivate'
+  // are 'task', 'taskloop', and 'target' (the other replayable-eligible
+  // directives -- 'target_enter_data' etc. -- do not admit
+  // 'firstprivate' at all).  Reject 'saved' on any other directive, as
+  // it can never become a replayable construct.
+  OpenMPDirectiveKind CurDir = DSAStack->getCurrentDirective();
+  if (FPKind == OMPC_FIRSTPRIVATE_saved && !isOpenMPTaskingDirective(CurDir) &&
+      !isOpenMPTargetExecutionDirective(CurDir)) {
+    Diag(FPKindLoc, diag::err_omp_firstprivate_saved_wrong_directive);
+    return nullptr;
+  }
+
   SmallVector<Expr *, 8> Vars;
   SmallVector<Expr *, 8> PrivateCopies;
   SmallVector<Expr *, 8> Inits;
diff --git a/clang/test/OpenMP/taskgraph_firstprivate_saved_ast_print.cpp b/clang/test/OpenMP/taskgraph_firstprivate_saved_ast_print.cpp
index faf6b7a2936ae..df358df14c82f 100644
--- a/clang/test/OpenMP/taskgraph_firstprivate_saved_ast_print.cpp
+++ b/clang/test/OpenMP/taskgraph_firstprivate_saved_ast_print.cpp
@@ -21,21 +21,59 @@ void firstprivate_saved() {
   #pragma omp task firstprivate(a)
   { (void)a; }
 
+  // 'saved' on an omp task lexically inside a taskgraph.
+  // CHECK: #pragma omp taskgraph
   // CHECK: #pragma omp task firstprivate(saved: a)
-  #pragma omp task firstprivate(saved: a)
-  { (void)a; }
+  #pragma omp taskgraph
+  {
+    #pragma omp task firstprivate(saved: a)
+    { (void)a; }
+  }
 
+  // Multiple variables.
+  // CHECK: #pragma omp taskgraph
   // CHECK: #pragma omp task firstprivate(saved: a,b,c)
-  #pragma omp task firstprivate(saved: a, b, c)
-  { (void)a; (void)b; (void)c; }
+  #pragma omp taskgraph
+  {
+    #pragma omp task firstprivate(saved: a, b, c)
+    { (void)a; (void)b; (void)c; }
+  }
 
+  // Mixed with another clause.
+  // CHECK: #pragma omp taskgraph
   // CHECK: #pragma omp task firstprivate(saved: a) shared(b)
-  #pragma omp task firstprivate(saved: a) shared(b)
-  { (void)a; (void)b; }
+  #pragma omp taskgraph
+  {
+    #pragma omp task firstprivate(saved: a) shared(b)
+    { (void)a; (void)b; }
+  }
 
+  // 'saved' on an omp taskloop lexically inside a taskgraph.
+  // CHECK: #pragma omp taskgraph
   // CHECK: #pragma omp taskloop firstprivate(saved: a)
-  #pragma omp taskloop firstprivate(saved: a)
+  #pragma omp taskgraph
+  {
+    #pragma omp taskloop firstprivate(saved: a)
+    for (int i = 0; i < 4; ++i) (void)a;
+  }
+
+  // 'saved' on a replayable omp task outside any taskgraph - also legal.
+  // CHECK: #pragma omp task replayable firstprivate(saved: a)
+  #pragma omp task replayable firstprivate(saved: a)
+  { (void)a; }
+
+  // 'saved' on a replayable omp taskloop outside any taskgraph - also legal.
+  // CHECK: #pragma omp taskloop replayable firstprivate(saved: a)
+  #pragma omp taskloop replayable firstprivate(saved: a)
   for (int i = 0; i < 4; ++i) (void)a;
+
+  // 'saved' on a non-lexically-nested task (dynamic nesting via a call into
+  // a function from a taskgraph region is the runtime use case) - we accept
+  // any task/taskloop construct since the static check cannot prove dynamic
+  // nesting.
+  // CHECK: #pragma omp task firstprivate(saved: a)
+  #pragma omp task firstprivate(saved: a)
+  { (void)a; }
 }
 
 #endif
diff --git a/clang/test/OpenMP/taskgraph_firstprivate_saved_messages.cpp b/clang/test/OpenMP/taskgraph_firstprivate_saved_messages.cpp
new file mode 100644
index 0000000000000..5197d16cd0421
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_firstprivate_saved_messages.cpp
@@ -0,0 +1,115 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -fsyntax-only -verify %s
+
+// Tests the OpenMP 6.0 'saved' modifier on the 'firstprivate' clause.  The
+// modifier is meaningful only on constructs that create tasks or taskloops
+// (the units of work that can participate in taskgraph replay).  Every other
+// directive that admits a 'firstprivate' clause must reject it.
+
+void unknown_modifier() {
+  int a = 0;
+  // The diagnostic comes from the generic "expected <list> in OpenMP clause"
+  // path and enumerates the legal modifier names ('saved' in OpenMP 6.0).
+  #pragma omp task firstprivate(bogus: a) // expected-error {{expected 'saved' in OpenMP clause 'firstprivate'}}
+  { (void)a; }
+}
+
+void rejected_on_non_tasking_constructs() {
+  int a = 0;
+  int b[8];
+
+  // parallel
+  #pragma omp parallel firstprivate(saved: a) // expected-error {{'saved' modifier on 'firstprivate' clause is only allowed on a 'task', 'taskloop', or 'target' construct}}
+  { (void)a; }
+
+  // for (worksharing)
+  #pragma omp parallel
+  {
+    #pragma omp for firstprivate(saved: a) // expected-error {{'saved' modifier on 'firstprivate' clause is only allowed on a 'task', 'taskloop', or 'target' construct}}
+    for (int i = 0; i < 4; ++i) (void)a;
+  }
+
+  // sections
+  #pragma omp parallel
+  {
+    #pragma omp sections firstprivate(saved: a) // expected-error {{'saved' modifier on 'firstprivate' clause is only allowed on a 'task', 'taskloop', or 'target' construct}}
+    {
+      (void)a;
+    }
+  }
+
+  // single
+  #pragma omp parallel
+  {
+    #pragma omp single firstprivate(saved: a) // expected-error {{'saved' modifier on 'firstprivate' clause is only allowed on a 'task', 'taskloop', or 'target' construct}}
+    { (void)a; }
+  }
+
+  // teams (inside target -- the inner directive is a standalone 'teams',
+  // not a 'target teams' combined directive, so it is not a target
+  // execution directive in its own right).
+  #pragma omp target
+  #pragma omp teams firstprivate(saved: a) // expected-error {{'saved' modifier on 'firstprivate' clause is only allowed on a 'task', 'taskloop', or 'target' construct}}
+  { (void)a; }
+
+  // distribute (inside teams) -- standalone 'distribute' is not a target
+  // execution directive either.
+  #pragma omp target teams
+  #pragma omp distribute firstprivate(saved: a) // expected-error {{'saved' modifier on 'firstprivate' clause is only allowed on a 'task', 'taskloop', or 'target' construct}}
+  for (int i = 0; i < 4; ++i) (void)a;
+}
+
+void accepted_on_task_taskloop_and_target() {
+  int a = 0;
+
+  // Bare task (no enclosing taskgraph, no replayable clause): accepted.
+  // Per OpenMP 6.0 [7.2] the 'saved' modifier silently has no effect on a
+  // non-replayable construct; we only enforce the directive-kind check
+  // statically.  In this implementation a bare task / taskloop is never
+  // recorded into a taskgraph -- recording requires either lexical
+  // nesting inside a '#pragma omp taskgraph' or an explicit 'replayable'
+  // clause -- so the modifier is a well-defined no-op here.
+  #pragma omp task firstprivate(saved: a)
+  { (void)a; }
+
+  // Bare taskloop: accepted, same rationale.
+  #pragma omp taskloop firstprivate(saved: a)
+  for (int i = 0; i < 4; ++i) (void)a;
+
+  // Replayable task: explicitly opted-in for replay.
+  #pragma omp task replayable firstprivate(saved: a)
+  { (void)a; }
+
+  // Replayable taskloop.
+  #pragma omp taskloop replayable firstprivate(saved: a)
+  for (int i = 0; i < 4; ++i) (void)a;
+
+  // Task lexically nested inside a taskgraph.
+  #pragma omp taskgraph
+  {
+    #pragma omp task firstprivate(saved: a)
+    { (void)a; }
+  }
+
+  // Bare target: accepted on the same well-formed-but-no-effect grounds
+  // as a bare task.  Per OpenMP 6.0 [14.6] the 'target' construct admits
+  // both 'firstprivate' and 'replayable', so 'saved' is meaningful as
+  // soon as a 'replayable' clause is added or the construct is nested
+  // inside a 'taskgraph' region.
+  #pragma omp target firstprivate(saved: a)
+  { (void)a; }
+
+  // Replayable target: explicitly opted-in for replay.
+  #pragma omp target replayable firstprivate(saved: a)
+  { (void)a; }
+
+  // Combined target construct (target + parallel): accepted because the
+  // composite directive is a target execution directive in its own
+  // right, so the captured snapshot at the target boundary belongs to a
+  // construct that may participate in taskgraph replay.
+  #pragma omp target parallel firstprivate(saved: a)
+  { (void)a; }
+
+  // Combined target teams distribute parallel for: same rationale.
+  #pragma omp target teams distribute parallel for firstprivate(saved: a)
+  for (int i = 0; i < 4; ++i) (void)a;
+}

>From 2bc758b7dbf88a4b86c0c63823b7916be1a7dc6c Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Wed, 27 May 2026 10:23:14 -0500
Subject: [PATCH 20/26] [OpenMP] Constructor and destructor fixes for tasks
 cloned for recorded taskgraphs

The OpenMP 6.0 'saved' firstprivate modifier (see [7.2] together
with [14.3]) requires each list item to be snapshotted once at
recording time and observable on every replay of the recorded
task.  libomp reuses the same task descriptor across every replay
of a taskgraph-owned task, so the '.kmp_privates.t' tail struct
that holds the firstprivate values is also the natural home for
the saved data environment.  Getting that right needs two
changes, which this patch lands together: the destructor of each
list item must fire exactly once at end-of-taskgraph (not after
every replay), and non-trivially-copyable list items must be
re-constructed per replay so that copy constructors and inner
self-references are respected.

On the runtime side, move the per-task destructor-thunk
invocation from __kmp_task_finish (which previously fired it at
the end of every replay, leaving the saved snapshot in a
destructed state for the next replay) to __kmp_taskgraph_free,
so it fires exactly once per task at end-of-taskgraph.  Skip
taskwait nodes (record_map entries with task == nullptr) in that
loop while we are there, to avoid a latent nullptr dereference
that the existing tests do not exercise.

On the compiler side, the runtime previously cloned each replay's
task descriptor with a bitwise memcpy in __kmp_taskgraph_clone_task,
and a FIXME noted that this silently corrupts firstprivate list items
whose type is not trivially copyable (self-referential structs, types
with user-defined copy constructors / destructors, types holding inner
pointers into themselves).  Emit a per-task clone helper

  void __omp_task_clone.NN(kmp_task_t *dst, kmp_task_t *src,
                           int lastpriv);

modelled on emitTaskDupFunction and reusing emitPrivatesInit (now
extended with a tri-state PrivatesInitMode of Normal / ForDup /
ForClone), which re-runs the copy constructor of each
firstprivate list item into the freshly allocated descriptor's
'.kmp_privates.t'.  Tasks whose firstprivates are all trivially
copyable still rely on the runtime's memcpy fast-path and emit no
clone helper.  emitTaskCall passes the helper to the runtime as
the new trailing argument of __kmpc_taskgraph_task (null when no
helper is needed).

Two previously-XFAIL'd taskgraph runtime tests
(taskgraph_replayable_saved_stack_depth.cpp and
taskgraph_shared_stack_depth.cpp) now pass and are un-XFAIL'd, and other
tests have been added to cover new functionality.

Assisted-By: Claude Opus 4.7

Pull Request: https://github.com/llvm/llvm-project/pull/200408
---
 clang/lib/CodeGen/CGOpenMPRuntime.cpp         | 115 +++++++++++++-
 clang/lib/CodeGen/CGOpenMPRuntime.h           |   4 +
 ...taskgraph_firstprivate_saved_ast_print.cpp |  67 ++++++++
 .../OpenMP/taskgraph_task_clone_codegen.cpp   |  51 ++++++
 .../include/llvm/Frontend/OpenMP/OMPKinds.def |   2 +-
 openmp/runtime/src/kmp.h                      |   2 +-
 openmp/runtime/src/kmp_tasking.cpp            |  57 ++++++-
 ...askgraph_firstprivate_saved_nontrivial.cpp | 146 ++++++++++++++++++
 ..._firstprivate_saved_nontrivial_selfref.cpp |  97 ++++++++++++
 .../taskgraph_firstprivate_saved_static.cpp   | 100 ++++++++++++
 ...taskgraph_replayable_saved_stack_depth.cpp |   1 -
 .../taskgraph_shared_stack_depth.cpp          |  16 +-
 12 files changed, 641 insertions(+), 17 deletions(-)
 create mode 100644 clang/test/OpenMP/taskgraph_task_clone_codegen.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_firstprivate_saved_nontrivial.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_firstprivate_saved_nontrivial_selfref.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_firstprivate_saved_static.cpp

diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
index 8bb2c5c8d7897..b715380b4d88c 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp
@@ -3700,14 +3700,34 @@ emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc,
 }
 
 /// Emit initialization for private variables in task-based directives.
+/// Selects where \c emitPrivatesInit should read the initial value of each
+/// non-trivial firstprivate copy from.
+enum class PrivatesInitMode {
+  /// Initialize using the captured original lvalues in the caller IR (i.e.
+  /// at task-allocation time).
+  Normal,
+  /// Reading from the source task's \c shareds region. Used by the taskloop
+  /// task-dup function to seed sibling tasks.
+  ForDup,
+  /// Reading from the source task's \c .kmp_privates.t region (the field at
+  /// the same index as the destination). Used by the taskgraph clone
+  /// function to seed the persistent clone from the original task's
+  /// already-initialized snapshot. Works uniformly for captured and
+  /// static-storage firstprivates because no capture lookup is needed.
+  ForClone,
+};
+
 static void emitPrivatesInit(CodeGenFunction &CGF,
                              const OMPExecutableDirective &D,
                              Address KmpTaskSharedsPtr, LValue TDBase,
                              const RecordDecl *KmpTaskTWithPrivatesQTyRD,
                              QualType SharedsTy, QualType SharedsPtrTy,
                              const OMPTaskDataTy &Data,
-                             ArrayRef<PrivateDataTy> Privates, bool ForDup) {
+                             ArrayRef<PrivateDataTy> Privates,
+                             PrivatesInitMode Mode, LValue SrcPrivatesBase) {
   ASTContext &C = CGF.getContext();
+  const bool ForDup = Mode == PrivatesInitMode::ForDup;
+  const bool ForClone = Mode == PrivatesInitMode::ForClone;
   auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
   LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI);
   OpenMPDirectiveKind Kind = isOpenMPTaskLoopDirective(D.getDirectiveKind())
@@ -3739,8 +3759,11 @@ static void emitPrivatesInit(CodeGenFunction &CGF,
     }
     const VarDecl *VD = Pair.second.PrivateCopy;
     const Expr *Init = VD->getAnyInitializer();
-    if (Init && (!ForDup || (isa<CXXConstructExpr>(Init) &&
-                             !CGF.isTrivialInitializer(Init)))) {
+    // ForDup and ForClone only re-initialize non-trivial firstprivates; the
+    // surrounding runtime memcpy is sufficient for trivially-copyable ones.
+    const bool NonTrivialOnly = ForDup || ForClone;
+    if (Init && (!NonTrivialOnly || (isa<CXXConstructExpr>(Init) &&
+                                     !CGF.isTrivialInitializer(Init)))) {
       LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI);
       if (const VarDecl *Elem = Pair.second.PrivateElemInit) {
         const VarDecl *OriginalVD = Pair.second.Original;
@@ -3760,6 +3783,9 @@ static void emitPrivatesInit(CodeGenFunction &CGF,
                  "Expected artificial target data variable.");
           SharedRefLValue =
               CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type);
+        } else if (ForClone) {
+          // Source is the same field on the origin task's privates record.
+          SharedRefLValue = CGF.EmitLValueForField(SrcPrivatesBase, *FI);
         } else if (ForDup) {
           SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField);
           SharedRefLValue = CGF.MakeAddrLValue(
@@ -3910,11 +3936,74 @@ emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc,
         CGF.Int8Ty, CGM.getNaturalTypeAlignment(SharedsTy));
   }
   emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
-                   SharedsTy, SharedsPtrTy, Data, Privates, /*ForDup=*/true);
+                   SharedsTy, SharedsPtrTy, Data, Privates,
+                   PrivatesInitMode::ForDup, /*SrcPrivatesBase=*/LValue());
   CGF.FinishFunction();
   return TaskDup;
 }
 
+/// Emit task_clone function (for re-initializing non-trivially-copyable
+/// firstprivate copies when cloning a task into a taskgraph record).
+/// \code
+/// void __omp_task_clone(kmp_task_t *task_dst, const kmp_task_t *task_src,
+///                       int /*unused*/) {
+///   // copy-construct each non-trivial firstprivate from
+///   // task_src->.kmp_privates.t into task_dst->.kmp_privates.t.
+/// }
+/// \endcode
+/// The (unused) third parameter is present so that the function shares the
+/// same calling convention as the existing taskloop \c task_dup callback
+/// (\c p_task_dup_t in the runtime), letting the runtime invoke either via
+/// a single function-pointer type.
+static llvm::Value *emitTaskCloneFunction(
+    CodeGenModule &CGM, SourceLocation Loc, const OMPExecutableDirective &D,
+    QualType KmpTaskTWithPrivatesPtrQTy,
+    const RecordDecl *KmpTaskTWithPrivatesQTyRD, QualType SharedsTy,
+    QualType SharedsPtrTy, const OMPTaskDataTy &Data,
+    ArrayRef<PrivateDataTy> Privates) {
+  ASTContext &C = CGM.getContext();
+  auto *DstArg = ImplicitParamDecl::Create(
+      C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpTaskTWithPrivatesPtrQTy,
+      ImplicitParamKind::Other);
+  auto *SrcArg = ImplicitParamDecl::Create(
+      C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpTaskTWithPrivatesPtrQTy,
+      ImplicitParamKind::Other);
+  auto *UnusedArg =
+      ImplicitParamDecl::Create(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy,
+                                ImplicitParamKind::Other);
+  FunctionArgList Args{DstArg, SrcArg, UnusedArg};
+  const auto &FnInfo =
+      CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args);
+  llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
+  std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_clone", ""});
+  auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,
+                                    Name, &CGM.getModule());
+  CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo);
+  if (!CGM.getCodeGenOpts().SampleProfileFile.empty())
+    Fn->addFnAttr("sample-profile-suffix-elision-policy", "selected");
+  Fn->setDoesNotRecurse();
+  CodeGenFunction CGF(CGM);
+  CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc);
+
+  LValue DstBase = CGF.EmitLoadOfPointerLValue(
+      CGF.GetAddrOfLocalVar(DstArg),
+      KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
+  LValue SrcBase = CGF.EmitLoadOfPointerLValue(
+      CGF.GetAddrOfLocalVar(SrcArg),
+      KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>());
+  // Address the .kmp_privates.t sub-record of the source task; the
+  // destination's privates record is located via DstBase inside
+  // emitPrivatesInit.
+  auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
+  LValue SrcPrivatesBase = CGF.EmitLValueForField(SrcBase, *PrivatesFI);
+
+  emitPrivatesInit(CGF, D, /*KmpTaskSharedsPtr=*/Address::invalid(), DstBase,
+                   KmpTaskTWithPrivatesQTyRD, SharedsTy, SharedsPtrTy, Data,
+                   Privates, PrivatesInitMode::ForClone, SrcPrivatesBase);
+  CGF.FinishFunction();
+  return Fn;
+}
+
 /// Checks if destructor function is required to be generated.
 /// \return true if cleanups are required, false otherwise.
 static bool
@@ -4416,7 +4505,7 @@ CGOpenMPRuntime::TaskResultTy CGOpenMPRuntime::emitTaskInit(
   if (!Privates.empty()) {
     emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD,
                      SharedsTy, SharedsPtrTy, Data, Privates,
-                     /*ForDup=*/false);
+                     PrivatesInitMode::Normal, /*SrcPrivatesBase=*/LValue());
     if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&
         (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) {
       Result.TaskDupFn = emitTaskDupFunction(
@@ -4424,6 +4513,16 @@ CGOpenMPRuntime::TaskResultTy CGOpenMPRuntime::emitTaskInit(
           KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
           /*WithLastIter=*/!Data.LastprivateVars.empty());
     }
+    // For plain tasks (not taskloops) that have at least one non-trivially
+    // copyable firstprivate, emit a clone function so that the runtime can
+    // re-initialize those fields when the task is recorded into a taskgraph.
+    // Taskloops already cover the same need via their TaskDupFn.
+    if (!isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&
+        checkInitIsRequired(CGF, Privates)) {
+      Result.TaskCloneFn = emitTaskCloneFunction(
+          CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
+          SharedsTy, SharedsPtrTy, Data, Privates);
+    }
   }
   // Fields of union "kmp_cmplrdata_t" for destructors and priority.
   enum { Priority = 0, Destructors = 1 };
@@ -4972,7 +5071,7 @@ void CGOpenMPRuntime::emitTaskCall(
                                                   PrePostActionTy &) {
     llvm::Value *ThreadId = getThreadID(CGF, Loc);
     llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc);
-    std::array<llvm::Value *, 9> TGTaskArgs;
+    std::array<llvm::Value *, 10> TGTaskArgs;
     std::array<llvm::Value *, 3> TaskAllocArgs;
     TaskResultTy Result = emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy,
                                        Shareds, Data, true, TaskAllocArgs);
@@ -5006,6 +5105,10 @@ void CGOpenMPRuntime::emitTaskCall(
     TGTaskArgs[8] = RelocFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
                                   RelocFn, CGM.VoidPtrTy)
                             : llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
+    TGTaskArgs[9] = Result.TaskCloneFn
+                        ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(
+                              Result.TaskCloneFn, CGF.VoidPtrTy)
+                        : llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
     CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
                             CGM.getModule(), OMPRTL___kmpc_taskgraph_task),
                         TGTaskArgs);
diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.h b/clang/lib/CodeGen/CGOpenMPRuntime.h
index 821c1ca9bcffd..b330291f53eff 100644
--- a/clang/lib/CodeGen/CGOpenMPRuntime.h
+++ b/clang/lib/CodeGen/CGOpenMPRuntime.h
@@ -557,6 +557,10 @@ class CGOpenMPRuntime {
     LValue TDBase;
     const RecordDecl *KmpTaskTQTyRD = nullptr;
     llvm::Value *TaskDupFn = nullptr;
+    /// Compiler-emitted helper that re-initializes any non-trivially-copyable
+    /// firstprivate fields after the runtime bitwise-clones a task into a
+    /// taskgraph record. Null when no such helper is required.
+    llvm::Value *TaskCloneFn = nullptr;
   };
   /// Emit task region for the task directive. The task region is emitted in
   /// several steps:
diff --git a/clang/test/OpenMP/taskgraph_firstprivate_saved_ast_print.cpp b/clang/test/OpenMP/taskgraph_firstprivate_saved_ast_print.cpp
index df358df14c82f..23cfbdddb3f68 100644
--- a/clang/test/OpenMP/taskgraph_firstprivate_saved_ast_print.cpp
+++ b/clang/test/OpenMP/taskgraph_firstprivate_saved_ast_print.cpp
@@ -76,4 +76,71 @@ void firstprivate_saved() {
   { (void)a; }
 }
 
+// Per OpenMP 6.0 [14.3], a 'firstprivate' clause with the 'saved' modifier on
+// a replayable construct may include variables with static storage duration;
+// they are copied into the saved data environment of the taskgraph record.
+// This covers file-scope statics, static-local variables, static data
+// members, and const-qualified statics, all of which Sema accepts and Clang
+// codegen places into the per-task '.kmp_privates.t' tail struct.
+
+static int FileScopeStatic = 100;
+static const int FileScopeConstStatic = 200;
+
+struct WithStaticMember {
+  static int StaticMember;
+  static const int StaticConstMember = 400;
+};
+int WithStaticMember::StaticMember = 0;
+
+void firstprivate_saved_statics() {
+  static int LocalStatic = 300;
+  static const int LocalConstStatic = 500;
+
+  // CHECK-LABEL: void firstprivate_saved_statics
+  // CHECK: #pragma omp task firstprivate(saved: FileScopeStatic)
+  #pragma omp task firstprivate(saved: FileScopeStatic)
+  { (void)FileScopeStatic; }
+
+  // CHECK: #pragma omp task firstprivate(saved: FileScopeConstStatic)
+  #pragma omp task firstprivate(saved: FileScopeConstStatic)
+  { (void)FileScopeConstStatic; }
+
+  // CHECK: #pragma omp task firstprivate(saved: LocalStatic)
+  #pragma omp task firstprivate(saved: LocalStatic)
+  { (void)LocalStatic; }
+
+  // CHECK: #pragma omp task firstprivate(saved: LocalConstStatic)
+  #pragma omp task firstprivate(saved: LocalConstStatic)
+  { (void)LocalConstStatic; }
+
+  // CHECK: #pragma omp task firstprivate(saved: WithStaticMember::StaticMember)
+  #pragma omp task firstprivate(saved: WithStaticMember::StaticMember)
+  { (void)WithStaticMember::StaticMember; }
+
+  // CHECK: #pragma omp task firstprivate(saved: WithStaticMember::StaticConstMember)
+  #pragma omp task firstprivate(saved: WithStaticMember::StaticConstMember)
+  { (void)WithStaticMember::StaticConstMember; }
+
+  // Multiple statics in a single clause, mixed with a non-static.
+  int local_int = 0;
+  // CHECK: #pragma omp task firstprivate(saved: FileScopeStatic,LocalStatic,WithStaticMember::StaticMember,local_int)
+  #pragma omp task firstprivate(saved:                                         \
+                                FileScopeStatic, LocalStatic,                  \
+                                WithStaticMember::StaticMember, local_int)
+  {
+    (void)FileScopeStatic;
+    (void)LocalStatic;
+    (void)WithStaticMember::StaticMember;
+    (void)local_int;
+  }
+
+  // Same on a 'taskloop' construct.
+  // CHECK: #pragma omp taskloop firstprivate(saved: FileScopeStatic,LocalConstStatic)
+  #pragma omp taskloop firstprivate(saved: FileScopeStatic, LocalConstStatic)
+  for (int i = 0; i < 4; ++i) {
+    (void)FileScopeStatic;
+    (void)LocalConstStatic;
+  }
+}
+
 #endif
diff --git a/clang/test/OpenMP/taskgraph_task_clone_codegen.cpp b/clang/test/OpenMP/taskgraph_task_clone_codegen.cpp
new file mode 100644
index 0000000000000..9451ad3ba9110
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_task_clone_codegen.cpp
@@ -0,0 +1,51 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+// Verifies that for a 'firstprivate' on a task inside a taskgraph whose
+// type has a non-trivial copy constructor, the compiler emits a dedicated
+// '.omp_task_clone.' helper and passes it to __kmpc_taskgraph_task in
+// the trailing argument slot.  The helper re-runs the copy constructor
+// from the origin task's '.kmp_privates.t' field into the clone's, so
+// that the runtime memcpy does not produce a torn copy of a non-
+// trivially-copyable object.
+
+struct NonTrivial {
+  int v;
+  int *self;
+  NonTrivial();
+  NonTrivial(const NonTrivial &other);
+  ~NonTrivial();
+};
+
+void run() {
+  NonTrivial nt;
+#pragma omp taskgraph
+  {
+#pragma omp task firstprivate(nt)
+    {
+      (void)nt.v;
+    }
+  }
+}
+
+// The clone helper is passed as the trailing pointer argument to
+// __kmpc_taskgraph_task (10 total: ident, gtid, task, flags, sizes...,
+// ndeps, deps, relocation, clone).
+// CHECK: call i32 @__kmpc_taskgraph_task(ptr {{[^,]+}}, i32 {{[^,]+}}, ptr {{[^,]+}}, i32 {{[^,]+}}, i64 {{[^,]+}}, i64 {{[^,]+}}, i32 {{[^,]+}}, ptr {{[^,]+}}, ptr {{[^,]+}}, ptr @.omp_task_clone.)
+
+// The clone helper has the same calling convention as the existing
+// taskloop task-dup callback so that the runtime can dispatch through
+// a single function-pointer type; the third parameter is unused here.
+// The body indexes the same .kmp_privates.t field on both source and
+// destination tasks and invokes NonTrivial's copy constructor.
+// CHECK: define internal void @.omp_task_clone.(ptr noundef %{{[^,]+}}, ptr noundef %{{[^,]+}}, i32 noundef %{{[^,]+}})
+// CHECK: getelementptr inbounds {{.*}} %struct.kmp_task_t_with_privates,
+// CHECK: getelementptr inbounds {{.*}} %struct.kmp_task_t_with_privates,
+// CHECK: getelementptr inbounds {{.*}} %struct..kmp_privates.t,
+// CHECK: getelementptr inbounds {{.*}} %struct..kmp_privates.t,
+// CHECK: call void @_ZN10NonTrivialC1ERKS_(
+
+#endif
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
index 6ff0235c04546..35f18380e14f6 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
@@ -360,7 +360,7 @@ __OMP_RTL(__kmpc_taskgroup, false, Void, IdentPtr, Int32)
 __OMP_RTL(__kmpc_taskgraph, false, Void, IdentPtr, Int32, VoidPtrPtr, SizeTy,
           Int32, Int32, VoidPtr, VoidPtr)
 __OMP_RTL(__kmpc_taskgraph_task, false, Int32, IdentPtr, Int32, VoidPtr, Int32,
-          SizeTy, SizeTy, Int32, VoidPtr, VoidPtr)
+          SizeTy, SizeTy, Int32, VoidPtr, VoidPtr, VoidPtr)
 __OMP_RTL(__kmpc_taskgraph_taskloop, false, Int32, IdentPtr, Int32, VoidPtr,
           Int32, Int32, Int64Ptr, Int64Ptr, Int64,
           Int32, Int32, Int64, Int32, VoidPtr, VoidPtr)
diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index 4d81d0afcfebe..429b74ef297a1 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -4534,7 +4534,7 @@ KMP_EXPORT void __kmpc_taskgraph(ident_t *loc_ref, kmp_int32 gtid,
 KMP_EXPORT kmp_uint32 __kmpc_taskgraph_task(
     ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *new_task, kmp_int32 flags,
     size_t sizeof_kmp_task_t, size_t sizeof_shareds, kmp_int32 ndeps,
-    kmp_depend_info_t *dep_list, kmp_task_relocate_t reloc);
+    kmp_depend_info_t *dep_list, kmp_task_relocate_t reloc, void *task_clone);
 KMP_EXPORT kmp_uint32 __kmpc_taskgraph_taskloop(
     ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *new_task, kmp_int32 flags,
     kmp_int32 if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
diff --git a/openmp/runtime/src/kmp_tasking.cpp b/openmp/runtime/src/kmp_tasking.cpp
index 2b02dd98b4640..93af002e9f9c1 100644
--- a/openmp/runtime/src/kmp_tasking.cpp
+++ b/openmp/runtime/src/kmp_tasking.cpp
@@ -838,8 +838,22 @@ static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task,
      placed here, since at this point other tasks might have been released
      hence overlapping the destructor invocations with some other work in the
      released tasks.  The OpenMP spec is not specific on when the destructors
-     are invoked, so we should be free to choose. */
-  if (UNLIKELY(taskdata->td_flags.destructors_thunk)) {
+     are invoked, so we should be free to choose.
+
+     For tasks owned by a taskgraph record, the same task descriptor (and
+     therefore the same .kmp_privates.t storage) is reused for every replay.
+     Firing the per-task destructor here would destruct the per-task
+     'firstprivate' copies (e.g. the snapshot used to realise the OpenMP 6.0
+     'firstprivate(saved: ...)' modifier) on the first replay completion,
+     leaving subsequent replays observing destructed state.  Defer the
+     destructor invocation to __kmp_taskgraph_free, which fires it exactly
+     once per task at end-of-taskgraph. */
+  bool defer_destructors_to_taskgraph_free = false;
+#if OMP_TASKGRAPH_EXPERIMENTAL
+  defer_destructors_to_taskgraph_free = is_taskgraph;
+#endif
+  if (UNLIKELY(taskdata->td_flags.destructors_thunk) &&
+      !defer_destructors_to_taskgraph_free) {
     kmp_routine_entry_t destr_thunk = task->data1.destructors;
     KMP_ASSERT(destr_thunk);
     destr_thunk(gtid, task);
@@ -5905,7 +5919,22 @@ static void __kmp_taskgraph_free(kmp_int32 gtid, kmp_taskgraph_record_t *rec,
   __kmp_taskgraph_free_region_metadata(thread, rec->root);
 
   for (size_t task = 0; task < rec->num_tasks; task++) {
-    kmp_taskdata *taskdata = KMP_TASK_TO_TASKDATA(rec->record_map[task].task);
+    // Skip entries that don't have an associated task (e.g. taskwait nodes
+    // recorded by __kmpc_taskgraph_taskwait).
+    if (rec->record_map[task].task == nullptr)
+      continue;
+    kmp_task_t *taskptr = rec->record_map[task].task;
+    kmp_taskdata *taskdata = KMP_TASK_TO_TASKDATA(taskptr);
+    // Fire the per-task destructor thunk exactly once here, at end-of-
+    // taskgraph.  __kmp_task_finish deliberately skips the thunk for
+    // taskgraph-owned tasks so that the per-replay state (in particular the
+    // 'firstprivate(saved: ...)' snapshot held in .kmp_privates.t) is not
+    // destructed between replays.
+    if (UNLIKELY(taskdata->td_flags.destructors_thunk)) {
+      kmp_routine_entry_t destr_thunk = taskptr->data1.destructors;
+      KMP_ASSERT(destr_thunk);
+      destr_thunk(gtid, taskptr);
+    }
     // Setting this here keeps an assertion in __kmp_free_task happy: the
     // clone may never have been replayed, in which case 'complete' will be
     // zero here, as initialized.
@@ -5955,14 +5984,18 @@ static kmp_taskgraph_header_t *__kmp_taskgraph_header_alloc(kmp_int32 gtid) {
 
 // Clone a (new) task that has had its private variables and shared variables
 // initialised already.
+//
+// The bitwise memcpy below is sufficient for trivially-copyable firstprivate
+// fields stored in the task's .kmp_privates.t region.  For
+// non-trivially-copyable firstprivates, the caller (\c __kmpc_taskgraph_task)
+// invokes a compiler-emitted \c task_clone thunk after we return, to re-run
+// the copy constructors with the source taken from the origin task's already
+// initialised privates record.
 static kmp_task_t *__kmp_taskgraph_clone_task(kmp_info_t *thread,
                                               kmp_taskgraph_record_t *taskgraph,
                                               kmp_task_t *orig,
                                               size_t sizeof_kmp_task_t,
                                               size_t sizeof_shareds) {
-  // FIXME: This should use a "taskdup" function like taskloops in cases where
-  // private variables are not trivially copyable.  For now, do it by plain
-  // bitwise copy.
   kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(orig);
   size_t shareds_offset = sizeof(kmp_taskdata_t) + sizeof_kmp_task_t;
   shareds_offset = __kmp_round_up_to_val(shareds_offset, sizeof(kmp_uint64));
@@ -6132,7 +6165,8 @@ kmp_uint32 __kmpc_taskgraph_task(ident_t *loc_ref, kmp_int32 gtid,
                                  size_t sizeof_kmp_task_t,
                                  size_t sizeof_shareds, kmp_int32 ndeps,
                                  kmp_depend_info_t *dep_list,
-                                 kmp_task_relocate_t relocate) {
+                                 kmp_task_relocate_t relocate,
+                                 void *task_clone) {
   kmp_info_t *thread = __kmp_threads[gtid];
   kmp_taskgroup_t *taskgroup = thread->th.th_current_task->td_taskgroup;
   kmp_taskgraph_record_t *rec = __kmp_taskgraph_or_parent_recording(taskgroup);
@@ -6142,6 +6176,15 @@ kmp_uint32 __kmpc_taskgraph_task(ident_t *loc_ref, kmp_int32 gtid,
     if (status == KMP_TDG_RECORDING) {
       kmp_task_t *cloned_task = __kmp_taskgraph_clone_task(
           thread, rec, new_task, sizeof_kmp_task_t, sizeof_shareds);
+      // If the compiler emitted a task-clone helper, run it now so that any
+      // non-trivially-copyable firstprivate fields are copy-constructed from
+      // the origin task's privates record instead of bitwise-copied.  Shares
+      // its calling convention with the taskloop \c task_dup callback; the
+      // \c lastpriv argument is unused for plain tasks.
+      if (task_clone) {
+        p_task_dup_t clone_fn = (p_task_dup_t)task_clone;
+        clone_fn(cloned_task, new_task, /*lastpriv=*/0);
+      }
       kmp_taskgraph_node_t *node = __kmp_taskgraph_node_alloc(rec, cloned_task);
       if (taskgroup->taskgraph.reduce_input) {
         node->reduce_input = taskgroup->taskgraph.reduce_input;
diff --git a/openmp/runtime/test/taskgraph/taskgraph_firstprivate_saved_nontrivial.cpp b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_saved_nontrivial.cpp
new file mode 100644
index 0000000000000..a10f306d8d3cb
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_saved_nontrivial.cpp
@@ -0,0 +1,146 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// Verifies the destructor-lifecycle contract for a non-trivially-copyable
+// 'firstprivate(saved: ...)' list item on a replayable task.
+//
+// The saved snapshot is held in '.kmp_privates.t' of the task descriptor
+// that the taskgraph record persists across replays, so the
+// compiler-emitted per-task destructor thunk must NOT fire at the end of
+// each replay (otherwise subsequent replays would observe a destroyed
+// object).  This is verified observationally: the destructor here writes
+// a recognisable sentinel value (-1) into the slot, so any replay that
+// saw a destroyed snapshot would observe -1 rather than the original
+// saved value.
+//
+// Construction/destruction balance is verified at program exit via a
+// global guard whose destructor fires after libomp's static-record
+// teardown has run, at which point the saved-firstprivate snapshot must
+// also have been destroyed exactly once.
+
+#include <cstdio>
+#include <cstdlib>
+
+struct Tracker {
+  static int Ctors;
+  static int CopyCtors;
+  static int Dtors;
+
+  int Value;
+
+  explicit Tracker(int V) : Value(V) { ++Ctors; }
+  Tracker(const Tracker &Other) : Value(Other.Value) { ++CopyCtors; }
+  // Use a sentinel value on destruction so that any read-after-destroy of
+  // the saved snapshot becomes observable.
+  ~Tracker() {
+    ++Dtors;
+    Value = -1;
+  }
+};
+
+int Tracker::Ctors = 0;
+int Tracker::CopyCtors = 0;
+int Tracker::Dtors = 0;
+
+// 'Failed' is shared across tasks and threads so we can record a fail
+// status from within the task body and across the program-exit guard.
+static int Failed = 0;
+
+__attribute__((noinline)) static void run_taskgraph_nontrivial(int seed) {
+  Tracker Local(seed);
+  int observed = 0;
+
+#pragma omp taskgraph graph_id(927)
+  {
+#pragma omp task firstprivate(saved : Local) shared(observed)
+    {
+      // Each replay must observe the value captured at recording time
+      // (which was 11).  If the per-replay destructor deferral were
+      // missing, the second and subsequent replays would observe the
+      // sentinel -1 written by Tracker::~Tracker().  Do not mutate the
+      // snapshot here: we want each replay to read the same value.
+      observed = Local.Value;
+    }
+  }
+
+  if (observed != 11) {
+    std::fprintf(stderr, "FAIL replay observed=%d expected=11 seed=%d\n",
+                 observed, seed);
+    Failed = 1;
+  }
+}
+
+// Final accounting fires at program exit.  Two snapshots are
+// copy-constructed exactly once each at recording time:
+//   1. The "original" task's '.kmp_privates.t' snapshot (initialised in IR
+//      at task allocation, then destructed when the original task finishes).
+//   2. The "clone" task's '.kmp_privates.t' snapshot (initialised by the
+//      compiler-emitted task-clone helper invoked from
+//      __kmpc_taskgraph_task; its destructor is deferred to
+//      __kmp_taskgraph_free, which is not driven by program exit in
+//      libomp today, hence the asymmetric expected dtor count below).
+struct ExitGuard {
+  ~ExitGuard() {
+    bool ok = true;
+    if (Tracker::CopyCtors != 2) {
+      std::fprintf(stderr,
+                   "FAIL (exit) CopyCtors=%d expected=2 (one for the "
+                   "original task, one for the persistent taskgraph clone)\n",
+                   Tracker::CopyCtors);
+      ok = false;
+    }
+    // Expected destruction count at program exit:
+    //   - Local objects: one ctor + one dtor per call to
+    //     run_taskgraph_nontrivial.
+    //   - The original task's snapshot: destructed when the task body
+    //     finishes (its task is not taskgraph-owned).
+    //   - The clone task's snapshot is deferred to __kmp_taskgraph_free,
+    //     which is not yet driven at program exit, so it does not
+    //     contribute to the dtor count here.
+    int expected_dtors = Tracker::Ctors + /* orig task snapshot */ 1;
+    if (Tracker::Dtors != expected_dtors) {
+      std::fprintf(stderr,
+                   "FAIL (exit) ctor/dtor imbalance ctors=%d copyctors=%d "
+                   "dtors=%d expected_dtors=%d\n",
+                   Tracker::Ctors, Tracker::CopyCtors, Tracker::Dtors,
+                   expected_dtors);
+      ok = false;
+    }
+
+    if (!ok || Failed) {
+      std::fprintf(stderr, "FAIL firstprivate(saved) non-trivial lifecycle\n");
+      std::_Exit(1);
+    }
+
+    std::fprintf(stderr,
+                 "PASS firstprivate(saved) non-trivial lifecycle "
+                 "ctors=%d copyctors=%d dtors=%d\n",
+                 Tracker::Ctors, Tracker::CopyCtors, Tracker::Dtors);
+  }
+};
+
+// Global guard whose destructor fires at program exit, after all OpenMP
+// teardown has occurred and the taskgraph record has been freed.
+static ExitGuard Guard;
+
+int main() {
+  // Recording run.  Inside the function we create Local(11), then the saved
+  // firstprivate copy-constructs that into the task's slot (one copyctor).
+  run_taskgraph_nontrivial(11);
+
+  // Replay runs.  Each replay reuses the recorded task descriptor and its
+  // in-place saved snapshot.  No additional copy-construction occurs, and
+  // no destruction occurs until end-of-taskgraph.  The body observes the
+  // snapshot value (11) on every replay -- the per-call seed argument is
+  // intentionally ignored by the task body to demonstrate that the saved
+  // snapshot drives what the task sees, not the call-site argument.
+  for (int i = 0; i < 5; ++i)
+    run_taskgraph_nontrivial(42 + i);
+
+  return 0;
+}
+
+// CHECK: PASS firstprivate(saved) non-trivial lifecycle
diff --git a/openmp/runtime/test/taskgraph/taskgraph_firstprivate_saved_nontrivial_selfref.cpp b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_saved_nontrivial_selfref.cpp
new file mode 100644
index 0000000000000..dfad59bb4c149
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_saved_nontrivial_selfref.cpp
@@ -0,0 +1,97 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// Verifies that taskgraph cloning copy-constructs non-trivially-copyable
+// 'firstprivate(saved: ...)' list items rather than bitwise-copying them.
+//
+// 'SelfRef' carries an internal pointer that the constructor wires back to
+// its own member.  Any bitwise clone of the task descriptor (the historical
+// behaviour) would leave the cloned object's internal pointer aimed at the
+// original (which goes out of scope when the recording call returns), so
+// the first replay would dereference dangling memory.  When the compiler-
+// emitted task-clone helper runs, it copy-constructs the cloned snapshot
+// in-place, repairing the internal pointer to refer to the clone's own
+// storage.  Subsequent replays therefore observe a consistent value.
+
+#include <cstdio>
+#include <cstdlib>
+
+struct SelfRef {
+  int Value;
+  // Self-pointer that the (copy) constructor steers at our own 'Value'
+  // member.  Bitwise duplication leaves this dangling.
+  int *Inside;
+
+  explicit SelfRef(int V) : Value(V), Inside(&Value) {}
+  SelfRef(const SelfRef &Other) : Value(Other.Value), Inside(&Value) {}
+  ~SelfRef() {
+    // Poison the self pointer so any dangling read after destruction is
+    // observable as a (likely) crash or wrong value.
+    Inside = nullptr;
+    Value = -1;
+  }
+};
+
+static int Failed = 0;
+
+__attribute__((noinline)) static void run_taskgraph_selfref(int seed) {
+  SelfRef Local(seed);
+  int observed_via_self = 0;
+  int observed_value = 0;
+
+#pragma omp taskgraph graph_id(3142)
+  {
+#pragma omp task firstprivate(saved : Local)                                   \
+    shared(observed_via_self, observed_value)
+    {
+      // Read through the self pointer.  This is exactly the operation that
+      // bitwise cloning would break: the cloned task's 'Inside' would point
+      // at the recording-time stack frame, not at the cloned 'Value'.
+      observed_via_self = *Local.Inside;
+      observed_value = Local.Value;
+    }
+  }
+
+  if (observed_value != 7 || observed_via_self != 7) {
+    std::fprintf(stderr,
+                 "FAIL seed=%d observed_value=%d observed_via_self=%d "
+                 "(expected both = 7)\n",
+                 seed, observed_value, observed_via_self);
+    Failed = 1;
+  }
+}
+
+struct ExitGuard {
+  ~ExitGuard() {
+    if (Failed) {
+      std::fprintf(stderr, "FAIL non-trivial taskgraph clone\n");
+      std::_Exit(1);
+    }
+    std::fprintf(stderr, "PASS non-trivial taskgraph clone\n");
+  }
+};
+
+static ExitGuard Guard;
+
+int main() {
+  // Recording captures Local(7) into the task's '.kmp_privates.t' slot, then
+  // the runtime clones that descriptor.  After this returns 'Local' is gone,
+  // so any cloned 'Inside' pointer that still aimed at &Local.Value would
+  // dangle.  The clone helper must copy-construct the clone in place to
+  // repair the self pointer.
+  run_taskgraph_selfref(7);
+
+  // Replays.  Each one must observe the saved 7 both directly and through
+  // the (repaired) self-referencing pointer.  Different seed values are
+  // intentionally ignored by the task body: only the saved snapshot drives
+  // what the task sees.
+  for (int i = 0; i < 5; ++i)
+    run_taskgraph_selfref(42 + i);
+
+  return 0;
+}
+
+// CHECK: PASS non-trivial taskgraph clone
diff --git a/openmp/runtime/test/taskgraph/taskgraph_firstprivate_saved_static.cpp b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_saved_static.cpp
new file mode 100644
index 0000000000000..ae613e2bebef2
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_firstprivate_saved_static.cpp
@@ -0,0 +1,100 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// OpenMP 6.0 [14.3]: the 'saved' modifier on a 'firstprivate' clause of a
+// replayable construct extends the saved data environment to also include
+// copies of variables with static storage duration that appear in the
+// clause.  Within the current Clang/libomp implementation those snapshots
+// live in the per-task '.kmp_privates.t' tail struct, which is allocated
+// once at recording time and reused for every replay of the recorded task.
+// This test exercises four flavours of static-storage list items and
+// verifies that the saved snapshot is what each replay observes, regardless
+// of any subsequent mutation of the underlying static variable between
+// taskgraph encounters.
+
+#include <cstdio>
+
+static int FileScopeStaticInt = 100;
+static const int FileScopeConstStaticInt = 200;
+
+struct WithStaticMember {
+  static int StaticMember;
+  static const int StaticConstMember = 400;
+};
+int WithStaticMember::StaticMember = 300;
+// Out-of-line definition is required (pre-C++17) because the saved
+// firstprivate slot captures the static by-value via odr-use.
+const int WithStaticMember::StaticConstMember;
+
+__attribute__((noinline)) static void
+run_taskgraph_saved_static(int *out_fs, int *out_fsc, int *out_local,
+                           int *out_member, int *out_member_const) {
+  static int LocalStaticInt = 500;
+
+#pragma omp taskgraph graph_id(811)
+  {
+#pragma omp task firstprivate(saved : FileScopeStaticInt,                      \
+                                  FileScopeConstStaticInt, LocalStaticInt,     \
+                                  WithStaticMember::StaticMember,              \
+                                  WithStaticMember::StaticConstMember)         \
+    shared(out_fs, out_fsc, out_local, out_member, out_member_const)
+    {
+      *out_fs = FileScopeStaticInt;
+      *out_fsc = FileScopeConstStaticInt;
+      *out_local = LocalStaticInt;
+      *out_member = WithStaticMember::StaticMember;
+      *out_member_const = WithStaticMember::StaticConstMember;
+    }
+  }
+}
+
+int main() {
+  bool failed = false;
+  int fs = -1, fsc = -1, local = -1, member = -1, member_const = -1;
+
+  // First call: recording.  Each captured-by-saved static is snapshotted
+  // into the task's '.kmp_privates.t' slot at this point.
+  run_taskgraph_saved_static(&fs, &fsc, &local, &member, &member_const);
+  if (fs != 100 || fsc != 200 || local != 500 || member != 300 ||
+      member_const != 400) {
+    std::fprintf(stderr,
+                 "FAIL initial record fs=%d fsc=%d local=%d member=%d "
+                 "member_const=%d\n",
+                 fs, fsc, local, member, member_const);
+    failed = true;
+  }
+
+  // Mutate the underlying non-const statics.  Because the task's firstprivate
+  // slots were snapshotted with 'saved:' at recording, every subsequent
+  // replay must continue to observe the recorded values (100, 500, 300) for
+  // the non-const statics, not the mutated values.
+  FileScopeStaticInt = 11;
+  WithStaticMember::StaticMember = 13;
+  // LocalStaticInt is not visible here; we can rely on the fact that the
+  // function-local static is also snapshotted at recording.
+
+  for (int i = 0; i < 4; ++i) {
+    fs = fsc = local = member = member_const = -1;
+    run_taskgraph_saved_static(&fs, &fsc, &local, &member, &member_const);
+    if (fs != 100 || fsc != 200 || local != 500 || member != 300 ||
+        member_const != 400) {
+      std::fprintf(stderr,
+                   "FAIL replay %d fs=%d fsc=%d local=%d member=%d "
+                   "member_const=%d\n",
+                   i, fs, fsc, local, member, member_const);
+      failed = true;
+    }
+  }
+
+  if (failed)
+    return 1;
+
+  std::fprintf(stderr,
+               "PASS firstprivate(saved) statics persist across replays\n");
+  return 0;
+}
+
+// CHECK: PASS firstprivate(saved) statics persist across replays
diff --git a/openmp/runtime/test/taskgraph/taskgraph_replayable_saved_stack_depth.cpp b/openmp/runtime/test/taskgraph/taskgraph_replayable_saved_stack_depth.cpp
index 922cb85a53eec..aa4eec17ad091 100644
--- a/openmp/runtime/test/taskgraph/taskgraph_replayable_saved_stack_depth.cpp
+++ b/openmp/runtime/test/taskgraph/taskgraph_replayable_saved_stack_depth.cpp
@@ -1,7 +1,6 @@
 // clang-format off
 // RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t && env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
 // REQUIRES: omp_taskgraph_experimental
-// XFAIL: *
 // clang-format on
 
 #include <cstdio>
diff --git a/openmp/runtime/test/taskgraph/taskgraph_shared_stack_depth.cpp b/openmp/runtime/test/taskgraph/taskgraph_shared_stack_depth.cpp
index ad579e8ed1b3d..533cd929170fb 100644
--- a/openmp/runtime/test/taskgraph/taskgraph_shared_stack_depth.cpp
+++ b/openmp/runtime/test/taskgraph/taskgraph_shared_stack_depth.cpp
@@ -2,9 +2,23 @@
 // RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
 // RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
 // REQUIRES: omp_taskgraph_experimental
-// XFAIL: *
 // clang-format on
 
+// This test exercises a non-lexically-nested replayable task that captures
+// 'payload' and 'seed' as 'firstprivate' (without the OpenMP 6.0 'saved'
+// modifier).  Per OpenMP 6.0 [14.3], when a variable referenced in a
+// replayable construct does not exist in the enclosing data environment
+// of the taskgraph construct, the saved data environment in the
+// taskgraph record is used as the enclosing data environment for that
+// variable.  Both 'payload' and 'seed' live in run_taskgraph's stack
+// frame, not in main's, so the spec mandates the saved-data-environment
+// fallback - i.e. the snapshot captured at recording time is what every
+// replay observes, independent of the call-stack depth of the replay.
+//
+// In the current libomp implementation that snapshot lives in the
+// task descriptor's '.kmp_privates.t' tail struct, which persists for
+// the lifetime of the taskgraph record, so this test passes naturally.
+
 #include <cstdio>
 
 static volatile int StackSink = 0;

>From 88ceb8cbbc7c0e1817d90d161a5323187523099c Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Thu, 28 May 2026 05:03:38 -0500
Subject: [PATCH 21/26] [OpenMP] Add taskgraph template interaction codegen
 tests

Add two Clang codegen tests for the OpenMP 6.0 taskgraph
directive exercising its interaction with templates -- in
particular checking that different instantiations of the same
template get distinct taskgraph handles.

Assisted-By: Claude Opus 4.7

Pull Request: https://github.com/llvm/llvm-project/pull/200409
---
 .../taskgraph_template_handle_codegen.cpp     | 36 +++++++++
 .../OpenMP/taskgraph_templates_codegen.cpp    | 73 +++++++++++++++++++
 2 files changed, 109 insertions(+)
 create mode 100644 clang/test/OpenMP/taskgraph_template_handle_codegen.cpp
 create mode 100644 clang/test/OpenMP/taskgraph_templates_codegen.cpp

diff --git a/clang/test/OpenMP/taskgraph_template_handle_codegen.cpp b/clang/test/OpenMP/taskgraph_template_handle_codegen.cpp
new file mode 100644
index 0000000000000..0874fe58edb03
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_template_handle_codegen.cpp
@@ -0,0 +1,36 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -o - | FileCheck %s
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+template <typename T>
+void templ_handle(T &x) {
+#pragma omp taskgraph
+  {
+#pragma omp task shared(x)
+    {
+      x += 1;
+    }
+  }
+}
+
+int main() {
+  int i = 0;
+  long l = 0;
+  templ_handle(i);
+  templ_handle(l);
+  return 0;
+}
+
+// CHECK-DAG: @[[H1:.omp.taskgraph.handle[^ ]*]] = internal global ptr null
+// CHECK-DAG: @[[H2:.omp.taskgraph.handle[^ ]*]] = internal global ptr null
+
+// CHECK-LABEL: define linkonce_odr {{.*}} @_Z12templ_handleIiEvRT_(
+// CHECK: call void @__kmpc_taskgraph(ptr {{[^,]+}}, i32 {{[^,]+}}, ptr @[[H1]], i64 0, i32 0, i32 0, ptr {{[^,]+}}, ptr {{[^)]+}})
+
+// CHECK-LABEL: define linkonce_odr {{.*}} @_Z12templ_handleIlEvRT_(
+// CHECK-NOT: ptr @[[H1]]
+// CHECK: call void @__kmpc_taskgraph(ptr {{[^,]+}}, i32 {{[^,]+}}, ptr @[[H2]], i64 0, i32 0, i32 0, ptr {{[^,]+}}, ptr {{[^)]+}})
+
+#endif
diff --git a/clang/test/OpenMP/taskgraph_templates_codegen.cpp b/clang/test/OpenMP/taskgraph_templates_codegen.cpp
new file mode 100644
index 0000000000000..16765bb078f4e
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_templates_codegen.cpp
@@ -0,0 +1,73 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+// Exercises taskgraph codegen with orthogonal language/runtime features:
+//  - C++ templates + non-trivial firstprivate cloning.
+//  - taskgroup task_reduction / task in_reduction inside taskgraph.
+//  - taskwait depend(...) inside taskgraph (task-generating path).
+
+template <typename T>
+struct Box {
+  T v;
+  Box();
+  Box(const Box &);
+  ~Box();
+};
+
+template <typename T>
+void templated_clone(T seed) {
+  Box<T> B;
+  B.v = seed;
+#pragma omp taskgraph
+  {
+#pragma omp task firstprivate(B)
+    {
+      (void)B.v;
+    }
+
+#pragma omp task replayable(false) firstprivate(B)
+    {
+      (void)B.v;
+    }
+  }
+}
+
+template <typename T>
+T templated_task_reduction(T seed) {
+  T Acc = seed;
+#pragma omp taskgraph
+  {
+#pragma omp taskgroup task_reduction(+: Acc)
+    {
+#pragma omp task in_reduction(+: Acc)
+      {
+        Acc += seed;
+      }
+    }
+#pragma omp taskwait depend(in: Acc)
+  }
+  return Acc;
+}
+
+int main() {
+  templated_clone<int>(7);
+  return templated_task_reduction<int>(1);
+}
+
+// The taskgraph task entry uses the taskgraph runtime path and carries a
+// clone helper for non-trivial firstprivate copies.
+// CHECK: call i32 @__kmpc_taskgraph_task(
+// CHECK-SAME: ptr @.omp_task_clone.)
+// CHECK: define internal void @.omp_task_clone.(
+
+// task_reduction in a taskgraph uses the dedicated taskgraph reduction init.
+// CHECK: call ptr @__kmpc_taskgraph_taskred_init(
+
+// taskwait depend(...) inside taskgraph uses the dedicated taskgraph taskwait
+// entry point instead of the generic taskwait runtime path.
+// CHECK: call void @__kmpc_taskgraph_taskwait(
+
+#endif

>From 5335688307a7c5cfa969fc92b25b2ac72cbc176b Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Thu, 11 Jun 2026 08:52:19 -0500
Subject: [PATCH 22/26] [OpenMP] Fix hang and crash on empty taskgraphs

Empty taskgraphs caused crashes (certain code relied on entry and exit
regions being distinct, for instance).  This patch contains a trivial fix.

Pull Request: https://github.com/llvm/llvm-project/pull/216327
---
 openmp/runtime/src/kmp_taskdeps.cpp | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/openmp/runtime/src/kmp_taskdeps.cpp b/openmp/runtime/src/kmp_taskdeps.cpp
index fe7c630b567b0..10a8e2bd544ed 100644
--- a/openmp/runtime/src/kmp_taskdeps.cpp
+++ b/openmp/runtime/src/kmp_taskdeps.cpp
@@ -2942,6 +2942,15 @@ kmp_int32 __kmp_build_taskgraph(kmp_int32 gtid,
   kmp_dephash_t *hash = __kmp_dephash_create(thread, current_taskdata);
   bool dep_barrier = false;
 
+  // An empty taskgraph isn't useful and causes analysis problems below, so
+  // catch it early.
+  if (numnodes == 0) {
+    taskgraph->root = nullptr;
+    taskgraph->alloc_root = nullptr;
+    KMP_ATOMIC_ST_REL(&taskgraph->status, KMP_TDG_READY);
+    return 0;
+  }
+
   // We need to take special care to align the all_depnodes array to the cache
   // line size, because kmp_depnode_t is marked as 64-byte aligned and
   // otherwise the compiler might generate faulting memory accesses based on

>From 451943838bd2ac6d1a18fd6c06f0f8de0a76f381 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Fri, 24 Jul 2026 14:03:36 -0500
Subject: [PATCH 23/26] [OpenMP] Add taskgraph regression test for out-of-line
 task replay deadlock

Adds a regression test for a taskgraph deadlock that occurred when
non-replayable tasks were encountered during taskgraph recording --
not lexically nested within the taskgraph construct.

Assisted-By: Claude Opus 4.8

Pull Request: https://github.com/llvm/llvm-project/pull/216328
---
 .../test/taskgraph/taskgraph_deadlock_1.cpp   | 53 +++++++++++++++++++
 1 file changed, 53 insertions(+)
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_deadlock_1.cpp

diff --git a/openmp/runtime/test/taskgraph/taskgraph_deadlock_1.cpp b/openmp/runtime/test/taskgraph/taskgraph_deadlock_1.cpp
new file mode 100644
index 0000000000000..ad0c5eaf9e61a
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_deadlock_1.cpp
@@ -0,0 +1,53 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// Regression test for a taskgraph deadlock.  Non-lexically scoped tasks
+// (without the 'replayable' clause) called while recording a taskgraph would
+// previously cause lockups.
+// The test should run to completion without crashing.
+
+#include <cstdio>
+
+int x = 0;
+int y = 10;
+int z = 20;
+
+__attribute__((noinline)) static void workOne() {
+#pragma omp task depend(inout : x)
+  {}
+}
+
+__attribute__((noinline)) static void workTwo() {
+#pragma omp task depend(inout : y, z)
+  {}
+}
+
+__attribute__((noinline)) static void workThree() {
+#pragma omp task depend(in : y, z) depend(out : x)
+  {}
+}
+
+int main() {
+#pragma omp parallel
+#pragma omp single
+  {
+    for (int i = 0; i < 10; ++i) {
+      // Would deadlock if the tasks are created out-of-line (not inlined) and
+      // the recorded graph is replayed on subsequent iterations.
+#pragma omp taskgraph
+      {
+        workOne();
+        workTwo();
+        workThree();
+      }
+    }
+  }
+
+  std::fprintf(stderr, "PASS taskgraph deadlock repro completed\n");
+  return 0;
+}
+
+// CHECK: PASS taskgraph deadlock repro completed

>From 2c2454d3827816bf5327f9f752f4609a0c97b3c7 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Fri, 24 Jul 2026 14:20:31 -0500
Subject: [PATCH 24/26] [Clang][OpenMP] Fix capture of reduction variables
 under nested default(firstprivate)

isOpenMPCapturedByRef() decided whether a scalar is captured by copy for a
given capture Level, but consulted the default data-sharing attribute of the
innermost directive (getDefaultDSA()) instead of the one at that Level
(getDefaultDSA(Level)).

When a construct with default(firstprivate) (or default(private)) is nested in
an enclosing captured region within the same function -- e.g. a taskloop with a
reduction inside a 'parallel' region -- this made the enclosing region capture
the reduction variable by copy.  The reduction then combined into that copy and
its result never propagated back to the original variable (it read as 0).

Query the default attribute at the capture Level being considered, matching the
Level already used by the neighbouring hasExplicitDSA()/isLoopControlVariable()
checks.  For top-level captures Level is the top of stack, so behaviour is
unchanged there.

This bug is independent of the 'taskgraph' directive; add a CodeGen regression
test (taskloop_reduction_default_firstprivate_codegen.cpp) exercising it with a
taskloop reduction under default(firstprivate) nested in a plain parallel/single
region.

Assisted-By: Claude Opus 4.8

Pull Request: https://github.com/llvm/llvm-project/pull/216329
---
 clang/lib/Sema/SemaOpenMP.cpp                 | 10 +++--
 ...reduction_default_firstprivate_codegen.cpp | 44 +++++++++++++++++++
 2 files changed, 51 insertions(+), 3 deletions(-)
 create mode 100644 clang/test/OpenMP/taskloop_reduction_default_firstprivate_codegen.cpp

diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index f0e4788a602be..3c809117e25a6 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -2354,9 +2354,13 @@ bool SemaOpenMP::isOpenMPCapturedByRef(const ValueDecl *D, unsigned Level,
         !(isa<OMPCapturedExprDecl>(D) && !D->hasAttr<OMPCaptureNoInitAttr>() &&
           !cast<OMPCapturedExprDecl>(D)->getInit()->isGLValue()) &&
         // If the variable is implicitly firstprivate and scalar - capture by
-        // copy
-        !((DSAStack->getDefaultDSA() == DSA_firstprivate ||
-           DSAStack->getDefaultDSA() == DSA_private) &&
+        // copy.  Query the default data-sharing attribute at the capture level
+        // being considered rather than at the innermost directive: a nested
+        // construct's default(firstprivate)/default(private) must not force an
+        // enclosing captured region (e.g. 'taskgraph') to capture a reduction
+        // variable by copy, which would sever the reduction's write-back.
+        !((DSAStack->getDefaultDSA(Level) == DSA_firstprivate ||
+           DSAStack->getDefaultDSA(Level) == DSA_private) &&
           !DSAStack->hasExplicitDSA(
               D, [](OpenMPClauseKind K, bool) { return K != OMPC_unknown; },
               Level) &&
diff --git a/clang/test/OpenMP/taskloop_reduction_default_firstprivate_codegen.cpp b/clang/test/OpenMP/taskloop_reduction_default_firstprivate_codegen.cpp
new file mode 100644
index 0000000000000..0bbd7593c337f
--- /dev/null
+++ b/clang/test/OpenMP/taskloop_reduction_default_firstprivate_codegen.cpp
@@ -0,0 +1,44 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
+// expected-no-diagnostics
+
+// A taskloop with default(firstprivate) and a reduction, nested inside an
+// enclosing captured region (here 'parallel'/'single', with no 'taskgraph'
+// involved).  The reduction variable 'res' must be captured *by reference* by
+// the enclosing parallel region (its address forwarded to the outlined
+// function) so the reduction result propagates back to it.
+//
+// This is the non-taskgraph form of the same bug fixed for taskgraph: when
+// deciding the enclosing region's capture kind, Sema consulted the innermost
+// directive's default(firstprivate) instead of the default at the capture
+// level, capturing 'res' by copy and severing the reduction write-back (the
+// caller then read the unmodified original, i.e. 0).
+
+#ifndef HEADER
+#define HEADER
+
+int run(int seed) {
+  int x = seed;
+  int res = 0;
+
+#pragma omp parallel
+#pragma omp single
+  {
+#pragma omp taskloop num_tasks(4) default(firstprivate) reduction(+ : res)
+    for (int i = 0; i < 8; ++i)
+      res += x + i;
+  }
+
+  return res;
+}
+
+#endif
+
+// CHECK-LABEL: define {{.*}}@_Z3runi(
+// The reduction variable is captured BY REFERENCE by the enclosing parallel
+// region: its address (not its value) is forwarded to the outlined function.
+// CHECK:         call {{.*}}@__kmpc_fork_call(ptr {{.*}}, i32 2, ptr @_Z3runi.omp_outlined, ptr %[[RES:[0-9a-z._]+]], ptr %{{[0-9a-z._]+}})
+// The caller reads the (reduced) value straight back out of that same storage.
+// CHECK:         %[[RET:.*]] = load i32, ptr %[[RES]], align 4
+// CHECK-NEXT:    ret i32 %[[RET]]

>From b1bae6eceeb73f534519715bbae0f1023f9d6335 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Fri, 24 Jul 2026 14:20:41 -0500
Subject: [PATCH 25/26] [OpenMP] Add taskgraph tests for the 'default' clause
 on replayable constructs

Adds tests covering default(none), default(shared) and default(firstprivate) on
replayable tasks and taskloops inside a taskgraph.  default(none) on a
replayable construct previously crashed Clang's Sema because the implicit-DSA
check had no case for the 'replayable' clause.

Beyond compiling, the tests verify the runtime record/replay semantics under
each default value: shared list items are relocated to (and re-read from) the
current call on every replay, while plain firstprivate items and
firstprivate(saved) items are snapshotted at recording and persist across
replays even after the underlying static is mutated.  The firstprivate(saved)
and plain-firstprivate list items are aggregates / known-size arrays so the
aggregate save path (rather than the always-inlined intptr-sized scalar path)
is exercised.  The default(firstprivate) taskloop additionally uses a reduction
as a regression guard for the enclosing-region reduction-variable capture fix.

Each encounter enters the taskgraph at a different call-stack depth, with the
abandoned frames clobbered in between, so a replay that used the recorded
shareds pointer cannot land on the live objects by accident.

Assisted-By: Claude Opus 4.8

Pull Request: https://github.com/llvm/llvm-project/pull/216330
---
 .../taskgraph_default_firstprivate.cpp        | 124 +++++++++++++++
 ...askgraph_default_firstprivate_taskloop.cpp | 145 ++++++++++++++++++
 .../taskgraph/taskgraph_default_none_1.cpp    | 114 ++++++++++++++
 .../taskgraph_default_none_taskloop.cpp       | 105 +++++++++++++
 .../taskgraph/taskgraph_default_shared.cpp    |  88 +++++++++++
 .../taskgraph_default_shared_taskloop.cpp     |  85 ++++++++++
 6 files changed, 661 insertions(+)
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_default_firstprivate.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_default_firstprivate_taskloop.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_default_none_1.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_default_none_taskloop.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_default_shared.cpp
 create mode 100644 openmp/runtime/test/taskgraph/taskgraph_default_shared_taskloop.cpp

diff --git a/openmp/runtime/test/taskgraph/taskgraph_default_firstprivate.cpp b/openmp/runtime/test/taskgraph/taskgraph_default_firstprivate.cpp
new file mode 100644
index 0000000000000..f61b7e2cf212c
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_default_firstprivate.cpp
@@ -0,0 +1,124 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// default(firstprivate) on a replayable task.  Every variable referenced in the
+// body without an explicit attribute becomes (plain) firstprivate; here that is
+// the automatic array 'local_arr'.  'SavedOff' is captured with the 'saved'
+// modifier and 'result' is explicitly shared so the outcome is observable.
+//
+// Record/replay semantics being verified:
+//   - a plain firstprivate is snapshotted at *recording* and reused verbatim on
+//     every replay (local_arr keeps its recorded values even though later calls
+//     pass a different seed);
+//   - firstprivate(saved) likewise keeps the value snapshotted at recording,
+//     even after the underlying static is mutated between encounters;
+//   - the shared 'result' is relocated to the *current* call's frame on each
+//     replay (so replays write to the live object, not the recorded one).
+//
+// The taskgraph is entered through call_with_depth() at a different recursion
+// depth for the recording and for each replay, and clobber_stack() overwrites
+// the abandoned frames in between.  This is what makes the relocation clause
+// above load-bearing: if every encounter reused a single frame at one address,
+// a replay writing through the recorded (stale) shareds pointer would still
+// land on the live object by accident and the relocation path would not be
+// exercised at all.
+//
+// 'result' is an int in the frame rather than an int* supplied by the caller so
+// that a missing relocation fails as a stale write to a dead frame (leaving
+// 'result' at -1) instead of a wild store through a clobbered pointer.
+//
+// Both firstprivate list items are aggregates (an automatic array and a static
+// struct) so the aggregate save path is exercised rather than the
+// always-inlined intptr-sized scalar path.
+
+#include <cstdio>
+
+struct Off {
+  int a, b, c, d;
+};
+
+static Off SavedOff = {1, 2, 3, 4}; // a+b+c+d == 10
+
+static volatile int StackSink = 0;
+
+__attribute__((noinline)) static void clobber_stack(int base) {
+  volatile int scratch[4096];
+
+  for (int i = 0; i < 4096; ++i)
+    scratch[i] = base + i;
+
+  StackSink += scratch[base & 63];
+}
+
+__attribute__((noinline)) static int run_default_firstprivate(int seed) {
+  int x = seed; // only used to anchor a dependence
+  int local_arr[3] = {seed, seed + 1, seed + 2};
+  int result = -1;
+
+#pragma omp taskgraph graph_id(9105)
+  {
+#pragma omp task default(firstprivate) shared(result)                          \
+    firstprivate(saved : SavedOff) depend(inout : x)
+    {
+      result = local_arr[0] + local_arr[1] + local_arr[2] + SavedOff.a +
+               SavedOff.b + SavedOff.c + SavedOff.d;
+    }
+  }
+
+  return result;
+}
+
+__attribute__((noinline)) static int call_with_depth(int seed, int depth) {
+  volatile int padding[128];
+
+  for (int i = 0; i < 128; ++i)
+    padding[i] = seed + depth + i;
+
+  StackSink += padding[(seed + depth) & 127];
+
+  if (depth == 0)
+    return run_default_firstprivate(seed);
+  return call_with_depth(seed, depth - 1);
+}
+
+int main() {
+  bool failed = false;
+
+  // Recording: local_arr == {1,2,3} (sum 6), SavedOff snapshot sum == 10.
+  const int a = call_with_depth(1, 0); // 6 + 10
+  if (a != 16) {
+    std::fprintf(stderr, "FAIL default(firstprivate) record a=%d expected=16\n",
+                 a);
+    failed = true;
+  }
+
+  // Mutate the static; the saved snapshot must keep the recorded value, and the
+  // plain firstprivate array must keep its recorded {1,2,3}.  A correct replay
+  // therefore still yields 16 -- and it must land in the frame of the current
+  // call, which sits at a different stack depth than the recording, proving the
+  // shared capture was relocated.
+  SavedOff = Off{9, 9, 9, 9};
+
+  clobber_stack(100 * 1000);
+  const int b = call_with_depth(100, 3); // frozen: 6 + 10
+  clobber_stack(5 * 1000);
+  const int c = call_with_depth(5, 6); // frozen: 6 + 10
+  if (b != 16 || c != 16) {
+    std::fprintf(stderr,
+                 "FAIL default(firstprivate) replay b=%d c=%d expected=16/16\n",
+                 b, c);
+    failed = true;
+  }
+
+  if (failed)
+    return 1;
+
+  std::fprintf(stderr, "PASS default(firstprivate) task a=%d b=%d c=%d\n", a, b,
+               c);
+  return 0;
+}
+
+// CHECK: PASS default(firstprivate) task a=16 b=16 c=16
diff --git a/openmp/runtime/test/taskgraph/taskgraph_default_firstprivate_taskloop.cpp b/openmp/runtime/test/taskgraph/taskgraph_default_firstprivate_taskloop.cpp
new file mode 100644
index 0000000000000..25081e4df7aee
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_default_firstprivate_taskloop.cpp
@@ -0,0 +1,145 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// default(firstprivate) on a replayable taskloop.  'x' becomes (plain)
+// firstprivate and is therefore snapshotted at recording and reused on every
+// replay; 'SavedBump' is captured via firstprivate(saved) (an aggregate, so the
+// aggregate save path is exercised); the loop iteration variable is
+// predetermined private; results are written both through an explicitly shared
+// output array (which must be relocated to the current call on replay) and a
+// reduction.
+//
+// The reduction here is a regression guard: default(firstprivate) previously
+// caused the enclosing taskgraph region to capture the reduction variable by
+// copy, so the reduced value never propagated back to the caller (it read as
+// 0).
+//
+// The shared output array is a local of run_default_firstprivate_taskloop
+// rather than the caller's array, and the taskgraph is entered through
+// call_with_depth() at a different recursion depth per encounter (with
+// clobber_stack() overwriting the abandoned frames in between).  That is what
+// makes the relocation of the shared array load-bearing: reached at one fixed
+// depth the frame would recur at one address and a replay writing through the
+// recorded (stale) shareds pointer would hit the live array by accident.  Using
+// a local rather than a caller-supplied pointer also means a missing relocation
+// shows up as a stale write to a dead frame (leaving the -1 initialiser behind)
+// instead of a wild store through a clobbered pointer.
+
+#include <cstdio>
+
+struct Bump {
+  int a, b;
+};
+
+static Bump SavedBump = {5, 7}; // a+b == 12
+
+static volatile int StackSink = 0;
+
+__attribute__((noinline)) static void clobber_stack(int base) {
+  volatile int scratch[4096];
+
+  for (int i = 0; i < 4096; ++i)
+    scratch[i] = base + i;
+
+  StackSink += scratch[base & 63];
+}
+
+__attribute__((noinline)) static int
+run_default_firstprivate_taskloop(int seed, int *out_arr) {
+  int x = seed;
+  int total = 0;
+  int local_out[8];
+  for (int i = 0; i < 8; ++i)
+    local_out[i] = -1;
+
+#pragma omp taskgraph graph_id(9106)
+  {
+#pragma omp taskloop replayable num_tasks(4) default(firstprivate)             \
+    shared(local_out) firstprivate(saved : SavedBump) reduction(+ : total)
+    for (int i = 0; i < 8; ++i) {
+      int v = x + i + SavedBump.a + SavedBump.b;
+      local_out[i] = v;
+      total += v;
+    }
+  }
+
+  for (int i = 0; i < 8; ++i)
+    out_arr[i] = local_out[i];
+
+  return total;
+}
+
+__attribute__((noinline)) static int call_with_depth(int seed, int *out_arr,
+                                                     int depth) {
+  volatile int padding[128];
+
+  for (int i = 0; i < 128; ++i)
+    padding[i] = seed + depth + i;
+
+  StackSink += padding[(seed + depth) & 127];
+
+  if (depth == 0)
+    return run_default_firstprivate_taskloop(seed, out_arr);
+  return call_with_depth(seed, out_arr, depth - 1);
+}
+
+static bool check(const int *arr, int base, const char *tag) {
+  for (int i = 0; i < 8; ++i) {
+    if (arr[i] != base + i) {
+      std::fprintf(stderr,
+                   "FAIL default(firstprivate) taskloop %s arr[%d]=%d "
+                   "expected=%d\n",
+                   tag, i, arr[i], base + i);
+      return false;
+    }
+  }
+  return true;
+}
+
+int main() {
+  bool ok = true;
+  int o1[8], o2[8];
+  for (int i = 0; i < 8; ++i) {
+    o1[i] = -1;
+    o2[i] = -1;
+  }
+
+  // Recording: out_arr[i] = x(10) + i + 12 = 22 + i; total = sum(22+i) = 204.
+  const int t1 = call_with_depth(10, o1, 0);
+  ok &= check(o1, 22, "record");
+  if (t1 != 204) {
+    std::fprintf(stderr,
+                 "FAIL default(firstprivate) taskloop record total=%d "
+                 "expected=204\n",
+                 t1);
+    ok = false;
+  }
+
+  // Mutate the static; the plain firstprivate 'x' (frozen at 10) and the saved
+  // 'SavedBump' (frozen at sum 12) must keep their recorded values, so the
+  // replay still produces 22 + i (into the current call's array, reached at a
+  // different stack depth) and the reduction total is again 204.
+  SavedBump = Bump{100, 100};
+
+  clobber_stack(99 * 1000);
+  const int t2 = call_with_depth(99, o2, 4);
+  ok &= check(o2, 22, "replay");
+  if (t2 != 204) {
+    std::fprintf(stderr,
+                 "FAIL default(firstprivate) taskloop replay total=%d "
+                 "expected=204\n",
+                 t2);
+    ok = false;
+  }
+
+  if (!ok)
+    return 1;
+
+  std::fprintf(stderr, "PASS default(firstprivate) taskloop total=%d\n", t1);
+  return 0;
+}
+
+// CHECK: PASS default(firstprivate) taskloop total=204
diff --git a/openmp/runtime/test/taskgraph/taskgraph_default_none_1.cpp b/openmp/runtime/test/taskgraph/taskgraph_default_none_1.cpp
new file mode 100644
index 0000000000000..24448d92fdf7d
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_default_none_1.cpp
@@ -0,0 +1,114 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// A replayable task with default(none) forces every referenced variable to be
+// given an explicit data-sharing attribute.  This used to crash Clang's Sema
+// (llvm_unreachable "Unexpected clause") because the implicit-DSA check for
+// default(none) walked the clause list and had no case for the taskgraph
+// 'replayable' clause.  Beyond just compiling, this exercises the two runtime
+// mechanisms that must keep working under default(none):
+//   - shared-by-reference relocation: the shared 'x'/'out' locals live on a
+//     stack frame that moves between the recording call and each replay call,
+//     and the shared 'x' must be re-read from the *current* call;
+//   - firstprivate(saved) snapshotting of a variable that is mutated between
+//     encounters.  A static ARRAY (larger than a pointer) is used on purpose so
+//     the aggregate save path -- rather than the always-inlined intptr-sized
+//     scalar path -- is exercised.
+//
+// The frame only moves because each encounter enters the taskgraph through
+// call_with_depth() at a different recursion depth, with clobber_stack()
+// overwriting the abandoned frames in between.  Called straight from main() the
+// frame would recur at one address every time and a replay dereferencing the
+// recorded (stale) shareds pointer would still find the live 'x'/'out', so the
+// relocation half of the above would go untested.
+
+#include <cstdio>
+
+static int SavedTable[5] = {2, 4, 6, 8, 10}; // sum == 30
+
+static volatile int StackSink = 0;
+
+__attribute__((noinline)) static void clobber_stack(int base) {
+  volatile int scratch[4096];
+
+  for (int i = 0; i < 4096; ++i)
+    scratch[i] = base + i;
+
+  StackSink += scratch[base & 63];
+}
+
+__attribute__((noinline)) static int sum_table(const int *t) {
+  int s = 0;
+  for (int i = 0; i < 5; ++i)
+    s += t[i];
+  return s;
+}
+
+__attribute__((noinline)) static int run_default_none(int seed) {
+  int x = seed;
+  int out = -1;
+
+#pragma omp taskgraph graph_id(9101)
+  {
+#pragma omp task default(none) shared(x, out) firstprivate(saved : SavedTable) \
+    depend(inout : x)
+    {
+      out = x + sum_table(SavedTable);
+    }
+  }
+
+  return out;
+}
+
+__attribute__((noinline)) static int call_with_depth(int seed, int depth) {
+  volatile int padding[128];
+
+  for (int i = 0; i < 128; ++i)
+    padding[i] = seed + depth + i;
+
+  StackSink += padding[(seed + depth) & 127];
+
+  if (depth == 0)
+    return run_default_none(seed);
+  return call_with_depth(seed, depth - 1);
+}
+
+int main() {
+  bool failed = false;
+
+  // Recording: snapshots SavedTable (sum 30) into the task's saved slot.
+  const int first = call_with_depth(1, 0); // 1 + 30
+  if (first != 31) {
+    std::fprintf(stderr, "FAIL default(none) record out=%d expected=31\n",
+                 first);
+    failed = true;
+  }
+
+  // Mutate the whole array; the saved snapshot (sum 30) must keep winning while
+  // the shared 'x' is re-read from the current call.
+  for (int i = 0; i < 5; ++i)
+    SavedTable[i] = 100;
+
+  clobber_stack(50 * 1000);
+  const int second = call_with_depth(50, 3); // 50 + 30
+  clobber_stack(7 * 1000);
+  const int third = call_with_depth(7, 6); // 7 + 30
+  if (second != 80 || third != 37) {
+    std::fprintf(
+        stderr, "FAIL default(none) replay second=%d third=%d expected=80/37\n",
+        second, third);
+    failed = true;
+  }
+
+  if (failed)
+    return 1;
+
+  std::fprintf(stderr, "PASS default(none) task first=%d second=%d third=%d\n",
+               first, second, third);
+  return 0;
+}
+
+// CHECK: PASS default(none) task first=31 second=80 third=37
diff --git a/openmp/runtime/test/taskgraph/taskgraph_default_none_taskloop.cpp b/openmp/runtime/test/taskgraph/taskgraph_default_none_taskloop.cpp
new file mode 100644
index 0000000000000..0b072c0773ff7
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_default_none_taskloop.cpp
@@ -0,0 +1,105 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// default(none) on a replayable taskloop.  As with the task variant this used
+// to crash Sema.  The loop iteration variable is predetermined private and so
+// needs no explicit clause; everything else is spelled out.  The shared 'x'
+// local is relocated (and re-read) between the recording call and the replays,
+// 'res' is a reduction, and 'SavedBias' is snapshotted with firstprivate(saved)
+// and must survive being mutated between encounters.  An aggregate (struct) is
+// used for the saved list item so the aggregate save path is exercised rather
+// than the always-inlined intptr-sized scalar path.
+//
+// Each encounter enters the taskgraph through call_with_depth() at a different
+// recursion depth, with clobber_stack() overwriting the abandoned frames in
+// between, so the frame holding 'x' and 'res' really does move.  Entering from
+// main() directly would reuse one frame at one address, and a replay reading
+// 'x' through the recorded (stale) shareds pointer would still find the live
+// value by accident.
+
+#include <cstdio>
+
+struct Bias3 {
+  int p, q, r;
+};
+
+static Bias3 SavedBias = {1, 2, 3}; // p+q+r == 6
+
+static volatile int StackSink = 0;
+
+__attribute__((noinline)) static void clobber_stack(int base) {
+  volatile int scratch[4096];
+
+  for (int i = 0; i < 4096; ++i)
+    scratch[i] = base + i;
+
+  StackSink += scratch[base & 63];
+}
+
+__attribute__((noinline)) static int run_default_none_taskloop(int seed) {
+  int x = seed;
+  int res = 0;
+
+#pragma omp taskgraph graph_id(9102)
+  {
+#pragma omp taskloop replayable num_tasks(8) default(none) shared(x)           \
+    firstprivate(saved : SavedBias) reduction(+ : res)
+    for (int i = 0; i < 16; ++i)
+      res += x + i + SavedBias.p + SavedBias.q + SavedBias.r;
+  }
+
+  return res;
+}
+
+__attribute__((noinline)) static int call_with_depth(int seed, int depth) {
+  volatile int padding[128];
+
+  for (int i = 0; i < 128; ++i)
+    padding[i] = seed + depth + i;
+
+  StackSink += padding[(seed + depth) & 127];
+
+  if (depth == 0)
+    return run_default_none_taskloop(seed);
+  return call_with_depth(seed, depth - 1);
+}
+
+int main() {
+  bool failed = false;
+
+  // sum_{i=0}^{15} (x + i + 6) = 16*x + 120 + 96 = 16*x + 216.
+  const int first = call_with_depth(1, 0); // 16 + 216
+  if (first != 232) {
+    std::fprintf(stderr,
+                 "FAIL default(none) taskloop record res=%d expected=232\n",
+                 first);
+    failed = true;
+  }
+
+  SavedBias = Bias3{100, 100, 100};
+
+  clobber_stack(100 * 1000);
+  const int second = call_with_depth(100, 3); // 1600 + 216
+  clobber_stack(5 * 1000);
+  const int third = call_with_depth(5, 6); // 80 + 216
+  if (second != 1816 || third != 296) {
+    std::fprintf(stderr,
+                 "FAIL default(none) taskloop replay second=%d third=%d "
+                 "expected=1816/296\n",
+                 second, third);
+    failed = true;
+  }
+
+  if (failed)
+    return 1;
+
+  std::fprintf(stderr,
+               "PASS default(none) taskloop first=%d second=%d third=%d\n",
+               first, second, third);
+  return 0;
+}
+
+// CHECK: PASS default(none) taskloop first=232 second=1816 third=296
diff --git a/openmp/runtime/test/taskgraph/taskgraph_default_shared.cpp b/openmp/runtime/test/taskgraph/taskgraph_default_shared.cpp
new file mode 100644
index 0000000000000..3a2a0a1ed5189
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_default_shared.cpp
@@ -0,0 +1,88 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// default(shared) on a replayable task: 'x' and 'out' get an implicit shared
+// attribute rather than being listed explicitly.  Because they are shared by
+// reference and live on a stack frame that moves between the recording call and
+// the replay calls, this specifically exercises shared-by-reference relocation
+// under an implicit (default) data-sharing attribute.
+//
+// The frame only actually moves because the taskgraph is entered through
+// call_with_depth() at a different recursion depth per encounter, with
+// clobber_stack() overwriting the abandoned frames in between.  Calling
+// run_default_shared() directly from main() would reuse one frame at one
+// address on every encounter, so a replay reading and writing through the
+// recorded (stale) shareds pointer would still hit the live objects by accident
+// and relocation would not be exercised at all.
+
+#include <cstdio>
+
+static volatile int StackSink = 0;
+
+__attribute__((noinline)) static void clobber_stack(int base) {
+  volatile int scratch[4096];
+
+  for (int i = 0; i < 4096; ++i)
+    scratch[i] = base + i;
+
+  StackSink += scratch[base & 63];
+}
+
+__attribute__((noinline)) static int run_default_shared(int seed) {
+  int x = seed;
+  int out = -1;
+
+#pragma omp taskgraph graph_id(9103)
+  {
+#pragma omp task default(shared) depend(inout : x)
+    {
+      x += 5;
+      out = x;
+    }
+  }
+
+  return out;
+}
+
+__attribute__((noinline)) static int call_with_depth(int seed, int depth) {
+  volatile int padding[128];
+
+  for (int i = 0; i < 128; ++i)
+    padding[i] = seed + depth + i;
+
+  StackSink += padding[(seed + depth) & 127];
+
+  if (depth == 0)
+    return run_default_shared(seed);
+  return call_with_depth(seed, depth - 1);
+}
+
+int main() {
+  bool failed = false;
+
+  const int first = call_with_depth(1, 0);
+  clobber_stack(100 * 1000);
+  const int second = call_with_depth(100, 3);
+  clobber_stack(5 * 1000);
+  const int third = call_with_depth(5, 6);
+  if (first != 6 || second != 105 || third != 10) {
+    std::fprintf(stderr,
+                 "FAIL default(shared) task first=%d second=%d third=%d "
+                 "expected=6/105/10\n",
+                 first, second, third);
+    failed = true;
+  }
+
+  if (failed)
+    return 1;
+
+  std::fprintf(stderr,
+               "PASS default(shared) task first=%d second=%d third=%d\n", first,
+               second, third);
+  return 0;
+}
+
+// CHECK: PASS default(shared) task first=6 second=105 third=10
diff --git a/openmp/runtime/test/taskgraph/taskgraph_default_shared_taskloop.cpp b/openmp/runtime/test/taskgraph/taskgraph_default_shared_taskloop.cpp
new file mode 100644
index 0000000000000..ec917f2348aad
--- /dev/null
+++ b/openmp/runtime/test/taskgraph/taskgraph_default_shared_taskloop.cpp
@@ -0,0 +1,85 @@
+// clang-format off
+// RUN: %clangXX %flags %openmp_flags -fopenmp-version=60 %s -o %t
+// RUN: env OMP_NUM_THREADS=4 %libomp-run 2>&1 | FileCheck %s
+// REQUIRES: omp_taskgraph_experimental
+// clang-format on
+
+// default(shared) on a replayable taskloop: 'x' is implicitly shared and
+// relocated across replays; 'res' is an explicit reduction (which overrides the
+// default) and the loop variable is predetermined private.
+//
+// Both 'x' and 'res' live in run_default_shared_taskloop's frame, and the
+// taskgraph is entered through call_with_depth() at a different recursion depth
+// per encounter (with clobber_stack() overwriting the abandoned frames in
+// between) so that frame genuinely moves.  Entering from main() directly would
+// reuse one frame at one address, letting a replay that read 'x' through the
+// recorded (stale) shareds pointer land on the live object by accident, so the
+// relocation claim above would not be tested.
+
+#include <cstdio>
+
+static volatile int StackSink = 0;
+
+__attribute__((noinline)) static void clobber_stack(int base) {
+  volatile int scratch[4096];
+
+  for (int i = 0; i < 4096; ++i)
+    scratch[i] = base + i;
+
+  StackSink += scratch[base & 63];
+}
+
+__attribute__((noinline)) static int run_default_shared_taskloop(int seed) {
+  int x = seed;
+  int res = 0;
+
+#pragma omp taskgraph graph_id(9104)
+  {
+#pragma omp taskloop replayable num_tasks(8) default(shared) reduction(+ : res)
+    for (int i = 0; i < 16; ++i)
+      res += x + i;
+  }
+
+  return res;
+}
+
+__attribute__((noinline)) static int call_with_depth(int seed, int depth) {
+  volatile int padding[128];
+
+  for (int i = 0; i < 128; ++i)
+    padding[i] = seed + depth + i;
+
+  StackSink += padding[(seed + depth) & 127];
+
+  if (depth == 0)
+    return run_default_shared_taskloop(seed);
+  return call_with_depth(seed, depth - 1);
+}
+
+int main() {
+  bool failed = false;
+
+  // sum_{i=0}^{15} (x + i) = 16*x + 120.
+  const int first = call_with_depth(1, 0); // 16 + 120
+  clobber_stack(100 * 1000);
+  const int second = call_with_depth(100, 3); // 1600 + 120
+  clobber_stack(5 * 1000);
+  const int third = call_with_depth(5, 6); // 80 + 120
+  if (first != 136 || second != 1720 || third != 200) {
+    std::fprintf(stderr,
+                 "FAIL default(shared) taskloop first=%d second=%d third=%d "
+                 "expected=136/1720/200\n",
+                 first, second, third);
+    failed = true;
+  }
+
+  if (failed)
+    return 1;
+
+  std::fprintf(stderr,
+               "PASS default(shared) taskloop first=%d second=%d third=%d\n",
+               first, second, third);
+  return 0;
+}
+
+// CHECK: PASS default(shared) taskloop first=136 second=1720 third=200

>From c37e0ea39fff7ec2eba94e8a33327d410c993da4 Mon Sep 17 00:00:00 2001
From: Julian Brown <julian.brown at amd.com>
Date: Fri, 24 Jul 2026 14:25:51 -0500
Subject: [PATCH 26/26] [Clang][OpenMP] Add codegen test for reduction capture
 under nested default(firstprivate)

Add a CodeGen regression test covering a taskloop with default(firstprivate)
and a reduction nested inside a 'taskgraph' region. It asserts the reduction
variable is captured by reference by the enclosing taskgraph (its address is
stored into the captured record) and that the caller reads the reduced value
back out of that same storage. This guards the Sema fix that consults the
capture level's default DSA rather than the innermost directive's.

Assisted-By: Claude Opus 4.8

Pull Request: https://github.com/llvm/llvm-project/pull/216331
---
 ...default_firstprivate_reduction_codegen.cpp | 42 +++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100644 clang/test/OpenMP/taskgraph_default_firstprivate_reduction_codegen.cpp

diff --git a/clang/test/OpenMP/taskgraph_default_firstprivate_reduction_codegen.cpp b/clang/test/OpenMP/taskgraph_default_firstprivate_reduction_codegen.cpp
new file mode 100644
index 0000000000000..75c27202ae680
--- /dev/null
+++ b/clang/test/OpenMP/taskgraph_default_firstprivate_reduction_codegen.cpp
@@ -0,0 +1,42 @@
+// RUN: %clang_cc1 -verify -fopenmp -fopenmp-version=60 -x c++ -triple x86_64-unknown-unknown -emit-llvm %s -fexceptions -fcxx-exceptions -o - | FileCheck %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -x c++ -std=c++11 -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -emit-pch -o %t %s
+// RUN: %clang_cc1 -fopenmp -fopenmp-version=60 -x c++ -triple x86_64-unknown-unknown -fexceptions -fcxx-exceptions -std=c++11 -include-pch %t -verify %s -emit-llvm -o - | FileCheck %s
+// expected-no-diagnostics
+
+// A taskloop with default(firstprivate) and a reduction, nested inside a
+// 'taskgraph' region.  The reduction variable 'res' must be captured *by
+// reference* by the enclosing taskgraph region (its address stored into the
+// captured record) so the reduction result propagates back to it.  A previous
+// bug consulted the innermost directive's default(firstprivate) when deciding
+// the enclosing region's capture kind and captured 'res' by copy, severing the
+// write-back (the caller then read the unmodified original, i.e. 0).
+
+#ifndef HEADER
+#define HEADER
+
+int run(int seed) {
+  int x = seed;
+  int res = 0;
+
+#pragma omp taskgraph graph_id(1)
+  {
+#pragma omp taskloop replayable num_tasks(4) default(firstprivate) reduction(+ : res)
+    for (int i = 0; i < 8; ++i)
+      res += x + i;
+  }
+
+  return res;
+}
+
+#endif
+
+// CHECK-LABEL: define {{.*}}@_Z3runi(
+// CHECK:         %[[CAP:.*]] = alloca %struct.anon, align 8
+// The reduction variable is captured BY REFERENCE (a pointer is stored into the
+// taskgraph captured record's first field), not by copy.
+// CHECK:         %[[FIELD:.*]] = getelementptr inbounds nuw %struct.anon, ptr %[[CAP]], i32 0, i32 0
+// CHECK-NEXT:    store ptr %[[RES:.*]], ptr %[[FIELD]], align 8
+// CHECK:         call void @__kmpc_taskgraph(
+// The caller reads the (reduced) value straight back out of that same storage.
+// CHECK:         %[[RET:.*]] = load i32, ptr %[[RES]], align 4
+// CHECK-NEXT:    ret i32 %[[RET]]


From openmp-commits at lists.llvm.org  Fri Aug 14 07:45:13 2026
From: openmp-commits at lists.llvm.org (Julian Brown via Openmp-commits)
Date: Fri, 14 Aug 2026 07:45:13 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [OpenMP] OpenMP 6.0
 taskgraph support (WIP) (PR #188765)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/188765@github.com>
Message-ID: <6a7f29f9.fae8f0c5.94af1.8470@mx.google.com>


jtb20 wrote:

I've uploaded a new version of host-based taskgraph support in this issue, and also as an "spr" (https://spacedentist.github.io/spr/) stack, though the update didn't seem to go entirely smoothly -- this workflow appears awkward when patches get moved around, merged and so forth. The following list might not be entirely right...

Pending [Clang][OpenMP] Add codegen test for reduction capture under nested default(firstprivate) https://github.com/llvm/llvm-project/pull/216331
Pending [OpenMP] Add taskgraph tests for the 'default' clause on replayable constructs https://github.com/llvm/llvm-project/pull/216330
Pending [Clang][OpenMP] Fix capture of reduction variables under nested default(firstprivate) https://github.com/llvm/llvm-project/pull/216329
Pending [OpenMP] Add taskgraph regression test for out-of-line task replay deadlock https://github.com/llvm/llvm-project/pull/216328
Pending [OpenMP] Fix hang and crash on empty taskgraphs https://github.com/llvm/llvm-project/pull/216327
Pending [OpenMP] Runtime support for graph_id and graph_reset clauses https://github.com/llvm/llvm-project/pull/216325
Pending [OpenMP] New taskgraph runtime implementation https://github.com/llvm/llvm-project/pull/216324
Pending [OpenMP] Add taskgraph template interaction codegen tests https://github.com/llvm/llvm-project/pull/200409
Pending [OpenMP] Constructor and destructor fixes for tasks cloned for recorded taskgraphs https://github.com/llvm/llvm-project/pull/200408
Pending [Clang][OpenMP] Restrict 'saved' firstprivate modifier to admissible directives https://github.com/llvm/llvm-project/pull/200407
Pending [Clang][OpenMP] Parsing, serialisation and deserialisation for the 'saved' firstprivate modifier https://github.com/llvm/llvm-project/pull/200406
Pending [OpenMP] Extend taskgraph shared-data relocation to taskloop https://github.com/llvm/llvm-project/pull/200405
Pending [OpenMP] Relocate task shared-by-ref captures across taskgraph replay https://github.com/llvm/llvm-project/pull/200404
Pending [OpenMP] Widen taskgraph graph_id to uintptr_t https://github.com/llvm/llvm-project/pull/200403
Pending [OpenMP] Make __kmp_replay_taskgraph static (NFC) https://github.com/llvm/llvm-project/pull/200402
Pending [OpenMP] Add test for concurrent taskgraph re-record sanity check https://github.com/llvm/llvm-project/pull/195077
Pending [OpenMP] Add sanity check for concurrent taskgraph reset https://github.com/llvm/llvm-project/pull/195076
Pending [OpenMP] New runtime tests for graph_id/graph_reset clauses (taskgraph) https://github.com/llvm/llvm-project/pull/195075
Pending [OpenMP] Runtime support for graph_id and graph_reset clauses (for taskgraph) https://github.com/llvm/llvm-project/pull/195074
Pending [OpenMP] OpenMP 6.0 "taskgraph" support, add new tests https://github.com/llvm/llvm-project/pull/194057
Pending [OpenMP] Add libomp_debug feature for tests https://github.com/llvm/llvm-project/pull/194056
Pending [OpenMP] OpenMP 6.0 "taskgraph" support, additional tests from previous implementation https://github.com/llvm/llvm-project/pull/194055
Pending [OpenMP] Add tests for 'replayable' clause https://github.com/llvm/llvm-project/pull/194054
Pending [OpenMP] Add 'replayable' clause (for 'taskgraph' support) https://github.com/llvm/llvm-project/pull/194053
Pending [OpenMP] New Clang tests for 'taskgraph' directive https://github.com/llvm/llvm-project/pull/194052
Pending [OpenMP] Support 'taskgraph' semantics via new libomp API entry points https://github.com/llvm/llvm-project/pull/194051
Accepted [OpenMP] Add OpenMP 6.0 taskgraph parsing/trivial semantics https://github.com/llvm/llvm-project/pull/194050
Pending [OpenMP] Add graph_id/graph_reset tests https://github.com/llvm/llvm-project/pull/194049
Pending [OpenMP] Add graph_id and graph_reset clause support (for taskgraph directive) https://github.com/llvm/llvm-project/pull/194048
Accepted [OpenMP] Mostly remove experimental taskgraph support from runtime https://github.com/llvm/llvm-project/pull/194046

https://github.com/llvm/llvm-project/pull/188765

From openmp-commits at lists.llvm.org  Fri Aug 14 08:18:12 2026
From: openmp-commits at lists.llvm.org (Spencer Bryngelson via Openmp-commits)
Date: Fri, 14 Aug 2026 08:18:12 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMP][DeviceRTL] Fix the
 chunked static distribution dropping iterations (PR #216117)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/216117@github.com>
Message-ID: <6a7f31b4.6c8668f2.1a23c2.411c@mx.google.com>


https://github.com/sbryngelson edited https://github.com/llvm/llvm-project/pull/216117

From openmp-commits at lists.llvm.org  Fri Aug 14 08:50:05 2026
From: openmp-commits at lists.llvm.org (Michael Klemm via Openmp-commits)
Date: Fri, 14 Aug 2026 08:50:05 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [libomp] Fix hang when a fatal error is
 raised before library registration (PR #215988)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/215988@github.com>
Message-ID: <6a7f392d.26b50d05.336427.3268@mx.google.com>


https://github.com/mjklemm edited https://github.com/llvm/llvm-project/pull/215988

From openmp-commits at lists.llvm.org  Fri Aug 14 08:50:05 2026
From: openmp-commits at lists.llvm.org (Michael Klemm via Openmp-commits)
Date: Fri, 14 Aug 2026 08:50:05 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [libomp] Fix hang when a fatal error is
 raised before library registration (PR #215988)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/215988@github.com>
Message-ID: <6a7f392d.d2cb4adc.10b298.0495@mx.google.com>


https://github.com/mjklemm approved this pull request.

LGTM

https://github.com/llvm/llvm-project/pull/215988

From openmp-commits at lists.llvm.org  Fri Aug 14 08:50:05 2026
From: openmp-commits at lists.llvm.org (Michael Klemm via Openmp-commits)
Date: Fri, 14 Aug 2026 08:50:05 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [libomp] Fix hang when a fatal error is
 raised before library registration (PR #215988)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/215988@github.com>
Message-ID: <6a7f392d.46c6f052.195070.c1bd@mx.google.com>


================
@@ -6918,6 +6927,13 @@ void __kmp_register_library_startup(void) {
 
 void __kmp_unregister_library(void) {
 
+  // The library can be torn down before it ever registered itself, e.g. when
+  // __kmp_abort_process() runs for a fatal error raised during environment
+  // parsing. There is nothing to unregister then, and __kmp_registration_str
+  // is still NULL, so the strcmp() below would dereference it.
+  if (__kmp_registration_flag == 0)
+    return;
----------------
mjklemm wrote:

This might be a race condition, but I think fixing the race conditiion is beyond the scope of this PR.

https://github.com/llvm/llvm-project/pull/215988

From openmp-commits at lists.llvm.org  Fri Aug 14 09:50:23 2026
From: openmp-commits at lists.llvm.org (Johannes Doerfert via Openmp-commits)
Date: Fri, 14 Aug 2026 09:50:23 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMP][DeviceRTL] Fix the
 chunked static distribution dropping iterations (PR #216117)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/216117@github.com>
Message-ID: <6a7f474f.d1496011.60cc2.d0be@mx.google.com>


================
@@ -0,0 +1,75 @@
+// Chunked static distribution: thread T owns iterations [T*chunk, T*chunk+chunk)
+// within each block chunk. The index used to start each thread has to account
+// for the chunk, and the block chunk has to cover one chunk per thread.
+//
+// The entry is called directly because no C/C++ construct reaches it: clang
+// emits __kmpc_for_static_init_4, only flang emits this one.
----------------
jdoerfert wrote:

Until someone finally changes clang's codegen :(

https://github.com/llvm/llvm-project/pull/216117

From openmp-commits at lists.llvm.org  Fri Aug 14 09:50:23 2026
From: openmp-commits at lists.llvm.org (Johannes Doerfert via Openmp-commits)
Date: Fri, 14 Aug 2026 09:50:23 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMP][DeviceRTL] Fix the
 chunked static distribution dropping iterations (PR #216117)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/216117@github.com>
Message-ID: <6a7f474f.871e6cae.1644c3.2adf@mx.google.com>


================
@@ -0,0 +1,75 @@
+// Chunked static distribution: thread T owns iterations [T*chunk, T*chunk+chunk)
+// within each block chunk. The index used to start each thread has to account
+// for the chunk, and the block chunk has to cover one chunk per thread.
+//
+// The entry is called directly because no C/C++ construct reaches it: clang
+// emits __kmpc_for_static_init_4, only flang emits this one.
+//
+// RUN: %libomptarget-compile-run-and-check-generic
+// REQUIRES: gpu
+
+#include <omp.h>
+#include <stdio.h>
+
+#define N 32
+#define NT 8
+#define CHUNK 4
+
+struct Args {
+  int *Tid;
+};
+
+#pragma omp begin declare target
+extern void __kmpc_distribute_for_static_loop_4u(
+    void *Loc, void (*Fn)(unsigned, void *), void *Arg, unsigned NumIters,
+    unsigned NumThreads, unsigned BlockChunk, unsigned ThreadChunk,
+    unsigned char OneIterationPerThread);
+
+__attribute__((noinline)) static void body(unsigned I, void *A) {
+  ((struct Args *)A)->Tid[I] = omp_get_thread_num();
+}
+#pragma omp end declare target
+
+// For the host fallback copy only; the device uses the runtime's definition.
+#ifndef __AMDGCN__
+void __kmpc_distribute_for_static_loop_4u(
+    void *Loc, void (*Fn)(unsigned, void *), void *Arg, unsigned NumIters,
+    unsigned NumThreads, unsigned BlockChunk, unsigned ThreadChunk,
+    unsigned char OneIterationPerThread) {
+  for (unsigned I = 0; I < NumIters; ++I)
+    Fn(I, Arg);
----------------
jdoerfert wrote:

Did you test this for the host?
Shouldn't we need to honor chunks here as well?

https://github.com/llvm/llvm-project/pull/216117

From openmp-commits at lists.llvm.org  Fri Aug 14 09:50:23 2026
From: openmp-commits at lists.llvm.org (Johannes Doerfert via Openmp-commits)
Date: Fri, 14 Aug 2026 09:50:23 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMP][DeviceRTL] Fix the
 chunked static distribution dropping iterations (PR #216117)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/216117@github.com>
Message-ID: <6a7f474f.7d702ed2.24827e.9fbe@mx.google.com>


================
@@ -0,0 +1,75 @@
+// Chunked static distribution: thread T owns iterations [T*chunk, T*chunk+chunk)
+// within each block chunk. The index used to start each thread has to account
+// for the chunk, and the block chunk has to cover one chunk per thread.
+//
+// The entry is called directly because no C/C++ construct reaches it: clang
+// emits __kmpc_for_static_init_4, only flang emits this one.
+//
+// RUN: %libomptarget-compile-run-and-check-generic
+// REQUIRES: gpu
+
+#include <omp.h>
+#include <stdio.h>
+
+#define N 32
+#define NT 8
+#define CHUNK 4
+
+struct Args {
+  int *Tid;
+};
----------------
jdoerfert wrote:

Why 2 levels of indirection? Can't we just return an array with omp_get_thread_num values without having an Args type and a pointer indirection in there?

https://github.com/llvm/llvm-project/pull/216117

From openmp-commits at lists.llvm.org  Fri Aug 14 09:50:23 2026
From: openmp-commits at lists.llvm.org (Johannes Doerfert via Openmp-commits)
Date: Fri, 14 Aug 2026 09:50:23 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMP][DeviceRTL] Fix the
 chunked static distribution dropping iterations (PR #216117)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/216117@github.com>
Message-ID: <6a7f474f.471ae13c.1a9730.4748@mx.google.com>


================
@@ -0,0 +1,75 @@
+// Chunked static distribution: thread T owns iterations [T*chunk, T*chunk+chunk)
+// within each block chunk. The index used to start each thread has to account
+// for the chunk, and the block chunk has to cover one chunk per thread.
+//
+// The entry is called directly because no C/C++ construct reaches it: clang
+// emits __kmpc_for_static_init_4, only flang emits this one.
+//
+// RUN: %libomptarget-compile-run-and-check-generic
+// REQUIRES: gpu
+
+#include <omp.h>
+#include <stdio.h>
+
+#define N 32
+#define NT 8
+#define CHUNK 4
+
+struct Args {
+  int *Tid;
+};
+
+#pragma omp begin declare target
+extern void __kmpc_distribute_for_static_loop_4u(
+    void *Loc, void (*Fn)(unsigned, void *), void *Arg, unsigned NumIters,
+    unsigned NumThreads, unsigned BlockChunk, unsigned ThreadChunk,
+    unsigned char OneIterationPerThread);
+
+__attribute__((noinline)) static void body(unsigned I, void *A) {
+  ((struct Args *)A)->Tid[I] = omp_get_thread_num();
+}
+#pragma omp end declare target
+
+// For the host fallback copy only; the device uses the runtime's definition.
+#ifndef __AMDGCN__
----------------
jdoerfert wrote:

What about NVIDIA? Intel soon?
You can disable codegen for the fallback via a clang cmd arg, or always provide the host version here restricted to host only or made weak.

https://github.com/llvm/llvm-project/pull/216117

From openmp-commits at lists.llvm.org  Fri Aug 14 09:50:23 2026
From: openmp-commits at lists.llvm.org (Johannes Doerfert via Openmp-commits)
Date: Fri, 14 Aug 2026 09:50:23 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMP][DeviceRTL] Fix the
 chunked static distribution dropping iterations (PR #216117)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/216117@github.com>
Message-ID: <6a7f474f.d76d81af.1a7b23.26eb@mx.google.com>


================
@@ -0,0 +1,75 @@
+// Chunked static distribution: thread T owns iterations [T*chunk, T*chunk+chunk)
+// within each block chunk. The index used to start each thread has to account
+// for the chunk, and the block chunk has to cover one chunk per thread.
+//
+// The entry is called directly because no C/C++ construct reaches it: clang
+// emits __kmpc_for_static_init_4, only flang emits this one.
+//
+// RUN: %libomptarget-compile-run-and-check-generic
+// REQUIRES: gpu
+
+#include <omp.h>
+#include <stdio.h>
+
+#define N 32
+#define NT 8
+#define CHUNK 4
+
+struct Args {
+  int *Tid;
+};
+
+#pragma omp begin declare target
+extern void __kmpc_distribute_for_static_loop_4u(
+    void *Loc, void (*Fn)(unsigned, void *), void *Arg, unsigned NumIters,
+    unsigned NumThreads, unsigned BlockChunk, unsigned ThreadChunk,
+    unsigned char OneIterationPerThread);
+
+__attribute__((noinline)) static void body(unsigned I, void *A) {
+  ((struct Args *)A)->Tid[I] = omp_get_thread_num();
----------------
jdoerfert wrote:

Removing the indirection should result in something like
```
((int*)A)[I] = omp_get_...();
```

https://github.com/llvm/llvm-project/pull/216117

From openmp-commits at lists.llvm.org  Sun Aug 16 18:31:45 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Sun, 16 Aug 2026 18:31:45 -0700 (PDT)
Subject: [Openmp-commits] [flang] [llvm] [openmp] [OpenMP][Flang] Add
 integer-kind wrappers to omp_lib (PR #213505)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213505@github.com>
Message-ID: <6a826481.b1b3e0a1.325f8e.9eda@mx.google.com>


https://github.com/shivaramaarao updated https://github.com/llvm/llvm-project/pull/213505

>From 8ea27bcd0b4efff2ff3ebe0232f92b26115b3eca Mon Sep 17 00:00:00 2001
From: Shivarama Rao <shivarama.rao at amd.com>
Date: Sun, 2 Aug 2026 10:20:39 +0530
Subject: [PATCH] [OpenMP][Flang] Add integer-kind wrappers to omp_lib

Many OpenMP routines in omp_lib only accept integer(kind=omp_integer_kind),
which is c_int. Calls with integer(1), integer(2), or integer(8) arguments
therefore fail Flang's type checking. These wrapper function interfaces
are added to omp_lib module file to resolve this issue. The implementation
functions are added omp_lib_impl module file as a submodule.

fixes issue #123948

Co-authored-by: Cursor <cursoragent at cursor.com>
---
 flang-rt/lib/runtime/CMakeLists.txt           |  12 +
 .../Lower/OpenMP/omp-lib-integer-wrappers.f90 | 237 +++++++
 .../test/Lower/OpenMP/omp-lib-num-threads.f90 |   2 +-
 openmp/module/CMakeLists.txt                  |  39 +-
 openmp/module/omp_lib.F90.var                 | 626 ++++++++++++++++--
 openmp/module/omp_lib_impl.F90.var            | 465 +++++++++++++
 6 files changed, 1331 insertions(+), 50 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/omp-lib-integer-wrappers.f90
 create mode 100644 openmp/module/omp_lib_impl.F90.var

diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index ae9a6e7fb7188..4cd3090113af3 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -322,6 +322,13 @@ else ()
   set(sources ${supported_sources} ${host_sources} ${f128_sources})
 endif ()
 
+# Include the Fortran OpenMP module implementation (omp_lib.o) in
+# libflang_rt.runtime when the OpenMP runtime is also being built.
+# Skip for GPU-default targets: omp_lib is a host-side Fortran module.
+if (TARGET libomp-mod AND NOT "${LLVM_DEFAULT_TARGET_TRIPLE}" MATCHES "^amdgcn|^nvptx")
+  list(APPEND sources "$<TARGET_OBJECTS:libomp-mod>")
+endif ()
+
 
 if (NOT WIN32)
   add_flangrt_library(flang_rt.runtime STATIC SHARED
@@ -333,6 +340,11 @@ if (NOT WIN32)
 
   enable_cuda_compilation(flang_rt.runtime "${sources}")
 
+  if (TARGET libomp-mod AND TARGET flang_rt.runtime.shared AND TARGET omp)
+    # Resolve bind(c) references from omp_lib when building the shared runtime.
+    target_link_libraries(flang_rt.runtime.shared PUBLIC omp)
+  endif ()
+
   # Select a default runtime, which is used for unit and regression tests.
   if (TARGET flang_rt.runtime.default)
     get_target_property(default_target flang_rt.runtime.default ALIASED_TARGET)
diff --git a/flang/test/Lower/OpenMP/omp-lib-integer-wrappers.f90 b/flang/test/Lower/OpenMP/omp-lib-integer-wrappers.f90
new file mode 100644
index 0000000000000..9881fe56261fa
--- /dev/null
+++ b/flang/test/Lower/OpenMP/omp-lib-integer-wrappers.f90
@@ -0,0 +1,237 @@
+! REQUIRES: openmp_runtime
+
+! RUN: %flang_fc1 -emit-hlfir %openmp_flags %s -o - 2>&1 | FileCheck %s
+! RUN: bbc %openmp_flags -emit-hlfir -o - %s 2>&1 | FileCheck %s
+! RUN: %flang_fc1 -emit-fir %openmp_flags %s -o - 2>&1 | FileCheck %s
+! RUN: bbc -emit-fir %openmp_flags -o - %s 2>&1 | FileCheck %s
+!
+! Test that omp_lib integer-kind wrapper procedures lower to module
+! procedures instead of failing intrinsic resolution.
+
+program main
+  use omp_lib
+  integer(1) :: i1 = 1_1
+  integer(2) :: i2 = 2_2
+  integer(4) :: i4 = 4_4
+  integer(8) :: i8 = 8_8
+  integer(omp_integer_kind) :: ires
+  integer(omp_sched_kind) :: sched_kind
+  integer(kmp_affinity_mask_kind) :: mask
+
+  call omp_set_num_threads(i1)
+  call omp_set_num_threads(i2)
+  call omp_set_num_threads(i4)
+  call omp_set_num_threads(i8)
+
+  call omp_set_max_active_levels(i1)
+  call omp_set_max_active_levels(i2)
+  call omp_set_max_active_levels(i4)
+  call omp_set_max_active_levels(i8)
+
+  call omp_set_default_device(i1)
+  call omp_set_default_device(i2)
+  call omp_set_default_device(i4)
+  call omp_set_default_device(i8)
+
+  call omp_set_num_teams(i1)
+  call omp_set_num_teams(i2)
+  call omp_set_num_teams(i4)
+  call omp_set_num_teams(i8)
+
+  call omp_set_teams_thread_limit(i1)
+  call omp_set_teams_thread_limit(i2)
+  call omp_set_teams_thread_limit(i4)
+  call omp_set_teams_thread_limit(i8)
+
+  call kmp_set_stacksize(i1)
+  call kmp_set_stacksize(i2)
+  call kmp_set_stacksize(i4)
+  call kmp_set_stacksize(i8)
+
+  call kmp_set_blocktime(i1)
+  call kmp_set_blocktime(i2)
+  call kmp_set_blocktime(i4)
+  call kmp_set_blocktime(i8)
+
+  call kmp_set_library(i1)
+  call kmp_set_library(i2)
+  call kmp_set_library(i4)
+  call kmp_set_library(i8)
+
+  call kmp_set_disp_num_buffers(i1)
+  call kmp_set_disp_num_buffers(i2)
+  call kmp_set_disp_num_buffers(i4)
+  call kmp_set_disp_num_buffers(i8)
+
+  ires = omp_get_ancestor_thread_num(i1)
+  ires = omp_get_ancestor_thread_num(i2)
+  ires = omp_get_ancestor_thread_num(i4)
+  ires = omp_get_ancestor_thread_num(i8)
+
+  ires = omp_get_team_size(i1)
+  ires = omp_get_team_size(i2)
+  ires = omp_get_team_size(i4)
+  ires = omp_get_team_size(i8)
+
+  ires = omp_get_place_num_procs(i1)
+  ires = omp_get_place_num_procs(i2)
+  ires = omp_get_place_num_procs(i4)
+  ires = omp_get_place_num_procs(i8)
+
+  call omp_set_schedule(omp_sched_static, i1)
+  call omp_set_schedule(omp_sched_static, i2)
+  call omp_set_schedule(omp_sched_static, i4)
+  call omp_set_schedule(omp_sched_static, i8)
+
+  call omp_get_schedule(sched_kind, i1)
+  call omp_get_schedule(sched_kind, i2)
+  call omp_get_schedule(sched_kind, i4)
+  call omp_get_schedule(sched_kind, i8)
+
+  ires = omp_pause_resource(omp_pause_soft, i1)
+  ires = omp_pause_resource(omp_pause_soft, i2)
+  ires = omp_pause_resource(omp_pause_soft, i4)
+  ires = omp_pause_resource(omp_pause_soft, i8)
+
+  call kmp_create_affinity_mask(mask)
+  ires = kmp_set_affinity_mask_proc(i1, mask)
+  ires = kmp_set_affinity_mask_proc(i2, mask)
+  ires = kmp_set_affinity_mask_proc(i4, mask)
+  ires = kmp_set_affinity_mask_proc(i8, mask)
+
+  ires = kmp_unset_affinity_mask_proc(i1, mask)
+  ires = kmp_unset_affinity_mask_proc(i2, mask)
+  ires = kmp_unset_affinity_mask_proc(i4, mask)
+  ires = kmp_unset_affinity_mask_proc(i8, mask)
+
+  ires = kmp_get_affinity_mask_proc(i1, mask)
+  ires = kmp_get_affinity_mask_proc(i2, mask)
+  ires = kmp_get_affinity_mask_proc(i4, mask)
+  ires = kmp_get_affinity_mask_proc(i8, mask)
+
+  block
+    integer(4) :: ids4(1)
+    integer(8) :: ids8(1)
+    integer(4) :: parts4(1)
+    integer(8) :: parts8(1)
+    call omp_get_place_proc_ids(i4, ids4)
+    call omp_get_place_proc_ids(i8, ids8)
+    call omp_get_partition_place_nums(parts4)
+    call omp_get_partition_place_nums(parts8)
+  end block
+end program
+
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_max_active_levels
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_default_device
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_num_teams
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_teams_thread_limit
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_stacksize
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_blocktime
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_library
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_disp_num_buffers
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_ancestor_thread_num
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_team_size
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_place_num_procs
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_schedule
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_schedule
+!CHECK-NOT: not yet implemented: intrinsic: omp_pause_resource
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_affinity_mask_proc
+!CHECK-NOT: not yet implemented: intrinsic: kmp_unset_affinity_mask_proc
+!CHECK-NOT: not yet implemented: intrinsic: kmp_get_affinity_mask_proc
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_place_proc_ids
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_partition_place_nums
+
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i1
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i2
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i4
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_max_active_levels_i1
+!CHECK: fir.call @_QMomp_libPomp_set_max_active_levels_i2
+!CHECK: fir.call @_QMomp_libPomp_set_max_active_levels_i4
+!CHECK: fir.call @_QMomp_libPomp_set_max_active_levels_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_default_device_i1
+!CHECK: fir.call @_QMomp_libPomp_set_default_device_i2
+!CHECK: fir.call @_QMomp_libPomp_set_default_device_i4
+!CHECK: fir.call @_QMomp_libPomp_set_default_device_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_num_teams_i1
+!CHECK: fir.call @_QMomp_libPomp_set_num_teams_i2
+!CHECK: fir.call @_QMomp_libPomp_set_num_teams_i4
+!CHECK: fir.call @_QMomp_libPomp_set_num_teams_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_teams_thread_limit_i1
+!CHECK: fir.call @_QMomp_libPomp_set_teams_thread_limit_i2
+!CHECK: fir.call @_QMomp_libPomp_set_teams_thread_limit_i4
+!CHECK: fir.call @_QMomp_libPomp_set_teams_thread_limit_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_stacksize_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_stacksize_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_stacksize_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_stacksize_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_blocktime_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_blocktime_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_blocktime_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_blocktime_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_library_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_library_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_library_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_library_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_disp_num_buffers_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_disp_num_buffers_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_disp_num_buffers_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_disp_num_buffers_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_ancestor_thread_num_i1
+!CHECK: fir.call @_QMomp_libPomp_get_ancestor_thread_num_i2
+!CHECK: fir.call @_QMomp_libPomp_get_ancestor_thread_num_i4
+!CHECK: fir.call @_QMomp_libPomp_get_ancestor_thread_num_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_team_size_i1
+!CHECK: fir.call @_QMomp_libPomp_get_team_size_i2
+!CHECK: fir.call @_QMomp_libPomp_get_team_size_i4
+!CHECK: fir.call @_QMomp_libPomp_get_team_size_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_place_num_procs_i1
+!CHECK: fir.call @_QMomp_libPomp_get_place_num_procs_i2
+!CHECK: fir.call @_QMomp_libPomp_get_place_num_procs_i4
+!CHECK: fir.call @_QMomp_libPomp_get_place_num_procs_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_schedule_i1
+!CHECK: fir.call @_QMomp_libPomp_set_schedule_i2
+!CHECK: fir.call @_QMomp_libPomp_set_schedule_i4
+!CHECK: fir.call @_QMomp_libPomp_set_schedule_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_schedule_i1
+!CHECK: fir.call @_QMomp_libPomp_get_schedule_i2
+!CHECK: fir.call @_QMomp_libPomp_get_schedule_i4
+!CHECK: fir.call @_QMomp_libPomp_get_schedule_i8
+
+!CHECK: fir.call @_QMomp_libPomp_pause_resource_i1
+!CHECK: fir.call @_QMomp_libPomp_pause_resource_i2
+!CHECK: fir.call @_QMomp_libPomp_pause_resource_i4
+!CHECK: fir.call @_QMomp_libPomp_pause_resource_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_affinity_mask_proc_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_affinity_mask_proc_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_affinity_mask_proc_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_affinity_mask_proc_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_unset_affinity_mask_proc_i1
+!CHECK: fir.call @_QMomp_libPkmp_unset_affinity_mask_proc_i2
+!CHECK: fir.call @_QMomp_libPkmp_unset_affinity_mask_proc_i4
+!CHECK: fir.call @_QMomp_libPkmp_unset_affinity_mask_proc_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_get_affinity_mask_proc_i1
+!CHECK: fir.call @_QMomp_libPkmp_get_affinity_mask_proc_i2
+!CHECK: fir.call @_QMomp_libPkmp_get_affinity_mask_proc_i4
+!CHECK: fir.call @_QMomp_libPkmp_get_affinity_mask_proc_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_place_proc_ids_i4
+!CHECK: fir.call @_QMomp_libPomp_get_place_proc_ids_i8
+!CHECK: fir.call @_QMomp_libPomp_get_partition_place_nums_i4
+!CHECK: fir.call @_QMomp_libPomp_get_partition_place_nums_i8
diff --git a/flang/test/Lower/OpenMP/omp-lib-num-threads.f90 b/flang/test/Lower/OpenMP/omp-lib-num-threads.f90
index 6e76dce473057..a2ec031a62d88 100644
--- a/flang/test/Lower/OpenMP/omp-lib-num-threads.f90
+++ b/flang/test/Lower/OpenMP/omp-lib-num-threads.f90
@@ -21,5 +21,5 @@ program main
 
 !CHECK-NOT: not yet implemented: intrinsic: omp_set_num_threads
 !CHECK-NOT: not yet implemented: intrinsic: omp_get_num_threads
-!CHECK: fir.call @omp_set_num_threads
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i4
 !CHECK: fir.call @omp_get_num_threads
diff --git a/openmp/module/CMakeLists.txt b/openmp/module/CMakeLists.txt
index 2d9829e84849a..b788f1b26dfe2 100644
--- a/openmp/module/CMakeLists.txt
+++ b/openmp/module/CMakeLists.txt
@@ -9,13 +9,16 @@
 # Build the module files if a Fortran compiler is available.
 
 configure_file(omp_lib.F90.var "${CMAKE_CURRENT_BINARY_DIR}/omp_lib.F90" @ONLY)
+configure_file(omp_lib_impl.F90.var "${CMAKE_CURRENT_BINARY_DIR}/omp_lib_impl.F90" @ONLY)
 configure_file(omp_lib.h.var "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}/omp_lib.h" @ONLY)
 
-# One compilation step creates both, omp_lib.mod and omp_lib_kinds.mod. Only
-# these files are used, the object file itself can be discarded.
+# omp_lib.F90 declares the interfaces and creates both omp_lib.mod and
+# omp_lib_kinds.mod; omp_lib_impl.F90 holds the bodies of the separate module
+# procedures and creates omp_lib-omp_lib_impl.mod.
 # TODO: Adding it to libomp ($<TARGET_OBJECTS:libomp-mod>) would allow implementing Fortran API in Fortran
 add_library(libomp-mod OBJECT
   "${CMAKE_CURRENT_BINARY_DIR}/omp_lib.F90"
+  "${CMAKE_CURRENT_BINARY_DIR}/omp_lib_impl.F90"
 )
 set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Fortran Modules")
 
@@ -34,6 +37,38 @@ endif ()
 
 flang_module_target(libomp-mod PUBLIC)
 
+# flang_module_target(PUBLIC) emits the .mod files directly into the toolchain
+# resource directory, but that directory is also Flang's intrinsic module
+# directory, and when resolving the parent of a SUBMODULE Flang searches only
+# the non-intrinsic module path. omp_lib.mod would therefore be invisible to
+# omp_lib_impl.F90. Emit the module files into a private directory instead,
+# where the submodule can find its parent, and publish the importable ones
+# afterwards. omp_lib-omp_lib_impl.mod is only needed to compile further
+# submodules of omp_lib, so it stays private.
+set(libomp_mod_dir "${CMAKE_CURRENT_BINARY_DIR}/mod/${CMAKE_CFG_INTDIR}")
+set_target_properties(libomp-mod PROPERTIES
+  Fortran_MODULE_DIRECTORY "${libomp_mod_dir}"
+)
+
+set(libomp_public_mods
+  "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}/omp_lib.mod"
+  "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}/omp_lib_kinds.mod"
+)
+add_custom_command(OUTPUT ${libomp_public_mods}
+  COMMAND "${CMAKE_COMMAND}" -E copy_if_different
+          "${libomp_mod_dir}/omp_lib.mod"
+          "${libomp_mod_dir}/omp_lib_kinds.mod"
+          "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}"
+  DEPENDS libomp-mod
+  COMMENT "Publishing omp_lib module files"
+)
+
+# libomp-mod is an OBJECT library, so the module files it writes are opaque to
+# CMake. Dependees on omp_lib.mod must add a target-level dependency on this
+# barrier to ensure the module files exist before they are used.
+add_custom_target(libomp-mod.barrier ALL DEPENDS ${libomp_public_mods})
+set_target_properties(libomp-mod.barrier PROPERTIES FOLDER "OpenMP/Fortran Modules")
+
 install(FILES
   "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}/omp_lib.h"
   DESTINATION "${RUNTIMES_INSTALL_RESOURCE_MOD_PATH}"
diff --git a/openmp/module/omp_lib.F90.var b/openmp/module/omp_lib.F90.var
index a41b2faad9d3d..f27ea19fbe8c5 100644
--- a/openmp/module/omp_lib.F90.var
+++ b/openmp/module/omp_lib.F90.var
@@ -63,6 +63,27 @@
         ! to be exported by this MODULE.
         private
 
+        private :: kmp_get_affinity_mask_proc_c
+        private :: kmp_set_affinity_mask_proc_c
+        private :: kmp_set_blocktime_c
+        private :: kmp_set_disp_num_buffers_c
+        private :: kmp_set_library_c
+        private :: kmp_set_stacksize_c
+        private :: kmp_unset_affinity_mask_proc_c
+        private :: omp_get_ancestor_thread_num_c
+        private :: omp_get_partition_place_nums_c
+        private :: omp_get_place_num_procs_c
+        private :: omp_get_place_proc_ids_c
+        private :: omp_get_schedule_c
+        private :: omp_get_team_size_c
+        private :: omp_pause_resource_c
+        private :: omp_set_default_device_c
+        private :: omp_set_max_active_levels_c
+        private :: omp_set_num_teams_c
+        private :: omp_set_num_threads_c
+        private :: omp_set_schedule_c
+        private :: omp_set_teams_thread_limit_c
+
         ! Re-export definitions in omp_lib_kinds
         public :: omp_integer_kind
         public :: omp_logical_kind
@@ -250,10 +271,11 @@
 !         *** omp_* entry points
 !         ***
 
-          subroutine omp_set_num_threads(num_threads) bind(c)
+          subroutine omp_set_num_threads_c(num_threads) bind(c,                &
+              name='omp_set_num_threads')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: num_threads
-          end subroutine omp_set_num_threads
+          end subroutine omp_set_num_threads_c
 
           subroutine omp_set_dynamic(dynamic_threads) bind(c)
             use omp_lib_kinds
@@ -310,10 +332,11 @@
             integer (kind=omp_integer_kind) omp_get_thread_limit
           end function omp_get_thread_limit
 
-          subroutine omp_set_max_active_levels(max_levels) bind(c)
+          subroutine omp_set_max_active_levels_c(max_levels) bind(c,           &
+              name='omp_set_max_active_levels')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: max_levels
-          end subroutine omp_set_max_active_levels
+          end subroutine omp_set_max_active_levels_c
 
           function omp_get_max_active_levels() bind(c)
             use omp_lib_kinds
@@ -330,29 +353,32 @@
             integer (kind=omp_integer_kind) omp_get_active_level
           end function omp_get_active_level
 
-          function omp_get_ancestor_thread_num(level) bind(c)
+          function omp_get_ancestor_thread_num_c(level) bind(c,                &
+              name='omp_get_ancestor_thread_num')
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) omp_get_ancestor_thread_num
+            integer (kind=omp_integer_kind) omp_get_ancestor_thread_num_c
             integer (kind=omp_integer_kind), value :: level
-          end function omp_get_ancestor_thread_num
+          end function omp_get_ancestor_thread_num_c
 
-          function omp_get_team_size(level) bind(c)
+          function omp_get_team_size_c(level) bind(c, name='omp_get_team_size')
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) omp_get_team_size
+            integer (kind=omp_integer_kind) omp_get_team_size_c
             integer (kind=omp_integer_kind), value :: level
-          end function omp_get_team_size
+          end function omp_get_team_size_c
 
-          subroutine omp_set_schedule(kind, chunk_size) bind(c)
+          subroutine omp_set_schedule_c(kind, chunk_size) bind(c,              &
+              name='omp_set_schedule')
             use omp_lib_kinds
             integer (kind=omp_sched_kind), value :: kind
             integer (kind=omp_integer_kind), value :: chunk_size
-          end subroutine omp_set_schedule
+          end subroutine omp_set_schedule_c
 
-          subroutine omp_get_schedule(kind, chunk_size) bind(c)
+          subroutine omp_get_schedule_c(kind, chunk_size) bind(c,              &
+              name='omp_get_schedule')
             use omp_lib_kinds
             integer (kind=omp_sched_kind) kind
             integer (kind=omp_integer_kind) chunk_size
-          end subroutine omp_get_schedule
+          end subroutine omp_get_schedule_c
 
           function omp_get_proc_bind() bind(c)
             use omp_lib_kinds
@@ -364,17 +390,19 @@
             integer (kind=omp_integer_kind) omp_get_num_places
           end function omp_get_num_places
 
-          function omp_get_place_num_procs(place_num) bind(c)
+          function omp_get_place_num_procs_c(place_num) bind(c,                &
+              name='omp_get_place_num_procs')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: place_num
-            integer (kind=omp_integer_kind) omp_get_place_num_procs
-          end function omp_get_place_num_procs
+            integer (kind=omp_integer_kind) omp_get_place_num_procs_c
+          end function omp_get_place_num_procs_c
 
-          subroutine omp_get_place_proc_ids(place_num, ids) bind(c)
+          subroutine omp_get_place_proc_ids_c(place_num, ids) bind(c,          &
+              name='omp_get_place_proc_ids')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: place_num
             integer (kind=omp_integer_kind) ids(*)
-          end subroutine omp_get_place_proc_ids
+          end subroutine omp_get_place_proc_ids_c
 
           function omp_get_place_num() bind(c)
             use omp_lib_kinds
@@ -386,10 +414,11 @@
             integer (kind=omp_integer_kind) omp_get_partition_num_places
           end function omp_get_partition_num_places
 
-          subroutine omp_get_partition_place_nums(place_nums) bind(c)
+          subroutine omp_get_partition_place_nums_c(place_nums) bind(c,        &
+              name='omp_get_partition_place_nums')
             use omp_lib_kinds
             integer (kind=omp_integer_kind) place_nums(*)
-          end subroutine omp_get_partition_place_nums
+          end subroutine omp_get_partition_place_nums_c
 
           function omp_get_wtime() bind(c)
             use omp_lib_kinds
@@ -406,10 +435,11 @@
             integer (kind=omp_integer_kind) omp_get_default_device
           end function omp_get_default_device
 
-          subroutine omp_set_default_device(device_num) bind(c)
+          subroutine omp_set_default_device_c(device_num) bind(c,              &
+              name='omp_set_default_device')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: device_num
-          end subroutine omp_set_default_device
+          end subroutine omp_set_default_device_c
 
           function omp_get_num_devices() bind(c)
             use omp_lib_kinds
@@ -446,12 +476,13 @@
             integer (kind=omp_integer_kind) omp_get_device_num
           end function omp_get_device_num
 
-          function omp_pause_resource(kind, device_num) bind(c)
+          function omp_pause_resource_c(kind, device_num) bind(c,              &
+              name='omp_pause_resource')
             use omp_lib_kinds
             integer (kind=omp_pause_resource_kind), value :: kind
             integer (kind=omp_integer_kind), value :: device_num
-            integer (kind=omp_integer_kind) omp_pause_resource
-          end function omp_pause_resource
+            integer (kind=omp_integer_kind) omp_pause_resource_c
+          end function omp_pause_resource_c
 
           function omp_pause_resource_all(kind) bind(c)
             use omp_lib_kinds
@@ -640,20 +671,22 @@
             integer (kind=kmp_size_t_kind) :: omp_capture_affinity
           end function omp_capture_affinity
 
-          subroutine omp_set_num_teams(num_teams) bind(c)
+          subroutine omp_set_num_teams_c(num_teams) bind(c,                    &
+              name='omp_set_num_teams')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: num_teams
-          end subroutine omp_set_num_teams
+          end subroutine omp_set_num_teams_c
 
           function omp_get_max_teams() bind(c)
             use omp_lib_kinds
             integer (kind=omp_integer_kind) omp_get_max_teams
           end function omp_get_max_teams
 
-          subroutine omp_set_teams_thread_limit(thread_limit) bind(c)
+          subroutine omp_set_teams_thread_limit_c(thread_limit) bind(c,        &
+              name='omp_set_teams_thread_limit')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: thread_limit
-          end subroutine omp_set_teams_thread_limit
+          end subroutine omp_set_teams_thread_limit_c
 
           function omp_get_teams_thread_limit() bind(c)
             use omp_lib_kinds
@@ -937,20 +970,20 @@
 !         *** kmp_* entry points
 !         ***
 
-          subroutine kmp_set_stacksize(size) bind(c)
+          subroutine kmp_set_stacksize_c(size) bind(c, name='kmp_set_stacksize')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: size
-          end subroutine kmp_set_stacksize
+          end subroutine kmp_set_stacksize_c
 
           subroutine kmp_set_stacksize_s(size) bind(c)
             use omp_lib_kinds
             integer (kind=kmp_size_t_kind), value :: size
           end subroutine kmp_set_stacksize_s
 
-          subroutine kmp_set_blocktime(msec) bind(c)
+          subroutine kmp_set_blocktime_c(msec) bind(c, name='kmp_set_blocktime')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: msec
-          end subroutine kmp_set_blocktime
+          end subroutine kmp_set_blocktime_c
 
           subroutine kmp_set_library_serial() bind(c)
           end subroutine kmp_set_library_serial
@@ -961,10 +994,10 @@
           subroutine kmp_set_library_throughput() bind(c)
           end subroutine kmp_set_library_throughput
 
-          subroutine kmp_set_library(libnum) bind(c)
+          subroutine kmp_set_library_c(libnum) bind(c, name='kmp_set_library')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: libnum
-          end subroutine kmp_set_library
+          end subroutine kmp_set_library_c
 
           subroutine kmp_set_defaults(string)
             character (len=*) :: string
@@ -990,10 +1023,11 @@
             integer (kind=omp_integer_kind) kmp_get_library
           end function kmp_get_library
 
-          subroutine kmp_set_disp_num_buffers(num) bind(c)
+          subroutine kmp_set_disp_num_buffers_c(num) bind(c,                   &
+              name='kmp_set_disp_num_buffers')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: num
-          end subroutine kmp_set_disp_num_buffers
+          end subroutine kmp_set_disp_num_buffers_c
 
           function kmp_set_affinity(mask) bind(c)
             use omp_lib_kinds
@@ -1022,26 +1056,29 @@
             integer (kind=kmp_affinity_mask_kind) mask
           end subroutine kmp_destroy_affinity_mask
 
-          function kmp_set_affinity_mask_proc(proc, mask) bind(c)
+          function kmp_set_affinity_mask_proc_c(proc, mask) bind(c,            &
+              name='kmp_set_affinity_mask_proc')
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) kmp_set_affinity_mask_proc
+            integer (kind=omp_integer_kind) kmp_set_affinity_mask_proc_c
             integer (kind=omp_integer_kind), value :: proc
             integer (kind=kmp_affinity_mask_kind) mask
-          end function kmp_set_affinity_mask_proc
+          end function kmp_set_affinity_mask_proc_c
 
-          function kmp_unset_affinity_mask_proc(proc, mask) bind(c)
+          function kmp_unset_affinity_mask_proc_c(proc, mask) bind(c,          &
+              name='kmp_unset_affinity_mask_proc')
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) kmp_unset_affinity_mask_proc
+            integer (kind=omp_integer_kind) kmp_unset_affinity_mask_proc_c
             integer (kind=omp_integer_kind), value :: proc
             integer (kind=kmp_affinity_mask_kind) mask
-          end function kmp_unset_affinity_mask_proc
+          end function kmp_unset_affinity_mask_proc_c
 
-          function kmp_get_affinity_mask_proc(proc, mask) bind(c)
+          function kmp_get_affinity_mask_proc_c(proc, mask) bind(c,            &
+              name='kmp_get_affinity_mask_proc')
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) kmp_get_affinity_mask_proc
+            integer (kind=omp_integer_kind) kmp_get_affinity_mask_proc_c
             integer (kind=omp_integer_kind), value :: proc
             integer (kind=kmp_affinity_mask_kind) mask
-          end function kmp_get_affinity_mask_proc
+          end function kmp_get_affinity_mask_proc_c
 
           function kmp_malloc(size) bind(c)
             use omp_lib_kinds
@@ -1150,6 +1187,98 @@
 
         end interface
 
+        ! Generic interfaces for multiple integer kinds
+        interface omp_set_num_threads
+          module procedure omp_set_num_threads_i1, omp_set_num_threads_i2,     &
+              omp_set_num_threads_i4, omp_set_num_threads_i8
+        end interface omp_set_num_threads
+        interface omp_set_max_active_levels
+          module procedure omp_set_max_active_levels_i1,                       &
+              omp_set_max_active_levels_i2, omp_set_max_active_levels_i4,      &
+              omp_set_max_active_levels_i8
+        end interface omp_set_max_active_levels
+        interface omp_set_default_device
+          module procedure omp_set_default_device_i1,                          &
+              omp_set_default_device_i2, omp_set_default_device_i4,            &
+              omp_set_default_device_i8
+        end interface omp_set_default_device
+        interface omp_set_num_teams
+          module procedure omp_set_num_teams_i1, omp_set_num_teams_i2,         &
+              omp_set_num_teams_i4, omp_set_num_teams_i8
+        end interface omp_set_num_teams
+        interface omp_set_teams_thread_limit
+          module procedure omp_set_teams_thread_limit_i1,                      &
+              omp_set_teams_thread_limit_i2, omp_set_teams_thread_limit_i4,    &
+              omp_set_teams_thread_limit_i8
+        end interface omp_set_teams_thread_limit
+        interface kmp_set_stacksize
+          module procedure kmp_set_stacksize_i1, kmp_set_stacksize_i2,         &
+              kmp_set_stacksize_i4, kmp_set_stacksize_i8
+        end interface kmp_set_stacksize
+        interface kmp_set_blocktime
+          module procedure kmp_set_blocktime_i1, kmp_set_blocktime_i2,         &
+              kmp_set_blocktime_i4, kmp_set_blocktime_i8
+        end interface kmp_set_blocktime
+        interface kmp_set_library
+          module procedure kmp_set_library_i1, kmp_set_library_i2,             &
+              kmp_set_library_i4, kmp_set_library_i8
+        end interface kmp_set_library
+        interface kmp_set_disp_num_buffers
+          module procedure kmp_set_disp_num_buffers_i1,                        &
+              kmp_set_disp_num_buffers_i2, kmp_set_disp_num_buffers_i4,        &
+              kmp_set_disp_num_buffers_i8
+        end interface kmp_set_disp_num_buffers
+        interface omp_get_ancestor_thread_num
+          module procedure omp_get_ancestor_thread_num_i1,                     &
+              omp_get_ancestor_thread_num_i2, omp_get_ancestor_thread_num_i4,  &
+              omp_get_ancestor_thread_num_i8
+        end interface omp_get_ancestor_thread_num
+        interface omp_get_team_size
+          module procedure omp_get_team_size_i1, omp_get_team_size_i2,         &
+              omp_get_team_size_i4, omp_get_team_size_i8
+        end interface omp_get_team_size
+        interface omp_get_place_num_procs
+          module procedure omp_get_place_num_procs_i1,                         &
+              omp_get_place_num_procs_i2, omp_get_place_num_procs_i4,          &
+              omp_get_place_num_procs_i8
+        end interface omp_get_place_num_procs
+        interface omp_set_schedule
+          module procedure omp_set_schedule_i1, omp_set_schedule_i2,           &
+              omp_set_schedule_i4, omp_set_schedule_i8
+        end interface omp_set_schedule
+        interface omp_get_schedule
+          module procedure omp_get_schedule_i1, omp_get_schedule_i2,           &
+              omp_get_schedule_i4, omp_get_schedule_i8
+        end interface omp_get_schedule
+        interface omp_pause_resource
+          module procedure omp_pause_resource_i1, omp_pause_resource_i2,       &
+              omp_pause_resource_i4, omp_pause_resource_i8
+        end interface omp_pause_resource
+        interface omp_get_place_proc_ids
+          module procedure omp_get_place_proc_ids_i4, omp_get_place_proc_ids_i8
+        end interface omp_get_place_proc_ids
+        interface omp_get_partition_place_nums
+          module procedure omp_get_partition_place_nums_i4,                    &
+              omp_get_partition_place_nums_i8
+        end interface omp_get_partition_place_nums
+        interface kmp_set_affinity_mask_proc
+          module procedure kmp_set_affinity_mask_proc_i1,                      &
+              kmp_set_affinity_mask_proc_i2, kmp_set_affinity_mask_proc_i4,    &
+              kmp_set_affinity_mask_proc_i8
+        end interface kmp_set_affinity_mask_proc
+        interface kmp_unset_affinity_mask_proc
+          module procedure kmp_unset_affinity_mask_proc_i1,                    &
+              kmp_unset_affinity_mask_proc_i2,                                 &
+              kmp_unset_affinity_mask_proc_i4,                                 &
+              kmp_unset_affinity_mask_proc_i8
+        end interface kmp_unset_affinity_mask_proc
+        interface kmp_get_affinity_mask_proc
+          module procedure kmp_get_affinity_mask_proc_i1,                      &
+              kmp_get_affinity_mask_proc_i2, kmp_get_affinity_mask_proc_i4,    &
+              kmp_get_affinity_mask_proc_i8
+        end interface kmp_get_affinity_mask_proc
+
+
         ! make the above routine definitions public
         public :: omp_set_num_threads
         public :: omp_set_dynamic
@@ -1277,4 +1406,407 @@
         public :: omp_get_interop_type_desc
         public :: omp_get_interop_rc_desc
 
+
+
+        ! Submodule procedure declarations for integer-kind wrappers.
+        interface
+          module subroutine omp_set_num_threads_i1(num_threads)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: num_threads
+          end subroutine omp_set_num_threads_i1
+          module subroutine omp_set_num_threads_i2(num_threads)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: num_threads
+          end subroutine omp_set_num_threads_i2
+          module subroutine omp_set_num_threads_i4(num_threads)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: num_threads
+          end subroutine omp_set_num_threads_i4
+          module subroutine omp_set_num_threads_i8(num_threads)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: num_threads
+          end subroutine omp_set_num_threads_i8
+          module subroutine omp_set_max_active_levels_i1(max_levels)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: max_levels
+          end subroutine omp_set_max_active_levels_i1
+          module subroutine omp_set_max_active_levels_i2(max_levels)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: max_levels
+          end subroutine omp_set_max_active_levels_i2
+          module subroutine omp_set_max_active_levels_i4(max_levels)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: max_levels
+          end subroutine omp_set_max_active_levels_i4
+          module subroutine omp_set_max_active_levels_i8(max_levels)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: max_levels
+          end subroutine omp_set_max_active_levels_i8
+          module subroutine omp_set_default_device_i1(device_num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: device_num
+          end subroutine omp_set_default_device_i1
+          module subroutine omp_set_default_device_i2(device_num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: device_num
+          end subroutine omp_set_default_device_i2
+          module subroutine omp_set_default_device_i4(device_num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: device_num
+          end subroutine omp_set_default_device_i4
+          module subroutine omp_set_default_device_i8(device_num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: device_num
+          end subroutine omp_set_default_device_i8
+          module subroutine omp_set_num_teams_i1(num_teams)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: num_teams
+          end subroutine omp_set_num_teams_i1
+          module subroutine omp_set_num_teams_i2(num_teams)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: num_teams
+          end subroutine omp_set_num_teams_i2
+          module subroutine omp_set_num_teams_i4(num_teams)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: num_teams
+          end subroutine omp_set_num_teams_i4
+          module subroutine omp_set_num_teams_i8(num_teams)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: num_teams
+          end subroutine omp_set_num_teams_i8
+          module subroutine omp_set_teams_thread_limit_i1(thread_limit)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: thread_limit
+          end subroutine omp_set_teams_thread_limit_i1
+          module subroutine omp_set_teams_thread_limit_i2(thread_limit)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: thread_limit
+          end subroutine omp_set_teams_thread_limit_i2
+          module subroutine omp_set_teams_thread_limit_i4(thread_limit)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: thread_limit
+          end subroutine omp_set_teams_thread_limit_i4
+          module subroutine omp_set_teams_thread_limit_i8(thread_limit)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: thread_limit
+          end subroutine omp_set_teams_thread_limit_i8
+          module subroutine kmp_set_stacksize_i1(size)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: size
+          end subroutine kmp_set_stacksize_i1
+          module subroutine kmp_set_stacksize_i2(size)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: size
+          end subroutine kmp_set_stacksize_i2
+          module subroutine kmp_set_stacksize_i4(size)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: size
+          end subroutine kmp_set_stacksize_i4
+          module subroutine kmp_set_stacksize_i8(size)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: size
+          end subroutine kmp_set_stacksize_i8
+          module subroutine kmp_set_blocktime_i1(msec)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: msec
+          end subroutine kmp_set_blocktime_i1
+          module subroutine kmp_set_blocktime_i2(msec)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: msec
+          end subroutine kmp_set_blocktime_i2
+          module subroutine kmp_set_blocktime_i4(msec)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: msec
+          end subroutine kmp_set_blocktime_i4
+          module subroutine kmp_set_blocktime_i8(msec)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: msec
+          end subroutine kmp_set_blocktime_i8
+          module subroutine kmp_set_library_i1(libnum)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: libnum
+          end subroutine kmp_set_library_i1
+          module subroutine kmp_set_library_i2(libnum)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: libnum
+          end subroutine kmp_set_library_i2
+          module subroutine kmp_set_library_i4(libnum)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: libnum
+          end subroutine kmp_set_library_i4
+          module subroutine kmp_set_library_i8(libnum)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: libnum
+          end subroutine kmp_set_library_i8
+          module subroutine kmp_set_disp_num_buffers_i1(num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: num
+          end subroutine kmp_set_disp_num_buffers_i1
+          module subroutine kmp_set_disp_num_buffers_i2(num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: num
+          end subroutine kmp_set_disp_num_buffers_i2
+          module subroutine kmp_set_disp_num_buffers_i4(num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: num
+          end subroutine kmp_set_disp_num_buffers_i4
+          module subroutine kmp_set_disp_num_buffers_i8(num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: num
+          end subroutine kmp_set_disp_num_buffers_i8
+          module function omp_get_ancestor_thread_num_i1(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_ancestor_thread_num_i1
+          end function omp_get_ancestor_thread_num_i1
+          module function omp_get_ancestor_thread_num_i2(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_ancestor_thread_num_i2
+          end function omp_get_ancestor_thread_num_i2
+          module function omp_get_ancestor_thread_num_i4(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_ancestor_thread_num_i4
+          end function omp_get_ancestor_thread_num_i4
+          module function omp_get_ancestor_thread_num_i8(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_ancestor_thread_num_i8
+          end function omp_get_ancestor_thread_num_i8
+          module function omp_get_team_size_i1(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_team_size_i1
+          end function omp_get_team_size_i1
+          module function omp_get_team_size_i2(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_team_size_i2
+          end function omp_get_team_size_i2
+          module function omp_get_team_size_i4(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_team_size_i4
+          end function omp_get_team_size_i4
+          module function omp_get_team_size_i8(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_team_size_i8
+          end function omp_get_team_size_i8
+          module function omp_get_place_num_procs_i1(place_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: place_num
+          integer(kind=omp_integer_kind) :: omp_get_place_num_procs_i1
+          end function omp_get_place_num_procs_i1
+          module function omp_get_place_num_procs_i2(place_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: place_num
+          integer(kind=omp_integer_kind) :: omp_get_place_num_procs_i2
+          end function omp_get_place_num_procs_i2
+          module function omp_get_place_num_procs_i4(place_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: place_num
+          integer(kind=omp_integer_kind) :: omp_get_place_num_procs_i4
+          end function omp_get_place_num_procs_i4
+          module function omp_get_place_num_procs_i8(place_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: place_num
+          integer(kind=omp_integer_kind) :: omp_get_place_num_procs_i8
+          end function omp_get_place_num_procs_i8
+          module subroutine omp_set_schedule_i1(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(kind=omp_sched_kind), value :: kind
+          integer(c_int8_t), value :: chunk_size
+          end subroutine omp_set_schedule_i1
+          module subroutine omp_set_schedule_i2(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(kind=omp_sched_kind), value :: kind
+          integer(c_int16_t), value :: chunk_size
+          end subroutine omp_set_schedule_i2
+          module subroutine omp_set_schedule_i4(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(kind=omp_sched_kind), value :: kind
+          integer(c_int), value :: chunk_size
+          end subroutine omp_set_schedule_i4
+          module subroutine omp_set_schedule_i8(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(kind=omp_sched_kind), value :: kind
+          integer(c_int64_t), value :: chunk_size
+          end subroutine omp_set_schedule_i8
+          module subroutine omp_get_schedule_i1(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(kind=omp_sched_kind), intent(out) :: kind
+          integer(c_int8_t), intent(out) :: chunk_size
+          end subroutine omp_get_schedule_i1
+          module subroutine omp_get_schedule_i2(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(kind=omp_sched_kind), intent(out) :: kind
+          integer(c_int16_t), intent(out) :: chunk_size
+          end subroutine omp_get_schedule_i2
+          module subroutine omp_get_schedule_i4(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(kind=omp_sched_kind), intent(out) :: kind
+          integer(c_int), intent(out) :: chunk_size
+          end subroutine omp_get_schedule_i4
+          module subroutine omp_get_schedule_i8(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(kind=omp_sched_kind), intent(out) :: kind
+          integer(c_int64_t), intent(out) :: chunk_size
+          end subroutine omp_get_schedule_i8
+          module function omp_pause_resource_i1(kind, device_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(kind=omp_pause_resource_kind), value :: kind
+          integer(c_int8_t), value :: device_num
+          integer(kind=omp_integer_kind) :: omp_pause_resource_i1
+          end function omp_pause_resource_i1
+          module function omp_pause_resource_i2(kind, device_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(kind=omp_pause_resource_kind), value :: kind
+          integer(c_int16_t), value :: device_num
+          integer(kind=omp_integer_kind) :: omp_pause_resource_i2
+          end function omp_pause_resource_i2
+          module function omp_pause_resource_i4(kind, device_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(kind=omp_pause_resource_kind), value :: kind
+          integer(c_int), value :: device_num
+          integer(kind=omp_integer_kind) :: omp_pause_resource_i4
+          end function omp_pause_resource_i4
+          module function omp_pause_resource_i8(kind, device_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(kind=omp_pause_resource_kind), value :: kind
+          integer(c_int64_t), value :: device_num
+          integer(kind=omp_integer_kind) :: omp_pause_resource_i8
+          end function omp_pause_resource_i8
+          module function kmp_set_affinity_mask_proc_i1(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_set_affinity_mask_proc_i1
+          end function kmp_set_affinity_mask_proc_i1
+          module function kmp_set_affinity_mask_proc_i2(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_set_affinity_mask_proc_i2
+          end function kmp_set_affinity_mask_proc_i2
+          module function kmp_set_affinity_mask_proc_i4(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_set_affinity_mask_proc_i4
+          end function kmp_set_affinity_mask_proc_i4
+          module function kmp_set_affinity_mask_proc_i8(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_set_affinity_mask_proc_i8
+          end function kmp_set_affinity_mask_proc_i8
+          module function kmp_unset_affinity_mask_proc_i1(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_unset_affinity_mask_proc_i1
+          end function kmp_unset_affinity_mask_proc_i1
+          module function kmp_unset_affinity_mask_proc_i2(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_unset_affinity_mask_proc_i2
+          end function kmp_unset_affinity_mask_proc_i2
+          module function kmp_unset_affinity_mask_proc_i4(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_unset_affinity_mask_proc_i4
+          end function kmp_unset_affinity_mask_proc_i4
+          module function kmp_unset_affinity_mask_proc_i8(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_unset_affinity_mask_proc_i8
+          end function kmp_unset_affinity_mask_proc_i8
+          module function kmp_get_affinity_mask_proc_i1(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_get_affinity_mask_proc_i1
+          end function kmp_get_affinity_mask_proc_i1
+          module function kmp_get_affinity_mask_proc_i2(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_get_affinity_mask_proc_i2
+          end function kmp_get_affinity_mask_proc_i2
+          module function kmp_get_affinity_mask_proc_i4(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_get_affinity_mask_proc_i4
+          end function kmp_get_affinity_mask_proc_i4
+          module function kmp_get_affinity_mask_proc_i8(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_get_affinity_mask_proc_i8
+          end function kmp_get_affinity_mask_proc_i8
+          module subroutine omp_get_place_proc_ids_i4(place_num, ids)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: place_num
+          integer(c_int), intent(out) :: ids(*)
+          end subroutine omp_get_place_proc_ids_i4
+          module subroutine omp_get_place_proc_ids_i8(place_num, ids)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: place_num
+          integer(c_int64_t), intent(out) :: ids(*)
+          end subroutine omp_get_place_proc_ids_i8
+          module subroutine omp_get_partition_place_nums_i4(place_nums)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), intent(out) :: place_nums(*)
+          end subroutine omp_get_partition_place_nums_i4
+          module subroutine omp_get_partition_place_nums_i8(place_nums)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), intent(out) :: place_nums(*)
+          end subroutine omp_get_partition_place_nums_i8
+        end interface
       end module omp_lib
diff --git a/openmp/module/omp_lib_impl.F90.var b/openmp/module/omp_lib_impl.F90.var
new file mode 100644
index 0000000000000..368551b6b3f32
--- /dev/null
+++ b/openmp/module/omp_lib_impl.F90.var
@@ -0,0 +1,465 @@
+! include/omp_lib_impl.f90.var
+!
+!//===----------------------------------------------------------------------===//
+!//
+!// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+!// See https://llvm.org/LICENSE.txt for license information.
+!// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+!//
+!//===----------------------------------------------------------------------===//
+!
+! Integer-kind wrapper procedures for omp_lib
+
+      submodule (omp_lib) omp_lib_impl
+        use omp_lib_kinds
+      contains
+
+        module procedure omp_set_num_threads_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_num_threads_c(int(num_threads, kind=c_int))
+        end procedure omp_set_num_threads_i1
+
+        module procedure omp_set_num_threads_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_num_threads_c(int(num_threads, kind=c_int))
+        end procedure omp_set_num_threads_i2
+
+        module procedure omp_set_num_threads_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_num_threads_c(int(num_threads, kind=c_int))
+        end procedure omp_set_num_threads_i4
+
+        module procedure omp_set_num_threads_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_num_threads_c(int(num_threads, kind=c_int))
+        end procedure omp_set_num_threads_i8
+
+        module procedure omp_set_max_active_levels_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_max_active_levels_c(int(max_levels, kind=c_int))
+        end procedure omp_set_max_active_levels_i1
+
+        module procedure omp_set_max_active_levels_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_max_active_levels_c(int(max_levels, kind=c_int))
+        end procedure omp_set_max_active_levels_i2
+
+        module procedure omp_set_max_active_levels_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_max_active_levels_c(int(max_levels, kind=c_int))
+        end procedure omp_set_max_active_levels_i4
+
+        module procedure omp_set_max_active_levels_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_max_active_levels_c(int(max_levels, kind=c_int))
+        end procedure omp_set_max_active_levels_i8
+
+        module procedure omp_set_default_device_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_default_device_c(int(device_num, kind=c_int))
+        end procedure omp_set_default_device_i1
+
+        module procedure omp_set_default_device_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_default_device_c(int(device_num, kind=c_int))
+        end procedure omp_set_default_device_i2
+
+        module procedure omp_set_default_device_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_default_device_c(int(device_num, kind=c_int))
+        end procedure omp_set_default_device_i4
+
+        module procedure omp_set_default_device_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_default_device_c(int(device_num, kind=c_int))
+        end procedure omp_set_default_device_i8
+
+        module procedure omp_set_num_teams_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_num_teams_c(int(num_teams, kind=c_int))
+        end procedure omp_set_num_teams_i1
+
+        module procedure omp_set_num_teams_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_num_teams_c(int(num_teams, kind=c_int))
+        end procedure omp_set_num_teams_i2
+
+        module procedure omp_set_num_teams_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_num_teams_c(int(num_teams, kind=c_int))
+        end procedure omp_set_num_teams_i4
+
+        module procedure omp_set_num_teams_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_num_teams_c(int(num_teams, kind=c_int))
+        end procedure omp_set_num_teams_i8
+
+        module procedure omp_set_teams_thread_limit_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_teams_thread_limit_c(int(thread_limit, kind=c_int))
+        end procedure omp_set_teams_thread_limit_i1
+
+        module procedure omp_set_teams_thread_limit_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_teams_thread_limit_c(int(thread_limit, kind=c_int))
+        end procedure omp_set_teams_thread_limit_i2
+
+        module procedure omp_set_teams_thread_limit_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_teams_thread_limit_c(int(thread_limit, kind=c_int))
+        end procedure omp_set_teams_thread_limit_i4
+
+        module procedure omp_set_teams_thread_limit_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_teams_thread_limit_c(int(thread_limit, kind=c_int))
+        end procedure omp_set_teams_thread_limit_i8
+
+        module procedure kmp_set_stacksize_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_stacksize_c(int(size, kind=c_int))
+        end procedure kmp_set_stacksize_i1
+
+        module procedure kmp_set_stacksize_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_stacksize_c(int(size, kind=c_int))
+        end procedure kmp_set_stacksize_i2
+
+        module procedure kmp_set_stacksize_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_stacksize_c(int(size, kind=c_int))
+        end procedure kmp_set_stacksize_i4
+
+        module procedure kmp_set_stacksize_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_stacksize_c(int(size, kind=c_int))
+        end procedure kmp_set_stacksize_i8
+
+        module procedure kmp_set_blocktime_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_blocktime_c(int(msec, kind=c_int))
+        end procedure kmp_set_blocktime_i1
+
+        module procedure kmp_set_blocktime_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_blocktime_c(int(msec, kind=c_int))
+        end procedure kmp_set_blocktime_i2
+
+        module procedure kmp_set_blocktime_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_blocktime_c(int(msec, kind=c_int))
+        end procedure kmp_set_blocktime_i4
+
+        module procedure kmp_set_blocktime_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_blocktime_c(int(msec, kind=c_int))
+        end procedure kmp_set_blocktime_i8
+
+        module procedure kmp_set_library_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_library_c(int(libnum, kind=c_int))
+        end procedure kmp_set_library_i1
+
+        module procedure kmp_set_library_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_library_c(int(libnum, kind=c_int))
+        end procedure kmp_set_library_i2
+
+        module procedure kmp_set_library_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_library_c(int(libnum, kind=c_int))
+        end procedure kmp_set_library_i4
+
+        module procedure kmp_set_library_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_library_c(int(libnum, kind=c_int))
+        end procedure kmp_set_library_i8
+
+        module procedure kmp_set_disp_num_buffers_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_disp_num_buffers_c(int(num, kind=c_int))
+        end procedure kmp_set_disp_num_buffers_i1
+
+        module procedure kmp_set_disp_num_buffers_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_disp_num_buffers_c(int(num, kind=c_int))
+        end procedure kmp_set_disp_num_buffers_i2
+
+        module procedure kmp_set_disp_num_buffers_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_disp_num_buffers_c(int(num, kind=c_int))
+        end procedure kmp_set_disp_num_buffers_i4
+
+        module procedure kmp_set_disp_num_buffers_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_disp_num_buffers_c(int(num, kind=c_int))
+        end procedure kmp_set_disp_num_buffers_i8
+
+        module procedure omp_get_ancestor_thread_num_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_ancestor_thread_num_i1 =                                     &
+          omp_get_ancestor_thread_num_c(int(level, kind=c_int))
+        end procedure omp_get_ancestor_thread_num_i1
+
+        module procedure omp_get_ancestor_thread_num_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_ancestor_thread_num_i2 =                                     &
+          omp_get_ancestor_thread_num_c(int(level, kind=c_int))
+        end procedure omp_get_ancestor_thread_num_i2
+
+        module procedure omp_get_ancestor_thread_num_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_ancestor_thread_num_i4 =                                     &
+          omp_get_ancestor_thread_num_c(int(level, kind=c_int))
+        end procedure omp_get_ancestor_thread_num_i4
+
+        module procedure omp_get_ancestor_thread_num_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_ancestor_thread_num_i8 =                                     &
+          omp_get_ancestor_thread_num_c(int(level, kind=c_int))
+        end procedure omp_get_ancestor_thread_num_i8
+
+        module procedure omp_get_team_size_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_team_size_i1 = omp_get_team_size_c(int(level, kind=c_int))
+        end procedure omp_get_team_size_i1
+
+        module procedure omp_get_team_size_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_team_size_i2 = omp_get_team_size_c(int(level, kind=c_int))
+        end procedure omp_get_team_size_i2
+
+        module procedure omp_get_team_size_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_team_size_i4 = omp_get_team_size_c(int(level, kind=c_int))
+        end procedure omp_get_team_size_i4
+
+        module procedure omp_get_team_size_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_team_size_i8 = omp_get_team_size_c(int(level, kind=c_int))
+        end procedure omp_get_team_size_i8
+
+        module procedure omp_get_place_num_procs_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_place_num_procs_i1 =                                         &
+          omp_get_place_num_procs_c(int(place_num, kind=c_int))
+        end procedure omp_get_place_num_procs_i1
+
+        module procedure omp_get_place_num_procs_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_place_num_procs_i2 =                                         &
+          omp_get_place_num_procs_c(int(place_num, kind=c_int))
+        end procedure omp_get_place_num_procs_i2
+
+        module procedure omp_get_place_num_procs_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_place_num_procs_i4 =                                         &
+          omp_get_place_num_procs_c(int(place_num, kind=c_int))
+        end procedure omp_get_place_num_procs_i4
+
+        module procedure omp_get_place_num_procs_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_place_num_procs_i8 =                                         &
+          omp_get_place_num_procs_c(int(place_num, kind=c_int))
+        end procedure omp_get_place_num_procs_i8
+
+        module procedure omp_set_schedule_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_schedule_c(kind, int(chunk_size, kind=c_int))
+        end procedure omp_set_schedule_i1
+
+        module procedure omp_set_schedule_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_schedule_c(kind, int(chunk_size, kind=c_int))
+        end procedure omp_set_schedule_i2
+
+        module procedure omp_set_schedule_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_schedule_c(kind, int(chunk_size, kind=c_int))
+        end procedure omp_set_schedule_i4
+
+        module procedure omp_set_schedule_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_schedule_c(kind, int(chunk_size, kind=c_int))
+        end procedure omp_set_schedule_i8
+
+        module procedure omp_get_schedule_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          integer(c_int) :: kind_c, chunk_c
+          call omp_get_schedule_c(kind_c, chunk_c)
+          kind = kind_c
+          chunk_size = chunk_c
+        end procedure omp_get_schedule_i1
+
+        module procedure omp_get_schedule_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          integer(c_int) :: kind_c, chunk_c
+          call omp_get_schedule_c(kind_c, chunk_c)
+          kind = kind_c
+          chunk_size = chunk_c
+        end procedure omp_get_schedule_i2
+
+        module procedure omp_get_schedule_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          integer(c_int) :: kind_c, chunk_c
+          call omp_get_schedule_c(kind_c, chunk_c)
+          kind = kind_c
+          chunk_size = chunk_c
+        end procedure omp_get_schedule_i4
+
+        module procedure omp_get_schedule_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          integer(c_int) :: kind_c, chunk_c
+          call omp_get_schedule_c(kind_c, chunk_c)
+          kind = kind_c
+          chunk_size = chunk_c
+        end procedure omp_get_schedule_i8
+
+        module procedure omp_pause_resource_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_pause_resource_i1 =                                              &
+          omp_pause_resource_c(kind, int(device_num, kind=c_int))
+        end procedure omp_pause_resource_i1
+
+        module procedure omp_pause_resource_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_pause_resource_i2 =                                              &
+          omp_pause_resource_c(kind, int(device_num, kind=c_int))
+        end procedure omp_pause_resource_i2
+
+        module procedure omp_pause_resource_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_pause_resource_i4 =                                              &
+          omp_pause_resource_c(kind, int(device_num, kind=c_int))
+        end procedure omp_pause_resource_i4
+
+        module procedure omp_pause_resource_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_pause_resource_i8 =                                              &
+          omp_pause_resource_c(kind, int(device_num, kind=c_int))
+        end procedure omp_pause_resource_i8
+
+        module procedure kmp_set_affinity_mask_proc_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_set_affinity_mask_proc_i1 =                                      &
+          kmp_set_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_set_affinity_mask_proc_i1
+
+        module procedure kmp_set_affinity_mask_proc_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_set_affinity_mask_proc_i2 =                                      &
+          kmp_set_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_set_affinity_mask_proc_i2
+
+        module procedure kmp_set_affinity_mask_proc_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_set_affinity_mask_proc_i4 =                                      &
+          kmp_set_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_set_affinity_mask_proc_i4
+
+        module procedure kmp_set_affinity_mask_proc_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_set_affinity_mask_proc_i8 =                                      &
+          kmp_set_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_set_affinity_mask_proc_i8
+
+        module procedure kmp_unset_affinity_mask_proc_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_unset_affinity_mask_proc_i1 =                                    &
+          kmp_unset_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_unset_affinity_mask_proc_i1
+
+        module procedure kmp_unset_affinity_mask_proc_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_unset_affinity_mask_proc_i2 =                                    &
+          kmp_unset_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_unset_affinity_mask_proc_i2
+
+        module procedure kmp_unset_affinity_mask_proc_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_unset_affinity_mask_proc_i4 =                                    &
+          kmp_unset_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_unset_affinity_mask_proc_i4
+
+        module procedure kmp_unset_affinity_mask_proc_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_unset_affinity_mask_proc_i8 =                                    &
+          kmp_unset_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_unset_affinity_mask_proc_i8
+
+        module procedure kmp_get_affinity_mask_proc_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_get_affinity_mask_proc_i1 =                                      &
+          kmp_get_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_get_affinity_mask_proc_i1
+
+        module procedure kmp_get_affinity_mask_proc_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_get_affinity_mask_proc_i2 =                                      &
+          kmp_get_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_get_affinity_mask_proc_i2
+
+        module procedure kmp_get_affinity_mask_proc_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_get_affinity_mask_proc_i4 =                                      &
+          kmp_get_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_get_affinity_mask_proc_i4
+
+        module procedure kmp_get_affinity_mask_proc_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_get_affinity_mask_proc_i8 =                                      &
+          kmp_get_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_get_affinity_mask_proc_i8
+
+        module procedure omp_get_place_proc_ids_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          integer(c_int) :: n, i
+          integer(c_int), allocatable :: ids_c(:)
+          n = omp_get_place_num_procs_c(int(place_num, kind=c_int))
+          allocate(ids_c(n))
+          call omp_get_place_proc_ids_c(int(place_num, kind=c_int), ids_c)
+          do i = 1, n
+          ids(i) = ids_c(i)
+          end do
+          deallocate(ids_c)
+        end procedure omp_get_place_proc_ids_i4
+
+        module procedure omp_get_place_proc_ids_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          integer(c_int) :: n, i
+          integer(c_int), allocatable :: ids_c(:)
+          n = omp_get_place_num_procs_c(int(place_num, kind=c_int))
+          allocate(ids_c(n))
+          call omp_get_place_proc_ids_c(int(place_num, kind=c_int), ids_c)
+          do i = 1, n
+          ids(i) = ids_c(i)
+          end do
+          deallocate(ids_c)
+        end procedure omp_get_place_proc_ids_i8
+
+        module procedure omp_get_partition_place_nums_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          integer(c_int) :: n, i
+          integer(c_int), allocatable :: place_nums_c(:)
+          n = omp_get_partition_num_places()
+          allocate(place_nums_c(n))
+          call omp_get_partition_place_nums_c(place_nums_c)
+          do i = 1, n
+          place_nums(i) = place_nums_c(i)
+          end do
+          deallocate(place_nums_c)
+        end procedure omp_get_partition_place_nums_i4
+
+        module procedure omp_get_partition_place_nums_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          integer(c_int) :: n, i
+          integer(c_int), allocatable :: place_nums_c(:)
+          n = omp_get_partition_num_places()
+          allocate(place_nums_c(n))
+          call omp_get_partition_place_nums_c(place_nums_c)
+          do i = 1, n
+          place_nums(i) = place_nums_c(i)
+          end do
+          deallocate(place_nums_c)
+        end procedure omp_get_partition_place_nums_i8
+
+      end submodule omp_lib_impl


From openmp-commits at lists.llvm.org  Sun Aug 16 18:35:29 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Sun, 16 Aug 2026 18:35:29 -0700 (PDT)
Subject: [Openmp-commits] [flang] [llvm] [openmp] [OpenMP][Flang] Add
 integer-kind wrappers to omp_lib (PR #213505)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213505@github.com>
Message-ID: <6a826561.af4834b4.1ff9ff.770a@mx.google.com>


https://github.com/shivaramaarao edited https://github.com/llvm/llvm-project/pull/213505

From openmp-commits at lists.llvm.org  Sun Aug 16 23:15:36 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Sun, 16 Aug 2026 23:15:36 -0700 (PDT)
Subject: [Openmp-commits] [flang] [llvm] [openmp] [OpenMP][Flang] Add
 integer-kind wrappers to omp_lib (PR #213505)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213505@github.com>
Message-ID: <6a82a708.3cce8983.a1350.174f@mx.google.com>


https://github.com/shivaramaarao updated https://github.com/llvm/llvm-project/pull/213505

>From a175dbf4810aaade3602d3080cd50495407ae828 Mon Sep 17 00:00:00 2001
From: Shivarama Rao <shivarama.rao at amd.com>
Date: Sun, 2 Aug 2026 10:20:39 +0530
Subject: [PATCH] [OpenMP][Flang] Add integer-kind wrappers to omp_lib

Many OpenMP routines in omp_lib only accept integer(kind=omp_integer_kind),
which is c_int. Calls with integer(1), integer(2), or integer(8) arguments
therefore fail Flang's type checking. These wrapper function interfaces
are added to omp_lib module file to resolve this issue. The implementation
functions are added omp_lib_impl module file as a submodule.

fixes issue #123948

Co-authored-by: Cursor <cursoragent at cursor.com>
---
 flang-rt/lib/runtime/CMakeLists.txt           |  12 +
 .../Lower/OpenMP/omp-lib-integer-wrappers.f90 | 237 +++++++
 .../test/Lower/OpenMP/omp-lib-num-threads.f90 |   2 +-
 llvm/runtimes/CMakeLists.txt                  |   2 +
 openmp/module/CMakeLists.txt                  |  39 +-
 openmp/module/omp_lib.F90.var                 | 626 ++++++++++++++++--
 openmp/module/omp_lib_impl.F90.var            | 465 +++++++++++++
 openmp/runtime/cmake/LibompExports.cmake      |   2 +-
 8 files changed, 1334 insertions(+), 51 deletions(-)
 create mode 100644 flang/test/Lower/OpenMP/omp-lib-integer-wrappers.f90
 create mode 100644 openmp/module/omp_lib_impl.F90.var

diff --git a/flang-rt/lib/runtime/CMakeLists.txt b/flang-rt/lib/runtime/CMakeLists.txt
index ae9a6e7fb7188..4cd3090113af3 100644
--- a/flang-rt/lib/runtime/CMakeLists.txt
+++ b/flang-rt/lib/runtime/CMakeLists.txt
@@ -322,6 +322,13 @@ else ()
   set(sources ${supported_sources} ${host_sources} ${f128_sources})
 endif ()
 
+# Include the Fortran OpenMP module implementation (omp_lib.o) in
+# libflang_rt.runtime when the OpenMP runtime is also being built.
+# Skip for GPU-default targets: omp_lib is a host-side Fortran module.
+if (TARGET libomp-mod AND NOT "${LLVM_DEFAULT_TARGET_TRIPLE}" MATCHES "^amdgcn|^nvptx")
+  list(APPEND sources "$<TARGET_OBJECTS:libomp-mod>")
+endif ()
+
 
 if (NOT WIN32)
   add_flangrt_library(flang_rt.runtime STATIC SHARED
@@ -333,6 +340,11 @@ if (NOT WIN32)
 
   enable_cuda_compilation(flang_rt.runtime "${sources}")
 
+  if (TARGET libomp-mod AND TARGET flang_rt.runtime.shared AND TARGET omp)
+    # Resolve bind(c) references from omp_lib when building the shared runtime.
+    target_link_libraries(flang_rt.runtime.shared PUBLIC omp)
+  endif ()
+
   # Select a default runtime, which is used for unit and regression tests.
   if (TARGET flang_rt.runtime.default)
     get_target_property(default_target flang_rt.runtime.default ALIASED_TARGET)
diff --git a/flang/test/Lower/OpenMP/omp-lib-integer-wrappers.f90 b/flang/test/Lower/OpenMP/omp-lib-integer-wrappers.f90
new file mode 100644
index 0000000000000..9881fe56261fa
--- /dev/null
+++ b/flang/test/Lower/OpenMP/omp-lib-integer-wrappers.f90
@@ -0,0 +1,237 @@
+! REQUIRES: openmp_runtime
+
+! RUN: %flang_fc1 -emit-hlfir %openmp_flags %s -o - 2>&1 | FileCheck %s
+! RUN: bbc %openmp_flags -emit-hlfir -o - %s 2>&1 | FileCheck %s
+! RUN: %flang_fc1 -emit-fir %openmp_flags %s -o - 2>&1 | FileCheck %s
+! RUN: bbc -emit-fir %openmp_flags -o - %s 2>&1 | FileCheck %s
+!
+! Test that omp_lib integer-kind wrapper procedures lower to module
+! procedures instead of failing intrinsic resolution.
+
+program main
+  use omp_lib
+  integer(1) :: i1 = 1_1
+  integer(2) :: i2 = 2_2
+  integer(4) :: i4 = 4_4
+  integer(8) :: i8 = 8_8
+  integer(omp_integer_kind) :: ires
+  integer(omp_sched_kind) :: sched_kind
+  integer(kmp_affinity_mask_kind) :: mask
+
+  call omp_set_num_threads(i1)
+  call omp_set_num_threads(i2)
+  call omp_set_num_threads(i4)
+  call omp_set_num_threads(i8)
+
+  call omp_set_max_active_levels(i1)
+  call omp_set_max_active_levels(i2)
+  call omp_set_max_active_levels(i4)
+  call omp_set_max_active_levels(i8)
+
+  call omp_set_default_device(i1)
+  call omp_set_default_device(i2)
+  call omp_set_default_device(i4)
+  call omp_set_default_device(i8)
+
+  call omp_set_num_teams(i1)
+  call omp_set_num_teams(i2)
+  call omp_set_num_teams(i4)
+  call omp_set_num_teams(i8)
+
+  call omp_set_teams_thread_limit(i1)
+  call omp_set_teams_thread_limit(i2)
+  call omp_set_teams_thread_limit(i4)
+  call omp_set_teams_thread_limit(i8)
+
+  call kmp_set_stacksize(i1)
+  call kmp_set_stacksize(i2)
+  call kmp_set_stacksize(i4)
+  call kmp_set_stacksize(i8)
+
+  call kmp_set_blocktime(i1)
+  call kmp_set_blocktime(i2)
+  call kmp_set_blocktime(i4)
+  call kmp_set_blocktime(i8)
+
+  call kmp_set_library(i1)
+  call kmp_set_library(i2)
+  call kmp_set_library(i4)
+  call kmp_set_library(i8)
+
+  call kmp_set_disp_num_buffers(i1)
+  call kmp_set_disp_num_buffers(i2)
+  call kmp_set_disp_num_buffers(i4)
+  call kmp_set_disp_num_buffers(i8)
+
+  ires = omp_get_ancestor_thread_num(i1)
+  ires = omp_get_ancestor_thread_num(i2)
+  ires = omp_get_ancestor_thread_num(i4)
+  ires = omp_get_ancestor_thread_num(i8)
+
+  ires = omp_get_team_size(i1)
+  ires = omp_get_team_size(i2)
+  ires = omp_get_team_size(i4)
+  ires = omp_get_team_size(i8)
+
+  ires = omp_get_place_num_procs(i1)
+  ires = omp_get_place_num_procs(i2)
+  ires = omp_get_place_num_procs(i4)
+  ires = omp_get_place_num_procs(i8)
+
+  call omp_set_schedule(omp_sched_static, i1)
+  call omp_set_schedule(omp_sched_static, i2)
+  call omp_set_schedule(omp_sched_static, i4)
+  call omp_set_schedule(omp_sched_static, i8)
+
+  call omp_get_schedule(sched_kind, i1)
+  call omp_get_schedule(sched_kind, i2)
+  call omp_get_schedule(sched_kind, i4)
+  call omp_get_schedule(sched_kind, i8)
+
+  ires = omp_pause_resource(omp_pause_soft, i1)
+  ires = omp_pause_resource(omp_pause_soft, i2)
+  ires = omp_pause_resource(omp_pause_soft, i4)
+  ires = omp_pause_resource(omp_pause_soft, i8)
+
+  call kmp_create_affinity_mask(mask)
+  ires = kmp_set_affinity_mask_proc(i1, mask)
+  ires = kmp_set_affinity_mask_proc(i2, mask)
+  ires = kmp_set_affinity_mask_proc(i4, mask)
+  ires = kmp_set_affinity_mask_proc(i8, mask)
+
+  ires = kmp_unset_affinity_mask_proc(i1, mask)
+  ires = kmp_unset_affinity_mask_proc(i2, mask)
+  ires = kmp_unset_affinity_mask_proc(i4, mask)
+  ires = kmp_unset_affinity_mask_proc(i8, mask)
+
+  ires = kmp_get_affinity_mask_proc(i1, mask)
+  ires = kmp_get_affinity_mask_proc(i2, mask)
+  ires = kmp_get_affinity_mask_proc(i4, mask)
+  ires = kmp_get_affinity_mask_proc(i8, mask)
+
+  block
+    integer(4) :: ids4(1)
+    integer(8) :: ids8(1)
+    integer(4) :: parts4(1)
+    integer(8) :: parts8(1)
+    call omp_get_place_proc_ids(i4, ids4)
+    call omp_get_place_proc_ids(i8, ids8)
+    call omp_get_partition_place_nums(parts4)
+    call omp_get_partition_place_nums(parts8)
+  end block
+end program
+
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_max_active_levels
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_default_device
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_num_teams
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_teams_thread_limit
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_stacksize
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_blocktime
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_library
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_disp_num_buffers
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_ancestor_thread_num
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_team_size
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_place_num_procs
+!CHECK-NOT: not yet implemented: intrinsic: omp_set_schedule
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_schedule
+!CHECK-NOT: not yet implemented: intrinsic: omp_pause_resource
+!CHECK-NOT: not yet implemented: intrinsic: kmp_set_affinity_mask_proc
+!CHECK-NOT: not yet implemented: intrinsic: kmp_unset_affinity_mask_proc
+!CHECK-NOT: not yet implemented: intrinsic: kmp_get_affinity_mask_proc
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_place_proc_ids
+!CHECK-NOT: not yet implemented: intrinsic: omp_get_partition_place_nums
+
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i1
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i2
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i4
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_max_active_levels_i1
+!CHECK: fir.call @_QMomp_libPomp_set_max_active_levels_i2
+!CHECK: fir.call @_QMomp_libPomp_set_max_active_levels_i4
+!CHECK: fir.call @_QMomp_libPomp_set_max_active_levels_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_default_device_i1
+!CHECK: fir.call @_QMomp_libPomp_set_default_device_i2
+!CHECK: fir.call @_QMomp_libPomp_set_default_device_i4
+!CHECK: fir.call @_QMomp_libPomp_set_default_device_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_num_teams_i1
+!CHECK: fir.call @_QMomp_libPomp_set_num_teams_i2
+!CHECK: fir.call @_QMomp_libPomp_set_num_teams_i4
+!CHECK: fir.call @_QMomp_libPomp_set_num_teams_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_teams_thread_limit_i1
+!CHECK: fir.call @_QMomp_libPomp_set_teams_thread_limit_i2
+!CHECK: fir.call @_QMomp_libPomp_set_teams_thread_limit_i4
+!CHECK: fir.call @_QMomp_libPomp_set_teams_thread_limit_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_stacksize_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_stacksize_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_stacksize_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_stacksize_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_blocktime_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_blocktime_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_blocktime_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_blocktime_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_library_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_library_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_library_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_library_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_disp_num_buffers_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_disp_num_buffers_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_disp_num_buffers_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_disp_num_buffers_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_ancestor_thread_num_i1
+!CHECK: fir.call @_QMomp_libPomp_get_ancestor_thread_num_i2
+!CHECK: fir.call @_QMomp_libPomp_get_ancestor_thread_num_i4
+!CHECK: fir.call @_QMomp_libPomp_get_ancestor_thread_num_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_team_size_i1
+!CHECK: fir.call @_QMomp_libPomp_get_team_size_i2
+!CHECK: fir.call @_QMomp_libPomp_get_team_size_i4
+!CHECK: fir.call @_QMomp_libPomp_get_team_size_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_place_num_procs_i1
+!CHECK: fir.call @_QMomp_libPomp_get_place_num_procs_i2
+!CHECK: fir.call @_QMomp_libPomp_get_place_num_procs_i4
+!CHECK: fir.call @_QMomp_libPomp_get_place_num_procs_i8
+
+!CHECK: fir.call @_QMomp_libPomp_set_schedule_i1
+!CHECK: fir.call @_QMomp_libPomp_set_schedule_i2
+!CHECK: fir.call @_QMomp_libPomp_set_schedule_i4
+!CHECK: fir.call @_QMomp_libPomp_set_schedule_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_schedule_i1
+!CHECK: fir.call @_QMomp_libPomp_get_schedule_i2
+!CHECK: fir.call @_QMomp_libPomp_get_schedule_i4
+!CHECK: fir.call @_QMomp_libPomp_get_schedule_i8
+
+!CHECK: fir.call @_QMomp_libPomp_pause_resource_i1
+!CHECK: fir.call @_QMomp_libPomp_pause_resource_i2
+!CHECK: fir.call @_QMomp_libPomp_pause_resource_i4
+!CHECK: fir.call @_QMomp_libPomp_pause_resource_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_set_affinity_mask_proc_i1
+!CHECK: fir.call @_QMomp_libPkmp_set_affinity_mask_proc_i2
+!CHECK: fir.call @_QMomp_libPkmp_set_affinity_mask_proc_i4
+!CHECK: fir.call @_QMomp_libPkmp_set_affinity_mask_proc_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_unset_affinity_mask_proc_i1
+!CHECK: fir.call @_QMomp_libPkmp_unset_affinity_mask_proc_i2
+!CHECK: fir.call @_QMomp_libPkmp_unset_affinity_mask_proc_i4
+!CHECK: fir.call @_QMomp_libPkmp_unset_affinity_mask_proc_i8
+
+!CHECK: fir.call @_QMomp_libPkmp_get_affinity_mask_proc_i1
+!CHECK: fir.call @_QMomp_libPkmp_get_affinity_mask_proc_i2
+!CHECK: fir.call @_QMomp_libPkmp_get_affinity_mask_proc_i4
+!CHECK: fir.call @_QMomp_libPkmp_get_affinity_mask_proc_i8
+
+!CHECK: fir.call @_QMomp_libPomp_get_place_proc_ids_i4
+!CHECK: fir.call @_QMomp_libPomp_get_place_proc_ids_i8
+!CHECK: fir.call @_QMomp_libPomp_get_partition_place_nums_i4
+!CHECK: fir.call @_QMomp_libPomp_get_partition_place_nums_i8
diff --git a/flang/test/Lower/OpenMP/omp-lib-num-threads.f90 b/flang/test/Lower/OpenMP/omp-lib-num-threads.f90
index 6e76dce473057..a2ec031a62d88 100644
--- a/flang/test/Lower/OpenMP/omp-lib-num-threads.f90
+++ b/flang/test/Lower/OpenMP/omp-lib-num-threads.f90
@@ -21,5 +21,5 @@ program main
 
 !CHECK-NOT: not yet implemented: intrinsic: omp_set_num_threads
 !CHECK-NOT: not yet implemented: intrinsic: omp_get_num_threads
-!CHECK: fir.call @omp_set_num_threads
+!CHECK: fir.call @_QMomp_libPomp_set_num_threads_i4
 !CHECK: fir.call @omp_get_num_threads
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index 02d58618299a1..99af0db8d3403 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -302,6 +302,7 @@ function(runtime_default_target)
     # OpenMP tests. install-libomp-mod is a separate distribution component
     # so that install-openmp does not require building flang.
     list(APPEND extra_targets "libomp-mod")
+    list(APPEND extra_targets "libomp-mod-barrier")
     list(APPEND extra_targets "install-libomp-mod")
     list(APPEND extra_targets "install-libomp-mod-stripped")
 
@@ -864,6 +865,7 @@ ${_flag_values}\
     endif ()
     if (TARGET libomp-mod)
       add_dependencies(flang-rt-test-deps libomp-mod)
+      add_dependencies(flang-rt-test-deps libomp-mod-barrier)
     endif ()
   endif ()
 endif()
diff --git a/openmp/module/CMakeLists.txt b/openmp/module/CMakeLists.txt
index 2d9829e84849a..b2531e94eb7be 100644
--- a/openmp/module/CMakeLists.txt
+++ b/openmp/module/CMakeLists.txt
@@ -9,13 +9,16 @@
 # Build the module files if a Fortran compiler is available.
 
 configure_file(omp_lib.F90.var "${CMAKE_CURRENT_BINARY_DIR}/omp_lib.F90" @ONLY)
+configure_file(omp_lib_impl.F90.var "${CMAKE_CURRENT_BINARY_DIR}/omp_lib_impl.F90" @ONLY)
 configure_file(omp_lib.h.var "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}/omp_lib.h" @ONLY)
 
-# One compilation step creates both, omp_lib.mod and omp_lib_kinds.mod. Only
-# these files are used, the object file itself can be discarded.
+# omp_lib.F90 declares the interfaces and creates both omp_lib.mod and
+# omp_lib_kinds.mod; omp_lib_impl.F90 holds the bodies of the separate module
+# procedures and creates omp_lib-omp_lib_impl.mod.
 # TODO: Adding it to libomp ($<TARGET_OBJECTS:libomp-mod>) would allow implementing Fortran API in Fortran
 add_library(libomp-mod OBJECT
   "${CMAKE_CURRENT_BINARY_DIR}/omp_lib.F90"
+  "${CMAKE_CURRENT_BINARY_DIR}/omp_lib_impl.F90"
 )
 set_target_properties(libomp-mod PROPERTIES FOLDER "OpenMP/Fortran Modules")
 
@@ -34,6 +37,38 @@ endif ()
 
 flang_module_target(libomp-mod PUBLIC)
 
+# flang_module_target(PUBLIC) emits the .mod files directly into the toolchain
+# resource directory, but that directory is also Flang's intrinsic module
+# directory, and when resolving the parent of a SUBMODULE Flang searches only
+# the non-intrinsic module path. omp_lib.mod would therefore be invisible to
+# omp_lib_impl.F90. Emit the module files into a private directory instead,
+# where the submodule can find its parent, and publish the importable ones
+# afterwards. omp_lib-omp_lib_impl.mod is only needed to compile further
+# submodules of omp_lib, so it stays private.
+set(libomp_mod_dir "${CMAKE_CURRENT_BINARY_DIR}/mod/${CMAKE_CFG_INTDIR}")
+set_target_properties(libomp-mod PROPERTIES
+  Fortran_MODULE_DIRECTORY "${libomp_mod_dir}"
+)
+
+set(libomp_public_mods
+  "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}/omp_lib.mod"
+  "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}/omp_lib_kinds.mod"
+)
+add_custom_command(OUTPUT ${libomp_public_mods}
+  COMMAND "${CMAKE_COMMAND}" -E copy_if_different
+          "${libomp_mod_dir}/omp_lib.mod"
+          "${libomp_mod_dir}/omp_lib_kinds.mod"
+          "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}"
+  DEPENDS libomp-mod
+  COMMENT "Publishing omp_lib module files"
+)
+
+# libomp-mod is an OBJECT library, so the module files it writes are opaque to
+# CMake. Dependees on omp_lib.mod must add a target-level dependency on this
+# barrier to ensure the module files exist before they are used.
+add_custom_target(libomp-mod-barrier ALL DEPENDS ${libomp_public_mods})
+set_target_properties(libomp-mod-barrier PROPERTIES FOLDER "OpenMP/Fortran Modules")
+
 install(FILES
   "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}/omp_lib.h"
   DESTINATION "${RUNTIMES_INSTALL_RESOURCE_MOD_PATH}"
diff --git a/openmp/module/omp_lib.F90.var b/openmp/module/omp_lib.F90.var
index a41b2faad9d3d..f27ea19fbe8c5 100644
--- a/openmp/module/omp_lib.F90.var
+++ b/openmp/module/omp_lib.F90.var
@@ -63,6 +63,27 @@
         ! to be exported by this MODULE.
         private
 
+        private :: kmp_get_affinity_mask_proc_c
+        private :: kmp_set_affinity_mask_proc_c
+        private :: kmp_set_blocktime_c
+        private :: kmp_set_disp_num_buffers_c
+        private :: kmp_set_library_c
+        private :: kmp_set_stacksize_c
+        private :: kmp_unset_affinity_mask_proc_c
+        private :: omp_get_ancestor_thread_num_c
+        private :: omp_get_partition_place_nums_c
+        private :: omp_get_place_num_procs_c
+        private :: omp_get_place_proc_ids_c
+        private :: omp_get_schedule_c
+        private :: omp_get_team_size_c
+        private :: omp_pause_resource_c
+        private :: omp_set_default_device_c
+        private :: omp_set_max_active_levels_c
+        private :: omp_set_num_teams_c
+        private :: omp_set_num_threads_c
+        private :: omp_set_schedule_c
+        private :: omp_set_teams_thread_limit_c
+
         ! Re-export definitions in omp_lib_kinds
         public :: omp_integer_kind
         public :: omp_logical_kind
@@ -250,10 +271,11 @@
 !         *** omp_* entry points
 !         ***
 
-          subroutine omp_set_num_threads(num_threads) bind(c)
+          subroutine omp_set_num_threads_c(num_threads) bind(c,                &
+              name='omp_set_num_threads')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: num_threads
-          end subroutine omp_set_num_threads
+          end subroutine omp_set_num_threads_c
 
           subroutine omp_set_dynamic(dynamic_threads) bind(c)
             use omp_lib_kinds
@@ -310,10 +332,11 @@
             integer (kind=omp_integer_kind) omp_get_thread_limit
           end function omp_get_thread_limit
 
-          subroutine omp_set_max_active_levels(max_levels) bind(c)
+          subroutine omp_set_max_active_levels_c(max_levels) bind(c,           &
+              name='omp_set_max_active_levels')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: max_levels
-          end subroutine omp_set_max_active_levels
+          end subroutine omp_set_max_active_levels_c
 
           function omp_get_max_active_levels() bind(c)
             use omp_lib_kinds
@@ -330,29 +353,32 @@
             integer (kind=omp_integer_kind) omp_get_active_level
           end function omp_get_active_level
 
-          function omp_get_ancestor_thread_num(level) bind(c)
+          function omp_get_ancestor_thread_num_c(level) bind(c,                &
+              name='omp_get_ancestor_thread_num')
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) omp_get_ancestor_thread_num
+            integer (kind=omp_integer_kind) omp_get_ancestor_thread_num_c
             integer (kind=omp_integer_kind), value :: level
-          end function omp_get_ancestor_thread_num
+          end function omp_get_ancestor_thread_num_c
 
-          function omp_get_team_size(level) bind(c)
+          function omp_get_team_size_c(level) bind(c, name='omp_get_team_size')
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) omp_get_team_size
+            integer (kind=omp_integer_kind) omp_get_team_size_c
             integer (kind=omp_integer_kind), value :: level
-          end function omp_get_team_size
+          end function omp_get_team_size_c
 
-          subroutine omp_set_schedule(kind, chunk_size) bind(c)
+          subroutine omp_set_schedule_c(kind, chunk_size) bind(c,              &
+              name='omp_set_schedule')
             use omp_lib_kinds
             integer (kind=omp_sched_kind), value :: kind
             integer (kind=omp_integer_kind), value :: chunk_size
-          end subroutine omp_set_schedule
+          end subroutine omp_set_schedule_c
 
-          subroutine omp_get_schedule(kind, chunk_size) bind(c)
+          subroutine omp_get_schedule_c(kind, chunk_size) bind(c,              &
+              name='omp_get_schedule')
             use omp_lib_kinds
             integer (kind=omp_sched_kind) kind
             integer (kind=omp_integer_kind) chunk_size
-          end subroutine omp_get_schedule
+          end subroutine omp_get_schedule_c
 
           function omp_get_proc_bind() bind(c)
             use omp_lib_kinds
@@ -364,17 +390,19 @@
             integer (kind=omp_integer_kind) omp_get_num_places
           end function omp_get_num_places
 
-          function omp_get_place_num_procs(place_num) bind(c)
+          function omp_get_place_num_procs_c(place_num) bind(c,                &
+              name='omp_get_place_num_procs')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: place_num
-            integer (kind=omp_integer_kind) omp_get_place_num_procs
-          end function omp_get_place_num_procs
+            integer (kind=omp_integer_kind) omp_get_place_num_procs_c
+          end function omp_get_place_num_procs_c
 
-          subroutine omp_get_place_proc_ids(place_num, ids) bind(c)
+          subroutine omp_get_place_proc_ids_c(place_num, ids) bind(c,          &
+              name='omp_get_place_proc_ids')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: place_num
             integer (kind=omp_integer_kind) ids(*)
-          end subroutine omp_get_place_proc_ids
+          end subroutine omp_get_place_proc_ids_c
 
           function omp_get_place_num() bind(c)
             use omp_lib_kinds
@@ -386,10 +414,11 @@
             integer (kind=omp_integer_kind) omp_get_partition_num_places
           end function omp_get_partition_num_places
 
-          subroutine omp_get_partition_place_nums(place_nums) bind(c)
+          subroutine omp_get_partition_place_nums_c(place_nums) bind(c,        &
+              name='omp_get_partition_place_nums')
             use omp_lib_kinds
             integer (kind=omp_integer_kind) place_nums(*)
-          end subroutine omp_get_partition_place_nums
+          end subroutine omp_get_partition_place_nums_c
 
           function omp_get_wtime() bind(c)
             use omp_lib_kinds
@@ -406,10 +435,11 @@
             integer (kind=omp_integer_kind) omp_get_default_device
           end function omp_get_default_device
 
-          subroutine omp_set_default_device(device_num) bind(c)
+          subroutine omp_set_default_device_c(device_num) bind(c,              &
+              name='omp_set_default_device')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: device_num
-          end subroutine omp_set_default_device
+          end subroutine omp_set_default_device_c
 
           function omp_get_num_devices() bind(c)
             use omp_lib_kinds
@@ -446,12 +476,13 @@
             integer (kind=omp_integer_kind) omp_get_device_num
           end function omp_get_device_num
 
-          function omp_pause_resource(kind, device_num) bind(c)
+          function omp_pause_resource_c(kind, device_num) bind(c,              &
+              name='omp_pause_resource')
             use omp_lib_kinds
             integer (kind=omp_pause_resource_kind), value :: kind
             integer (kind=omp_integer_kind), value :: device_num
-            integer (kind=omp_integer_kind) omp_pause_resource
-          end function omp_pause_resource
+            integer (kind=omp_integer_kind) omp_pause_resource_c
+          end function omp_pause_resource_c
 
           function omp_pause_resource_all(kind) bind(c)
             use omp_lib_kinds
@@ -640,20 +671,22 @@
             integer (kind=kmp_size_t_kind) :: omp_capture_affinity
           end function omp_capture_affinity
 
-          subroutine omp_set_num_teams(num_teams) bind(c)
+          subroutine omp_set_num_teams_c(num_teams) bind(c,                    &
+              name='omp_set_num_teams')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: num_teams
-          end subroutine omp_set_num_teams
+          end subroutine omp_set_num_teams_c
 
           function omp_get_max_teams() bind(c)
             use omp_lib_kinds
             integer (kind=omp_integer_kind) omp_get_max_teams
           end function omp_get_max_teams
 
-          subroutine omp_set_teams_thread_limit(thread_limit) bind(c)
+          subroutine omp_set_teams_thread_limit_c(thread_limit) bind(c,        &
+              name='omp_set_teams_thread_limit')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: thread_limit
-          end subroutine omp_set_teams_thread_limit
+          end subroutine omp_set_teams_thread_limit_c
 
           function omp_get_teams_thread_limit() bind(c)
             use omp_lib_kinds
@@ -937,20 +970,20 @@
 !         *** kmp_* entry points
 !         ***
 
-          subroutine kmp_set_stacksize(size) bind(c)
+          subroutine kmp_set_stacksize_c(size) bind(c, name='kmp_set_stacksize')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: size
-          end subroutine kmp_set_stacksize
+          end subroutine kmp_set_stacksize_c
 
           subroutine kmp_set_stacksize_s(size) bind(c)
             use omp_lib_kinds
             integer (kind=kmp_size_t_kind), value :: size
           end subroutine kmp_set_stacksize_s
 
-          subroutine kmp_set_blocktime(msec) bind(c)
+          subroutine kmp_set_blocktime_c(msec) bind(c, name='kmp_set_blocktime')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: msec
-          end subroutine kmp_set_blocktime
+          end subroutine kmp_set_blocktime_c
 
           subroutine kmp_set_library_serial() bind(c)
           end subroutine kmp_set_library_serial
@@ -961,10 +994,10 @@
           subroutine kmp_set_library_throughput() bind(c)
           end subroutine kmp_set_library_throughput
 
-          subroutine kmp_set_library(libnum) bind(c)
+          subroutine kmp_set_library_c(libnum) bind(c, name='kmp_set_library')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: libnum
-          end subroutine kmp_set_library
+          end subroutine kmp_set_library_c
 
           subroutine kmp_set_defaults(string)
             character (len=*) :: string
@@ -990,10 +1023,11 @@
             integer (kind=omp_integer_kind) kmp_get_library
           end function kmp_get_library
 
-          subroutine kmp_set_disp_num_buffers(num) bind(c)
+          subroutine kmp_set_disp_num_buffers_c(num) bind(c,                   &
+              name='kmp_set_disp_num_buffers')
             use omp_lib_kinds
             integer (kind=omp_integer_kind), value :: num
-          end subroutine kmp_set_disp_num_buffers
+          end subroutine kmp_set_disp_num_buffers_c
 
           function kmp_set_affinity(mask) bind(c)
             use omp_lib_kinds
@@ -1022,26 +1056,29 @@
             integer (kind=kmp_affinity_mask_kind) mask
           end subroutine kmp_destroy_affinity_mask
 
-          function kmp_set_affinity_mask_proc(proc, mask) bind(c)
+          function kmp_set_affinity_mask_proc_c(proc, mask) bind(c,            &
+              name='kmp_set_affinity_mask_proc')
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) kmp_set_affinity_mask_proc
+            integer (kind=omp_integer_kind) kmp_set_affinity_mask_proc_c
             integer (kind=omp_integer_kind), value :: proc
             integer (kind=kmp_affinity_mask_kind) mask
-          end function kmp_set_affinity_mask_proc
+          end function kmp_set_affinity_mask_proc_c
 
-          function kmp_unset_affinity_mask_proc(proc, mask) bind(c)
+          function kmp_unset_affinity_mask_proc_c(proc, mask) bind(c,          &
+              name='kmp_unset_affinity_mask_proc')
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) kmp_unset_affinity_mask_proc
+            integer (kind=omp_integer_kind) kmp_unset_affinity_mask_proc_c
             integer (kind=omp_integer_kind), value :: proc
             integer (kind=kmp_affinity_mask_kind) mask
-          end function kmp_unset_affinity_mask_proc
+          end function kmp_unset_affinity_mask_proc_c
 
-          function kmp_get_affinity_mask_proc(proc, mask) bind(c)
+          function kmp_get_affinity_mask_proc_c(proc, mask) bind(c,            &
+              name='kmp_get_affinity_mask_proc')
             use omp_lib_kinds
-            integer (kind=omp_integer_kind) kmp_get_affinity_mask_proc
+            integer (kind=omp_integer_kind) kmp_get_affinity_mask_proc_c
             integer (kind=omp_integer_kind), value :: proc
             integer (kind=kmp_affinity_mask_kind) mask
-          end function kmp_get_affinity_mask_proc
+          end function kmp_get_affinity_mask_proc_c
 
           function kmp_malloc(size) bind(c)
             use omp_lib_kinds
@@ -1150,6 +1187,98 @@
 
         end interface
 
+        ! Generic interfaces for multiple integer kinds
+        interface omp_set_num_threads
+          module procedure omp_set_num_threads_i1, omp_set_num_threads_i2,     &
+              omp_set_num_threads_i4, omp_set_num_threads_i8
+        end interface omp_set_num_threads
+        interface omp_set_max_active_levels
+          module procedure omp_set_max_active_levels_i1,                       &
+              omp_set_max_active_levels_i2, omp_set_max_active_levels_i4,      &
+              omp_set_max_active_levels_i8
+        end interface omp_set_max_active_levels
+        interface omp_set_default_device
+          module procedure omp_set_default_device_i1,                          &
+              omp_set_default_device_i2, omp_set_default_device_i4,            &
+              omp_set_default_device_i8
+        end interface omp_set_default_device
+        interface omp_set_num_teams
+          module procedure omp_set_num_teams_i1, omp_set_num_teams_i2,         &
+              omp_set_num_teams_i4, omp_set_num_teams_i8
+        end interface omp_set_num_teams
+        interface omp_set_teams_thread_limit
+          module procedure omp_set_teams_thread_limit_i1,                      &
+              omp_set_teams_thread_limit_i2, omp_set_teams_thread_limit_i4,    &
+              omp_set_teams_thread_limit_i8
+        end interface omp_set_teams_thread_limit
+        interface kmp_set_stacksize
+          module procedure kmp_set_stacksize_i1, kmp_set_stacksize_i2,         &
+              kmp_set_stacksize_i4, kmp_set_stacksize_i8
+        end interface kmp_set_stacksize
+        interface kmp_set_blocktime
+          module procedure kmp_set_blocktime_i1, kmp_set_blocktime_i2,         &
+              kmp_set_blocktime_i4, kmp_set_blocktime_i8
+        end interface kmp_set_blocktime
+        interface kmp_set_library
+          module procedure kmp_set_library_i1, kmp_set_library_i2,             &
+              kmp_set_library_i4, kmp_set_library_i8
+        end interface kmp_set_library
+        interface kmp_set_disp_num_buffers
+          module procedure kmp_set_disp_num_buffers_i1,                        &
+              kmp_set_disp_num_buffers_i2, kmp_set_disp_num_buffers_i4,        &
+              kmp_set_disp_num_buffers_i8
+        end interface kmp_set_disp_num_buffers
+        interface omp_get_ancestor_thread_num
+          module procedure omp_get_ancestor_thread_num_i1,                     &
+              omp_get_ancestor_thread_num_i2, omp_get_ancestor_thread_num_i4,  &
+              omp_get_ancestor_thread_num_i8
+        end interface omp_get_ancestor_thread_num
+        interface omp_get_team_size
+          module procedure omp_get_team_size_i1, omp_get_team_size_i2,         &
+              omp_get_team_size_i4, omp_get_team_size_i8
+        end interface omp_get_team_size
+        interface omp_get_place_num_procs
+          module procedure omp_get_place_num_procs_i1,                         &
+              omp_get_place_num_procs_i2, omp_get_place_num_procs_i4,          &
+              omp_get_place_num_procs_i8
+        end interface omp_get_place_num_procs
+        interface omp_set_schedule
+          module procedure omp_set_schedule_i1, omp_set_schedule_i2,           &
+              omp_set_schedule_i4, omp_set_schedule_i8
+        end interface omp_set_schedule
+        interface omp_get_schedule
+          module procedure omp_get_schedule_i1, omp_get_schedule_i2,           &
+              omp_get_schedule_i4, omp_get_schedule_i8
+        end interface omp_get_schedule
+        interface omp_pause_resource
+          module procedure omp_pause_resource_i1, omp_pause_resource_i2,       &
+              omp_pause_resource_i4, omp_pause_resource_i8
+        end interface omp_pause_resource
+        interface omp_get_place_proc_ids
+          module procedure omp_get_place_proc_ids_i4, omp_get_place_proc_ids_i8
+        end interface omp_get_place_proc_ids
+        interface omp_get_partition_place_nums
+          module procedure omp_get_partition_place_nums_i4,                    &
+              omp_get_partition_place_nums_i8
+        end interface omp_get_partition_place_nums
+        interface kmp_set_affinity_mask_proc
+          module procedure kmp_set_affinity_mask_proc_i1,                      &
+              kmp_set_affinity_mask_proc_i2, kmp_set_affinity_mask_proc_i4,    &
+              kmp_set_affinity_mask_proc_i8
+        end interface kmp_set_affinity_mask_proc
+        interface kmp_unset_affinity_mask_proc
+          module procedure kmp_unset_affinity_mask_proc_i1,                    &
+              kmp_unset_affinity_mask_proc_i2,                                 &
+              kmp_unset_affinity_mask_proc_i4,                                 &
+              kmp_unset_affinity_mask_proc_i8
+        end interface kmp_unset_affinity_mask_proc
+        interface kmp_get_affinity_mask_proc
+          module procedure kmp_get_affinity_mask_proc_i1,                      &
+              kmp_get_affinity_mask_proc_i2, kmp_get_affinity_mask_proc_i4,    &
+              kmp_get_affinity_mask_proc_i8
+        end interface kmp_get_affinity_mask_proc
+
+
         ! make the above routine definitions public
         public :: omp_set_num_threads
         public :: omp_set_dynamic
@@ -1277,4 +1406,407 @@
         public :: omp_get_interop_type_desc
         public :: omp_get_interop_rc_desc
 
+
+
+        ! Submodule procedure declarations for integer-kind wrappers.
+        interface
+          module subroutine omp_set_num_threads_i1(num_threads)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: num_threads
+          end subroutine omp_set_num_threads_i1
+          module subroutine omp_set_num_threads_i2(num_threads)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: num_threads
+          end subroutine omp_set_num_threads_i2
+          module subroutine omp_set_num_threads_i4(num_threads)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: num_threads
+          end subroutine omp_set_num_threads_i4
+          module subroutine omp_set_num_threads_i8(num_threads)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: num_threads
+          end subroutine omp_set_num_threads_i8
+          module subroutine omp_set_max_active_levels_i1(max_levels)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: max_levels
+          end subroutine omp_set_max_active_levels_i1
+          module subroutine omp_set_max_active_levels_i2(max_levels)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: max_levels
+          end subroutine omp_set_max_active_levels_i2
+          module subroutine omp_set_max_active_levels_i4(max_levels)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: max_levels
+          end subroutine omp_set_max_active_levels_i4
+          module subroutine omp_set_max_active_levels_i8(max_levels)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: max_levels
+          end subroutine omp_set_max_active_levels_i8
+          module subroutine omp_set_default_device_i1(device_num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: device_num
+          end subroutine omp_set_default_device_i1
+          module subroutine omp_set_default_device_i2(device_num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: device_num
+          end subroutine omp_set_default_device_i2
+          module subroutine omp_set_default_device_i4(device_num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: device_num
+          end subroutine omp_set_default_device_i4
+          module subroutine omp_set_default_device_i8(device_num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: device_num
+          end subroutine omp_set_default_device_i8
+          module subroutine omp_set_num_teams_i1(num_teams)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: num_teams
+          end subroutine omp_set_num_teams_i1
+          module subroutine omp_set_num_teams_i2(num_teams)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: num_teams
+          end subroutine omp_set_num_teams_i2
+          module subroutine omp_set_num_teams_i4(num_teams)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: num_teams
+          end subroutine omp_set_num_teams_i4
+          module subroutine omp_set_num_teams_i8(num_teams)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: num_teams
+          end subroutine omp_set_num_teams_i8
+          module subroutine omp_set_teams_thread_limit_i1(thread_limit)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: thread_limit
+          end subroutine omp_set_teams_thread_limit_i1
+          module subroutine omp_set_teams_thread_limit_i2(thread_limit)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: thread_limit
+          end subroutine omp_set_teams_thread_limit_i2
+          module subroutine omp_set_teams_thread_limit_i4(thread_limit)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: thread_limit
+          end subroutine omp_set_teams_thread_limit_i4
+          module subroutine omp_set_teams_thread_limit_i8(thread_limit)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: thread_limit
+          end subroutine omp_set_teams_thread_limit_i8
+          module subroutine kmp_set_stacksize_i1(size)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: size
+          end subroutine kmp_set_stacksize_i1
+          module subroutine kmp_set_stacksize_i2(size)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: size
+          end subroutine kmp_set_stacksize_i2
+          module subroutine kmp_set_stacksize_i4(size)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: size
+          end subroutine kmp_set_stacksize_i4
+          module subroutine kmp_set_stacksize_i8(size)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: size
+          end subroutine kmp_set_stacksize_i8
+          module subroutine kmp_set_blocktime_i1(msec)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: msec
+          end subroutine kmp_set_blocktime_i1
+          module subroutine kmp_set_blocktime_i2(msec)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: msec
+          end subroutine kmp_set_blocktime_i2
+          module subroutine kmp_set_blocktime_i4(msec)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: msec
+          end subroutine kmp_set_blocktime_i4
+          module subroutine kmp_set_blocktime_i8(msec)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: msec
+          end subroutine kmp_set_blocktime_i8
+          module subroutine kmp_set_library_i1(libnum)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: libnum
+          end subroutine kmp_set_library_i1
+          module subroutine kmp_set_library_i2(libnum)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: libnum
+          end subroutine kmp_set_library_i2
+          module subroutine kmp_set_library_i4(libnum)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: libnum
+          end subroutine kmp_set_library_i4
+          module subroutine kmp_set_library_i8(libnum)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: libnum
+          end subroutine kmp_set_library_i8
+          module subroutine kmp_set_disp_num_buffers_i1(num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: num
+          end subroutine kmp_set_disp_num_buffers_i1
+          module subroutine kmp_set_disp_num_buffers_i2(num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: num
+          end subroutine kmp_set_disp_num_buffers_i2
+          module subroutine kmp_set_disp_num_buffers_i4(num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: num
+          end subroutine kmp_set_disp_num_buffers_i4
+          module subroutine kmp_set_disp_num_buffers_i8(num)
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: num
+          end subroutine kmp_set_disp_num_buffers_i8
+          module function omp_get_ancestor_thread_num_i1(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_ancestor_thread_num_i1
+          end function omp_get_ancestor_thread_num_i1
+          module function omp_get_ancestor_thread_num_i2(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_ancestor_thread_num_i2
+          end function omp_get_ancestor_thread_num_i2
+          module function omp_get_ancestor_thread_num_i4(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_ancestor_thread_num_i4
+          end function omp_get_ancestor_thread_num_i4
+          module function omp_get_ancestor_thread_num_i8(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_ancestor_thread_num_i8
+          end function omp_get_ancestor_thread_num_i8
+          module function omp_get_team_size_i1(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_team_size_i1
+          end function omp_get_team_size_i1
+          module function omp_get_team_size_i2(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_team_size_i2
+          end function omp_get_team_size_i2
+          module function omp_get_team_size_i4(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_team_size_i4
+          end function omp_get_team_size_i4
+          module function omp_get_team_size_i8(level)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: level
+          integer(kind=omp_integer_kind) :: omp_get_team_size_i8
+          end function omp_get_team_size_i8
+          module function omp_get_place_num_procs_i1(place_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: place_num
+          integer(kind=omp_integer_kind) :: omp_get_place_num_procs_i1
+          end function omp_get_place_num_procs_i1
+          module function omp_get_place_num_procs_i2(place_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: place_num
+          integer(kind=omp_integer_kind) :: omp_get_place_num_procs_i2
+          end function omp_get_place_num_procs_i2
+          module function omp_get_place_num_procs_i4(place_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: place_num
+          integer(kind=omp_integer_kind) :: omp_get_place_num_procs_i4
+          end function omp_get_place_num_procs_i4
+          module function omp_get_place_num_procs_i8(place_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: place_num
+          integer(kind=omp_integer_kind) :: omp_get_place_num_procs_i8
+          end function omp_get_place_num_procs_i8
+          module subroutine omp_set_schedule_i1(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(kind=omp_sched_kind), value :: kind
+          integer(c_int8_t), value :: chunk_size
+          end subroutine omp_set_schedule_i1
+          module subroutine omp_set_schedule_i2(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(kind=omp_sched_kind), value :: kind
+          integer(c_int16_t), value :: chunk_size
+          end subroutine omp_set_schedule_i2
+          module subroutine omp_set_schedule_i4(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(kind=omp_sched_kind), value :: kind
+          integer(c_int), value :: chunk_size
+          end subroutine omp_set_schedule_i4
+          module subroutine omp_set_schedule_i8(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(kind=omp_sched_kind), value :: kind
+          integer(c_int64_t), value :: chunk_size
+          end subroutine omp_set_schedule_i8
+          module subroutine omp_get_schedule_i1(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(kind=omp_sched_kind), intent(out) :: kind
+          integer(c_int8_t), intent(out) :: chunk_size
+          end subroutine omp_get_schedule_i1
+          module subroutine omp_get_schedule_i2(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(kind=omp_sched_kind), intent(out) :: kind
+          integer(c_int16_t), intent(out) :: chunk_size
+          end subroutine omp_get_schedule_i2
+          module subroutine omp_get_schedule_i4(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(kind=omp_sched_kind), intent(out) :: kind
+          integer(c_int), intent(out) :: chunk_size
+          end subroutine omp_get_schedule_i4
+          module subroutine omp_get_schedule_i8(kind, chunk_size)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(kind=omp_sched_kind), intent(out) :: kind
+          integer(c_int64_t), intent(out) :: chunk_size
+          end subroutine omp_get_schedule_i8
+          module function omp_pause_resource_i1(kind, device_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(kind=omp_pause_resource_kind), value :: kind
+          integer(c_int8_t), value :: device_num
+          integer(kind=omp_integer_kind) :: omp_pause_resource_i1
+          end function omp_pause_resource_i1
+          module function omp_pause_resource_i2(kind, device_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(kind=omp_pause_resource_kind), value :: kind
+          integer(c_int16_t), value :: device_num
+          integer(kind=omp_integer_kind) :: omp_pause_resource_i2
+          end function omp_pause_resource_i2
+          module function omp_pause_resource_i4(kind, device_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(kind=omp_pause_resource_kind), value :: kind
+          integer(c_int), value :: device_num
+          integer(kind=omp_integer_kind) :: omp_pause_resource_i4
+          end function omp_pause_resource_i4
+          module function omp_pause_resource_i8(kind, device_num)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(kind=omp_pause_resource_kind), value :: kind
+          integer(c_int64_t), value :: device_num
+          integer(kind=omp_integer_kind) :: omp_pause_resource_i8
+          end function omp_pause_resource_i8
+          module function kmp_set_affinity_mask_proc_i1(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_set_affinity_mask_proc_i1
+          end function kmp_set_affinity_mask_proc_i1
+          module function kmp_set_affinity_mask_proc_i2(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_set_affinity_mask_proc_i2
+          end function kmp_set_affinity_mask_proc_i2
+          module function kmp_set_affinity_mask_proc_i4(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_set_affinity_mask_proc_i4
+          end function kmp_set_affinity_mask_proc_i4
+          module function kmp_set_affinity_mask_proc_i8(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_set_affinity_mask_proc_i8
+          end function kmp_set_affinity_mask_proc_i8
+          module function kmp_unset_affinity_mask_proc_i1(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_unset_affinity_mask_proc_i1
+          end function kmp_unset_affinity_mask_proc_i1
+          module function kmp_unset_affinity_mask_proc_i2(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_unset_affinity_mask_proc_i2
+          end function kmp_unset_affinity_mask_proc_i2
+          module function kmp_unset_affinity_mask_proc_i4(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_unset_affinity_mask_proc_i4
+          end function kmp_unset_affinity_mask_proc_i4
+          module function kmp_unset_affinity_mask_proc_i8(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_unset_affinity_mask_proc_i8
+          end function kmp_unset_affinity_mask_proc_i8
+          module function kmp_get_affinity_mask_proc_i1(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int8_t
+          integer(c_int8_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_get_affinity_mask_proc_i1
+          end function kmp_get_affinity_mask_proc_i1
+          module function kmp_get_affinity_mask_proc_i2(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int16_t
+          integer(c_int16_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_get_affinity_mask_proc_i2
+          end function kmp_get_affinity_mask_proc_i2
+          module function kmp_get_affinity_mask_proc_i4(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_get_affinity_mask_proc_i4
+          end function kmp_get_affinity_mask_proc_i4
+          module function kmp_get_affinity_mask_proc_i8(proc, mask)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: proc
+          integer(kind=kmp_affinity_mask_kind) :: mask
+          integer(kind=omp_integer_kind) :: kmp_get_affinity_mask_proc_i8
+          end function kmp_get_affinity_mask_proc_i8
+          module subroutine omp_get_place_proc_ids_i4(place_num, ids)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), value :: place_num
+          integer(c_int), intent(out) :: ids(*)
+          end subroutine omp_get_place_proc_ids_i4
+          module subroutine omp_get_place_proc_ids_i8(place_num, ids)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), value :: place_num
+          integer(c_int64_t), intent(out) :: ids(*)
+          end subroutine omp_get_place_proc_ids_i8
+          module subroutine omp_get_partition_place_nums_i4(place_nums)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int
+          integer(c_int), intent(out) :: place_nums(*)
+          end subroutine omp_get_partition_place_nums_i4
+          module subroutine omp_get_partition_place_nums_i8(place_nums)
+          use omp_lib_kinds
+          use, intrinsic :: iso_c_binding, only: c_int, c_int64_t
+          integer(c_int64_t), intent(out) :: place_nums(*)
+          end subroutine omp_get_partition_place_nums_i8
+        end interface
       end module omp_lib
diff --git a/openmp/module/omp_lib_impl.F90.var b/openmp/module/omp_lib_impl.F90.var
new file mode 100644
index 0000000000000..368551b6b3f32
--- /dev/null
+++ b/openmp/module/omp_lib_impl.F90.var
@@ -0,0 +1,465 @@
+! include/omp_lib_impl.f90.var
+!
+!//===----------------------------------------------------------------------===//
+!//
+!// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+!// See https://llvm.org/LICENSE.txt for license information.
+!// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+!//
+!//===----------------------------------------------------------------------===//
+!
+! Integer-kind wrapper procedures for omp_lib
+
+      submodule (omp_lib) omp_lib_impl
+        use omp_lib_kinds
+      contains
+
+        module procedure omp_set_num_threads_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_num_threads_c(int(num_threads, kind=c_int))
+        end procedure omp_set_num_threads_i1
+
+        module procedure omp_set_num_threads_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_num_threads_c(int(num_threads, kind=c_int))
+        end procedure omp_set_num_threads_i2
+
+        module procedure omp_set_num_threads_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_num_threads_c(int(num_threads, kind=c_int))
+        end procedure omp_set_num_threads_i4
+
+        module procedure omp_set_num_threads_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_num_threads_c(int(num_threads, kind=c_int))
+        end procedure omp_set_num_threads_i8
+
+        module procedure omp_set_max_active_levels_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_max_active_levels_c(int(max_levels, kind=c_int))
+        end procedure omp_set_max_active_levels_i1
+
+        module procedure omp_set_max_active_levels_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_max_active_levels_c(int(max_levels, kind=c_int))
+        end procedure omp_set_max_active_levels_i2
+
+        module procedure omp_set_max_active_levels_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_max_active_levels_c(int(max_levels, kind=c_int))
+        end procedure omp_set_max_active_levels_i4
+
+        module procedure omp_set_max_active_levels_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_max_active_levels_c(int(max_levels, kind=c_int))
+        end procedure omp_set_max_active_levels_i8
+
+        module procedure omp_set_default_device_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_default_device_c(int(device_num, kind=c_int))
+        end procedure omp_set_default_device_i1
+
+        module procedure omp_set_default_device_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_default_device_c(int(device_num, kind=c_int))
+        end procedure omp_set_default_device_i2
+
+        module procedure omp_set_default_device_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_default_device_c(int(device_num, kind=c_int))
+        end procedure omp_set_default_device_i4
+
+        module procedure omp_set_default_device_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_default_device_c(int(device_num, kind=c_int))
+        end procedure omp_set_default_device_i8
+
+        module procedure omp_set_num_teams_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_num_teams_c(int(num_teams, kind=c_int))
+        end procedure omp_set_num_teams_i1
+
+        module procedure omp_set_num_teams_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_num_teams_c(int(num_teams, kind=c_int))
+        end procedure omp_set_num_teams_i2
+
+        module procedure omp_set_num_teams_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_num_teams_c(int(num_teams, kind=c_int))
+        end procedure omp_set_num_teams_i4
+
+        module procedure omp_set_num_teams_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_num_teams_c(int(num_teams, kind=c_int))
+        end procedure omp_set_num_teams_i8
+
+        module procedure omp_set_teams_thread_limit_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_teams_thread_limit_c(int(thread_limit, kind=c_int))
+        end procedure omp_set_teams_thread_limit_i1
+
+        module procedure omp_set_teams_thread_limit_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_teams_thread_limit_c(int(thread_limit, kind=c_int))
+        end procedure omp_set_teams_thread_limit_i2
+
+        module procedure omp_set_teams_thread_limit_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_teams_thread_limit_c(int(thread_limit, kind=c_int))
+        end procedure omp_set_teams_thread_limit_i4
+
+        module procedure omp_set_teams_thread_limit_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_teams_thread_limit_c(int(thread_limit, kind=c_int))
+        end procedure omp_set_teams_thread_limit_i8
+
+        module procedure kmp_set_stacksize_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_stacksize_c(int(size, kind=c_int))
+        end procedure kmp_set_stacksize_i1
+
+        module procedure kmp_set_stacksize_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_stacksize_c(int(size, kind=c_int))
+        end procedure kmp_set_stacksize_i2
+
+        module procedure kmp_set_stacksize_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_stacksize_c(int(size, kind=c_int))
+        end procedure kmp_set_stacksize_i4
+
+        module procedure kmp_set_stacksize_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_stacksize_c(int(size, kind=c_int))
+        end procedure kmp_set_stacksize_i8
+
+        module procedure kmp_set_blocktime_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_blocktime_c(int(msec, kind=c_int))
+        end procedure kmp_set_blocktime_i1
+
+        module procedure kmp_set_blocktime_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_blocktime_c(int(msec, kind=c_int))
+        end procedure kmp_set_blocktime_i2
+
+        module procedure kmp_set_blocktime_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_blocktime_c(int(msec, kind=c_int))
+        end procedure kmp_set_blocktime_i4
+
+        module procedure kmp_set_blocktime_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_blocktime_c(int(msec, kind=c_int))
+        end procedure kmp_set_blocktime_i8
+
+        module procedure kmp_set_library_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_library_c(int(libnum, kind=c_int))
+        end procedure kmp_set_library_i1
+
+        module procedure kmp_set_library_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_library_c(int(libnum, kind=c_int))
+        end procedure kmp_set_library_i2
+
+        module procedure kmp_set_library_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_library_c(int(libnum, kind=c_int))
+        end procedure kmp_set_library_i4
+
+        module procedure kmp_set_library_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_library_c(int(libnum, kind=c_int))
+        end procedure kmp_set_library_i8
+
+        module procedure kmp_set_disp_num_buffers_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_disp_num_buffers_c(int(num, kind=c_int))
+        end procedure kmp_set_disp_num_buffers_i1
+
+        module procedure kmp_set_disp_num_buffers_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_disp_num_buffers_c(int(num, kind=c_int))
+        end procedure kmp_set_disp_num_buffers_i2
+
+        module procedure kmp_set_disp_num_buffers_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_disp_num_buffers_c(int(num, kind=c_int))
+        end procedure kmp_set_disp_num_buffers_i4
+
+        module procedure kmp_set_disp_num_buffers_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call kmp_set_disp_num_buffers_c(int(num, kind=c_int))
+        end procedure kmp_set_disp_num_buffers_i8
+
+        module procedure omp_get_ancestor_thread_num_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_ancestor_thread_num_i1 =                                     &
+          omp_get_ancestor_thread_num_c(int(level, kind=c_int))
+        end procedure omp_get_ancestor_thread_num_i1
+
+        module procedure omp_get_ancestor_thread_num_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_ancestor_thread_num_i2 =                                     &
+          omp_get_ancestor_thread_num_c(int(level, kind=c_int))
+        end procedure omp_get_ancestor_thread_num_i2
+
+        module procedure omp_get_ancestor_thread_num_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_ancestor_thread_num_i4 =                                     &
+          omp_get_ancestor_thread_num_c(int(level, kind=c_int))
+        end procedure omp_get_ancestor_thread_num_i4
+
+        module procedure omp_get_ancestor_thread_num_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_ancestor_thread_num_i8 =                                     &
+          omp_get_ancestor_thread_num_c(int(level, kind=c_int))
+        end procedure omp_get_ancestor_thread_num_i8
+
+        module procedure omp_get_team_size_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_team_size_i1 = omp_get_team_size_c(int(level, kind=c_int))
+        end procedure omp_get_team_size_i1
+
+        module procedure omp_get_team_size_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_team_size_i2 = omp_get_team_size_c(int(level, kind=c_int))
+        end procedure omp_get_team_size_i2
+
+        module procedure omp_get_team_size_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_team_size_i4 = omp_get_team_size_c(int(level, kind=c_int))
+        end procedure omp_get_team_size_i4
+
+        module procedure omp_get_team_size_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_team_size_i8 = omp_get_team_size_c(int(level, kind=c_int))
+        end procedure omp_get_team_size_i8
+
+        module procedure omp_get_place_num_procs_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_place_num_procs_i1 =                                         &
+          omp_get_place_num_procs_c(int(place_num, kind=c_int))
+        end procedure omp_get_place_num_procs_i1
+
+        module procedure omp_get_place_num_procs_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_place_num_procs_i2 =                                         &
+          omp_get_place_num_procs_c(int(place_num, kind=c_int))
+        end procedure omp_get_place_num_procs_i2
+
+        module procedure omp_get_place_num_procs_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_place_num_procs_i4 =                                         &
+          omp_get_place_num_procs_c(int(place_num, kind=c_int))
+        end procedure omp_get_place_num_procs_i4
+
+        module procedure omp_get_place_num_procs_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_get_place_num_procs_i8 =                                         &
+          omp_get_place_num_procs_c(int(place_num, kind=c_int))
+        end procedure omp_get_place_num_procs_i8
+
+        module procedure omp_set_schedule_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_schedule_c(kind, int(chunk_size, kind=c_int))
+        end procedure omp_set_schedule_i1
+
+        module procedure omp_set_schedule_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_schedule_c(kind, int(chunk_size, kind=c_int))
+        end procedure omp_set_schedule_i2
+
+        module procedure omp_set_schedule_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_schedule_c(kind, int(chunk_size, kind=c_int))
+        end procedure omp_set_schedule_i4
+
+        module procedure omp_set_schedule_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          call omp_set_schedule_c(kind, int(chunk_size, kind=c_int))
+        end procedure omp_set_schedule_i8
+
+        module procedure omp_get_schedule_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          integer(c_int) :: kind_c, chunk_c
+          call omp_get_schedule_c(kind_c, chunk_c)
+          kind = kind_c
+          chunk_size = chunk_c
+        end procedure omp_get_schedule_i1
+
+        module procedure omp_get_schedule_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          integer(c_int) :: kind_c, chunk_c
+          call omp_get_schedule_c(kind_c, chunk_c)
+          kind = kind_c
+          chunk_size = chunk_c
+        end procedure omp_get_schedule_i2
+
+        module procedure omp_get_schedule_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          integer(c_int) :: kind_c, chunk_c
+          call omp_get_schedule_c(kind_c, chunk_c)
+          kind = kind_c
+          chunk_size = chunk_c
+        end procedure omp_get_schedule_i4
+
+        module procedure omp_get_schedule_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          integer(c_int) :: kind_c, chunk_c
+          call omp_get_schedule_c(kind_c, chunk_c)
+          kind = kind_c
+          chunk_size = chunk_c
+        end procedure omp_get_schedule_i8
+
+        module procedure omp_pause_resource_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_pause_resource_i1 =                                              &
+          omp_pause_resource_c(kind, int(device_num, kind=c_int))
+        end procedure omp_pause_resource_i1
+
+        module procedure omp_pause_resource_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_pause_resource_i2 =                                              &
+          omp_pause_resource_c(kind, int(device_num, kind=c_int))
+        end procedure omp_pause_resource_i2
+
+        module procedure omp_pause_resource_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_pause_resource_i4 =                                              &
+          omp_pause_resource_c(kind, int(device_num, kind=c_int))
+        end procedure omp_pause_resource_i4
+
+        module procedure omp_pause_resource_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          omp_pause_resource_i8 =                                              &
+          omp_pause_resource_c(kind, int(device_num, kind=c_int))
+        end procedure omp_pause_resource_i8
+
+        module procedure kmp_set_affinity_mask_proc_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_set_affinity_mask_proc_i1 =                                      &
+          kmp_set_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_set_affinity_mask_proc_i1
+
+        module procedure kmp_set_affinity_mask_proc_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_set_affinity_mask_proc_i2 =                                      &
+          kmp_set_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_set_affinity_mask_proc_i2
+
+        module procedure kmp_set_affinity_mask_proc_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_set_affinity_mask_proc_i4 =                                      &
+          kmp_set_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_set_affinity_mask_proc_i4
+
+        module procedure kmp_set_affinity_mask_proc_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_set_affinity_mask_proc_i8 =                                      &
+          kmp_set_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_set_affinity_mask_proc_i8
+
+        module procedure kmp_unset_affinity_mask_proc_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_unset_affinity_mask_proc_i1 =                                    &
+          kmp_unset_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_unset_affinity_mask_proc_i1
+
+        module procedure kmp_unset_affinity_mask_proc_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_unset_affinity_mask_proc_i2 =                                    &
+          kmp_unset_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_unset_affinity_mask_proc_i2
+
+        module procedure kmp_unset_affinity_mask_proc_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_unset_affinity_mask_proc_i4 =                                    &
+          kmp_unset_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_unset_affinity_mask_proc_i4
+
+        module procedure kmp_unset_affinity_mask_proc_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_unset_affinity_mask_proc_i8 =                                    &
+          kmp_unset_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_unset_affinity_mask_proc_i8
+
+        module procedure kmp_get_affinity_mask_proc_i1
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_get_affinity_mask_proc_i1 =                                      &
+          kmp_get_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_get_affinity_mask_proc_i1
+
+        module procedure kmp_get_affinity_mask_proc_i2
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_get_affinity_mask_proc_i2 =                                      &
+          kmp_get_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_get_affinity_mask_proc_i2
+
+        module procedure kmp_get_affinity_mask_proc_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_get_affinity_mask_proc_i4 =                                      &
+          kmp_get_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_get_affinity_mask_proc_i4
+
+        module procedure kmp_get_affinity_mask_proc_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          kmp_get_affinity_mask_proc_i8 =                                      &
+          kmp_get_affinity_mask_proc_c(int(proc, kind=c_int), mask)
+        end procedure kmp_get_affinity_mask_proc_i8
+
+        module procedure omp_get_place_proc_ids_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          integer(c_int) :: n, i
+          integer(c_int), allocatable :: ids_c(:)
+          n = omp_get_place_num_procs_c(int(place_num, kind=c_int))
+          allocate(ids_c(n))
+          call omp_get_place_proc_ids_c(int(place_num, kind=c_int), ids_c)
+          do i = 1, n
+          ids(i) = ids_c(i)
+          end do
+          deallocate(ids_c)
+        end procedure omp_get_place_proc_ids_i4
+
+        module procedure omp_get_place_proc_ids_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          integer(c_int) :: n, i
+          integer(c_int), allocatable :: ids_c(:)
+          n = omp_get_place_num_procs_c(int(place_num, kind=c_int))
+          allocate(ids_c(n))
+          call omp_get_place_proc_ids_c(int(place_num, kind=c_int), ids_c)
+          do i = 1, n
+          ids(i) = ids_c(i)
+          end do
+          deallocate(ids_c)
+        end procedure omp_get_place_proc_ids_i8
+
+        module procedure omp_get_partition_place_nums_i4
+        use, intrinsic :: iso_c_binding, only: c_int
+          integer(c_int) :: n, i
+          integer(c_int), allocatable :: place_nums_c(:)
+          n = omp_get_partition_num_places()
+          allocate(place_nums_c(n))
+          call omp_get_partition_place_nums_c(place_nums_c)
+          do i = 1, n
+          place_nums(i) = place_nums_c(i)
+          end do
+          deallocate(place_nums_c)
+        end procedure omp_get_partition_place_nums_i4
+
+        module procedure omp_get_partition_place_nums_i8
+        use, intrinsic :: iso_c_binding, only: c_int
+          integer(c_int) :: n, i
+          integer(c_int), allocatable :: place_nums_c(:)
+          n = omp_get_partition_num_places()
+          allocate(place_nums_c(n))
+          call omp_get_partition_place_nums_c(place_nums_c)
+          do i = 1, n
+          place_nums(i) = place_nums_c(i)
+          end do
+          deallocate(place_nums_c)
+        end procedure omp_get_partition_place_nums_i8
+
+      end submodule omp_lib_impl
diff --git a/openmp/runtime/cmake/LibompExports.cmake b/openmp/runtime/cmake/LibompExports.cmake
index be7bc5f83d589..a2d4104da3773 100644
--- a/openmp/runtime/cmake/LibompExports.cmake
+++ b/openmp/runtime/cmake/LibompExports.cmake
@@ -72,7 +72,7 @@ if(LIBOMP_FORTRAN_MODULES)
     COMMAND ${CMAKE_COMMAND} -E copy "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}/omp_lib.mod" ${LIBOMP_EXPORTS_MOD_DIR}
     COMMAND ${CMAKE_COMMAND} -E copy "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}/omp_lib_kinds.mod" ${LIBOMP_EXPORTS_MOD_DIR}
   )
-  add_dependencies(omp libomp-mod)
+  add_dependencies(omp libomp-mod libomp-mod-barrier)
   add_custom_command(TARGET omp POST_BUILD
     COMMAND ${CMAKE_COMMAND} -E copy "${RUNTIMES_OUTPUT_RESOURCE_MOD_DIR}/omp_lib.h" ${LIBOMP_EXPORTS_CMN_DIR}
   )


From openmp-commits at lists.llvm.org  Mon Aug 17 00:22:42 2026
From: openmp-commits at lists.llvm.org (Robert Imschweiler via Openmp-commits)
Date: Mon, 17 Aug 2026 00:22:42 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [libomp] Fix hang when a fatal error is
 raised before library registration (PR #215988)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/215988@github.com>
Message-ID: <6a82b6c2.7270d2fd.2a8f15.1c9a@mx.google.com>


================
@@ -6918,6 +6927,13 @@ void __kmp_register_library_startup(void) {
 
 void __kmp_unregister_library(void) {
 
+  // The library can be torn down before it ever registered itself, e.g. when
+  // __kmp_abort_process() runs for a fatal error raised during environment
+  // parsing. There is nothing to unregister then, and __kmp_registration_str
+  // is still NULL, so the strcmp() below would dereference it.
+  if (__kmp_registration_flag == 0)
+    return;
----------------
ro-i wrote:

yeah, I'll do a follow-up

https://github.com/llvm/llvm-project/pull/215988

From openmp-commits at lists.llvm.org  Mon Aug 17 00:22:53 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Mon, 17 Aug 2026 00:22:53 -0700 (PDT)
Subject: [Openmp-commits] [openmp] a6d40b4 - [libomp] Fix hang when a fatal
 error is raised before library registration (#215988)
Message-ID: <6a82b6cd.1e1ac41d.26a5c9.81f4@mx.google.com>


Author: Robert Imschweiler
Date: 2026-08-17T09:22:49+02:00
New Revision: a6d40b40c42d95963f915234851855f456a55b53

URL: https://github.com/llvm/llvm-project/commit/a6d40b40c42d95963f915234851855f456a55b53
DIFF: https://github.com/llvm/llvm-project/commit/a6d40b40c42d95963f915234851855f456a55b53.diff

LOG: [libomp] Fix hang when a fatal error is raised before library registration (#215988)

Uncovered by debug-build testing, not triggered for non-debug builds due
to absence of KMP_DEBUG_ASSERT.

Claude assisted with this patch.

Added: 
    

Modified: 
    openmp/runtime/src/kmp_runtime.cpp

Removed: 
    


################################################################################
diff  --git a/openmp/runtime/src/kmp_runtime.cpp b/openmp/runtime/src/kmp_runtime.cpp
index 234a211bcfba3..f16bcf5e2cbbb 100644
--- a/openmp/runtime/src/kmp_runtime.cpp
+++ b/openmp/runtime/src/kmp_runtime.cpp
@@ -459,6 +459,15 @@ void __kmp_warn(char const *format, ...) {
 }
 
 void __kmp_abort_process() {
+  // A failed assertion or fatal error raised from inside the abort path itself
+  // re-enters this function on the same thread. __kmp_exit_lock is not
+  // recursive, so re-acquiring it below would hang the process instead of
+  // terminating it. Terminate directly on re-entry.
+  static KMP_THREAD_LOCAL bool aborting = false;
+  if (aborting)
+    abort();
+  aborting = true;
+
   // Later threads may stall here, but that's ok because abort() will kill them.
   __kmp_acquire_bootstrap_lock(&__kmp_exit_lock);
 
@@ -6918,6 +6927,13 @@ void __kmp_register_library_startup(void) {
 
 void __kmp_unregister_library(void) {
 
+  // The library can be torn down before it ever registered itself, e.g. when
+  // __kmp_abort_process() runs for a fatal error raised during environment
+  // parsing. There is nothing to unregister then, and __kmp_registration_str
+  // is still NULL, so the strcmp() below would dereference it.
+  if (__kmp_registration_flag == 0)
+    return;
+
   char *name = __kmp_reg_status_name();
   char *value = NULL;
 


        

From openmp-commits at lists.llvm.org  Mon Aug 17 00:22:56 2026
From: openmp-commits at lists.llvm.org (Robert Imschweiler via Openmp-commits)
Date: Mon, 17 Aug 2026 00:22:56 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [libomp] Fix hang when a fatal error is
 raised before library registration (PR #215988)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/215988@github.com>
Message-ID: <6a82b6d0.018c0331.51f41.102c@mx.google.com>


https://github.com/ro-i closed https://github.com/llvm/llvm-project/pull/215988

From openmp-commits at lists.llvm.org  Mon Aug 17 02:51:38 2026
From: openmp-commits at lists.llvm.org (Amit Tiwari via Openmp-commits)
Date: Mon, 17 Aug 2026 02:51:38 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [mlir] [openmp] [Clang][OpenMP]
 Canonicalize Intra-tiles in Loop Tiling (PR #191114)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/191114@github.com>
Message-ID: <6a82d9aa.160324b0.2834e3.5cda@mx.google.com>


https://github.com/loopacino updated https://github.com/llvm/llvm-project/pull/191114

>From cc69f5d3c05ae97a99cad8fa4f87aaffb6201063 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Apr 2026 07:07:16 -0400
Subject: [PATCH 01/19] nested-predicate

---
 clang/lib/Sema/SemaOpenMP.cpp | 167 ++++++++++++++++++++++++----------
 1 file changed, 121 insertions(+), 46 deletions(-)

diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 76bb0d38d428f..c8d75a0b333b4 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -15081,8 +15081,10 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
   // Create iteration variables for the generated loops.
   SmallVector<VarDecl *, 4> FloorIndVars;
   SmallVector<VarDecl *, 4> TileIndVars;
+  SmallVector<VarDecl *, 4> TileCntVars;
   FloorIndVars.resize(NumLoops);
   TileIndVars.resize(NumLoops);
+  TileCntVars.resize(NumLoops);
   for (unsigned I = 0; I < NumLoops; ++I) {
     OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
 
@@ -15102,27 +15104,101 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
       FloorIndVars[I] = FloorCntDecl;
     }
 
-    // Iteration variable for the tile (i.e. inner) loop.
+    // Logical iteration variable for the tile loop. Retains the meaning of
+    // the original logical iteration number (floor_iv + tile_cnt) so that
+    // LoopHelper.Updates can derive the original loop variable unchanged.
     {
-      std::string TileCntName =
+      std::string TileIVName =
           (Twine(".tile_") + llvm::utostr(I) + ".iv." + OrigVarName).str();
 
-      // Reuse the iteration variable created by checkOpenMPLoop. It is also
-      // used by the expressions to derive the original iteration variable's
-      // value from the logical iteration number.
-      auto *TileCntDecl = cast<VarDecl>(IterVarRef->getDecl());
-      TileCntDecl->setDeclName(
-          &SemaRef.PP.getIdentifierTable().get(TileCntName));
-      TileIndVars[I] = TileCntDecl;
+      auto *TileIVDecl = cast<VarDecl>(IterVarRef->getDecl());
+      TileIVDecl->setDeclName(&SemaRef.PP.getIdentifierTable().get(TileIVName));
+      TileIndVars[I] = TileIVDecl;
+    }
+
+    // Loop counter for the rectangular tile loop [0, TileSize).
+    {
+      std::string TileCntName =
+          (Twine(".tile.cnt.") + llvm::utostr(I) + ".iv." + OrigVarName).str();
+      VarDecl *TileCntDecl =
+          buildVarDecl(SemaRef, {}, CntTy, TileCntName, nullptr, OrigCntVar);
+      TileCntVars[I] = TileCntDecl;
     }
 
     addLoopPreInits(Context, LoopHelper, LoopStmts[I], OriginalInits[I],
                     PreInits);
+
+    // Declare the logical tile IV in PreInits so it is in scope for the
+    // entire loop nest (it will be assigned in each tile loop body).
+    Decl *TileIVDeclPtr = TileIndVars[I];
+    PreInits.push_back(new (Context) DeclStmt(
+        DeclGroupRef::Create(Context, &TileIVDeclPtr, 1), {}, {}));
   }
 
   // Once the original iteration values are set, append the innermost body.
   Stmt *Inner = Body;
 
+  // Build a combined validity predicate that guards the innermost body.
+  // For each tiled dimension, check that the logical iteration number
+  // (.tile.iv) is within the original trip count. This is required because the
+  // tile loop now has rectangular (constant) bounds and may overshoot on the
+  // remainder tile. The predicate is: .tile.iv.0 < N0 && .tile.iv.1 < N1 ...
+  //
+  // Optimization: if every dimension's trip count is a compile-time constant
+  // that is evenly divisible by the corresponding tile size (also a constant),
+  // then the remainder tile is empty and the predicate is trivially true.
+  {
+    bool PredicateNeeded = false;
+    for (unsigned I = 0; I < NumLoops; ++I) {
+      Expr *TSExpr = SizesClause->getSizesRefs()[I];
+      Expr *NExpr = LoopHelpers[I].NumIterations;
+      llvm::APSInt TileVal, TripVal;
+      bool TSConst =
+          !TSExpr->containsErrors() && TSExpr->isIntegerConstantExpr(Context);
+      bool NConst = NExpr->isIntegerConstantExpr(Context);
+      if (TSConst && NConst) {
+        Expr::EvalResult TSResult;
+        TSExpr->EvaluateAsInt(TSResult, Context);
+        TileVal = TSResult.Val.getInt();
+        Expr::EvalResult NResult;
+        NExpr->EvaluateAsInt(NResult, Context);
+        TripVal = NResult.Val.getInt();
+        if (TileVal.isStrictlyPositive() && (TripVal.srem(TileVal)).isZero())
+          continue;
+      }
+      PredicateNeeded = true;
+      break;
+    }
+
+    if (PredicateNeeded) {
+      Expr *CombinedPred = nullptr;
+      for (unsigned I = 0; I < NumLoops; ++I) {
+        auto *OrigCntVar = cast<DeclRefExpr>(LoopHelpers[I].Counters[0]);
+        QualType IVTy = LoopHelpers[I].NumIterations->getType();
+        Expr *TileIVRef = buildDeclRefExpr(SemaRef, TileIndVars[I], IVTy,
+                                           OrigCntVar->getExprLoc());
+        ExprResult DimPred =
+            SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(), BO_LT,
+                               TileIVRef, LoopHelpers[I].NumIterations);
+        if (!DimPred.isUsable())
+          return StmtError();
+        if (CombinedPred) {
+          ExprResult Combined =
+              SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(), BO_LAnd,
+                                 CombinedPred, DimPred.get());
+          if (!Combined.isUsable())
+            return StmtError();
+          CombinedPred = Combined.get();
+        } else {
+          CombinedPred = DimPred.get();
+        }
+      }
+      Inner = IfStmt::Create(
+          Context, SourceLocation(), IfStatementKind::Ordinary, nullptr,
+          nullptr, CombinedPred, SourceLocation(), SourceLocation(), Inner);
+    }
+  }
+
   auto MakeDimTileSize = [&SemaRef = this->SemaRef, &CopyTransformer, &Context,
                           SizesClause, CurScope](int I) -> Expr * {
     Expr *DimTileSizeExpr = SizesClause->getSizesRefs()[I];
@@ -15130,7 +15206,7 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     if (DimTileSizeExpr->containsErrors())
       return nullptr;
 
-    if (isa<ConstantExpr>(DimTileSizeExpr))
+    if (DimTileSizeExpr->isIntegerConstantExpr(Context))
       return AssertSuccess(CopyTransformer.TransformExpr(DimTileSizeExpr));
 
     // When the tile size is not a constant but a variable, it is possible to
@@ -15166,6 +15242,9 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
   };
 
   // Create tile loops from the inside to the outside.
+  // Each tile loop uses .tile.cnt as its counter with rectangular bounds
+  // [0, TileSize), and computes .tile.iv = .floor.iv + .tile.cnt to set
+  // the logical iteration number for LoopHelper.Updates.
   for (int I = NumLoops - 1; I >= 0; --I) {
     OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
     Expr *NumIterations = LoopHelper.NumIterations;
@@ -15176,70 +15255,65 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     // Commonly used variables. One of the constraints of an AST is that every
     // node object must appear at most once, hence we define a lambda that
     // creates a new AST node at every use.
+    auto MakeTileCntRef = [&SemaRef = this->SemaRef, &TileCntVars, I, IVTy,
+                           OrigCntVar]() {
+      return buildDeclRefExpr(SemaRef, TileCntVars[I], IVTy,
+                              OrigCntVar->getExprLoc());
+    };
     auto MakeTileIVRef = [&SemaRef = this->SemaRef, &TileIndVars, I, IVTy,
                           OrigCntVar]() {
       return buildDeclRefExpr(SemaRef, TileIndVars[I], IVTy,
                               OrigCntVar->getExprLoc());
     };
 
-    // For init-statement: auto .tile.iv = .floor.iv
+    // For init-statement: auto .tile.cnt = 0
     SemaRef.AddInitializerToDecl(
-        TileIndVars[I],
-        SemaRef
-            .DefaultLvalueConversion(
-                makeFloorIVRef(SemaRef, FloorIndVars, I, IVTy, OrigCntVar))
-            .get(),
+        TileCntVars[I],
+        SemaRef.ActOnIntegerConstant(LoopHelper.Init->getExprLoc(), 0).get(),
         /*DirectInit=*/false);
-    Decl *CounterDecl = TileIndVars[I];
+    Decl *CounterDecl = TileCntVars[I];
     StmtResult InitStmt = new (Context)
         DeclStmt(DeclGroupRef::Create(Context, &CounterDecl, 1),
                  OrigCntVar->getBeginLoc(), OrigCntVar->getEndLoc());
     if (!InitStmt.isUsable())
       return StmtError();
 
-    // For cond-expression:
-    //   .tile.iv < min(.floor.iv + DimTileSize, NumIterations)
+    // For cond-expression: .tile.cnt < DimTileSize  (rectangular bound)
     Expr *DimTileSize = MakeDimTileSize(I);
     if (!DimTileSize)
       return StmtError();
-    ExprResult EndOfTile = SemaRef.BuildBinOp(
-        CurScope, LoopHelper.Cond->getExprLoc(), BO_Add,
-        makeFloorIVRef(SemaRef, FloorIndVars, I, IVTy, OrigCntVar),
-        DimTileSize);
-    if (!EndOfTile.isUsable())
-      return StmtError();
-    ExprResult IsPartialTile =
-        SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
-                           NumIterations, EndOfTile.get());
-    if (!IsPartialTile.isUsable())
-      return StmtError();
-    ExprResult MinTileAndIterSpace = SemaRef.ActOnConditionalOp(
-        LoopHelper.Cond->getBeginLoc(), LoopHelper.Cond->getEndLoc(),
-        IsPartialTile.get(), NumIterations, EndOfTile.get());
-    if (!MinTileAndIterSpace.isUsable())
-      return StmtError();
     ExprResult CondExpr =
         SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
-                           MakeTileIVRef(), MinTileAndIterSpace.get());
+                           MakeTileCntRef(), DimTileSize);
     if (!CondExpr.isUsable())
       return StmtError();
 
-    // For incr-statement: ++.tile.iv
+    // For incr-statement: ++.tile.cnt
     ExprResult IncrStmt = SemaRef.BuildUnaryOp(
-        CurScope, LoopHelper.Inc->getExprLoc(), UO_PreInc, MakeTileIVRef());
+        CurScope, LoopHelper.Inc->getExprLoc(), UO_PreInc, MakeTileCntRef());
     if (!IncrStmt.isUsable())
       return StmtError();
 
-    // Statements to set the original iteration variable's value from the
-    // logical iteration number.
+    // Compute the logical iteration number:
+    //   .tile.iv = .floor.iv + .tile.cnt
+    ExprResult FloorPlusCnt = SemaRef.BuildBinOp(
+        CurScope, OrigCntVar->getExprLoc(), BO_Add,
+        makeFloorIVRef(SemaRef, FloorIndVars, I, IVTy, OrigCntVar),
+        MakeTileCntRef());
+    if (!FloorPlusCnt.isUsable())
+      return StmtError();
+    ExprResult TileIVAssign =
+        SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(), BO_Assign,
+                           MakeTileIVRef(), FloorPlusCnt.get());
+    if (!TileIVAssign.isUsable())
+      return StmtError();
+
     // Generated for loop is:
     // \code
-    // Original_for_init;
-    // for (auto .tile.iv = .floor.iv;
-    //      .tile.iv < min(.floor.iv + DimTileSize, NumIterations);
-    //      ++.tile.iv) {
-    //   Original_Body;
-    //   Original_counter_update;
+    // for (auto .tile.cnt = 0; .tile.cnt < DimTileSize; ++.tile.cnt) {
+    //   .tile.iv = .floor.iv + .tile.cnt;
+    //   Original_counter_update;  // derives orig var from .tile.iv
+    //   Inner;                    // predicated body or inner tile loops
     // }
     // \endcode
     // FIXME: If the innermost body is an loop itself, inserting these
@@ -15247,6 +15321,7 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     // for applying tiling again). If this is the case, sink the expressions
     // further into the inner loop.
     SmallVector<Stmt *, 4> BodyParts;
+    BodyParts.push_back(TileIVAssign.get());
     BodyParts.append(LoopHelper.Updates.begin(), LoopHelper.Updates.end());
     if (auto *SourceCXXFor = dyn_cast<CXXForRangeStmt>(LoopStmt))
       BodyParts.push_back(SourceCXXFor->getLoopVarStmt());

>From fbf1b6503562dedf0f6073263c2a9bc4c885b597 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Apr 2026 07:13:48 -0400
Subject: [PATCH 02/19] ir-builder

---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 64 ++++++++++++++---------
 1 file changed, 39 insertions(+), 25 deletions(-)

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index fc3812502fb2e..dba68179567e3 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -6984,7 +6984,7 @@ OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
   // Compute the trip counts of the floor loops.
   Builder.SetCurrentDebugLocation(DL);
   Builder.restoreIP(OutermostLoop->getPreheaderIP());
-  SmallVector<Value *, 4> FloorCompleteCount, FloorCount, FloorRems;
+  SmallVector<Value *, 4> FloorCount;
   for (int i = 0; i < NumLoops; ++i) {
     Value *TileSize = TileSizes[i];
     Value *OrigTripCount = OrigTripCounts[i];
@@ -7008,10 +7008,7 @@ OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
         Builder.CreateAdd(FloorCompleteTripCount, FloorTripOverflow,
                           "omp_floor" + Twine(i) + ".tripcount", true);
 
-    // Remember some values for later use.
-    FloorCompleteCount.push_back(FloorCompleteTripCount);
     FloorCount.push_back(FloorTripCount);
-    FloorRems.push_back(FloorTripRem);
   }
 
   // Generate the new loop nest, from the outermost to the innermost.
@@ -7055,23 +7052,10 @@ OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
 
   EmbeddNewLoops(FloorCount, "floor");
 
-  // Within the innermost floor loop, emit the code that computes the tile
-  // sizes.
-  Builder.SetInsertPoint(Enter->getTerminator());
-  SmallVector<Value *, 4> TileCounts;
-  for (int i = 0; i < NumLoops; ++i) {
-    CanonicalLoopInfo *FloorLoop = Result[i];
-    Value *TileSize = TileSizes[i];
-
-    Value *FloorIsEpilogue =
-        Builder.CreateICmpEQ(FloorLoop->getIndVar(), FloorCompleteCount[i]);
-    Value *TileTripCount =
-        Builder.CreateSelect(FloorIsEpilogue, FloorRems[i], TileSize);
-
-    TileCounts.push_back(TileTripCount);
-  }
-
-  // Create the tile loops.
+  // Create the tile loops with rectangular (constant) trip counts equal to
+  // the tile sizes. A validity predicate in the body guards against
+  // out-of-bounds iterations in the remainder tile.
+  SmallVector<Value *, 4> TileCounts(TileSizes.begin(), TileSizes.end());
   EmbeddNewLoops(TileCounts, "tile");
 
   // Insert the inbetween code into the body.
@@ -7100,6 +7084,7 @@ OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
   // Replace the original induction variable with an induction variable computed
   // from the tile and floor induction variables.
   Builder.restoreIP(Result.back()->getBodyIP());
+  SmallVector<Value *, 4> ShiftValues;
   for (int i = 0; i < NumLoops; ++i) {
     CanonicalLoopInfo *FloorLoop = Result[i];
     CanonicalLoopInfo *TileLoop = Result[NumLoops + i];
@@ -7111,18 +7096,47 @@ OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
     Value *Shift =
         Builder.CreateAdd(Scale, TileLoop->getIndVar(), {}, /*HasNUW=*/true);
     OrigIndVar->replaceAllUsesWith(Shift);
+    ShiftValues.push_back(Shift);
+  }
+
+  // Build a validity predicate: for each tiled dimension, check that the
+  // computed original index is within the original trip count. This guards
+  // against executing out-of-bounds iterations in the remainder tile.
+  Value *Pred = nullptr;
+  for (int i = 0; i < NumLoops; ++i) {
+    Value *DimPred = Builder.CreateICmpULT(ShiftValues[i], OrigTripCounts[i],
+                                           "omp_tile" + Twine(i) + ".inbounds");
+    Pred = Pred ? Builder.CreateAnd(Pred, DimPred) : DimPred;
   }
 
+  // Insert the conditional guard: split the body flow so that out-of-bounds
+  // iterations skip directly to a merge block before the tile latch.
+  BasicBlock *TileBodyBB = Builder.GetInsertBlock();
+  Instruction *BodyTerm = TileBodyBB->getTerminator();
+  BasicBlock *BodySucc = cast<BranchInst>(BodyTerm)->getSuccessor(0);
+  BasicBlock *TileLatch = Result.back()->getLatch();
+
+  BasicBlock *MergeBB =
+      BasicBlock::Create(F->getContext(), "omp_tile.body.merge", F, TileLatch);
+  Builder.SetInsertPoint(MergeBB);
+  Builder.CreateBr(TileLatch);
+
+  // Redirect the body chain's exit from the tile latch to the merge block.
+  for (BasicBlock *PredBB : llvm::make_early_inc_range(predecessors(TileLatch)))
+    if (PredBB != MergeBB)
+      PredBB->getTerminator()->replaceUsesOfWith(TileLatch, MergeBB);
+
+  // Replace the tile body's unconditional branch with a conditional one.
+  BodyTerm->eraseFromParent();
+  Builder.SetInsertPoint(TileBodyBB);
+  Builder.CreateCondBr(Pred, BodySucc, MergeBB);
+
   // Remove unused parts of the original loops.
   removeUnusedBlocksFromParent(OldControlBBs);
 
   for (CanonicalLoopInfo *L : Loops)
     L->invalidate();
 
-#ifndef NDEBUG
-  for (CanonicalLoopInfo *GenL : Result)
-    GenL->assertOK();
-#endif
   return Result;
 }
 

>From 28270e10df2128b1ccd2ad673ebe85c436c41b55 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Thu, 9 Apr 2026 02:27:14 -0400
Subject: [PATCH 03/19] UncondBrInst

---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index dba68179567e3..a38ecede63376 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -7113,7 +7113,7 @@ OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
   // iterations skip directly to a merge block before the tile latch.
   BasicBlock *TileBodyBB = Builder.GetInsertBlock();
   Instruction *BodyTerm = TileBodyBB->getTerminator();
-  BasicBlock *BodySucc = cast<BranchInst>(BodyTerm)->getSuccessor(0);
+  BasicBlock *BodySucc = cast<UncondBrInst>(BodyTerm)->getSuccessor(0);
   BasicBlock *TileLatch = Result.back()->getLatch();
 
   BasicBlock *MergeBB =

>From 08fa07fe6c0a451f05fd36ec9a019518b22e2a0d Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Sat, 11 Apr 2026 11:05:18 -0400
Subject: [PATCH 04/19] mlir-tests-rect-update

---
 .../test/Target/LLVMIR/openmp-cli-tile01.mlir |  18 +--
 .../test/Target/LLVMIR/openmp-cli-tile02.mlir |  30 +++--
 .../test/Target/LLVMIR/openmp-cli-tile03.mlir | 120 +++++++++---------
 3 files changed, 89 insertions(+), 79 deletions(-)

diff --git a/mlir/test/Target/LLVMIR/openmp-cli-tile01.mlir b/mlir/test/Target/LLVMIR/openmp-cli-tile01.mlir
index 0d559b69a3ad1..948a9815851a5 100644
--- a/mlir/test/Target/LLVMIR/openmp-cli-tile01.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-cli-tile01.mlir
@@ -38,8 +38,6 @@ llvm.func @tile_trivial_loop(%baseptr: !llvm.ptr, %tc: i32, %ts: i32) -> () {
 // CHECK-NEXT:    br i1 %[[OMP_FLOOR0_CMP:.+]], label %[[OMP_FLOOR0_BODY:.+]], label %[[OMP_FLOOR0_EXIT:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_FLOOR0_BODY]]:
-// CHECK-NEXT:    %[[TMP8:.+]] = icmp eq i32 %[[OMP_FLOOR0_IV:.+]], %[[TMP4:.+]]
-// CHECK-NEXT:    %[[TMP9:.+]] = select i1 %[[TMP8:.+]], i32 %[[TMP5:.+]], i32 %[[TMP2:.+]]
 // CHECK-NEXT:    br label %[[OMP_TILE0_PREHEADER:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_PREHEADER]]:
@@ -50,23 +48,27 @@ llvm.func @tile_trivial_loop(%baseptr: !llvm.ptr, %tc: i32, %ts: i32) -> () {
 // CHECK-NEXT:    br label %[[OMP_TILE0_COND:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_COND]]:
-// CHECK-NEXT:    %[[OMP_TILE0_CMP:.+]] = icmp ult i32 %[[OMP_TILE0_IV:.+]], %[[TMP9:.+]]
+// CHECK-NEXT:    %[[OMP_TILE0_CMP:.+]] = icmp ult i32 %[[OMP_TILE0_IV:.+]], %[[TMP2:.+]]
 // CHECK-NEXT:    br i1 %[[OMP_TILE0_CMP:.+]], label %[[OMP_TILE0_BODY:.+]], label %[[OMP_TILE0_EXIT:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_BODY]]:
-// CHECK-NEXT:    %[[TMP10:.+]] = mul nuw i32 %[[TMP2:.+]], %[[OMP_FLOOR0_IV:.+]]
-// CHECK-NEXT:    %[[TMP11:.+]] = add nuw i32 %[[TMP10:.+]], %[[OMP_TILE0_IV:.+]]
-// CHECK-NEXT:    br label %[[OMP_OMP_LOOP_BODY:.+]]
+// CHECK-NEXT:    %[[TMP8:.+]] = mul nuw i32 %[[TMP2:.+]], %[[OMP_FLOOR0_IV:.+]]
+// CHECK-NEXT:    %[[TMP9:.+]] = add nuw i32 %[[TMP8:.+]], %[[OMP_TILE0_IV:.+]]
+// CHECK-NEXT:    %[[OMP_TILE0_INBOUNDS:.+]] = icmp ult i32 %[[TMP9:.+]], %[[TMP1:.+]]
+// CHECK-NEXT:    br i1 %[[OMP_TILE0_INBOUNDS:.+]], label %[[OMP_OMP_LOOP_BODY:.+]], label %[[OMP_TILE_BODY_MERGE:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_OMP_LOOP_BODY]]:
 // CHECK-NEXT:    br label %[[OMP_LOOP_REGION:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_LOOP_REGION]]:
-// CHECK-NEXT:    %[[TMP12:.+]] = getelementptr inbounds float, ptr %[[TMP0:.+]], i32 %[[TMP11:.+]]
-// CHECK-NEXT:    store float 4.200000e+01, ptr %[[TMP12:.+]], align 4
+// CHECK-NEXT:    %[[TMP10:.+]] = getelementptr inbounds float, ptr %[[TMP0:.+]], i32 %[[TMP9:.+]]
+// CHECK-NEXT:    store float 4.200000e+01, ptr %[[TMP10:.+]], align 4
 // CHECK-NEXT:    br label %[[OMP_REGION_CONT:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_REGION_CONT]]:
+// CHECK-NEXT:    br label %[[OMP_TILE_BODY_MERGE:.+]]
+// CHECK-EMPTY:
+// CHECK-NEXT:  [[OMP_TILE_BODY_MERGE]]:
 // CHECK-NEXT:    br label %[[OMP_TILE0_INC:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_INC]]:
diff --git a/mlir/test/Target/LLVMIR/openmp-cli-tile02.mlir b/mlir/test/Target/LLVMIR/openmp-cli-tile02.mlir
index 22c2973164159..59ffa50037f4b 100644
--- a/mlir/test/Target/LLVMIR/openmp-cli-tile02.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-cli-tile02.mlir
@@ -79,10 +79,6 @@ llvm.func @tile_2d_loop(%baseptr: !llvm.ptr, %tc1: i32, %tc2: i32, %ts1: i32, %t
 // CHECK-NEXT:    br i1 %[[OMP_FLOOR1_CMP:.+]], label %[[OMP_FLOOR1_BODY:.+]], label %[[OMP_FLOOR1_EXIT:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_FLOOR1_BODY]]:
-// CHECK-NEXT:    %[[TMP14:.+]] = icmp eq i32 %[[OMP_FLOOR0_IV:.+]], %[[TMP6:.+]]
-// CHECK-NEXT:    %[[TMP15:.+]] = select i1 %[[TMP14:.+]], i32 %[[TMP7:.+]], i32 %[[TMP3:.+]]
-// CHECK-NEXT:    %[[TMP16:.+]] = icmp eq i32 %[[OMP_FLOOR1_IV:.+]], %[[TMP10:.+]]
-// CHECK-NEXT:    %[[TMP17:.+]] = select i1 %[[TMP16:.+]], i32 %[[TMP11:.+]], i32 %[[TMP4:.+]]
 // CHECK-NEXT:    br label %[[OMP_TILE0_PREHEADER:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_PREHEADER]]:
@@ -93,7 +89,7 @@ llvm.func @tile_2d_loop(%baseptr: !llvm.ptr, %tc1: i32, %tc2: i32, %ts1: i32, %t
 // CHECK-NEXT:    br label %[[OMP_TILE0_COND:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_COND]]:
-// CHECK-NEXT:    %[[OMP_TILE0_CMP:.+]] = icmp ult i32 %[[OMP_TILE0_IV:.+]], %[[TMP15:.+]]
+// CHECK-NEXT:    %[[OMP_TILE0_CMP:.+]] = icmp ult i32 %[[OMP_TILE0_IV:.+]], %[[TMP3:.+]]
 // CHECK-NEXT:    br i1 %[[OMP_TILE0_CMP:.+]], label %[[OMP_TILE0_BODY:.+]], label %[[OMP_TILE0_EXIT:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_BODY]]:
@@ -107,26 +103,32 @@ llvm.func @tile_2d_loop(%baseptr: !llvm.ptr, %tc1: i32, %tc2: i32, %ts1: i32, %t
 // CHECK-NEXT:    br label %[[OMP_TILE1_COND:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE1_COND]]:
-// CHECK-NEXT:    %[[OMP_TILE1_CMP:.+]] = icmp ult i32 %[[OMP_TILE1_IV:.+]], %[[TMP17:.+]]
+// CHECK-NEXT:    %[[OMP_TILE1_CMP:.+]] = icmp ult i32 %[[OMP_TILE1_IV:.+]], %[[TMP4:.+]]
 // CHECK-NEXT:    br i1 %[[OMP_TILE1_CMP:.+]], label %[[OMP_TILE1_BODY:.+]], label %[[OMP_TILE1_EXIT:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE1_BODY]]:
-// CHECK-NEXT:    %[[TMP18:.+]] = mul nuw i32 %[[TMP3:.+]], %[[OMP_FLOOR0_IV:.+]]
-// CHECK-NEXT:    %[[TMP19:.+]] = add nuw i32 %[[TMP18:.+]], %[[OMP_TILE0_IV:.+]]
-// CHECK-NEXT:    %[[TMP20:.+]] = mul nuw i32 %[[TMP4:.+]], %[[OMP_FLOOR1_IV:.+]]
-// CHECK-NEXT:    %[[TMP21:.+]] = add nuw i32 %[[TMP20:.+]], %[[OMP_TILE1_IV:.+]]
-// CHECK-NEXT:    br label %[[OMP_OMP_LOOP_BODY:.+]]
+// CHECK-NEXT:    %[[TMP14:.+]] = mul nuw i32 %[[TMP3:.+]], %[[OMP_FLOOR0_IV:.+]]
+// CHECK-NEXT:    %[[TMP15:.+]] = add nuw i32 %[[TMP14:.+]], %[[OMP_TILE0_IV:.+]]
+// CHECK-NEXT:    %[[TMP16:.+]] = mul nuw i32 %[[TMP4:.+]], %[[OMP_FLOOR1_IV:.+]]
+// CHECK-NEXT:    %[[TMP17:.+]] = add nuw i32 %[[TMP16:.+]], %[[OMP_TILE1_IV:.+]]
+// CHECK-NEXT:    %[[OMP_TILE0_INBOUNDS:.+]] = icmp ult i32 %[[TMP15:.+]], %[[TMP1:.+]]
+// CHECK-NEXT:    %[[OMP_TILE1_INBOUNDS:.+]] = icmp ult i32 %[[TMP17:.+]], %[[TMP2:.+]]
+// CHECK-NEXT:    %[[OMP_TILE_PRED:.+]] = and i1 %[[OMP_TILE0_INBOUNDS:.+]], %[[OMP_TILE1_INBOUNDS:.+]]
+// CHECK-NEXT:    br i1 %[[OMP_TILE_PRED:.+]], label %[[OMP_OMP_LOOP_BODY:.+]], label %[[OMP_TILE_BODY_MERGE:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_OMP_LOOP_BODY4]]:
 // CHECK-NEXT:    br label %[[OMP_LOOP_REGION12:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_LOOP_REGION12]]:
-// CHECK-NEXT:    %[[TMP22:.+]] = add i32 %[[TMP19:.+]], %[[TMP21:.+]]
-// CHECK-NEXT:    %[[TMP23:.+]] = getelementptr inbounds float, ptr %[[TMP0:.+]], i32 %[[TMP22:.+]]
-// CHECK-NEXT:    store float 4.200000e+01, ptr %[[TMP23:.+]], align 4
+// CHECK-NEXT:    %[[TMP18:.+]] = add i32 %[[TMP15:.+]], %[[TMP17:.+]]
+// CHECK-NEXT:    %[[TMP19:.+]] = getelementptr inbounds float, ptr %[[TMP0:.+]], i32 %[[TMP18:.+]]
+// CHECK-NEXT:    store float 4.200000e+01, ptr %[[TMP19:.+]], align 4
 // CHECK-NEXT:    br label %[[OMP_REGION_CONT11:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_REGION_CONT11]]:
+// CHECK-NEXT:    br label %[[OMP_TILE_BODY_MERGE:.+]]
+// CHECK-EMPTY:
+// CHECK-NEXT:  [[OMP_TILE_BODY_MERGE]]:
 // CHECK-NEXT:    br label %[[OMP_TILE1_INC:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE1_INC]]:
diff --git a/mlir/test/Target/LLVMIR/openmp-cli-tile03.mlir b/mlir/test/Target/LLVMIR/openmp-cli-tile03.mlir
index 0747ff8909362..fcc37af2ca8f3 100644
--- a/mlir/test/Target/LLVMIR/openmp-cli-tile03.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-cli-tile03.mlir
@@ -56,8 +56,6 @@ llvm.func @tile_composition(%baseptr: !llvm.ptr, %tc: i32, %ts: i32, %grid_ts: i
 // CHECK-NEXT:    br i1 %[[OMP_FLOOR0_CMP10:.+]], label %[[OMP_FLOOR0_BODY5:.+]], label %[[OMP_FLOOR0_EXIT7:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_FLOOR0_BODY5]]:
-// CHECK-NEXT:    %[[TMP14:.+]] = icmp eq i32 %[[OMP_FLOOR0_IV9:.+]], %[[TMP10:.+]]
-// CHECK-NEXT:    %[[TMP15:.+]] = select i1 %[[TMP14:.+]], i32 %[[TMP11:.+]], i32 %[[TMP3:.+]]
 // CHECK-NEXT:    br label %[[OMP_TILE0_PREHEADER12:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_PREHEADER12]]:
@@ -68,83 +66,88 @@ llvm.func @tile_composition(%baseptr: !llvm.ptr, %tc: i32, %ts: i32, %grid_ts: i
 // CHECK-NEXT:    br label %[[OMP_TILE0_COND14:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_COND14]]:
-// CHECK-NEXT:    %[[OMP_TILE0_CMP20:.+]] = icmp ult i32 %[[OMP_TILE0_IV19:.+]], %[[TMP15:.+]]
+// CHECK-NEXT:    %[[OMP_TILE0_CMP20:.+]] = icmp ult i32 %[[OMP_TILE0_IV19:.+]], %[[TMP3:.+]]
 // CHECK-NEXT:    br i1 %[[OMP_TILE0_CMP20:.+]], label %[[OMP_TILE0_BODY15:.+]], label %[[OMP_TILE0_EXIT17:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_BODY15]]:
-// CHECK-NEXT:    %[[TMP16:.+]] = mul nuw i32 %[[TMP3:.+]], %[[OMP_FLOOR0_IV9:.+]]
-// CHECK-NEXT:    %[[TMP17:.+]] = add nuw i32 %[[TMP16:.+]], %[[OMP_TILE0_IV19:.+]]
-// CHECK-NEXT:    br label %[[OMP_FLOOR0_BODY:.+]]
-// CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_FLOOR0_BODY]]:
-// CHECK-NEXT:    %[[TMP18:.+]] = icmp eq i32 %[[TMP17:.+]], %[[TMP6:.+]]
-// CHECK-NEXT:    %[[TMP19:.+]] = select i1 %[[TMP18:.+]], i32 %[[TMP7:.+]], i32 %[[TMP2:.+]]
-// CHECK-NEXT:    br label %[[OMP_TILE0_PREHEADER:.+]]
-// CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_PREHEADER]]:
-// CHECK-NEXT:    %[[TMP20:.+]] = udiv i32 %[[TMP19:.+]], %[[TMP4:.+]]
-// CHECK-NEXT:    %[[TMP21:.+]] = urem i32 %[[TMP19:.+]], %[[TMP4:.+]]
-// CHECK-NEXT:    %[[TMP22:.+]] = icmp ne i32 %[[TMP21:.+]], 0
-// CHECK-NEXT:    %[[TMP23:.+]] = zext i1 %[[TMP22:.+]] to i32
-// CHECK-NEXT:    %[[OMP_FLOOR0_TRIPCOUNT22:.+]] = add nuw i32 %[[TMP20:.+]], %[[TMP23:.+]]
-// CHECK-NEXT:    br label %[[OMP_FLOOR0_PREHEADER23:.+]]
-// CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_FLOOR0_PREHEADER23]]:
+// CHECK-NEXT:    %[[TMP14:.+]] = mul nuw i32 %[[TMP3:.+]], %[[OMP_FLOOR0_IV9:.+]]
+// CHECK-NEXT:    %[[TMP15:.+]] = add nuw i32 %[[TMP14:.+]], %[[OMP_TILE0_IV19:.+]]
+// CHECK-NEXT:    %[[OMP_TILE0_INBOUNDS22:.+]] = icmp ult i32 %[[TMP15:.+]], %[[OMP_FLOOR0_TRIPCOUNT:.+]]
+// CHECK-NEXT:    br i1 %[[OMP_TILE0_INBOUNDS22:.+]], label %[[OMP_FLOOR_INNER_ENTRY:.+]], label %[[OMP_TILE_BODY_MERGE23:.+]]
+// CHECK-EMPTY:
+// CHECK-NEXT:  [[OMP_FLOOR_INNER_ENTRY]]:
+// CHECK-NEXT:    br label %[[OMP_TILE0_PREHEADER_INNER:.+]]
+// CHECK-EMPTY:
+// CHECK-NEXT:  [[OMP_TILE0_PREHEADER_INNER]]:
+// CHECK-NEXT:    %[[TMP16:.+]] = udiv i32 %[[TMP2:.+]], %[[TMP4:.+]]
+// CHECK-NEXT:    %[[TMP17:.+]] = urem i32 %[[TMP2:.+]], %[[TMP4:.+]]
+// CHECK-NEXT:    %[[TMP18:.+]] = icmp ne i32 %[[TMP17:.+]], 0
+// CHECK-NEXT:    %[[TMP19:.+]] = zext i1 %[[TMP18:.+]] to i32
+// CHECK-NEXT:    %[[OMP_FLOOR0_TRIPCOUNT24:.+]] = add nuw i32 %[[TMP16:.+]], %[[TMP19:.+]]
+// CHECK-NEXT:    br label %[[OMP_FLOOR0_PREHEADER25:.+]]
+// CHECK-EMPTY:
+// CHECK-NEXT:  [[OMP_FLOOR0_PREHEADER25]]:
 // CHECK-NEXT:    br label %[[OMP_FLOOR0_HEADER:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_FLOOR0_HEADER]]:
-// CHECK-NEXT:    %[[OMP_FLOOR0_IV:.+]] = phi i32 [ 0, %[[OMP_FLOOR0_PREHEADER23:.+]] ], [ %[[OMP_FLOOR0_NEXT:.+]], %[[OMP_FLOOR0_INC:.+]] ]
+// CHECK-NEXT:    %[[OMP_FLOOR0_IV:.+]] = phi i32 [ 0, %[[OMP_FLOOR0_PREHEADER25:.+]] ], [ %[[OMP_FLOOR0_NEXT:.+]], %[[OMP_FLOOR0_INC:.+]] ]
 // CHECK-NEXT:    br label %[[OMP_FLOOR0_COND:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_FLOOR0_COND]]:
-// CHECK-NEXT:    %[[OMP_FLOOR0_CMP:.+]] = icmp ult i32 %[[OMP_FLOOR0_IV:.+]], %[[OMP_FLOOR0_TRIPCOUNT22:.+]]
-// CHECK-NEXT:    br i1 %[[OMP_FLOOR0_CMP:.+]], label %[[OMP_FLOOR0_BODY24:.+]], label %[[OMP_FLOOR0_EXIT:.+]]
+// CHECK-NEXT:    %[[OMP_FLOOR0_CMP:.+]] = icmp ult i32 %[[OMP_FLOOR0_IV:.+]], %[[OMP_FLOOR0_TRIPCOUNT24:.+]]
+// CHECK-NEXT:    br i1 %[[OMP_FLOOR0_CMP:.+]], label %[[OMP_FLOOR0_BODY26:.+]], label %[[OMP_FLOOR0_EXIT:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_FLOOR0_BODY24]]:
-// CHECK-NEXT:    %[[TMP24:.+]] = icmp eq i32 %[[OMP_FLOOR0_IV:.+]], %[[TMP20:.+]]
-// CHECK-NEXT:    %[[TMP25:.+]] = select i1 %[[TMP24:.+]], i32 %[[TMP21:.+]], i32 %[[TMP4:.+]]
-// CHECK-NEXT:    br label %[[OMP_TILE0_PREHEADER26:.+]]
+// CHECK-NEXT:  [[OMP_FLOOR0_BODY26]]:
+// CHECK-NEXT:    br label %[[OMP_TILE0_PREHEADER28:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_PREHEADER26]]:
-// CHECK-NEXT:    br label %[[OMP_TILE0_HEADER27:.+]]
+// CHECK-NEXT:  [[OMP_TILE0_PREHEADER28]]:
+// CHECK-NEXT:    br label %[[OMP_TILE0_HEADER29:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_HEADER27]]:
-// CHECK-NEXT:    %[[OMP_TILE0_IV33:.+]] = phi i32 [ 0, %[[OMP_TILE0_PREHEADER26:.+]] ], [ %[[OMP_TILE0_NEXT35:.+]], %[[OMP_TILE0_INC30:.+]] ]
-// CHECK-NEXT:    br label %[[OMP_TILE0_COND28:.+]]
+// CHECK-NEXT:  [[OMP_TILE0_HEADER29]]:
+// CHECK-NEXT:    %[[OMP_TILE0_IV35:.+]] = phi i32 [ 0, %[[OMP_TILE0_PREHEADER28:.+]] ], [ %[[OMP_TILE0_NEXT37:.+]], %[[OMP_TILE0_INC32:.+]] ]
+// CHECK-NEXT:    br label %[[OMP_TILE0_COND30:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_COND28]]:
-// CHECK-NEXT:    %[[OMP_TILE0_CMP34:.+]] = icmp ult i32 %[[OMP_TILE0_IV33:.+]], %[[TMP25:.+]]
-// CHECK-NEXT:    br i1 %[[OMP_TILE0_CMP34:.+]], label %[[OMP_TILE0_BODY29:.+]], label %[[OMP_TILE0_EXIT31:.+]]
+// CHECK-NEXT:  [[OMP_TILE0_COND30]]:
+// CHECK-NEXT:    %[[OMP_TILE0_CMP36:.+]] = icmp ult i32 %[[OMP_TILE0_IV35:.+]], %[[TMP4:.+]]
+// CHECK-NEXT:    br i1 %[[OMP_TILE0_CMP36:.+]], label %[[OMP_TILE0_BODY31:.+]], label %[[OMP_TILE0_EXIT33:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_BODY29]]:
-// CHECK-NEXT:    %[[TMP26:.+]] = mul nuw i32 %[[TMP4:.+]], %[[OMP_FLOOR0_IV:.+]]
-// CHECK-NEXT:    %[[TMP27:.+]] = add nuw i32 %[[TMP26:.+]], %[[OMP_TILE0_IV33:.+]]
-// CHECK-NEXT:    br label %[[OMP_TILE0_BODY:.+]]
+// CHECK-NEXT:  [[OMP_TILE0_BODY31]]:
+// CHECK-NEXT:    %[[TMP20:.+]] = mul nuw i32 %[[TMP4:.+]], %[[OMP_FLOOR0_IV:.+]]
+// CHECK-NEXT:    %[[TMP21:.+]] = add nuw i32 %[[TMP20:.+]], %[[OMP_TILE0_IV35:.+]]
+// CHECK-NEXT:    %[[OMP_TILE0_INBOUNDS38:.+]] = icmp ult i32 %[[TMP21:.+]], %[[TMP2:.+]]
+// CHECK-NEXT:    br i1 %[[OMP_TILE0_INBOUNDS38:.+]], label %[[OMP_TILE0_BODY_STORE:.+]], label %[[OMP_TILE_BODY_MERGE39:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_BODY]]:
-// CHECK-NEXT:    %[[TMP28:.+]] = mul nuw i32 %[[TMP2:.+]], %[[TMP17:.+]]
-// CHECK-NEXT:    %[[TMP29:.+]] = add nuw i32 %[[TMP28:.+]], %[[TMP27:.+]]
-// CHECK-NEXT:    br label %[[OMP_OMP_LOOP_BODY:.+]]
+// CHECK-NEXT:  [[OMP_TILE0_BODY_STORE]]:
+// CHECK-NEXT:    %[[TMP22:.+]] = mul nuw i32 %[[TMP2:.+]], %[[TMP15:.+]]
+// CHECK-NEXT:    %[[TMP23:.+]] = add nuw i32 %[[TMP22:.+]], %[[TMP21:.+]]
+// CHECK-NEXT:    %[[OMP_TILE0_INBOUNDS:.+]] = icmp ult i32 %[[TMP23:.+]], %[[TMP1:.+]]
+// CHECK-NEXT:    br i1 %[[OMP_TILE0_INBOUNDS:.+]], label %[[OMP_OMP_LOOP_BODY:.+]], label %[[OMP_TILE_BODY_MERGE:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_OMP_LOOP_BODY]]:
 // CHECK-NEXT:    br label %[[OMP_LOOP_REGION:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_LOOP_REGION]]:
-// CHECK-NEXT:    %[[TMP30:.+]] = getelementptr inbounds float, ptr %[[TMP0:.+]], i32 %[[TMP29:.+]]
-// CHECK-NEXT:    store float 4.200000e+01, ptr %[[TMP30:.+]], align 4
+// CHECK-NEXT:    %[[TMP24:.+]] = getelementptr inbounds float, ptr %[[TMP0:.+]], i32 %[[TMP23:.+]]
+// CHECK-NEXT:    store float 4.200000e+01, ptr %[[TMP24:.+]], align 4
 // CHECK-NEXT:    br label %[[OMP_REGION_CONT:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_REGION_CONT]]:
-// CHECK-NEXT:    br label %[[OMP_TILE0_INC30:.+]]
+// CHECK-NEXT:    br label %[[OMP_TILE_BODY_MERGE:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_INC30]]:
-// CHECK-NEXT:    %[[OMP_TILE0_NEXT35:.+]] = add nuw i32 %[[OMP_TILE0_IV33:.+]], 1
-// CHECK-NEXT:    br label %[[OMP_TILE0_HEADER27:.+]]
+// CHECK-NEXT:  [[OMP_TILE_BODY_MERGE]]:
+// CHECK-NEXT:    br label %[[OMP_TILE_BODY_MERGE39:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_EXIT31]]:
-// CHECK-NEXT:    br label %[[OMP_TILE0_AFTER32:.+]]
+// CHECK-NEXT:  [[OMP_TILE_BODY_MERGE39]]:
+// CHECK-NEXT:    br label %[[OMP_TILE0_INC32:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_AFTER32]]:
+// CHECK-NEXT:  [[OMP_TILE0_INC32]]:
+// CHECK-NEXT:    %[[OMP_TILE0_NEXT37:.+]] = add nuw i32 %[[OMP_TILE0_IV35:.+]], 1
+// CHECK-NEXT:    br label %[[OMP_TILE0_HEADER29:.+]]
+// CHECK-EMPTY:
+// CHECK-NEXT:  [[OMP_TILE0_EXIT33]]:
+// CHECK-NEXT:    br label %[[OMP_TILE0_AFTER34:.+]]
+// CHECK-EMPTY:
+// CHECK-NEXT:  [[OMP_TILE0_AFTER34]]:
 // CHECK-NEXT:    br label %[[OMP_FLOOR0_INC:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_FLOOR0_INC]]:
@@ -152,12 +155,15 @@ llvm.func @tile_composition(%baseptr: !llvm.ptr, %tc: i32, %ts: i32, %grid_ts: i
 // CHECK-NEXT:    br label %[[OMP_FLOOR0_HEADER:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_FLOOR0_EXIT]]:
-// CHECK-NEXT:    br label %[[OMP_FLOOR0_AFTER25:.+]]
+// CHECK-NEXT:    br label %[[OMP_FLOOR0_AFTER27:.+]]
+// CHECK-EMPTY:
+// CHECK-NEXT:  [[OMP_FLOOR0_AFTER27]]:
+// CHECK-NEXT:    br label %[[OMP_TILE0_AFTER_INNER:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_FLOOR0_AFTER25]]:
-// CHECK-NEXT:    br label %[[OMP_TILE0_AFTER:.+]]
+// CHECK-NEXT:  [[OMP_TILE0_AFTER_INNER]]:
+// CHECK-NEXT:    br label %[[OMP_TILE_BODY_MERGE23:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_AFTER]]:
+// CHECK-NEXT:  [[OMP_TILE_BODY_MERGE23]]:
 // CHECK-NEXT:    br label %[[OMP_TILE0_INC16:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_INC16]]:

>From c44c2822377ae53b790998dcb333cdf4cf3640b6 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Sat, 11 Apr 2026 11:12:42 -0400
Subject: [PATCH 05/19] tile-codegen

---
 clang/test/OpenMP/tile_codegen.cpp            | 2504 ++++++++---------
 .../OpenMP/tile_codegen_for_dependent.cpp     |  308 +-
 clang/test/OpenMP/tile_codegen_tile_for.cpp   |  417 ++-
 clang/test/OpenMP/tile_messages.cpp           |    2 +-
 4 files changed, 1439 insertions(+), 1792 deletions(-)

diff --git a/clang/test/OpenMP/tile_codegen.cpp b/clang/test/OpenMP/tile_codegen.cpp
index 85b004dc73f4f..1fd02d74a1209 100644
--- a/clang/test/OpenMP/tile_codegen.cpp
+++ b/clang/test/OpenMP/tile_codegen.cpp
@@ -56,7 +56,7 @@ extern "C" void foo4() {
 
 
 extern "C" void foo5() {
-#pragma omp for collapse(3)
+#pragma omp for collapse(2)
 #pragma omp tile sizes(5)
   for (int i = 7; i < 17; i += 3)
     for (int j = 7; j < 17; j += 3)
@@ -136,8 +136,9 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[I2:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[I:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
@@ -149,50 +150,47 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END12:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END11:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK1:       for.cond4:
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 5
-// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
-// CHECK1-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK1:       cond.true:
-// CHECK1-NEXT:    br label [[COND_END:%.*]]
-// CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP4]], 5
-// CHECK1-NEXT:    br label [[COND_END]]
-// CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
-// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body8:
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
-// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[I2]], align 8
-// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[TMP6]], align 4
-// CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[I2]], align 8
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP1]], 5
+// CHECK1-NEXT:    br i1 [[CMP5]], label [[FOR_BODY6:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body6:
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
+// CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP4]], 3
+// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
+// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[TMP5]], align 4
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP8:%.*]] = icmp slt i32 [[TMP6]], 4
+// CHECK1-NEXT:    br i1 [[CMP8]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK1:       if.then:
+// CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2]], !align [[META3]]
 // CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
 // CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP8]])
+// CHECK1-NEXT:    br label [[IF_END]]
+// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP9]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP3:![0-9]+]]
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP4:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC10:%.*]]
-// CHECK1:       for.inc10:
+// CHECK1-NEXT:    br label [[FOR_INC9:%.*]]
+// CHECK1:       for.inc9:
 // CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP10]], 5
-// CHECK1-NEXT:    store i32 [[ADD11]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP5:![0-9]+]]
-// CHECK1:       for.end12:
+// CHECK1-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP10]], 5
+// CHECK1-NEXT:    store i32 [[ADD10]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
+// CHECK1:       for.end11:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -207,8 +205,9 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -237,54 +236,49 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK1-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP8]], [[ADD5]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END15:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK1:       for.cond6:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[ADD7:%.*]] = add i32 [[TMP12]], 1
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD8:%.*]] = add i32 [[TMP13]], 5
-// CHECK1-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
-// CHECK1-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK1:       cond.true:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[ADD10:%.*]] = add i32 [[TMP14]], 1
-// CHECK1-NEXT:    br label [[COND_END:%.*]]
-// CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD11:%.*]] = add i32 [[TMP15]], 5
-// CHECK1-NEXT:    br label [[COND_END]]
-// CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP11]], [[COND]]
-// CHECK1-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body13:
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP17]], [[TMP18]]
-// CHECK1-NEXT:    [[ADD14:%.*]] = add i32 [[TMP16]], [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP19]])
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP10]], 5
+// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body8:
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD9:%.*]] = add i32 [[TMP11]], [[TMP12]]
+// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP14]], [[TMP15]]
+// CHECK1-NEXT:    [[ADD10:%.*]] = add i32 [[TMP13]], [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD10]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD11:%.*]] = add i32 [[TMP17]], 1
+// CHECK1-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP16]], [[ADD11]]
+// CHECK1-NEXT:    br i1 [[CMP12]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK1:       if.then:
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP18]])
+// CHECK1-NEXT:    br label [[IF_END]]
+// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP20]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP6:![0-9]+]]
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP19]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP7:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC15:%.*]]
-// CHECK1:       for.inc15:
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD16:%.*]] = add i32 [[TMP21]], 5
-// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP7:![0-9]+]]
-// CHECK1:       for.end17:
+// CHECK1-NEXT:    br label [[FOR_INC13:%.*]]
+// CHECK1:       for.inc13:
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD14:%.*]] = add i32 [[TMP20]], 5
+// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
+// CHECK1:       for.end15:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -295,11 +289,13 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -310,94 +306,86 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END30:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
 // CHECK1:       for.body:
 // CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK1:       for.cond1:
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 4
-// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END27:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END21:%.*]]
 // CHECK1:       for.body3:
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 [[TMP2]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK1:       for.cond4:
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], 5
-// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
-// CHECK1-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK1:       cond.true:
-// CHECK1-NEXT:    br label [[COND_END:%.*]]
-// CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP5]], 5
-// CHECK1-NEXT:    br label [[COND_END]]
-// CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP3]], [[COND]]
-// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END24:%.*]]
-// CHECK1:       for.body8:
-// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
-// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP2]], 5
+// CHECK1-NEXT:    br i1 [[CMP5]], label [[FOR_BODY6:%.*]], label [[FOR_END18:%.*]]
+// CHECK1:       for.body6:
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[TMP4]]
+// CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
+// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND8:%.*]]
+// CHECK1:       for.cond8:
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP6]], 5
+// CHECK1-NEXT:    br i1 [[CMP9]], label [[FOR_BODY10:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body10:
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND10:%.*]]
-// CHECK1:       for.cond10:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP9]], 5
-// CHECK1-NEXT:    [[CMP12:%.*]] = icmp slt i32 4, [[ADD11]]
-// CHECK1-NEXT:    br i1 [[CMP12]], label [[COND_TRUE13:%.*]], label [[COND_FALSE14:%.*]]
-// CHECK1:       cond.true13:
-// CHECK1-NEXT:    br label [[COND_END16:%.*]]
-// CHECK1:       cond.false14:
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP10]], 5
-// CHECK1-NEXT:    br label [[COND_END16]]
-// CHECK1:       cond.end16:
-// CHECK1-NEXT:    [[COND17:%.*]] = phi i32 [ 4, [[COND_TRUE13]] ], [ [[ADD15]], [[COND_FALSE14]] ]
-// CHECK1-NEXT:    [[CMP18:%.*]] = icmp slt i32 [[TMP8]], [[COND17]]
-// CHECK1-NEXT:    br i1 [[CMP18]], label [[FOR_BODY19:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body19:
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP7]], [[TMP8]]
+// CHECK1-NEXT:    store i32 [[ADD11]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[MUL12:%.*]] = mul nsw i32 [[TMP9]], 3
+// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i32 7, [[MUL12]]
+// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP10]], 4
+// CHECK1-NEXT:    br i1 [[CMP14]], label [[LAND_LHS_TRUE:%.*]], label [[IF_END:%.*]]
+// CHECK1:       land.lhs.true:
 // CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[TMP11]], 3
-// CHECK1-NEXT:    [[ADD21:%.*]] = add nsw i32 7, [[MUL20]]
-// CHECK1-NEXT:    store i32 [[ADD21]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[CMP15:%.*]] = icmp slt i32 [[TMP11]], 4
+// CHECK1-NEXT:    br i1 [[CMP15]], label [[IF_THEN:%.*]], label [[IF_END]]
+// CHECK1:       if.then:
 // CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
 // CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP12]], i32 noundef [[TMP13]])
+// CHECK1-NEXT:    br label [[IF_END]]
+// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND10]], !llvm.loop [[LOOP8:![0-9]+]]
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP9:![0-9]+]]
 // CHECK1:       for.end:
+// CHECK1-NEXT:    br label [[FOR_INC16:%.*]]
+// CHECK1:       for.inc16:
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC17:%.*]] = add nsw i32 [[TMP15]], 1
+// CHECK1-NEXT:    store i32 [[INC17]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP10:![0-9]+]]
+// CHECK1:       for.end18:
+// CHECK1-NEXT:    br label [[FOR_INC19:%.*]]
+// CHECK1:       for.inc19:
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP16]], 5
+// CHECK1-NEXT:    store i32 [[ADD20]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP11:![0-9]+]]
+// CHECK1:       for.end21:
 // CHECK1-NEXT:    br label [[FOR_INC22:%.*]]
 // CHECK1:       for.inc22:
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC23:%.*]] = add nsw i32 [[TMP15]], 1
-// CHECK1-NEXT:    store i32 [[INC23]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP9:![0-9]+]]
-// CHECK1:       for.end24:
-// CHECK1-NEXT:    br label [[FOR_INC25:%.*]]
-// CHECK1:       for.inc25:
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD26:%.*]] = add nsw i32 [[TMP16]], 5
-// CHECK1-NEXT:    store i32 [[ADD26]], ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP10:![0-9]+]]
-// CHECK1:       for.end27:
-// CHECK1-NEXT:    br label [[FOR_INC28:%.*]]
-// CHECK1:       for.inc28:
 // CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD29:%.*]] = add nsw i32 [[TMP17]], 5
-// CHECK1-NEXT:    store i32 [[ADD29]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP11:![0-9]+]]
-// CHECK1:       for.end30:
+// CHECK1-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP17]], 5
+// CHECK1-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP12:![0-9]+]]
+// CHECK1:       for.end24:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -407,15 +395,17 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
 // CHECK1-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK1-NEXT:    store i32 7, ptr [[J]], align 4
@@ -453,87 +443,79 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP7]], 4
-// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END32:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END22:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 [[TMP8]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND3:%.*]]
 // CHECK1:       for.cond3:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD4:%.*]] = add nsw i32 [[TMP10]], 5
-// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD4]]
-// CHECK1-NEXT:    br i1 [[CMP5]], label [[COND_TRUE6:%.*]], label [[COND_FALSE7:%.*]]
-// CHECK1:       cond.true6:
-// CHECK1-NEXT:    br label [[COND_END9:%.*]]
-// CHECK1:       cond.false7:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP11]], 5
-// CHECK1-NEXT:    br label [[COND_END9]]
-// CHECK1:       cond.end9:
-// CHECK1-NEXT:    [[COND10:%.*]] = phi i32 [ 4, [[COND_TRUE6]] ], [ [[ADD8]], [[COND_FALSE7]] ]
-// CHECK1-NEXT:    [[CMP11:%.*]] = icmp slt i32 [[TMP9]], [[COND10]]
-// CHECK1-NEXT:    br i1 [[CMP11]], label [[FOR_BODY12:%.*]], label [[FOR_END29:%.*]]
-// CHECK1:       for.body12:
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL13:%.*]] = mul nsw i32 [[TMP12]], 3
-// CHECK1-NEXT:    [[ADD14:%.*]] = add nsw i32 7, [[MUL13]]
-// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP8]], 5
+// CHECK1-NEXT:    br i1 [[CMP4]], label [[FOR_BODY5:%.*]], label [[FOR_END19:%.*]]
+// CHECK1:       for.body5:
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
+// CHECK1-NEXT:    store i32 [[ADD6]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL7:%.*]] = mul nsw i32 [[TMP11]], 3
+// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i32 7, [[MUL7]]
+// CHECK1-NEXT:    store i32 [[ADD8]], ptr [[I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND9:%.*]]
+// CHECK1:       for.cond9:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i32 [[TMP12]], 5
+// CHECK1-NEXT:    br i1 [[CMP10]], label [[FOR_BODY11:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body11:
 // CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    store i32 [[TMP13]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND15:%.*]]
-// CHECK1:       for.cond15:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP15]], 5
-// CHECK1-NEXT:    [[CMP17:%.*]] = icmp slt i32 4, [[ADD16]]
-// CHECK1-NEXT:    br i1 [[CMP17]], label [[COND_TRUE18:%.*]], label [[COND_FALSE19:%.*]]
-// CHECK1:       cond.true18:
-// CHECK1-NEXT:    br label [[COND_END21:%.*]]
-// CHECK1:       cond.false19:
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP16]], 5
-// CHECK1-NEXT:    br label [[COND_END21]]
-// CHECK1:       cond.end21:
-// CHECK1-NEXT:    [[COND22:%.*]] = phi i32 [ 4, [[COND_TRUE18]] ], [ [[ADD20]], [[COND_FALSE19]] ]
-// CHECK1-NEXT:    [[CMP23:%.*]] = icmp slt i32 [[TMP14]], [[COND22]]
-// CHECK1-NEXT:    br i1 [[CMP23]], label [[FOR_BODY24:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body24:
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP13]], [[TMP14]]
+// CHECK1-NEXT:    store i32 [[ADD12]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[MUL13:%.*]] = mul nsw i32 [[TMP15]], 3
+// CHECK1-NEXT:    [[ADD14:%.*]] = add nsw i32 7, [[MUL13]]
+// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP15:%.*]] = icmp slt i32 [[TMP16]], 4
+// CHECK1-NEXT:    br i1 [[CMP15]], label [[LAND_LHS_TRUE:%.*]], label [[IF_END:%.*]]
+// CHECK1:       land.lhs.true:
 // CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[MUL25:%.*]] = mul nsw i32 [[TMP17]], 3
-// CHECK1-NEXT:    [[ADD26:%.*]] = add nsw i32 7, [[MUL25]]
-// CHECK1-NEXT:    store i32 [[ADD26]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[CMP16:%.*]] = icmp slt i32 [[TMP17]], 4
+// CHECK1-NEXT:    br i1 [[CMP16]], label [[IF_THEN:%.*]], label [[IF_END]]
+// CHECK1:       if.then:
 // CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[J]], align 4
 // CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP18]], i32 noundef [[TMP19]])
+// CHECK1-NEXT:    br label [[IF_END]]
+// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP20]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND15]], !llvm.loop [[LOOP12:![0-9]+]]
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND9]], !llvm.loop [[LOOP13:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC27:%.*]]
-// CHECK1:       for.inc27:
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC28:%.*]] = add nsw i32 [[TMP21]], 1
-// CHECK1-NEXT:    store i32 [[INC28]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND3]], !llvm.loop [[LOOP13:![0-9]+]]
-// CHECK1:       for.end29:
-// CHECK1-NEXT:    br label [[FOR_INC30:%.*]]
-// CHECK1:       for.inc30:
+// CHECK1-NEXT:    br label [[FOR_INC17:%.*]]
+// CHECK1:       for.inc17:
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC18:%.*]] = add nsw i32 [[TMP21]], 1
+// CHECK1-NEXT:    store i32 [[INC18]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND3]], !llvm.loop [[LOOP14:![0-9]+]]
+// CHECK1:       for.end19:
+// CHECK1-NEXT:    br label [[FOR_INC20:%.*]]
+// CHECK1:       for.inc20:
 // CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD31:%.*]] = add nsw i32 [[TMP22]], 5
-// CHECK1-NEXT:    store i32 [[ADD31]], ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP14:![0-9]+]]
-// CHECK1:       for.end32:
+// CHECK1-NEXT:    [[ADD21:%.*]] = add nsw i32 [[TMP22]], 5
+// CHECK1-NEXT:    store i32 [[ADD21]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP15:![0-9]+]]
+// CHECK1:       for.end22:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
 // CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[ADD33:%.*]] = add nsw i32 [[TMP23]], 1
-// CHECK1-NEXT:    store i32 [[ADD33]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP23]], 1
+// CHECK1-NEXT:    store i32 [[ADD23]], ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -550,7 +532,9 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[_TMP1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
@@ -558,8 +542,8 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[K:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK1-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK1-NEXT:    store i32 7, ptr [[J]], align 4
@@ -606,87 +590,79 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP9]], 4
-// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY:%.*]], label [[FOR_END37:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY:%.*]], label [[FOR_END27:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND8:%.*]]
 // CHECK1:       for.cond8:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i32 [[TMP12]], 5
-// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i32 4, [[ADD9]]
-// CHECK1-NEXT:    br i1 [[CMP10]], label [[COND_TRUE11:%.*]], label [[COND_FALSE12:%.*]]
-// CHECK1:       cond.true11:
-// CHECK1-NEXT:    br label [[COND_END14:%.*]]
-// CHECK1:       cond.false12:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP13]], 5
-// CHECK1-NEXT:    br label [[COND_END14]]
-// CHECK1:       cond.end14:
-// CHECK1-NEXT:    [[COND15:%.*]] = phi i32 [ 4, [[COND_TRUE11]] ], [ [[ADD13]], [[COND_FALSE12]] ]
-// CHECK1-NEXT:    [[CMP16:%.*]] = icmp slt i32 [[TMP11]], [[COND15]]
-// CHECK1-NEXT:    br i1 [[CMP16]], label [[FOR_BODY17:%.*]], label [[FOR_END34:%.*]]
-// CHECK1:       for.body17:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL18:%.*]] = mul nsw i32 [[TMP14]], 3
-// CHECK1-NEXT:    [[ADD19:%.*]] = add nsw i32 7, [[MUL18]]
-// CHECK1-NEXT:    store i32 [[ADD19]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP10]], 5
+// CHECK1-NEXT:    br i1 [[CMP9]], label [[FOR_BODY10:%.*]], label [[FOR_END24:%.*]]
+// CHECK1:       for.body10:
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP11]], [[TMP12]]
+// CHECK1-NEXT:    store i32 [[ADD11]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL12:%.*]] = mul nsw i32 [[TMP13]], 3
+// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i32 7, [[MUL12]]
+// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND14:%.*]]
+// CHECK1:       for.cond14:
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[CMP15:%.*]] = icmp slt i32 [[TMP14]], 5
+// CHECK1-NEXT:    br i1 [[CMP15]], label [[FOR_BODY16:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body16:
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    store i32 [[TMP15]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND20:%.*]]
-// CHECK1:       for.cond20:
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD21:%.*]] = add nsw i32 [[TMP17]], 5
-// CHECK1-NEXT:    [[CMP22:%.*]] = icmp slt i32 4, [[ADD21]]
-// CHECK1-NEXT:    br i1 [[CMP22]], label [[COND_TRUE23:%.*]], label [[COND_FALSE24:%.*]]
-// CHECK1:       cond.true23:
-// CHECK1-NEXT:    br label [[COND_END26:%.*]]
-// CHECK1:       cond.false24:
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD25:%.*]] = add nsw i32 [[TMP18]], 5
-// CHECK1-NEXT:    br label [[COND_END26]]
-// CHECK1:       cond.end26:
-// CHECK1-NEXT:    [[COND27:%.*]] = phi i32 [ 4, [[COND_TRUE23]] ], [ [[ADD25]], [[COND_FALSE24]] ]
-// CHECK1-NEXT:    [[CMP28:%.*]] = icmp slt i32 [[TMP16]], [[COND27]]
-// CHECK1-NEXT:    br i1 [[CMP28]], label [[FOR_BODY29:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body29:
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD17:%.*]] = add nsw i32 [[TMP15]], [[TMP16]]
+// CHECK1-NEXT:    store i32 [[ADD17]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[MUL18:%.*]] = mul nsw i32 [[TMP17]], 3
+// CHECK1-NEXT:    [[ADD19:%.*]] = add nsw i32 7, [[MUL18]]
+// CHECK1-NEXT:    store i32 [[ADD19]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP20:%.*]] = icmp slt i32 [[TMP18]], 4
+// CHECK1-NEXT:    br i1 [[CMP20]], label [[LAND_LHS_TRUE:%.*]], label [[IF_END:%.*]]
+// CHECK1:       land.lhs.true:
 // CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[MUL30:%.*]] = mul nsw i32 [[TMP19]], 3
-// CHECK1-NEXT:    [[ADD31:%.*]] = add nsw i32 7, [[MUL30]]
-// CHECK1-NEXT:    store i32 [[ADD31]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[CMP21:%.*]] = icmp slt i32 [[TMP19]], 4
+// CHECK1-NEXT:    br i1 [[CMP21]], label [[IF_THEN:%.*]], label [[IF_END]]
+// CHECK1:       if.then:
 // CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[J]], align 4
 // CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP20]], i32 noundef [[TMP21]])
+// CHECK1-NEXT:    br label [[IF_END]]
+// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP22]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND20]], !llvm.loop [[LOOP15:![0-9]+]]
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND14]], !llvm.loop [[LOOP16:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC32:%.*]]
-// CHECK1:       for.inc32:
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC33:%.*]] = add nsw i32 [[TMP23]], 1
-// CHECK1-NEXT:    store i32 [[INC33]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP16:![0-9]+]]
-// CHECK1:       for.end34:
-// CHECK1-NEXT:    br label [[FOR_INC35:%.*]]
-// CHECK1:       for.inc35:
+// CHECK1-NEXT:    br label [[FOR_INC22:%.*]]
+// CHECK1:       for.inc22:
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC23:%.*]] = add nsw i32 [[TMP23]], 1
+// CHECK1-NEXT:    store i32 [[INC23]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP17:![0-9]+]]
+// CHECK1:       for.end24:
+// CHECK1-NEXT:    br label [[FOR_INC25:%.*]]
+// CHECK1:       for.inc25:
 // CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD36:%.*]] = add nsw i32 [[TMP24]], 5
-// CHECK1-NEXT:    store i32 [[ADD36]], ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP17:![0-9]+]]
-// CHECK1:       for.end37:
+// CHECK1-NEXT:    [[ADD26:%.*]] = add nsw i32 [[TMP24]], 5
+// CHECK1-NEXT:    store i32 [[ADD26]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP18:![0-9]+]]
+// CHECK1:       for.end27:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
 // CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[ADD38:%.*]] = add nsw i32 [[TMP25]], 1
-// CHECK1-NEXT:    store i32 [[ADD38]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[ADD28:%.*]] = add nsw i32 [[TMP25]], 1
+// CHECK1-NEXT:    store i32 [[ADD28]], ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -699,208 +675,101 @@ extern "C" void foo10(data_t data) {
 // CHECK1-LABEL: define dso_local void @foo5(
 // CHECK1-SAME: ) #[[ATTR0]] {
 // CHECK1-NEXT:  entry:
-// CHECK1-NEXT:    [[DOTOMP_IV:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[_TMP1:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[_TMP2:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_5:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I11:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I12:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[J13:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK1-NEXT:    store i32 7, ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP]], align 4
-// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP]], align 4
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], 5
-// CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
+// CHECK1-NEXT:    store i32 4, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
+// CHECK1-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], 4
 // CHECK1-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK1:       cond.true:
 // CHECK1-NEXT:    br label [[COND_END:%.*]]
 // CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP]], align 4
-// CHECK1-NEXT:    [[ADD4:%.*]] = add nsw i32 [[TMP3]], 5
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
 // CHECK1-NEXT:    br label [[COND_END]]
 // CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD4]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[SUB:%.*]] = sub i32 [[TMP4]], [[TMP5]]
-// CHECK1-NEXT:    [[SUB6:%.*]] = sub i32 [[SUB]], 1
-// CHECK1-NEXT:    [[ADD7:%.*]] = add i32 [[SUB6]], 1
-// CHECK1-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD7]], 1
-// CHECK1-NEXT:    [[CONV:%.*]] = zext i32 [[DIV]] to i64
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 1, [[CONV]]
-// CHECK1-NEXT:    [[MUL8:%.*]] = mul nsw i64 [[MUL]], 4
-// CHECK1-NEXT:    [[SUB9:%.*]] = sub nsw i64 [[MUL8]], 1
-// CHECK1-NEXT:    store i64 [[SUB9]], ptr [[DOTCAPTURE_EXPR_5]], align 8
-// CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    store i32 [[TMP6]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 7, ptr [[J]], align 4
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i32 [[TMP7]], [[TMP8]]
-// CHECK1-NEXT:    br i1 [[CMP10]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END:%.*]]
-// CHECK1:       omp.precond.then:
-// CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_5]], align 8
-// CHECK1-NEXT:    store i64 [[TMP9]], ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    store i64 1, ptr [[DOTOMP_STRIDE]], align 8
-// CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
-// CHECK1-NEXT:    call void @__kmpc_for_static_init_8(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i64 1, i64 1)
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_5]], align 8
-// CHECK1-NEXT:    [[CMP14:%.*]] = icmp sgt i64 [[TMP10]], [[TMP11]]
-// CHECK1-NEXT:    br i1 [[CMP14]], label [[COND_TRUE15:%.*]], label [[COND_FALSE16:%.*]]
-// CHECK1:       cond.true15:
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_5]], align 8
-// CHECK1-NEXT:    br label [[COND_END17:%.*]]
-// CHECK1:       cond.false16:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    br label [[COND_END17]]
-// CHECK1:       cond.end17:
-// CHECK1-NEXT:    [[COND18:%.*]] = phi i64 [ [[TMP12]], [[COND_TRUE15]] ], [ [[TMP13]], [[COND_FALSE16]] ]
-// CHECK1-NEXT:    store i64 [[COND18]], ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
-// CHECK1-NEXT:    store i64 [[TMP14]], ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[TMP2]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
+// CHECK1-NEXT:    store i32 [[TMP3]], ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK1:       omp.inner.for.cond:
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[CMP19:%.*]] = icmp sle i64 [[TMP15]], [[TMP16]]
-// CHECK1-NEXT:    br i1 [[CMP19]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], [[TMP5]]
+// CHECK1-NEXT:    br i1 [[CMP2]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
 // CHECK1:       omp.inner.for.body:
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[SUB20:%.*]] = sub i32 [[TMP18]], [[TMP19]]
-// CHECK1-NEXT:    [[SUB21:%.*]] = sub i32 [[SUB20]], 1
-// CHECK1-NEXT:    [[ADD22:%.*]] = add i32 [[SUB21]], 1
-// CHECK1-NEXT:    [[DIV23:%.*]] = udiv i32 [[ADD22]], 1
-// CHECK1-NEXT:    [[MUL24:%.*]] = mul i32 1, [[DIV23]]
-// CHECK1-NEXT:    [[MUL25:%.*]] = mul i32 [[MUL24]], 4
-// CHECK1-NEXT:    [[CONV26:%.*]] = zext i32 [[MUL25]] to i64
-// CHECK1-NEXT:    [[DIV27:%.*]] = sdiv i64 [[TMP17]], [[CONV26]]
-// CHECK1-NEXT:    [[MUL28:%.*]] = mul nsw i64 [[DIV27]], 5
-// CHECK1-NEXT:    [[ADD29:%.*]] = add nsw i64 0, [[MUL28]]
-// CHECK1-NEXT:    [[CONV30:%.*]] = trunc i64 [[ADD29]] to i32
-// CHECK1-NEXT:    store i32 [[CONV30]], ptr [[DOTFLOOR_0_IV_I11]], align 4
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[CONV31:%.*]] = sext i32 [[TMP20]] to i64
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP22:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[SUB32:%.*]] = sub i32 [[TMP23]], [[TMP24]]
-// CHECK1-NEXT:    [[SUB33:%.*]] = sub i32 [[SUB32]], 1
-// CHECK1-NEXT:    [[ADD34:%.*]] = add i32 [[SUB33]], 1
-// CHECK1-NEXT:    [[DIV35:%.*]] = udiv i32 [[ADD34]], 1
-// CHECK1-NEXT:    [[MUL36:%.*]] = mul i32 1, [[DIV35]]
-// CHECK1-NEXT:    [[MUL37:%.*]] = mul i32 [[MUL36]], 4
-// CHECK1-NEXT:    [[CONV38:%.*]] = zext i32 [[MUL37]] to i64
-// CHECK1-NEXT:    [[DIV39:%.*]] = sdiv i64 [[TMP22]], [[CONV38]]
-// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[SUB40:%.*]] = sub i32 [[TMP25]], [[TMP26]]
-// CHECK1-NEXT:    [[SUB41:%.*]] = sub i32 [[SUB40]], 1
-// CHECK1-NEXT:    [[ADD42:%.*]] = add i32 [[SUB41]], 1
-// CHECK1-NEXT:    [[DIV43:%.*]] = udiv i32 [[ADD42]], 1
-// CHECK1-NEXT:    [[MUL44:%.*]] = mul i32 1, [[DIV43]]
-// CHECK1-NEXT:    [[MUL45:%.*]] = mul i32 [[MUL44]], 4
-// CHECK1-NEXT:    [[CONV46:%.*]] = zext i32 [[MUL45]] to i64
-// CHECK1-NEXT:    [[MUL47:%.*]] = mul nsw i64 [[DIV39]], [[CONV46]]
-// CHECK1-NEXT:    [[SUB48:%.*]] = sub nsw i64 [[TMP21]], [[MUL47]]
-// CHECK1-NEXT:    [[DIV49:%.*]] = sdiv i64 [[SUB48]], 4
-// CHECK1-NEXT:    [[MUL50:%.*]] = mul nsw i64 [[DIV49]], 1
-// CHECK1-NEXT:    [[ADD51:%.*]] = add nsw i64 [[CONV31]], [[MUL50]]
-// CHECK1-NEXT:    [[CONV52:%.*]] = trunc i64 [[ADD51]] to i32
-// CHECK1-NEXT:    store i32 [[CONV52]], ptr [[DOTTILE_0_IV_I12]], align 4
-// CHECK1-NEXT:    [[TMP27:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP28:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[SUB53:%.*]] = sub i32 [[TMP29]], [[TMP30]]
-// CHECK1-NEXT:    [[SUB54:%.*]] = sub i32 [[SUB53]], 1
-// CHECK1-NEXT:    [[ADD55:%.*]] = add i32 [[SUB54]], 1
-// CHECK1-NEXT:    [[DIV56:%.*]] = udiv i32 [[ADD55]], 1
-// CHECK1-NEXT:    [[MUL57:%.*]] = mul i32 1, [[DIV56]]
-// CHECK1-NEXT:    [[MUL58:%.*]] = mul i32 [[MUL57]], 4
-// CHECK1-NEXT:    [[CONV59:%.*]] = zext i32 [[MUL58]] to i64
-// CHECK1-NEXT:    [[DIV60:%.*]] = sdiv i64 [[TMP28]], [[CONV59]]
-// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[SUB61:%.*]] = sub i32 [[TMP31]], [[TMP32]]
-// CHECK1-NEXT:    [[SUB62:%.*]] = sub i32 [[SUB61]], 1
-// CHECK1-NEXT:    [[ADD63:%.*]] = add i32 [[SUB62]], 1
-// CHECK1-NEXT:    [[DIV64:%.*]] = udiv i32 [[ADD63]], 1
-// CHECK1-NEXT:    [[MUL65:%.*]] = mul i32 1, [[DIV64]]
-// CHECK1-NEXT:    [[MUL66:%.*]] = mul i32 [[MUL65]], 4
-// CHECK1-NEXT:    [[CONV67:%.*]] = zext i32 [[MUL66]] to i64
-// CHECK1-NEXT:    [[MUL68:%.*]] = mul nsw i64 [[DIV60]], [[CONV67]]
-// CHECK1-NEXT:    [[SUB69:%.*]] = sub nsw i64 [[TMP27]], [[MUL68]]
-// CHECK1-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK1-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[SUB70:%.*]] = sub i32 [[TMP35]], [[TMP36]]
-// CHECK1-NEXT:    [[SUB71:%.*]] = sub i32 [[SUB70]], 1
-// CHECK1-NEXT:    [[ADD72:%.*]] = add i32 [[SUB71]], 1
-// CHECK1-NEXT:    [[DIV73:%.*]] = udiv i32 [[ADD72]], 1
-// CHECK1-NEXT:    [[MUL74:%.*]] = mul i32 1, [[DIV73]]
-// CHECK1-NEXT:    [[MUL75:%.*]] = mul i32 [[MUL74]], 4
-// CHECK1-NEXT:    [[CONV76:%.*]] = zext i32 [[MUL75]] to i64
-// CHECK1-NEXT:    [[DIV77:%.*]] = sdiv i64 [[TMP34]], [[CONV76]]
-// CHECK1-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK1-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[SUB78:%.*]] = sub i32 [[TMP37]], [[TMP38]]
-// CHECK1-NEXT:    [[SUB79:%.*]] = sub i32 [[SUB78]], 1
-// CHECK1-NEXT:    [[ADD80:%.*]] = add i32 [[SUB79]], 1
-// CHECK1-NEXT:    [[DIV81:%.*]] = udiv i32 [[ADD80]], 1
-// CHECK1-NEXT:    [[MUL82:%.*]] = mul i32 1, [[DIV81]]
-// CHECK1-NEXT:    [[MUL83:%.*]] = mul i32 [[MUL82]], 4
-// CHECK1-NEXT:    [[CONV84:%.*]] = zext i32 [[MUL83]] to i64
-// CHECK1-NEXT:    [[MUL85:%.*]] = mul nsw i64 [[DIV77]], [[CONV84]]
-// CHECK1-NEXT:    [[SUB86:%.*]] = sub nsw i64 [[TMP33]], [[MUL85]]
-// CHECK1-NEXT:    [[DIV87:%.*]] = sdiv i64 [[SUB86]], 4
-// CHECK1-NEXT:    [[MUL88:%.*]] = mul nsw i64 [[DIV87]], 4
-// CHECK1-NEXT:    [[SUB89:%.*]] = sub nsw i64 [[SUB69]], [[MUL88]]
-// CHECK1-NEXT:    [[MUL90:%.*]] = mul nsw i64 [[SUB89]], 3
-// CHECK1-NEXT:    [[ADD91:%.*]] = add nsw i64 7, [[MUL90]]
-// CHECK1-NEXT:    [[CONV92:%.*]] = trunc i64 [[ADD91]] to i32
-// CHECK1-NEXT:    store i32 [[CONV92]], ptr [[J13]], align 4
-// CHECK1-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTTILE_0_IV_I12]], align 4
-// CHECK1-NEXT:    [[MUL93:%.*]] = mul nsw i32 [[TMP39]], 3
-// CHECK1-NEXT:    [[ADD94:%.*]] = add nsw i32 7, [[MUL93]]
-// CHECK1-NEXT:    store i32 [[ADD94]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP41:%.*]] = load i32, ptr [[J13]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP40]], i32 noundef [[TMP41]])
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP6]], 5
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[DIV]], 5
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[DIV3:%.*]] = sdiv i32 [[TMP8]], 5
+// CHECK1-NEXT:    [[MUL4:%.*]] = mul nsw i32 [[DIV3]], 5
+// CHECK1-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP7]], [[MUL4]]
+// CHECK1-NEXT:    [[MUL5:%.*]] = mul nsw i32 [[SUB]], 1
+// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL5]]
+// CHECK1-NEXT:    store i32 [[ADD6]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
+// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL8:%.*]] = mul nsw i32 [[TMP11]], 3
+// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL8]]
+// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i32 [[TMP12]], 4
+// CHECK1-NEXT:    br i1 [[CMP10]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK1:       if.then:
+// CHECK1-NEXT:    store i32 7, ptr [[J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND:%.*]]
+// CHECK1:       for.cond:
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    [[CMP11:%.*]] = icmp slt i32 [[TMP13]], 17
+// CHECK1-NEXT:    br i1 [[CMP11]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body:
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP14]], i32 noundef [[TMP15]])
+// CHECK1-NEXT:    br label [[FOR_INC:%.*]]
+// CHECK1:       for.inc:
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP16]], 3
+// CHECK1-NEXT:    store i32 [[ADD12]], ptr [[J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP19:![0-9]+]]
+// CHECK1:       for.end:
+// CHECK1-NEXT:    br label [[IF_END]]
+// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP42:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[ADD95:%.*]] = add nsw i64 [[TMP42]], 1
-// CHECK1-NEXT:    store i64 [[ADD95]], ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP17]], 1
+// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
 // CHECK1:       omp.loop.exit:
 // CHECK1-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP0]])
-// CHECK1-NEXT:    br label [[OMP_PRECOND_END]]
-// CHECK1:       omp.precond.end:
 // CHECK1-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[TMP0]])
 // CHECK1-NEXT:    ret void
 //
@@ -920,12 +789,13 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
 // CHECK1-NEXT:    store i32 7, ptr [[I]], align 4
@@ -960,46 +830,43 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP7]], 5
 // CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 [[TMP8]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND:%.*]]
 // CHECK1:       for.cond:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD2:%.*]] = add nsw i32 [[TMP10]], 5
-// CHECK1-NEXT:    [[CMP3:%.*]] = icmp slt i32 4, [[ADD2]]
-// CHECK1-NEXT:    br i1 [[CMP3]], label [[COND_TRUE4:%.*]], label [[COND_FALSE5:%.*]]
-// CHECK1:       cond.true4:
-// CHECK1-NEXT:    br label [[COND_END7:%.*]]
-// CHECK1:       cond.false5:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP11]], 5
-// CHECK1-NEXT:    br label [[COND_END7]]
-// CHECK1:       cond.end7:
-// CHECK1-NEXT:    [[COND8:%.*]] = phi i32 [ 4, [[COND_TRUE4]] ], [ [[ADD6]], [[COND_FALSE5]] ]
-// CHECK1-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP9]], [[COND8]]
-// CHECK1-NEXT:    br i1 [[CMP9]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP8]], 5
+// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
 // CHECK1:       for.body:
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
+// CHECK1-NEXT:    store i32 [[ADD3]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL4:%.*]] = mul nsw i32 [[TMP11]], 3
+// CHECK1-NEXT:    [[ADD5:%.*]] = add nsw i32 7, [[MUL4]]
+// CHECK1-NEXT:    store i32 [[ADD5]], ptr [[I]], align 4
 // CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL10:%.*]] = mul nsw i32 [[TMP12]], 3
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 7, [[MUL10]]
-// CHECK1-NEXT:    store i32 [[ADD11]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[CMP6:%.*]] = icmp slt i32 [[TMP12]], 4
+// CHECK1-NEXT:    br i1 [[CMP6]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK1:       if.then:
 // CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP13]])
+// CHECK1-NEXT:    br label [[IF_END]]
+// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP18:![0-9]+]]
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP20:![0-9]+]]
 // CHECK1:       for.end:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP15]], 1
-// CHECK1-NEXT:    store i32 [[ADD12]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 [[TMP15]], 1
+// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -1024,8 +891,9 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[START_ADDR]], align 4
@@ -1049,53 +917,48 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK1-NEXT:    [[ADD5:%.*]] = add i32 [[TMP6]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP5]], [[ADD5]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END15:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK1:       for.cond6:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[ADD7:%.*]] = add i32 [[TMP9]], 1
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD8:%.*]] = add i32 [[TMP10]], 5
-// CHECK1-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
-// CHECK1-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK1:       cond.true:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[ADD10:%.*]] = add i32 [[TMP11]], 1
-// CHECK1-NEXT:    br label [[COND_END:%.*]]
-// CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD11:%.*]] = add i32 [[TMP12]], 5
-// CHECK1-NEXT:    br label [[COND_END]]
-// CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP8]], [[COND]]
-// CHECK1-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body13:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP14]], 3
-// CHECK1-NEXT:    [[ADD14:%.*]] = add i32 [[TMP13]], [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP15]])
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP7]], 5
+// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body8:
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD9:%.*]] = add i32 [[TMP8]], [[TMP9]]
+// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP11]], 3
+// CHECK1-NEXT:    [[ADD10:%.*]] = add i32 [[TMP10]], [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD10]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD11:%.*]] = add i32 [[TMP13]], 1
+// CHECK1-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP12]], [[ADD11]]
+// CHECK1-NEXT:    br i1 [[CMP12]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK1:       if.then:
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP14]])
+// CHECK1-NEXT:    br label [[IF_END]]
+// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP16]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP21:![0-9]+]]
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP15]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP23:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC15:%.*]]
-// CHECK1:       for.inc15:
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD16:%.*]] = add i32 [[TMP17]], 5
-// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP22:![0-9]+]]
-// CHECK1:       for.end17:
+// CHECK1-NEXT:    br label [[FOR_INC13:%.*]]
+// CHECK1:       for.inc13:
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD14:%.*]] = add i32 [[TMP16]], 5
+// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP24:![0-9]+]]
+// CHECK1:       for.end15:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -1104,8 +967,9 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
 // CHECK1-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
@@ -1113,78 +977,65 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END14:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK1:       for.cond1:
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], 0
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP2]], 0
 // CHECK1-NEXT:    br i1 [[CMP2]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK1:       cond.true:
 // CHECK1-NEXT:    br label [[COND_END:%.*]]
 // CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[A_ADDR]], align 4
 // CHECK1-NEXT:    br label [[COND_END]]
 // CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP5]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[COND]]
-// CHECK1-NEXT:    [[CMP3:%.*]] = icmp slt i32 4, [[ADD]]
-// CHECK1-NEXT:    br i1 [[CMP3]], label [[COND_TRUE4:%.*]], label [[COND_FALSE5:%.*]]
-// CHECK1:       cond.true4:
-// CHECK1-NEXT:    br label [[COND_END12:%.*]]
-// CHECK1:       cond.false5:
-// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    [[CMP6:%.*]] = icmp sle i32 [[TMP7]], 0
-// CHECK1-NEXT:    br i1 [[CMP6]], label [[COND_TRUE7:%.*]], label [[COND_FALSE8:%.*]]
-// CHECK1:       cond.true7:
-// CHECK1-NEXT:    br label [[COND_END9:%.*]]
-// CHECK1:       cond.false8:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    br label [[COND_END9]]
-// CHECK1:       cond.end9:
-// CHECK1-NEXT:    [[COND10:%.*]] = phi i32 [ 1, [[COND_TRUE7]] ], [ [[TMP8]], [[COND_FALSE8]] ]
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP6]], [[COND10]]
-// CHECK1-NEXT:    br label [[COND_END12]]
-// CHECK1:       cond.end12:
-// CHECK1-NEXT:    [[COND13:%.*]] = phi i32 [ 4, [[COND_TRUE4]] ], [ [[ADD11]], [[COND_END9]] ]
-// CHECK1-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP2]], [[COND13]]
-// CHECK1-NEXT:    br i1 [[CMP14]], label [[FOR_BODY15:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body15:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP9]], 3
-// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP10]])
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP3]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[TMP1]], [[COND]]
+// CHECK1-NEXT:    br i1 [[CMP3]], label [[FOR_BODY4:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body4:
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
+// CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
+// CHECK1-NEXT:    [[ADD5:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD5]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP6:%.*]] = icmp slt i32 [[TMP7]], 4
+// CHECK1-NEXT:    br i1 [[CMP6]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK1:       if.then:
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP8]])
+// CHECK1-NEXT:    br label [[IF_END]]
+// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP23:![0-9]+]]
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP9]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP25:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC17:%.*]]
-// CHECK1:       for.inc17:
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    [[CMP18:%.*]] = icmp sle i32 [[TMP12]], 0
-// CHECK1-NEXT:    br i1 [[CMP18]], label [[COND_TRUE19:%.*]], label [[COND_FALSE20:%.*]]
-// CHECK1:       cond.true19:
-// CHECK1-NEXT:    br label [[COND_END21:%.*]]
-// CHECK1:       cond.false20:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    br label [[COND_END21]]
-// CHECK1:       cond.end21:
-// CHECK1-NEXT:    [[COND22:%.*]] = phi i32 [ 1, [[COND_TRUE19]] ], [ [[TMP13]], [[COND_FALSE20]] ]
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP14]], [[COND22]]
-// CHECK1-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP24:![0-9]+]]
-// CHECK1:       for.end24:
+// CHECK1-NEXT:    br label [[FOR_INC7:%.*]]
+// CHECK1:       for.inc7:
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    [[CMP8:%.*]] = icmp sle i32 [[TMP10]], 0
+// CHECK1-NEXT:    br i1 [[CMP8]], label [[COND_TRUE9:%.*]], label [[COND_FALSE10:%.*]]
+// CHECK1:       cond.true9:
+// CHECK1-NEXT:    br label [[COND_END11:%.*]]
+// CHECK1:       cond.false10:
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    br label [[COND_END11]]
+// CHECK1:       cond.end11:
+// CHECK1-NEXT:    [[COND12:%.*]] = phi i32 [ 1, [[COND_TRUE9]] ], [ [[TMP11]], [[COND_FALSE10]] ]
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP12]], [[COND12]]
+// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP26:![0-9]+]]
+// CHECK1:       for.end14:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -1197,19 +1048,20 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK1-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
 // CHECK1-NEXT:    store ptr [[ARRAY]], ptr [[__RANGE2]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META27:![0-9]+]]
 // CHECK1-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP0]], i64 0, i64 0
 // CHECK1-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 12
 // CHECK1-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META27]]
 // CHECK1-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP1]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[ARRAYDECAY1]], ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META27]]
 // CHECK1-NEXT:    [[ARRAYDECAY2:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP2]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[ARRAYDECAY2]], ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__END2]], align 8
@@ -1232,56 +1084,51 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK1:       for.cond7:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
-// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
-// CHECK1-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK1:       cond.true:
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
-// CHECK1-NEXT:    br label [[COND_END:%.*]]
-// CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
-// CHECK1-NEXT:    br label [[COND_END]]
-// CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
-// CHECK1-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body14:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
+// CHECK1-NEXT:    br i1 [[CMP8]], label [[FOR_BODY9:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body9:
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
+// CHECK1-NEXT:    store i64 [[ADD10]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP12]], 1
+// CHECK1-NEXT:    [[ADD_PTR11:%.*]] = getelementptr inbounds double, ptr [[TMP11]], i64 [[MUL]]
+// CHECK1-NEXT:    store ptr [[ADD_PTR11]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP14:%.*]] = load double, ptr [[TMP13]], align 8
+// CHECK1-NEXT:    store double [[TMP14]], ptr [[V]], align 8
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
-// CHECK1-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
-// CHECK1-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
-// CHECK1-NEXT:    store double [[TMP17]], ptr [[V]], align 8
-// CHECK1-NEXT:    [[TMP18:%.*]] = load double, ptr [[V]], align 8
-// CHECK1-NEXT:    call void (...) @body(double noundef [[TMP18]])
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP16]], 1
+// CHECK1-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP15]], [[ADD12]]
+// CHECK1-NEXT:    br i1 [[CMP13]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK1:       if.then:
+// CHECK1-NEXT:    [[TMP17:%.*]] = load double, ptr [[V]], align 8
+// CHECK1-NEXT:    call void (...) @body(double noundef [[TMP17]])
+// CHECK1-NEXT:    br label [[IF_END]]
+// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP19]], 1
-// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP25:![0-9]+]]
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP18]], 1
+// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP28:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC16:%.*]]
-// CHECK1:       for.inc16:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP20]], 5
-// CHECK1-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP26:![0-9]+]]
-// CHECK1:       for.end18:
+// CHECK1-NEXT:    br label [[FOR_INC14:%.*]]
+// CHECK1:       for.inc14:
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD15:%.*]] = add nsw i64 [[TMP19]], 5
+// CHECK1-NEXT:    store i64 [[ADD15]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP29:![0-9]+]]
+// CHECK1:       for.end16:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -1295,20 +1142,21 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK1-NEXT:    store double 4.200000e+01, ptr [[C]], align 8
 // CHECK1-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
 // CHECK1-NEXT:    store ptr [[ARRAY]], ptr [[__RANGE2]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META27]]
 // CHECK1-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP0]], i64 0, i64 0
 // CHECK1-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 12
 // CHECK1-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META27]]
 // CHECK1-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP1]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[ARRAYDECAY1]], ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META27]]
 // CHECK1-NEXT:    [[ARRAYDECAY2:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP2]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[ARRAYDECAY2]], ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__END2]], align 8
@@ -1331,57 +1179,52 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK1:       for.cond7:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
-// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
-// CHECK1-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK1:       cond.true:
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
-// CHECK1-NEXT:    br label [[COND_END:%.*]]
-// CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
-// CHECK1-NEXT:    br label [[COND_END]]
-// CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
-// CHECK1-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body14:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
+// CHECK1-NEXT:    br i1 [[CMP8]], label [[FOR_BODY9:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body9:
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
+// CHECK1-NEXT:    store i64 [[ADD10]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP12]], 1
+// CHECK1-NEXT:    [[ADD_PTR11:%.*]] = getelementptr inbounds double, ptr [[TMP11]], i64 [[MUL]]
+// CHECK1-NEXT:    store ptr [[ADD_PTR11]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP14:%.*]] = load double, ptr [[TMP13]], align 8
+// CHECK1-NEXT:    store double [[TMP14]], ptr [[V]], align 8
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
-// CHECK1-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
-// CHECK1-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
-// CHECK1-NEXT:    store double [[TMP17]], ptr [[V]], align 8
-// CHECK1-NEXT:    [[TMP18:%.*]] = load double, ptr [[C]], align 8
-// CHECK1-NEXT:    [[TMP19:%.*]] = load double, ptr [[V]], align 8
-// CHECK1-NEXT:    call void (...) @body(double noundef [[TMP18]], double noundef [[TMP19]])
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP16]], 1
+// CHECK1-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP15]], [[ADD12]]
+// CHECK1-NEXT:    br i1 [[CMP13]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK1:       if.then:
+// CHECK1-NEXT:    [[TMP17:%.*]] = load double, ptr [[C]], align 8
+// CHECK1-NEXT:    [[TMP18:%.*]] = load double, ptr [[V]], align 8
+// CHECK1-NEXT:    call void (...) @body(double noundef [[TMP17]], double noundef [[TMP18]])
+// CHECK1-NEXT:    br label [[IF_END]]
+// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP20]], 1
-// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP27:![0-9]+]]
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP19]], 1
+// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP30:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC16:%.*]]
-// CHECK1:       for.inc16:
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP21]], 5
-// CHECK1-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP28:![0-9]+]]
-// CHECK1:       for.end18:
+// CHECK1-NEXT:    br label [[FOR_INC14:%.*]]
+// CHECK1:       for.inc14:
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD15:%.*]] = add nsw i64 [[TMP20]], 5
+// CHECK1-NEXT:    store i64 [[ADD15]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP31:![0-9]+]]
+// CHECK1:       for.end16:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -1414,8 +1257,9 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:  entry:
 // CHECK2-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[I2:%.*]] = alloca ptr, align 8
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK2-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK2-NEXT:    [[I:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
@@ -1427,50 +1271,47 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END12:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END11:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK2:       for.cond4:
-// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 5
-// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
-// CHECK2-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK2:       cond.true:
-// CHECK2-NEXT:    br label [[COND_END:%.*]]
-// CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP4]], 5
-// CHECK2-NEXT:    br label [[COND_END]]
-// CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
-// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body8:
-// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
-// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK2-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[I2]], align 8
-// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[TMP6]], align 4
-// CHECK2-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[I2]], align 8
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP1]], 5
+// CHECK2-NEXT:    br i1 [[CMP5]], label [[FOR_BODY6:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body6:
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
+// CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP4]], 3
+// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK2-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
+// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[TMP5]], align 4
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP8:%.*]] = icmp slt i32 [[TMP6]], 4
+// CHECK2-NEXT:    br i1 [[CMP8]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK2:       if.then:
+// CHECK2-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2]], !align [[META3]]
 // CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
 // CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP8]])
+// CHECK2-NEXT:    br label [[IF_END]]
+// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP9]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP3:![0-9]+]]
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP4:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC10:%.*]]
-// CHECK2:       for.inc10:
+// CHECK2-NEXT:    br label [[FOR_INC9:%.*]]
+// CHECK2:       for.inc9:
 // CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP10]], 5
-// CHECK2-NEXT:    store i32 [[ADD11]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP5:![0-9]+]]
-// CHECK2:       for.end12:
+// CHECK2-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP10]], 5
+// CHECK2-NEXT:    store i32 [[ADD10]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
+// CHECK2:       for.end11:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -1491,8 +1332,9 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -1521,54 +1363,49 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK2-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP8]], [[ADD5]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END15:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 [[TMP10]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK2:       for.cond6:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[ADD7:%.*]] = add i32 [[TMP12]], 1
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD8:%.*]] = add i32 [[TMP13]], 5
-// CHECK2-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
-// CHECK2-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK2:       cond.true:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[ADD10:%.*]] = add i32 [[TMP14]], 1
-// CHECK2-NEXT:    br label [[COND_END:%.*]]
-// CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD11:%.*]] = add i32 [[TMP15]], 5
-// CHECK2-NEXT:    br label [[COND_END]]
-// CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP11]], [[COND]]
-// CHECK2-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body13:
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP17]], [[TMP18]]
-// CHECK2-NEXT:    [[ADD14:%.*]] = add i32 [[TMP16]], [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP19]])
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP10]], 5
+// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body8:
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD9:%.*]] = add i32 [[TMP11]], [[TMP12]]
+// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP14]], [[TMP15]]
+// CHECK2-NEXT:    [[ADD10:%.*]] = add i32 [[TMP13]], [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD10]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD11:%.*]] = add i32 [[TMP17]], 1
+// CHECK2-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP16]], [[ADD11]]
+// CHECK2-NEXT:    br i1 [[CMP12]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK2:       if.then:
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP18]])
+// CHECK2-NEXT:    br label [[IF_END]]
+// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP20]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP6:![0-9]+]]
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP19]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP7:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC15:%.*]]
-// CHECK2:       for.inc15:
-// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD16:%.*]] = add i32 [[TMP21]], 5
-// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP7:![0-9]+]]
-// CHECK2:       for.end17:
+// CHECK2-NEXT:    br label [[FOR_INC13:%.*]]
+// CHECK2:       for.inc13:
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD14:%.*]] = add i32 [[TMP20]], 5
+// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
+// CHECK2:       for.end15:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -1582,20 +1419,21 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK2-NEXT:    store double 4.200000e+01, ptr [[C]], align 8
 // CHECK2-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
 // CHECK2-NEXT:    store ptr [[ARRAY]], ptr [[__RANGE2]], align 8
-// CHECK2-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META9:![0-9]+]]
 // CHECK2-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP0]], i64 0, i64 0
 // CHECK2-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 12
 // CHECK2-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
-// CHECK2-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META9]]
 // CHECK2-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP1]], i64 0, i64 0
 // CHECK2-NEXT:    store ptr [[ARRAYDECAY1]], ptr [[__BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META9]]
 // CHECK2-NEXT:    [[ARRAYDECAY2:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP2]], i64 0, i64 0
 // CHECK2-NEXT:    store ptr [[ARRAYDECAY2]], ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK2-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__END2]], align 8
@@ -1618,57 +1456,52 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK2:       for.cond7:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
-// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
-// CHECK2-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK2:       cond.true:
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
-// CHECK2-NEXT:    br label [[COND_END:%.*]]
-// CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
-// CHECK2-NEXT:    br label [[COND_END]]
-// CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
-// CHECK2-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body14:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
+// CHECK2-NEXT:    br i1 [[CMP8]], label [[FOR_BODY9:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body9:
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
+// CHECK2-NEXT:    store i64 [[ADD10]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP12]], 1
+// CHECK2-NEXT:    [[ADD_PTR11:%.*]] = getelementptr inbounds double, ptr [[TMP11]], i64 [[MUL]]
+// CHECK2-NEXT:    store ptr [[ADD_PTR11]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP14:%.*]] = load double, ptr [[TMP13]], align 8
+// CHECK2-NEXT:    store double [[TMP14]], ptr [[V]], align 8
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
-// CHECK2-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
-// CHECK2-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
-// CHECK2-NEXT:    store double [[TMP17]], ptr [[V]], align 8
-// CHECK2-NEXT:    [[TMP18:%.*]] = load double, ptr [[C]], align 8
-// CHECK2-NEXT:    [[TMP19:%.*]] = load double, ptr [[V]], align 8
-// CHECK2-NEXT:    call void (...) @body(double noundef [[TMP18]], double noundef [[TMP19]])
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP16]], 1
+// CHECK2-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP15]], [[ADD12]]
+// CHECK2-NEXT:    br i1 [[CMP13]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK2:       if.then:
+// CHECK2-NEXT:    [[TMP17:%.*]] = load double, ptr [[C]], align 8
+// CHECK2-NEXT:    [[TMP18:%.*]] = load double, ptr [[V]], align 8
+// CHECK2-NEXT:    call void (...) @body(double noundef [[TMP17]], double noundef [[TMP18]])
+// CHECK2-NEXT:    br label [[IF_END]]
+// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP20]], 1
-// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP8:![0-9]+]]
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP19]], 1
+// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP10:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC16:%.*]]
-// CHECK2:       for.inc16:
-// CHECK2-NEXT:    [[TMP21:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP21]], 5
-// CHECK2-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP9:![0-9]+]]
-// CHECK2:       for.end18:
+// CHECK2-NEXT:    br label [[FOR_INC14:%.*]]
+// CHECK2:       for.inc14:
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD15:%.*]] = add nsw i64 [[TMP20]], 5
+// CHECK2-NEXT:    store i64 [[ADD15]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP11:![0-9]+]]
+// CHECK2:       for.end16:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -1679,11 +1512,13 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -1694,94 +1529,86 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END30:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
 // CHECK2:       for.body:
 // CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK2:       for.cond1:
 // CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 4
-// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END27:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END21:%.*]]
 // CHECK2:       for.body3:
-// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 [[TMP2]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK2:       for.cond4:
-// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], 5
-// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
-// CHECK2-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK2:       cond.true:
-// CHECK2-NEXT:    br label [[COND_END:%.*]]
-// CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP5]], 5
-// CHECK2-NEXT:    br label [[COND_END]]
-// CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP3]], [[COND]]
-// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END24:%.*]]
-// CHECK2:       for.body8:
-// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
-// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP2]], 5
+// CHECK2-NEXT:    br i1 [[CMP5]], label [[FOR_BODY6:%.*]], label [[FOR_END18:%.*]]
+// CHECK2:       for.body6:
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[TMP4]]
+// CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
+// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND8:%.*]]
+// CHECK2:       for.cond8:
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP6]], 5
+// CHECK2-NEXT:    br i1 [[CMP9]], label [[FOR_BODY10:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body10:
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND10:%.*]]
-// CHECK2:       for.cond10:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP9]], 5
-// CHECK2-NEXT:    [[CMP12:%.*]] = icmp slt i32 4, [[ADD11]]
-// CHECK2-NEXT:    br i1 [[CMP12]], label [[COND_TRUE13:%.*]], label [[COND_FALSE14:%.*]]
-// CHECK2:       cond.true13:
-// CHECK2-NEXT:    br label [[COND_END16:%.*]]
-// CHECK2:       cond.false14:
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP10]], 5
-// CHECK2-NEXT:    br label [[COND_END16]]
-// CHECK2:       cond.end16:
-// CHECK2-NEXT:    [[COND17:%.*]] = phi i32 [ 4, [[COND_TRUE13]] ], [ [[ADD15]], [[COND_FALSE14]] ]
-// CHECK2-NEXT:    [[CMP18:%.*]] = icmp slt i32 [[TMP8]], [[COND17]]
-// CHECK2-NEXT:    br i1 [[CMP18]], label [[FOR_BODY19:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body19:
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP7]], [[TMP8]]
+// CHECK2-NEXT:    store i32 [[ADD11]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[MUL12:%.*]] = mul nsw i32 [[TMP9]], 3
+// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i32 7, [[MUL12]]
+// CHECK2-NEXT:    store i32 [[ADD13]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP10]], 4
+// CHECK2-NEXT:    br i1 [[CMP14]], label [[LAND_LHS_TRUE:%.*]], label [[IF_END:%.*]]
+// CHECK2:       land.lhs.true:
 // CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[TMP11]], 3
-// CHECK2-NEXT:    [[ADD21:%.*]] = add nsw i32 7, [[MUL20]]
-// CHECK2-NEXT:    store i32 [[ADD21]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[CMP15:%.*]] = icmp slt i32 [[TMP11]], 4
+// CHECK2-NEXT:    br i1 [[CMP15]], label [[IF_THEN:%.*]], label [[IF_END]]
+// CHECK2:       if.then:
 // CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
 // CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
 // CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP12]], i32 noundef [[TMP13]])
+// CHECK2-NEXT:    br label [[IF_END]]
+// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND10]], !llvm.loop [[LOOP10:![0-9]+]]
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP12:![0-9]+]]
 // CHECK2:       for.end:
+// CHECK2-NEXT:    br label [[FOR_INC16:%.*]]
+// CHECK2:       for.inc16:
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC17:%.*]] = add nsw i32 [[TMP15]], 1
+// CHECK2-NEXT:    store i32 [[INC17]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP13:![0-9]+]]
+// CHECK2:       for.end18:
+// CHECK2-NEXT:    br label [[FOR_INC19:%.*]]
+// CHECK2:       for.inc19:
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP16]], 5
+// CHECK2-NEXT:    store i32 [[ADD20]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP14:![0-9]+]]
+// CHECK2:       for.end21:
 // CHECK2-NEXT:    br label [[FOR_INC22:%.*]]
 // CHECK2:       for.inc22:
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC23:%.*]] = add nsw i32 [[TMP15]], 1
-// CHECK2-NEXT:    store i32 [[INC23]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP11:![0-9]+]]
-// CHECK2:       for.end24:
-// CHECK2-NEXT:    br label [[FOR_INC25:%.*]]
-// CHECK2:       for.inc25:
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD26:%.*]] = add nsw i32 [[TMP16]], 5
-// CHECK2-NEXT:    store i32 [[ADD26]], ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP12:![0-9]+]]
-// CHECK2:       for.end27:
-// CHECK2-NEXT:    br label [[FOR_INC28:%.*]]
-// CHECK2:       for.inc28:
 // CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD29:%.*]] = add nsw i32 [[TMP17]], 5
-// CHECK2-NEXT:    store i32 [[ADD29]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP13:![0-9]+]]
-// CHECK2:       for.end30:
+// CHECK2-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP17]], 5
+// CHECK2-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP15:![0-9]+]]
+// CHECK2:       for.end24:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -1791,15 +1618,17 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
 // CHECK2-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK2-NEXT:    store i32 7, ptr [[J]], align 4
@@ -1837,87 +1666,79 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP7]], 4
-// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END32:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END22:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 [[TMP8]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND3:%.*]]
 // CHECK2:       for.cond3:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD4:%.*]] = add nsw i32 [[TMP10]], 5
-// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD4]]
-// CHECK2-NEXT:    br i1 [[CMP5]], label [[COND_TRUE6:%.*]], label [[COND_FALSE7:%.*]]
-// CHECK2:       cond.true6:
-// CHECK2-NEXT:    br label [[COND_END9:%.*]]
-// CHECK2:       cond.false7:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP11]], 5
-// CHECK2-NEXT:    br label [[COND_END9]]
-// CHECK2:       cond.end9:
-// CHECK2-NEXT:    [[COND10:%.*]] = phi i32 [ 4, [[COND_TRUE6]] ], [ [[ADD8]], [[COND_FALSE7]] ]
-// CHECK2-NEXT:    [[CMP11:%.*]] = icmp slt i32 [[TMP9]], [[COND10]]
-// CHECK2-NEXT:    br i1 [[CMP11]], label [[FOR_BODY12:%.*]], label [[FOR_END29:%.*]]
-// CHECK2:       for.body12:
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL13:%.*]] = mul nsw i32 [[TMP12]], 3
-// CHECK2-NEXT:    [[ADD14:%.*]] = add nsw i32 7, [[MUL13]]
-// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP8]], 5
+// CHECK2-NEXT:    br i1 [[CMP4]], label [[FOR_BODY5:%.*]], label [[FOR_END19:%.*]]
+// CHECK2:       for.body5:
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
+// CHECK2-NEXT:    store i32 [[ADD6]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL7:%.*]] = mul nsw i32 [[TMP11]], 3
+// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i32 7, [[MUL7]]
+// CHECK2-NEXT:    store i32 [[ADD8]], ptr [[I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND9:%.*]]
+// CHECK2:       for.cond9:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i32 [[TMP12]], 5
+// CHECK2-NEXT:    br i1 [[CMP10]], label [[FOR_BODY11:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body11:
 // CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    store i32 [[TMP13]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND15:%.*]]
-// CHECK2:       for.cond15:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP15]], 5
-// CHECK2-NEXT:    [[CMP17:%.*]] = icmp slt i32 4, [[ADD16]]
-// CHECK2-NEXT:    br i1 [[CMP17]], label [[COND_TRUE18:%.*]], label [[COND_FALSE19:%.*]]
-// CHECK2:       cond.true18:
-// CHECK2-NEXT:    br label [[COND_END21:%.*]]
-// CHECK2:       cond.false19:
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP16]], 5
-// CHECK2-NEXT:    br label [[COND_END21]]
-// CHECK2:       cond.end21:
-// CHECK2-NEXT:    [[COND22:%.*]] = phi i32 [ 4, [[COND_TRUE18]] ], [ [[ADD20]], [[COND_FALSE19]] ]
-// CHECK2-NEXT:    [[CMP23:%.*]] = icmp slt i32 [[TMP14]], [[COND22]]
-// CHECK2-NEXT:    br i1 [[CMP23]], label [[FOR_BODY24:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body24:
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP13]], [[TMP14]]
+// CHECK2-NEXT:    store i32 [[ADD12]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[MUL13:%.*]] = mul nsw i32 [[TMP15]], 3
+// CHECK2-NEXT:    [[ADD14:%.*]] = add nsw i32 7, [[MUL13]]
+// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP15:%.*]] = icmp slt i32 [[TMP16]], 4
+// CHECK2-NEXT:    br i1 [[CMP15]], label [[LAND_LHS_TRUE:%.*]], label [[IF_END:%.*]]
+// CHECK2:       land.lhs.true:
 // CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[MUL25:%.*]] = mul nsw i32 [[TMP17]], 3
-// CHECK2-NEXT:    [[ADD26:%.*]] = add nsw i32 7, [[MUL25]]
-// CHECK2-NEXT:    store i32 [[ADD26]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[CMP16:%.*]] = icmp slt i32 [[TMP17]], 4
+// CHECK2-NEXT:    br i1 [[CMP16]], label [[IF_THEN:%.*]], label [[IF_END]]
+// CHECK2:       if.then:
 // CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
 // CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[J]], align 4
 // CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP18]], i32 noundef [[TMP19]])
+// CHECK2-NEXT:    br label [[IF_END]]
+// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP20]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND15]], !llvm.loop [[LOOP14:![0-9]+]]
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND9]], !llvm.loop [[LOOP16:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC27:%.*]]
-// CHECK2:       for.inc27:
-// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC28:%.*]] = add nsw i32 [[TMP21]], 1
-// CHECK2-NEXT:    store i32 [[INC28]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND3]], !llvm.loop [[LOOP15:![0-9]+]]
-// CHECK2:       for.end29:
-// CHECK2-NEXT:    br label [[FOR_INC30:%.*]]
-// CHECK2:       for.inc30:
+// CHECK2-NEXT:    br label [[FOR_INC17:%.*]]
+// CHECK2:       for.inc17:
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC18:%.*]] = add nsw i32 [[TMP21]], 1
+// CHECK2-NEXT:    store i32 [[INC18]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND3]], !llvm.loop [[LOOP17:![0-9]+]]
+// CHECK2:       for.end19:
+// CHECK2-NEXT:    br label [[FOR_INC20:%.*]]
+// CHECK2:       for.inc20:
 // CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD31:%.*]] = add nsw i32 [[TMP22]], 5
-// CHECK2-NEXT:    store i32 [[ADD31]], ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP16:![0-9]+]]
-// CHECK2:       for.end32:
+// CHECK2-NEXT:    [[ADD21:%.*]] = add nsw i32 [[TMP22]], 5
+// CHECK2-NEXT:    store i32 [[ADD21]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP18:![0-9]+]]
+// CHECK2:       for.end22:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
 // CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[ADD33:%.*]] = add nsw i32 [[TMP23]], 1
-// CHECK2-NEXT:    store i32 [[ADD33]], ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP23]], 1
+// CHECK2-NEXT:    store i32 [[ADD23]], ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -1934,7 +1755,9 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[_TMP1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
@@ -1942,8 +1765,8 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[K:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK2-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK2-NEXT:    store i32 7, ptr [[J]], align 4
@@ -1990,87 +1813,79 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP9]], 4
-// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY:%.*]], label [[FOR_END37:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY:%.*]], label [[FOR_END27:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 [[TMP10]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND8:%.*]]
 // CHECK2:       for.cond8:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i32 [[TMP12]], 5
-// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i32 4, [[ADD9]]
-// CHECK2-NEXT:    br i1 [[CMP10]], label [[COND_TRUE11:%.*]], label [[COND_FALSE12:%.*]]
-// CHECK2:       cond.true11:
-// CHECK2-NEXT:    br label [[COND_END14:%.*]]
-// CHECK2:       cond.false12:
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP13]], 5
-// CHECK2-NEXT:    br label [[COND_END14]]
-// CHECK2:       cond.end14:
-// CHECK2-NEXT:    [[COND15:%.*]] = phi i32 [ 4, [[COND_TRUE11]] ], [ [[ADD13]], [[COND_FALSE12]] ]
-// CHECK2-NEXT:    [[CMP16:%.*]] = icmp slt i32 [[TMP11]], [[COND15]]
-// CHECK2-NEXT:    br i1 [[CMP16]], label [[FOR_BODY17:%.*]], label [[FOR_END34:%.*]]
-// CHECK2:       for.body17:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL18:%.*]] = mul nsw i32 [[TMP14]], 3
-// CHECK2-NEXT:    [[ADD19:%.*]] = add nsw i32 7, [[MUL18]]
-// CHECK2-NEXT:    store i32 [[ADD19]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP10]], 5
+// CHECK2-NEXT:    br i1 [[CMP9]], label [[FOR_BODY10:%.*]], label [[FOR_END24:%.*]]
+// CHECK2:       for.body10:
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP11]], [[TMP12]]
+// CHECK2-NEXT:    store i32 [[ADD11]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL12:%.*]] = mul nsw i32 [[TMP13]], 3
+// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i32 7, [[MUL12]]
+// CHECK2-NEXT:    store i32 [[ADD13]], ptr [[I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND14:%.*]]
+// CHECK2:       for.cond14:
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[CMP15:%.*]] = icmp slt i32 [[TMP14]], 5
+// CHECK2-NEXT:    br i1 [[CMP15]], label [[FOR_BODY16:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body16:
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    store i32 [[TMP15]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND20:%.*]]
-// CHECK2:       for.cond20:
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD21:%.*]] = add nsw i32 [[TMP17]], 5
-// CHECK2-NEXT:    [[CMP22:%.*]] = icmp slt i32 4, [[ADD21]]
-// CHECK2-NEXT:    br i1 [[CMP22]], label [[COND_TRUE23:%.*]], label [[COND_FALSE24:%.*]]
-// CHECK2:       cond.true23:
-// CHECK2-NEXT:    br label [[COND_END26:%.*]]
-// CHECK2:       cond.false24:
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD25:%.*]] = add nsw i32 [[TMP18]], 5
-// CHECK2-NEXT:    br label [[COND_END26]]
-// CHECK2:       cond.end26:
-// CHECK2-NEXT:    [[COND27:%.*]] = phi i32 [ 4, [[COND_TRUE23]] ], [ [[ADD25]], [[COND_FALSE24]] ]
-// CHECK2-NEXT:    [[CMP28:%.*]] = icmp slt i32 [[TMP16]], [[COND27]]
-// CHECK2-NEXT:    br i1 [[CMP28]], label [[FOR_BODY29:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body29:
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD17:%.*]] = add nsw i32 [[TMP15]], [[TMP16]]
+// CHECK2-NEXT:    store i32 [[ADD17]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[MUL18:%.*]] = mul nsw i32 [[TMP17]], 3
+// CHECK2-NEXT:    [[ADD19:%.*]] = add nsw i32 7, [[MUL18]]
+// CHECK2-NEXT:    store i32 [[ADD19]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP20:%.*]] = icmp slt i32 [[TMP18]], 4
+// CHECK2-NEXT:    br i1 [[CMP20]], label [[LAND_LHS_TRUE:%.*]], label [[IF_END:%.*]]
+// CHECK2:       land.lhs.true:
 // CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[MUL30:%.*]] = mul nsw i32 [[TMP19]], 3
-// CHECK2-NEXT:    [[ADD31:%.*]] = add nsw i32 7, [[MUL30]]
-// CHECK2-NEXT:    store i32 [[ADD31]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[CMP21:%.*]] = icmp slt i32 [[TMP19]], 4
+// CHECK2-NEXT:    br i1 [[CMP21]], label [[IF_THEN:%.*]], label [[IF_END]]
+// CHECK2:       if.then:
 // CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[I]], align 4
 // CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[J]], align 4
 // CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP20]], i32 noundef [[TMP21]])
+// CHECK2-NEXT:    br label [[IF_END]]
+// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP22]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND20]], !llvm.loop [[LOOP17:![0-9]+]]
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND14]], !llvm.loop [[LOOP19:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC32:%.*]]
-// CHECK2:       for.inc32:
-// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC33:%.*]] = add nsw i32 [[TMP23]], 1
-// CHECK2-NEXT:    store i32 [[INC33]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP18:![0-9]+]]
-// CHECK2:       for.end34:
-// CHECK2-NEXT:    br label [[FOR_INC35:%.*]]
-// CHECK2:       for.inc35:
+// CHECK2-NEXT:    br label [[FOR_INC22:%.*]]
+// CHECK2:       for.inc22:
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC23:%.*]] = add nsw i32 [[TMP23]], 1
+// CHECK2-NEXT:    store i32 [[INC23]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP20:![0-9]+]]
+// CHECK2:       for.end24:
+// CHECK2-NEXT:    br label [[FOR_INC25:%.*]]
+// CHECK2:       for.inc25:
 // CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD36:%.*]] = add nsw i32 [[TMP24]], 5
-// CHECK2-NEXT:    store i32 [[ADD36]], ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP19:![0-9]+]]
-// CHECK2:       for.end37:
+// CHECK2-NEXT:    [[ADD26:%.*]] = add nsw i32 [[TMP24]], 5
+// CHECK2-NEXT:    store i32 [[ADD26]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP21:![0-9]+]]
+// CHECK2:       for.end27:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
 // CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[ADD38:%.*]] = add nsw i32 [[TMP25]], 1
-// CHECK2-NEXT:    store i32 [[ADD38]], ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[ADD28:%.*]] = add nsw i32 [[TMP25]], 1
+// CHECK2-NEXT:    store i32 [[ADD28]], ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -2083,208 +1898,101 @@ extern "C" void foo10(data_t data) {
 // CHECK2-LABEL: define dso_local void @foo5(
 // CHECK2-SAME: ) #[[ATTR1]] {
 // CHECK2-NEXT:  entry:
-// CHECK2-NEXT:    [[DOTOMP_IV:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[_TMP1:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[_TMP2:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_5:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I11:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I12:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[J13:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK2-NEXT:    store i32 7, ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP]], align 4
-// CHECK2-NEXT:    store i32 [[TMP1]], ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP]], align 4
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], 5
-// CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
+// CHECK2-NEXT:    store i32 4, ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
+// CHECK2-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], 4
 // CHECK2-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK2:       cond.true:
 // CHECK2-NEXT:    br label [[COND_END:%.*]]
 // CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP]], align 4
-// CHECK2-NEXT:    [[ADD4:%.*]] = add nsw i32 [[TMP3]], 5
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
 // CHECK2-NEXT:    br label [[COND_END]]
 // CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD4]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    store i32 [[COND]], ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[SUB:%.*]] = sub i32 [[TMP4]], [[TMP5]]
-// CHECK2-NEXT:    [[SUB6:%.*]] = sub i32 [[SUB]], 1
-// CHECK2-NEXT:    [[ADD7:%.*]] = add i32 [[SUB6]], 1
-// CHECK2-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD7]], 1
-// CHECK2-NEXT:    [[CONV:%.*]] = zext i32 [[DIV]] to i64
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 1, [[CONV]]
-// CHECK2-NEXT:    [[MUL8:%.*]] = mul nsw i64 [[MUL]], 4
-// CHECK2-NEXT:    [[SUB9:%.*]] = sub nsw i64 [[MUL8]], 1
-// CHECK2-NEXT:    store i64 [[SUB9]], ptr [[DOTCAPTURE_EXPR_5]], align 8
-// CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    store i32 [[TMP6]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 7, ptr [[J]], align 4
-// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i32 [[TMP7]], [[TMP8]]
-// CHECK2-NEXT:    br i1 [[CMP10]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END:%.*]]
-// CHECK2:       omp.precond.then:
-// CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_5]], align 8
-// CHECK2-NEXT:    store i64 [[TMP9]], ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    store i64 1, ptr [[DOTOMP_STRIDE]], align 8
-// CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
-// CHECK2-NEXT:    call void @__kmpc_for_static_init_8(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i64 1, i64 1)
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_5]], align 8
-// CHECK2-NEXT:    [[CMP14:%.*]] = icmp sgt i64 [[TMP10]], [[TMP11]]
-// CHECK2-NEXT:    br i1 [[CMP14]], label [[COND_TRUE15:%.*]], label [[COND_FALSE16:%.*]]
-// CHECK2:       cond.true15:
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_5]], align 8
-// CHECK2-NEXT:    br label [[COND_END17:%.*]]
-// CHECK2:       cond.false16:
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    br label [[COND_END17]]
-// CHECK2:       cond.end17:
-// CHECK2-NEXT:    [[COND18:%.*]] = phi i64 [ [[TMP12]], [[COND_TRUE15]] ], [ [[TMP13]], [[COND_FALSE16]] ]
-// CHECK2-NEXT:    store i64 [[COND18]], ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
-// CHECK2-NEXT:    store i64 [[TMP14]], ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[TMP2]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
+// CHECK2-NEXT:    store i32 [[TMP3]], ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK2:       omp.inner.for.cond:
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[CMP19:%.*]] = icmp sle i64 [[TMP15]], [[TMP16]]
-// CHECK2-NEXT:    br i1 [[CMP19]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], [[TMP5]]
+// CHECK2-NEXT:    br i1 [[CMP2]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
 // CHECK2:       omp.inner.for.body:
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[SUB20:%.*]] = sub i32 [[TMP18]], [[TMP19]]
-// CHECK2-NEXT:    [[SUB21:%.*]] = sub i32 [[SUB20]], 1
-// CHECK2-NEXT:    [[ADD22:%.*]] = add i32 [[SUB21]], 1
-// CHECK2-NEXT:    [[DIV23:%.*]] = udiv i32 [[ADD22]], 1
-// CHECK2-NEXT:    [[MUL24:%.*]] = mul i32 1, [[DIV23]]
-// CHECK2-NEXT:    [[MUL25:%.*]] = mul i32 [[MUL24]], 4
-// CHECK2-NEXT:    [[CONV26:%.*]] = zext i32 [[MUL25]] to i64
-// CHECK2-NEXT:    [[DIV27:%.*]] = sdiv i64 [[TMP17]], [[CONV26]]
-// CHECK2-NEXT:    [[MUL28:%.*]] = mul nsw i64 [[DIV27]], 5
-// CHECK2-NEXT:    [[ADD29:%.*]] = add nsw i64 0, [[MUL28]]
-// CHECK2-NEXT:    [[CONV30:%.*]] = trunc i64 [[ADD29]] to i32
-// CHECK2-NEXT:    store i32 [[CONV30]], ptr [[DOTFLOOR_0_IV_I11]], align 4
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[CONV31:%.*]] = sext i32 [[TMP20]] to i64
-// CHECK2-NEXT:    [[TMP21:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP22:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[SUB32:%.*]] = sub i32 [[TMP23]], [[TMP24]]
-// CHECK2-NEXT:    [[SUB33:%.*]] = sub i32 [[SUB32]], 1
-// CHECK2-NEXT:    [[ADD34:%.*]] = add i32 [[SUB33]], 1
-// CHECK2-NEXT:    [[DIV35:%.*]] = udiv i32 [[ADD34]], 1
-// CHECK2-NEXT:    [[MUL36:%.*]] = mul i32 1, [[DIV35]]
-// CHECK2-NEXT:    [[MUL37:%.*]] = mul i32 [[MUL36]], 4
-// CHECK2-NEXT:    [[CONV38:%.*]] = zext i32 [[MUL37]] to i64
-// CHECK2-NEXT:    [[DIV39:%.*]] = sdiv i64 [[TMP22]], [[CONV38]]
-// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[SUB40:%.*]] = sub i32 [[TMP25]], [[TMP26]]
-// CHECK2-NEXT:    [[SUB41:%.*]] = sub i32 [[SUB40]], 1
-// CHECK2-NEXT:    [[ADD42:%.*]] = add i32 [[SUB41]], 1
-// CHECK2-NEXT:    [[DIV43:%.*]] = udiv i32 [[ADD42]], 1
-// CHECK2-NEXT:    [[MUL44:%.*]] = mul i32 1, [[DIV43]]
-// CHECK2-NEXT:    [[MUL45:%.*]] = mul i32 [[MUL44]], 4
-// CHECK2-NEXT:    [[CONV46:%.*]] = zext i32 [[MUL45]] to i64
-// CHECK2-NEXT:    [[MUL47:%.*]] = mul nsw i64 [[DIV39]], [[CONV46]]
-// CHECK2-NEXT:    [[SUB48:%.*]] = sub nsw i64 [[TMP21]], [[MUL47]]
-// CHECK2-NEXT:    [[DIV49:%.*]] = sdiv i64 [[SUB48]], 4
-// CHECK2-NEXT:    [[MUL50:%.*]] = mul nsw i64 [[DIV49]], 1
-// CHECK2-NEXT:    [[ADD51:%.*]] = add nsw i64 [[CONV31]], [[MUL50]]
-// CHECK2-NEXT:    [[CONV52:%.*]] = trunc i64 [[ADD51]] to i32
-// CHECK2-NEXT:    store i32 [[CONV52]], ptr [[DOTTILE_0_IV_I12]], align 4
-// CHECK2-NEXT:    [[TMP27:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP28:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[SUB53:%.*]] = sub i32 [[TMP29]], [[TMP30]]
-// CHECK2-NEXT:    [[SUB54:%.*]] = sub i32 [[SUB53]], 1
-// CHECK2-NEXT:    [[ADD55:%.*]] = add i32 [[SUB54]], 1
-// CHECK2-NEXT:    [[DIV56:%.*]] = udiv i32 [[ADD55]], 1
-// CHECK2-NEXT:    [[MUL57:%.*]] = mul i32 1, [[DIV56]]
-// CHECK2-NEXT:    [[MUL58:%.*]] = mul i32 [[MUL57]], 4
-// CHECK2-NEXT:    [[CONV59:%.*]] = zext i32 [[MUL58]] to i64
-// CHECK2-NEXT:    [[DIV60:%.*]] = sdiv i64 [[TMP28]], [[CONV59]]
-// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK2-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[SUB61:%.*]] = sub i32 [[TMP31]], [[TMP32]]
-// CHECK2-NEXT:    [[SUB62:%.*]] = sub i32 [[SUB61]], 1
-// CHECK2-NEXT:    [[ADD63:%.*]] = add i32 [[SUB62]], 1
-// CHECK2-NEXT:    [[DIV64:%.*]] = udiv i32 [[ADD63]], 1
-// CHECK2-NEXT:    [[MUL65:%.*]] = mul i32 1, [[DIV64]]
-// CHECK2-NEXT:    [[MUL66:%.*]] = mul i32 [[MUL65]], 4
-// CHECK2-NEXT:    [[CONV67:%.*]] = zext i32 [[MUL66]] to i64
-// CHECK2-NEXT:    [[MUL68:%.*]] = mul nsw i64 [[DIV60]], [[CONV67]]
-// CHECK2-NEXT:    [[SUB69:%.*]] = sub nsw i64 [[TMP27]], [[MUL68]]
-// CHECK2-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK2-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[SUB70:%.*]] = sub i32 [[TMP35]], [[TMP36]]
-// CHECK2-NEXT:    [[SUB71:%.*]] = sub i32 [[SUB70]], 1
-// CHECK2-NEXT:    [[ADD72:%.*]] = add i32 [[SUB71]], 1
-// CHECK2-NEXT:    [[DIV73:%.*]] = udiv i32 [[ADD72]], 1
-// CHECK2-NEXT:    [[MUL74:%.*]] = mul i32 1, [[DIV73]]
-// CHECK2-NEXT:    [[MUL75:%.*]] = mul i32 [[MUL74]], 4
-// CHECK2-NEXT:    [[CONV76:%.*]] = zext i32 [[MUL75]] to i64
-// CHECK2-NEXT:    [[DIV77:%.*]] = sdiv i64 [[TMP34]], [[CONV76]]
-// CHECK2-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_3]], align 4
-// CHECK2-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[SUB78:%.*]] = sub i32 [[TMP37]], [[TMP38]]
-// CHECK2-NEXT:    [[SUB79:%.*]] = sub i32 [[SUB78]], 1
-// CHECK2-NEXT:    [[ADD80:%.*]] = add i32 [[SUB79]], 1
-// CHECK2-NEXT:    [[DIV81:%.*]] = udiv i32 [[ADD80]], 1
-// CHECK2-NEXT:    [[MUL82:%.*]] = mul i32 1, [[DIV81]]
-// CHECK2-NEXT:    [[MUL83:%.*]] = mul i32 [[MUL82]], 4
-// CHECK2-NEXT:    [[CONV84:%.*]] = zext i32 [[MUL83]] to i64
-// CHECK2-NEXT:    [[MUL85:%.*]] = mul nsw i64 [[DIV77]], [[CONV84]]
-// CHECK2-NEXT:    [[SUB86:%.*]] = sub nsw i64 [[TMP33]], [[MUL85]]
-// CHECK2-NEXT:    [[DIV87:%.*]] = sdiv i64 [[SUB86]], 4
-// CHECK2-NEXT:    [[MUL88:%.*]] = mul nsw i64 [[DIV87]], 4
-// CHECK2-NEXT:    [[SUB89:%.*]] = sub nsw i64 [[SUB69]], [[MUL88]]
-// CHECK2-NEXT:    [[MUL90:%.*]] = mul nsw i64 [[SUB89]], 3
-// CHECK2-NEXT:    [[ADD91:%.*]] = add nsw i64 7, [[MUL90]]
-// CHECK2-NEXT:    [[CONV92:%.*]] = trunc i64 [[ADD91]] to i32
-// CHECK2-NEXT:    store i32 [[CONV92]], ptr [[J13]], align 4
-// CHECK2-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTTILE_0_IV_I12]], align 4
-// CHECK2-NEXT:    [[MUL93:%.*]] = mul nsw i32 [[TMP39]], 3
-// CHECK2-NEXT:    [[ADD94:%.*]] = add nsw i32 7, [[MUL93]]
-// CHECK2-NEXT:    store i32 [[ADD94]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP41:%.*]] = load i32, ptr [[J13]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP40]], i32 noundef [[TMP41]])
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP6]], 5
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[DIV]], 5
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[DIV3:%.*]] = sdiv i32 [[TMP8]], 5
+// CHECK2-NEXT:    [[MUL4:%.*]] = mul nsw i32 [[DIV3]], 5
+// CHECK2-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP7]], [[MUL4]]
+// CHECK2-NEXT:    [[MUL5:%.*]] = mul nsw i32 [[SUB]], 1
+// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL5]]
+// CHECK2-NEXT:    store i32 [[ADD6]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
+// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL8:%.*]] = mul nsw i32 [[TMP11]], 3
+// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL8]]
+// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i32 [[TMP12]], 4
+// CHECK2-NEXT:    br i1 [[CMP10]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK2:       if.then:
+// CHECK2-NEXT:    store i32 7, ptr [[J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND:%.*]]
+// CHECK2:       for.cond:
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    [[CMP11:%.*]] = icmp slt i32 [[TMP13]], 17
+// CHECK2-NEXT:    br i1 [[CMP11]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body:
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP14]], i32 noundef [[TMP15]])
+// CHECK2-NEXT:    br label [[FOR_INC:%.*]]
+// CHECK2:       for.inc:
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP16]], 3
+// CHECK2-NEXT:    store i32 [[ADD12]], ptr [[J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP22:![0-9]+]]
+// CHECK2:       for.end:
+// CHECK2-NEXT:    br label [[IF_END]]
+// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
-// CHECK2-NEXT:    [[TMP42:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[ADD95:%.*]] = add nsw i64 [[TMP42]], 1
-// CHECK2-NEXT:    store i64 [[ADD95]], ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP17]], 1
+// CHECK2-NEXT:    store i32 [[ADD13]], ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
 // CHECK2:       omp.loop.exit:
 // CHECK2-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP0]])
-// CHECK2-NEXT:    br label [[OMP_PRECOND_END]]
-// CHECK2:       omp.precond.end:
 // CHECK2-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[TMP0]])
 // CHECK2-NEXT:    ret void
 //
@@ -2304,12 +2012,13 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK2-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
 // CHECK2-NEXT:    store i32 7, ptr [[I]], align 4
@@ -2344,46 +2053,43 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP7]], 5
 // CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 [[TMP8]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND:%.*]]
 // CHECK2:       for.cond:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD2:%.*]] = add nsw i32 [[TMP10]], 5
-// CHECK2-NEXT:    [[CMP3:%.*]] = icmp slt i32 4, [[ADD2]]
-// CHECK2-NEXT:    br i1 [[CMP3]], label [[COND_TRUE4:%.*]], label [[COND_FALSE5:%.*]]
-// CHECK2:       cond.true4:
-// CHECK2-NEXT:    br label [[COND_END7:%.*]]
-// CHECK2:       cond.false5:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP11]], 5
-// CHECK2-NEXT:    br label [[COND_END7]]
-// CHECK2:       cond.end7:
-// CHECK2-NEXT:    [[COND8:%.*]] = phi i32 [ 4, [[COND_TRUE4]] ], [ [[ADD6]], [[COND_FALSE5]] ]
-// CHECK2-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP9]], [[COND8]]
-// CHECK2-NEXT:    br i1 [[CMP9]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP8]], 5
+// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
 // CHECK2:       for.body:
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
+// CHECK2-NEXT:    store i32 [[ADD3]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL4:%.*]] = mul nsw i32 [[TMP11]], 3
+// CHECK2-NEXT:    [[ADD5:%.*]] = add nsw i32 7, [[MUL4]]
+// CHECK2-NEXT:    store i32 [[ADD5]], ptr [[I]], align 4
 // CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL10:%.*]] = mul nsw i32 [[TMP12]], 3
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 7, [[MUL10]]
-// CHECK2-NEXT:    store i32 [[ADD11]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[CMP6:%.*]] = icmp slt i32 [[TMP12]], 4
+// CHECK2-NEXT:    br i1 [[CMP6]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK2:       if.then:
 // CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[I]], align 4
 // CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP13]])
+// CHECK2-NEXT:    br label [[IF_END]]
+// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP20:![0-9]+]]
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP23:![0-9]+]]
 // CHECK2:       for.end:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP15]], 1
-// CHECK2-NEXT:    store i32 [[ADD12]], ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 [[TMP15]], 1
+// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -2397,8 +2103,9 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:  entry:
 // CHECK2-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
 // CHECK2-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
@@ -2406,78 +2113,65 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END14:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK2:       for.cond1:
-// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], 0
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP2]], 0
 // CHECK2-NEXT:    br i1 [[CMP2]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK2:       cond.true:
 // CHECK2-NEXT:    br label [[COND_END:%.*]]
 // CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[A_ADDR]], align 4
 // CHECK2-NEXT:    br label [[COND_END]]
 // CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP5]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[COND]]
-// CHECK2-NEXT:    [[CMP3:%.*]] = icmp slt i32 4, [[ADD]]
-// CHECK2-NEXT:    br i1 [[CMP3]], label [[COND_TRUE4:%.*]], label [[COND_FALSE5:%.*]]
-// CHECK2:       cond.true4:
-// CHECK2-NEXT:    br label [[COND_END12:%.*]]
-// CHECK2:       cond.false5:
-// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    [[CMP6:%.*]] = icmp sle i32 [[TMP7]], 0
-// CHECK2-NEXT:    br i1 [[CMP6]], label [[COND_TRUE7:%.*]], label [[COND_FALSE8:%.*]]
-// CHECK2:       cond.true7:
-// CHECK2-NEXT:    br label [[COND_END9:%.*]]
-// CHECK2:       cond.false8:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    br label [[COND_END9]]
-// CHECK2:       cond.end9:
-// CHECK2-NEXT:    [[COND10:%.*]] = phi i32 [ 1, [[COND_TRUE7]] ], [ [[TMP8]], [[COND_FALSE8]] ]
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP6]], [[COND10]]
-// CHECK2-NEXT:    br label [[COND_END12]]
-// CHECK2:       cond.end12:
-// CHECK2-NEXT:    [[COND13:%.*]] = phi i32 [ 4, [[COND_TRUE4]] ], [ [[ADD11]], [[COND_END9]] ]
-// CHECK2-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP2]], [[COND13]]
-// CHECK2-NEXT:    br i1 [[CMP14]], label [[FOR_BODY15:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body15:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP9]], 3
-// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP10]])
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP3]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[TMP1]], [[COND]]
+// CHECK2-NEXT:    br i1 [[CMP3]], label [[FOR_BODY4:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body4:
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
+// CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
+// CHECK2-NEXT:    [[ADD5:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD5]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP6:%.*]] = icmp slt i32 [[TMP7]], 4
+// CHECK2-NEXT:    br i1 [[CMP6]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK2:       if.then:
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP8]])
+// CHECK2-NEXT:    br label [[IF_END]]
+// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP23:![0-9]+]]
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP9]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP26:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC17:%.*]]
-// CHECK2:       for.inc17:
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    [[CMP18:%.*]] = icmp sle i32 [[TMP12]], 0
-// CHECK2-NEXT:    br i1 [[CMP18]], label [[COND_TRUE19:%.*]], label [[COND_FALSE20:%.*]]
-// CHECK2:       cond.true19:
-// CHECK2-NEXT:    br label [[COND_END21:%.*]]
-// CHECK2:       cond.false20:
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    br label [[COND_END21]]
-// CHECK2:       cond.end21:
-// CHECK2-NEXT:    [[COND22:%.*]] = phi i32 [ 1, [[COND_TRUE19]] ], [ [[TMP13]], [[COND_FALSE20]] ]
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP14]], [[COND22]]
-// CHECK2-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP24:![0-9]+]]
-// CHECK2:       for.end24:
+// CHECK2-NEXT:    br label [[FOR_INC7:%.*]]
+// CHECK2:       for.inc7:
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    [[CMP8:%.*]] = icmp sle i32 [[TMP10]], 0
+// CHECK2-NEXT:    br i1 [[CMP8]], label [[COND_TRUE9:%.*]], label [[COND_FALSE10:%.*]]
+// CHECK2:       cond.true9:
+// CHECK2-NEXT:    br label [[COND_END11:%.*]]
+// CHECK2:       cond.false10:
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    br label [[COND_END11]]
+// CHECK2:       cond.end11:
+// CHECK2-NEXT:    [[COND12:%.*]] = phi i32 [ 1, [[COND_TRUE9]] ], [ [[TMP11]], [[COND_FALSE10]] ]
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP12]], [[COND12]]
+// CHECK2-NEXT:    store i32 [[ADD13]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP27:![0-9]+]]
+// CHECK2:       for.end14:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -2490,19 +2184,20 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK2-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
 // CHECK2-NEXT:    store ptr [[ARRAY]], ptr [[__RANGE2]], align 8
-// CHECK2-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META9]]
 // CHECK2-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP0]], i64 0, i64 0
 // CHECK2-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 12
 // CHECK2-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
-// CHECK2-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META9]]
 // CHECK2-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP1]], i64 0, i64 0
 // CHECK2-NEXT:    store ptr [[ARRAYDECAY1]], ptr [[__BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META9]]
 // CHECK2-NEXT:    [[ARRAYDECAY2:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP2]], i64 0, i64 0
 // CHECK2-NEXT:    store ptr [[ARRAYDECAY2]], ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK2-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__END2]], align 8
@@ -2525,56 +2220,51 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK2:       for.cond7:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
-// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
-// CHECK2-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK2:       cond.true:
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
-// CHECK2-NEXT:    br label [[COND_END:%.*]]
-// CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
-// CHECK2-NEXT:    br label [[COND_END]]
-// CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
-// CHECK2-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body14:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
+// CHECK2-NEXT:    br i1 [[CMP8]], label [[FOR_BODY9:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body9:
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
+// CHECK2-NEXT:    store i64 [[ADD10]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP12]], 1
+// CHECK2-NEXT:    [[ADD_PTR11:%.*]] = getelementptr inbounds double, ptr [[TMP11]], i64 [[MUL]]
+// CHECK2-NEXT:    store ptr [[ADD_PTR11]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP14:%.*]] = load double, ptr [[TMP13]], align 8
+// CHECK2-NEXT:    store double [[TMP14]], ptr [[V]], align 8
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
-// CHECK2-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
-// CHECK2-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
-// CHECK2-NEXT:    store double [[TMP17]], ptr [[V]], align 8
-// CHECK2-NEXT:    [[TMP18:%.*]] = load double, ptr [[V]], align 8
-// CHECK2-NEXT:    call void (...) @body(double noundef [[TMP18]])
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP16]], 1
+// CHECK2-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP15]], [[ADD12]]
+// CHECK2-NEXT:    br i1 [[CMP13]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK2:       if.then:
+// CHECK2-NEXT:    [[TMP17:%.*]] = load double, ptr [[V]], align 8
+// CHECK2-NEXT:    call void (...) @body(double noundef [[TMP17]])
+// CHECK2-NEXT:    br label [[IF_END]]
+// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP19]], 1
-// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP25:![0-9]+]]
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP18]], 1
+// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP28:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC16:%.*]]
-// CHECK2:       for.inc16:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP20]], 5
-// CHECK2-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP26:![0-9]+]]
-// CHECK2:       for.end18:
+// CHECK2-NEXT:    br label [[FOR_INC14:%.*]]
+// CHECK2:       for.inc14:
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD15:%.*]] = add nsw i64 [[TMP19]], 5
+// CHECK2-NEXT:    store i64 [[ADD15]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP29:![0-9]+]]
+// CHECK2:       for.end16:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -2594,8 +2284,9 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[START_ADDR]], align 4
@@ -2619,53 +2310,48 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK2-NEXT:    [[ADD5:%.*]] = add i32 [[TMP6]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP5]], [[ADD5]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END15:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK2:       for.cond6:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[ADD7:%.*]] = add i32 [[TMP9]], 1
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD8:%.*]] = add i32 [[TMP10]], 5
-// CHECK2-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
-// CHECK2-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK2:       cond.true:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[ADD10:%.*]] = add i32 [[TMP11]], 1
-// CHECK2-NEXT:    br label [[COND_END:%.*]]
-// CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD11:%.*]] = add i32 [[TMP12]], 5
-// CHECK2-NEXT:    br label [[COND_END]]
-// CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP8]], [[COND]]
-// CHECK2-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body13:
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP14]], 3
-// CHECK2-NEXT:    [[ADD14:%.*]] = add i32 [[TMP13]], [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP15]])
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP7]], 5
+// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body8:
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD9:%.*]] = add i32 [[TMP8]], [[TMP9]]
+// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP11]], 3
+// CHECK2-NEXT:    [[ADD10:%.*]] = add i32 [[TMP10]], [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD10]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD11:%.*]] = add i32 [[TMP13]], 1
+// CHECK2-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP12]], [[ADD11]]
+// CHECK2-NEXT:    br i1 [[CMP12]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK2:       if.then:
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP14]])
+// CHECK2-NEXT:    br label [[IF_END]]
+// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP16]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP27:![0-9]+]]
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP15]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP30:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC15:%.*]]
-// CHECK2:       for.inc15:
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD16:%.*]] = add i32 [[TMP17]], 5
-// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP28:![0-9]+]]
-// CHECK2:       for.end17:
+// CHECK2-NEXT:    br label [[FOR_INC13:%.*]]
+// CHECK2:       for.inc13:
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD14:%.*]] = add i32 [[TMP16]], 5
+// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP31:![0-9]+]]
+// CHECK2:       for.end15:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -2676,53 +2362,61 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    ret void
 //
 //.
-// CHECK1: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]]}
-// CHECK1: [[META4]] = !{!"llvm.loop.mustprogress"}
-// CHECK1: [[LOOP5]] = distinct !{[[LOOP5]], [[META4]]}
-// CHECK1: [[LOOP6]] = distinct !{[[LOOP6]], [[META4]]}
-// CHECK1: [[LOOP7]] = distinct !{[[LOOP7]], [[META4]]}
-// CHECK1: [[LOOP8]] = distinct !{[[LOOP8]], [[META4]]}
-// CHECK1: [[LOOP9]] = distinct !{[[LOOP9]], [[META4]]}
-// CHECK1: [[LOOP10]] = distinct !{[[LOOP10]], [[META4]]}
-// CHECK1: [[LOOP11]] = distinct !{[[LOOP11]], [[META4]]}
-// CHECK1: [[LOOP12]] = distinct !{[[LOOP12]], [[META4]]}
-// CHECK1: [[LOOP13]] = distinct !{[[LOOP13]], [[META4]]}
-// CHECK1: [[LOOP14]] = distinct !{[[LOOP14]], [[META4]]}
-// CHECK1: [[LOOP15]] = distinct !{[[LOOP15]], [[META4]]}
-// CHECK1: [[LOOP16]] = distinct !{[[LOOP16]], [[META4]]}
-// CHECK1: [[LOOP17]] = distinct !{[[LOOP17]], [[META4]]}
-// CHECK1: [[LOOP18]] = distinct !{[[LOOP18]], [[META4]]}
-// CHECK1: [[LOOP21]] = distinct !{[[LOOP21]], [[META4]]}
-// CHECK1: [[LOOP22]] = distinct !{[[LOOP22]], [[META4]]}
-// CHECK1: [[LOOP23]] = distinct !{[[LOOP23]], [[META4]]}
-// CHECK1: [[LOOP24]] = distinct !{[[LOOP24]], [[META4]]}
-// CHECK1: [[LOOP25]] = distinct !{[[LOOP25]], [[META4]]}
-// CHECK1: [[LOOP26]] = distinct !{[[LOOP26]], [[META4]]}
-// CHECK1: [[LOOP27]] = distinct !{[[LOOP27]], [[META4]]}
-// CHECK1: [[LOOP28]] = distinct !{[[LOOP28]], [[META4]]}
+// CHECK1: [[META2]] = !{}
+// CHECK1: [[META3]] = !{i64 4}
+// CHECK1: [[LOOP4]] = distinct !{[[LOOP4]], [[META5:![0-9]+]]}
+// CHECK1: [[META5]] = !{!"llvm.loop.mustprogress"}
+// CHECK1: [[LOOP6]] = distinct !{[[LOOP6]], [[META5]]}
+// CHECK1: [[LOOP7]] = distinct !{[[LOOP7]], [[META5]]}
+// CHECK1: [[LOOP8]] = distinct !{[[LOOP8]], [[META5]]}
+// CHECK1: [[LOOP9]] = distinct !{[[LOOP9]], [[META5]]}
+// CHECK1: [[LOOP10]] = distinct !{[[LOOP10]], [[META5]]}
+// CHECK1: [[LOOP11]] = distinct !{[[LOOP11]], [[META5]]}
+// CHECK1: [[LOOP12]] = distinct !{[[LOOP12]], [[META5]]}
+// CHECK1: [[LOOP13]] = distinct !{[[LOOP13]], [[META5]]}
+// CHECK1: [[LOOP14]] = distinct !{[[LOOP14]], [[META5]]}
+// CHECK1: [[LOOP15]] = distinct !{[[LOOP15]], [[META5]]}
+// CHECK1: [[LOOP16]] = distinct !{[[LOOP16]], [[META5]]}
+// CHECK1: [[LOOP17]] = distinct !{[[LOOP17]], [[META5]]}
+// CHECK1: [[LOOP18]] = distinct !{[[LOOP18]], [[META5]]}
+// CHECK1: [[LOOP19]] = distinct !{[[LOOP19]], [[META5]]}
+// CHECK1: [[LOOP20]] = distinct !{[[LOOP20]], [[META5]]}
+// CHECK1: [[LOOP23]] = distinct !{[[LOOP23]], [[META5]]}
+// CHECK1: [[LOOP24]] = distinct !{[[LOOP24]], [[META5]]}
+// CHECK1: [[LOOP25]] = distinct !{[[LOOP25]], [[META5]]}
+// CHECK1: [[LOOP26]] = distinct !{[[LOOP26]], [[META5]]}
+// CHECK1: [[META27]] = !{i64 8}
+// CHECK1: [[LOOP28]] = distinct !{[[LOOP28]], [[META5]]}
+// CHECK1: [[LOOP29]] = distinct !{[[LOOP29]], [[META5]]}
+// CHECK1: [[LOOP30]] = distinct !{[[LOOP30]], [[META5]]}
+// CHECK1: [[LOOP31]] = distinct !{[[LOOP31]], [[META5]]}
 //.
-// CHECK2: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]]}
-// CHECK2: [[META4]] = !{!"llvm.loop.mustprogress"}
-// CHECK2: [[LOOP5]] = distinct !{[[LOOP5]], [[META4]]}
-// CHECK2: [[LOOP6]] = distinct !{[[LOOP6]], [[META4]]}
-// CHECK2: [[LOOP7]] = distinct !{[[LOOP7]], [[META4]]}
-// CHECK2: [[LOOP8]] = distinct !{[[LOOP8]], [[META4]]}
-// CHECK2: [[LOOP9]] = distinct !{[[LOOP9]], [[META4]]}
-// CHECK2: [[LOOP10]] = distinct !{[[LOOP10]], [[META4]]}
-// CHECK2: [[LOOP11]] = distinct !{[[LOOP11]], [[META4]]}
-// CHECK2: [[LOOP12]] = distinct !{[[LOOP12]], [[META4]]}
-// CHECK2: [[LOOP13]] = distinct !{[[LOOP13]], [[META4]]}
-// CHECK2: [[LOOP14]] = distinct !{[[LOOP14]], [[META4]]}
-// CHECK2: [[LOOP15]] = distinct !{[[LOOP15]], [[META4]]}
-// CHECK2: [[LOOP16]] = distinct !{[[LOOP16]], [[META4]]}
-// CHECK2: [[LOOP17]] = distinct !{[[LOOP17]], [[META4]]}
-// CHECK2: [[LOOP18]] = distinct !{[[LOOP18]], [[META4]]}
-// CHECK2: [[LOOP19]] = distinct !{[[LOOP19]], [[META4]]}
-// CHECK2: [[LOOP20]] = distinct !{[[LOOP20]], [[META4]]}
-// CHECK2: [[LOOP23]] = distinct !{[[LOOP23]], [[META4]]}
-// CHECK2: [[LOOP24]] = distinct !{[[LOOP24]], [[META4]]}
-// CHECK2: [[LOOP25]] = distinct !{[[LOOP25]], [[META4]]}
-// CHECK2: [[LOOP26]] = distinct !{[[LOOP26]], [[META4]]}
-// CHECK2: [[LOOP27]] = distinct !{[[LOOP27]], [[META4]]}
-// CHECK2: [[LOOP28]] = distinct !{[[LOOP28]], [[META4]]}
+// CHECK2: [[META2]] = !{}
+// CHECK2: [[META3]] = !{i64 4}
+// CHECK2: [[LOOP4]] = distinct !{[[LOOP4]], [[META5:![0-9]+]]}
+// CHECK2: [[META5]] = !{!"llvm.loop.mustprogress"}
+// CHECK2: [[LOOP6]] = distinct !{[[LOOP6]], [[META5]]}
+// CHECK2: [[LOOP7]] = distinct !{[[LOOP7]], [[META5]]}
+// CHECK2: [[LOOP8]] = distinct !{[[LOOP8]], [[META5]]}
+// CHECK2: [[META9]] = !{i64 8}
+// CHECK2: [[LOOP10]] = distinct !{[[LOOP10]], [[META5]]}
+// CHECK2: [[LOOP11]] = distinct !{[[LOOP11]], [[META5]]}
+// CHECK2: [[LOOP12]] = distinct !{[[LOOP12]], [[META5]]}
+// CHECK2: [[LOOP13]] = distinct !{[[LOOP13]], [[META5]]}
+// CHECK2: [[LOOP14]] = distinct !{[[LOOP14]], [[META5]]}
+// CHECK2: [[LOOP15]] = distinct !{[[LOOP15]], [[META5]]}
+// CHECK2: [[LOOP16]] = distinct !{[[LOOP16]], [[META5]]}
+// CHECK2: [[LOOP17]] = distinct !{[[LOOP17]], [[META5]]}
+// CHECK2: [[LOOP18]] = distinct !{[[LOOP18]], [[META5]]}
+// CHECK2: [[LOOP19]] = distinct !{[[LOOP19]], [[META5]]}
+// CHECK2: [[LOOP20]] = distinct !{[[LOOP20]], [[META5]]}
+// CHECK2: [[LOOP21]] = distinct !{[[LOOP21]], [[META5]]}
+// CHECK2: [[LOOP22]] = distinct !{[[LOOP22]], [[META5]]}
+// CHECK2: [[LOOP23]] = distinct !{[[LOOP23]], [[META5]]}
+// CHECK2: [[LOOP26]] = distinct !{[[LOOP26]], [[META5]]}
+// CHECK2: [[LOOP27]] = distinct !{[[LOOP27]], [[META5]]}
+// CHECK2: [[LOOP28]] = distinct !{[[LOOP28]], [[META5]]}
+// CHECK2: [[LOOP29]] = distinct !{[[LOOP29]], [[META5]]}
+// CHECK2: [[LOOP30]] = distinct !{[[LOOP30]], [[META5]]}
+// CHECK2: [[LOOP31]] = distinct !{[[LOOP31]], [[META5]]}
 //.
diff --git a/clang/test/OpenMP/tile_codegen_for_dependent.cpp b/clang/test/OpenMP/tile_codegen_for_dependent.cpp
index 820d33d15287b..aadc8e58bebfc 100644
--- a/clang/test/OpenMP/tile_codegen_for_dependent.cpp
+++ b/clang/test/OpenMP/tile_codegen_for_dependent.cpp
@@ -1,3 +1,4 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
 // Check code generation
 // RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fopenmp -emit-llvm %s -o - | FileCheck %s --check-prefix=IR
 
@@ -17,172 +18,151 @@
 extern "C" void body(...) {}
 
 
-// IR-LABEL: define {{.*}}@func(
-// IR-NEXT:  [[ENTRY:.*]]:
-// IR-NEXT:    %[[START_ADDR:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[END_ADDR:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[STEP_ADDR:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_IV:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[TMP:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[I:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_1:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTNEW_STEP:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_2:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_5:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_7:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTFLOOR_0_IV_I:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_LB:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_UB:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_STRIDE:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_IS_LAST:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTFLOOR_0_IV_I11:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTTILE_0_IV_I:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[TMP0:.+]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:.+]])
-// IR-NEXT:    store i32 %[[START:.+]], ptr %[[START_ADDR]], align 4
-// IR-NEXT:    store i32 %[[END:.+]], ptr %[[END_ADDR]], align 4
-// IR-NEXT:    store i32 %[[STEP:.+]], ptr %[[STEP_ADDR]], align 4
-// IR-NEXT:    %[[TMP1:.+]] = load i32, ptr %[[START_ADDR]], align 4
-// IR-NEXT:    store i32 %[[TMP1]], ptr %[[I]], align 4
-// IR-NEXT:    %[[TMP2:.+]] = load i32, ptr %[[START_ADDR]], align 4
-// IR-NEXT:    store i32 %[[TMP2]], ptr %[[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    %[[TMP3:.+]] = load i32, ptr %[[END_ADDR]], align 4
-// IR-NEXT:    store i32 %[[TMP3]], ptr %[[DOTCAPTURE_EXPR_1]], align 4
-// IR-NEXT:    %[[TMP4:.+]] = load i32, ptr %[[STEP_ADDR]], align 4
-// IR-NEXT:    store i32 %[[TMP4]], ptr %[[DOTNEW_STEP]], align 4
-// IR-NEXT:    %[[TMP5:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_1]], align 4
-// IR-NEXT:    %[[TMP6:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    %[[SUB:.+]] = sub i32 %[[TMP5]], %[[TMP6]]
-// IR-NEXT:    %[[SUB3:.+]] = sub i32 %[[SUB]], 1
-// IR-NEXT:    %[[TMP7:.+]] = load i32, ptr %[[DOTNEW_STEP]], align 4
-// IR-NEXT:    %[[ADD:.+]] = add i32 %[[SUB3]], %[[TMP7]]
-// IR-NEXT:    %[[TMP8:.+]] = load i32, ptr %[[DOTNEW_STEP]], align 4
-// IR-NEXT:    %[[DIV:.+]] = udiv i32 %[[ADD]], %[[TMP8]]
-// IR-NEXT:    %[[SUB4:.+]] = sub i32 %[[DIV]], 1
-// IR-NEXT:    store i32 %[[SUB4]], ptr %[[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    %[[TMP9:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    %[[ADD6:.+]] = add i32 %[[TMP9]], 1
-// IR-NEXT:    store i32 %[[ADD6]], ptr %[[DOTCAPTURE_EXPR_5]], align 4
-// IR-NEXT:    %[[TMP10:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_5]], align 4
-// IR-NEXT:    %[[SUB8:.+]] = sub i32 %[[TMP10]], -3
-// IR-NEXT:    %[[DIV9:.+]] = udiv i32 %[[SUB8]], 4
-// IR-NEXT:    %[[SUB10:.+]] = sub i32 %[[DIV9]], 1
-// IR-NEXT:    store i32 %[[SUB10]], ptr %[[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    store i32 0, ptr %[[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    %[[TMP11:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_5]], align 4
-// IR-NEXT:    %[[CMP:.+]] = icmp ult i32 0, %[[TMP11]]
-// IR-NEXT:    br i1 %[[CMP]], label %[[OMP_PRECOND_THEN:.+]], label %[[OMP_PRECOND_END:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_PRECOND_THEN]]:
-// IR-NEXT:    store i32 0, ptr %[[DOTOMP_LB]], align 4
-// IR-NEXT:    %[[TMP12:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    store i32 %[[TMP12]], ptr %[[DOTOMP_UB]], align 4
-// IR-NEXT:    store i32 1, ptr %[[DOTOMP_STRIDE]], align 4
-// IR-NEXT:    store i32 0, ptr %[[DOTOMP_IS_LAST]], align 4
-// IR-NEXT:    call void @__kmpc_for_static_init_4u(ptr @[[GLOB1:.+]], i32 %[[TMP0]], i32 34, ptr %[[DOTOMP_IS_LAST]], ptr %[[DOTOMP_LB]], ptr %[[DOTOMP_UB]], ptr %[[DOTOMP_STRIDE]], i32 1, i32 1)
-// IR-NEXT:    %[[TMP13:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
-// IR-NEXT:    %[[TMP14:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    %[[CMP12:.+]] = icmp ugt i32 %[[TMP13]], %[[TMP14]]
-// IR-NEXT:    br i1 %[[CMP12]], label %[[COND_TRUE:.+]], label %[[COND_FALSE:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_TRUE]]:
-// IR-NEXT:    %[[TMP15:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    br label %[[COND_END:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_FALSE]]:
-// IR-NEXT:    %[[TMP16:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
+// IR-LABEL: define dso_local void @func(
+// IR-SAME: i32 noundef [[START:%.*]], i32 noundef [[END:%.*]], i32 noundef [[STEP:%.*]]) #[[ATTR0:[0-9]+]] {
+// IR-NEXT:  [[ENTRY:.*:]]
+// IR-NEXT:    [[START_ADDR:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[TMP:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_5:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I11:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
+// IR-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
+// IR-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
+// IR-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// IR-NEXT:    store i32 [[TMP1]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// IR-NEXT:    store i32 [[TMP2]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// IR-NEXT:    store i32 [[TMP3]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// IR-NEXT:    store i32 [[TMP4]], ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    [[SUB:%.*]] = sub i32 [[TMP5]], [[TMP6]]
+// IR-NEXT:    [[SUB3:%.*]] = sub i32 [[SUB]], 1
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add i32 [[SUB3]], [[TMP7]]
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP8]]
+// IR-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
+// IR-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    [[ADD6:%.*]] = add i32 [[TMP9]], 1
+// IR-NEXT:    store i32 [[ADD6]], ptr [[DOTCAPTURE_EXPR_5]], align 4
+// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// IR-NEXT:    [[SUB8:%.*]] = sub i32 [[TMP10]], -3
+// IR-NEXT:    [[DIV9:%.*]] = udiv i32 [[SUB8]], 4
+// IR-NEXT:    [[SUB10:%.*]] = sub i32 [[DIV9]], 1
+// IR-NEXT:    store i32 [[SUB10]], ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// IR-NEXT:    [[CMP:%.*]] = icmp ult i32 0, [[TMP11]]
+// IR-NEXT:    br i1 [[CMP]], label %[[OMP_PRECOND_THEN:.*]], label %[[OMP_PRECOND_END:.*]]
+// IR:       [[OMP_PRECOND_THEN]]:
+// IR-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
+// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    store i32 [[TMP12]], ptr [[DOTOMP_UB]], align 4
+// IR-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
+// IR-NEXT:    call void @__kmpc_for_static_init_4u(ptr @[[GLOB1:[0-9]+]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
+// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    [[CMP12:%.*]] = icmp ugt i32 [[TMP13]], [[TMP14]]
+// IR-NEXT:    br i1 [[CMP12]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
 // IR-NEXT:    br label %[[COND_END]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_END]]:
-// IR-NEXT:    %[[COND:.+]] = phi i32 [ %[[TMP15]], %[[COND_TRUE]] ], [ %[[TMP16]], %[[COND_FALSE]] ]
-// IR-NEXT:    store i32 %[[COND]], ptr %[[DOTOMP_UB]], align 4
-// IR-NEXT:    %[[TMP17:.+]] = load i32, ptr %[[DOTOMP_LB]], align 4
-// IR-NEXT:    store i32 %[[TMP17]], ptr %[[DOTOMP_IV]], align 4
-// IR-NEXT:    br label %[[OMP_INNER_FOR_COND:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_INNER_FOR_COND]]:
-// IR-NEXT:    %[[TMP18:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
-// IR-NEXT:    %[[TMP19:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
-// IR-NEXT:    %[[ADD13:.+]] = add i32 %[[TMP19]], 1
-// IR-NEXT:    %[[CMP14:.+]] = icmp ult i32 %[[TMP18]], %[[ADD13]]
-// IR-NEXT:    br i1 %[[CMP14]], label %[[OMP_INNER_FOR_BODY:.+]], label %[[OMP_INNER_FOR_END:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_INNER_FOR_BODY]]:
-// IR-NEXT:    %[[TMP20:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
-// IR-NEXT:    %[[MUL:.+]] = mul i32 %[[TMP20]], 4
-// IR-NEXT:    %[[ADD15:.+]] = add i32 0, %[[MUL]]
-// IR-NEXT:    store i32 %[[ADD15]], ptr %[[DOTFLOOR_0_IV_I11]], align 4
-// IR-NEXT:    %[[TMP21:.+]] = load i32, ptr %[[DOTFLOOR_0_IV_I11]], align 4
-// IR-NEXT:    store i32 %[[TMP21]], ptr %[[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_COND]]:
-// IR-NEXT:    %[[TMP22:.+]] = load i32, ptr %[[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    %[[TMP23:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    %[[ADD16:.+]] = add i32 %[[TMP23]], 1
-// IR-NEXT:    %[[TMP24:.+]] = load i32, ptr %[[DOTFLOOR_0_IV_I11]], align 4
-// IR-NEXT:    %[[ADD17:.+]] = add i32 %[[TMP24]], 4
-// IR-NEXT:    %[[CMP18:.+]] = icmp ult i32 %[[ADD16]], %[[ADD17]]
-// IR-NEXT:    br i1 %[[CMP18]], label %[[COND_TRUE19:.+]], label %[[COND_FALSE21:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_TRUE19]]:
-// IR-NEXT:    %[[TMP25:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    %[[ADD20:.+]] = add i32 %[[TMP25]], 1
-// IR-NEXT:    br label %[[COND_END23:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_FALSE21]]:
-// IR-NEXT:    %[[TMP26:.+]] = load i32, ptr %[[DOTFLOOR_0_IV_I11]], align 4
-// IR-NEXT:    %[[ADD22:.+]] = add i32 %[[TMP26]], 4
-// IR-NEXT:    br label %[[COND_END23]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_END23]]:
-// IR-NEXT:    %[[COND24:.+]] = phi i32 [ %[[ADD20]], %[[COND_TRUE19]] ], [ %[[ADD22]], %[[COND_FALSE21]] ]
-// IR-NEXT:    %[[CMP25:.+]] = icmp ult i32 %[[TMP22]], %[[COND24]]
-// IR-NEXT:    br i1 %[[CMP25]], label %[[FOR_BODY:.+]], label %[[FOR_END:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_BODY]]:
-// IR-NEXT:    %[[TMP27:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    %[[TMP28:.+]] = load i32, ptr %[[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    %[[TMP29:.+]] = load i32, ptr %[[DOTNEW_STEP]], align 4
-// IR-NEXT:    %[[MUL26:.+]] = mul i32 %[[TMP28]], %[[TMP29]]
-// IR-NEXT:    %[[ADD27:.+]] = add i32 %[[TMP27]], %[[MUL26]]
-// IR-NEXT:    store i32 %[[ADD27]], ptr %[[I]], align 4
-// IR-NEXT:    %[[TMP30:.+]] = load i32, ptr %[[START_ADDR]], align 4
-// IR-NEXT:    %[[TMP31:.+]] = load i32, ptr %[[END_ADDR]], align 4
-// IR-NEXT:    %[[TMP32:.+]] = load i32, ptr %[[STEP_ADDR]], align 4
-// IR-NEXT:    %[[TMP33:.+]] = load i32, ptr %[[I]], align 4
-// IR-NEXT:    call void (...) @body(i32 noundef %[[TMP30]], i32 noundef %[[TMP31]], i32 noundef %[[TMP32]], i32 noundef %[[TMP33]])
-// IR-NEXT:    br label %[[FOR_INC:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_INC]]:
-// IR-NEXT:    %[[TMP34:.+]] = load i32, ptr %[[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    %[[INC:.+]] = add i32 %[[TMP34]], 1
-// IR-NEXT:    store i32 %[[INC]], ptr %[[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop ![[LOOP3:[0-9]+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_END]]:
-// IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_BODY_CONTINUE]]:
-// IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_INNER_FOR_INC]]:
-// IR-NEXT:    %[[TMP35:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
-// IR-NEXT:    %[[ADD28:.+]] = add i32 %[[TMP35]], 1
-// IR-NEXT:    store i32 %[[ADD28]], ptr %[[DOTOMP_IV]], align 4
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP15]], %[[COND_TRUE]] ], [ [[TMP16]], %[[COND_FALSE]] ]
+// IR-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
+// IR-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
+// IR-NEXT:    store i32 [[TMP17]], ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    br label %[[OMP_INNER_FOR_COND:.*]]
+// IR:       [[OMP_INNER_FOR_COND]]:
+// IR-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// IR-NEXT:    [[ADD13:%.*]] = add i32 [[TMP19]], 1
+// IR-NEXT:    [[CMP14:%.*]] = icmp ult i32 [[TMP18]], [[ADD13]]
+// IR-NEXT:    br i1 [[CMP14]], label %[[OMP_INNER_FOR_BODY:.*]], label %[[OMP_INNER_FOR_END:.*]]
+// IR:       [[OMP_INNER_FOR_BODY]]:
+// IR-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul i32 [[TMP20]], 4
+// IR-NEXT:    [[ADD15:%.*]] = add i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I11]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND:.*]]
+// IR:       [[FOR_COND]]:
+// IR-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[CMP16:%.*]] = icmp ult i32 [[TMP21]], 4
+// IR-NEXT:    br i1 [[CMP16]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY]]:
+// IR-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I11]], align 4
+// IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD17:%.*]] = add i32 [[TMP22]], [[TMP23]]
+// IR-NEXT:    store i32 [[ADD17]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[MUL18:%.*]] = mul i32 [[TMP25]], [[TMP26]]
+// IR-NEXT:    [[ADD19:%.*]] = add i32 [[TMP24]], [[MUL18]]
+// IR-NEXT:    store i32 [[ADD19]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    [[ADD20:%.*]] = add i32 [[TMP28]], 1
+// IR-NEXT:    [[CMP21:%.*]] = icmp ult i32 [[TMP27]], [[ADD20]]
+// IR-NEXT:    br i1 [[CMP21]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
+// IR:       [[IF_THEN]]:
+// IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void (...) @body(i32 noundef [[TMP29]], i32 noundef [[TMP30]], i32 noundef [[TMP31]], i32 noundef [[TMP32]])
+// IR-NEXT:    br label %[[IF_END]]
+// IR:       [[IF_END]]:
+// IR-NEXT:    br label %[[FOR_INC:.*]]
+// IR:       [[FOR_INC]]:
+// IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add i32 [[TMP33]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP2:![0-9]+]]
+// IR:       [[FOR_END]]:
+// IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.*]]
+// IR:       [[OMP_BODY_CONTINUE]]:
+// IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.*]]
+// IR:       [[OMP_INNER_FOR_INC]]:
+// IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[ADD22:%.*]] = add i32 [[TMP34]], 1
+// IR-NEXT:    store i32 [[ADD22]], ptr [[DOTOMP_IV]], align 4
 // IR-NEXT:    br label %[[OMP_INNER_FOR_COND]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_INNER_FOR_END]]:
-// IR-NEXT:    br label %[[OMP_LOOP_EXIT:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_LOOP_EXIT]]:
-// IR-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 %[[TMP0]])
+// IR:       [[OMP_INNER_FOR_END]]:
+// IR-NEXT:    br label %[[OMP_LOOP_EXIT:.*]]
+// IR:       [[OMP_LOOP_EXIT]]:
+// IR-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP0]])
 // IR-NEXT:    br label %[[OMP_PRECOND_END]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_PRECOND_END]]:
-// IR-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3:.+]], i32 %[[TMP0]])
+// IR:       [[OMP_PRECOND_END]]:
+// IR-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3:[0-9]+]], i32 [[TMP0]])
 // IR-NEXT:    ret void
-// IR-NEXT:  }
+//
 extern "C" void func(int start, int end, int step) {
 #pragma omp for
 #pragma omp tile sizes(4)
@@ -191,3 +171,7 @@ extern "C" void func(int start, int end, int step) {
 }
 
 #endif /* HEADER */
+//.
+// IR: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]]}
+// IR: [[META3]] = !{!"llvm.loop.mustprogress"}
+//.
diff --git a/clang/test/OpenMP/tile_codegen_tile_for.cpp b/clang/test/OpenMP/tile_codegen_tile_for.cpp
index 8a844c78c54a6..8153f76b7e1a9 100644
--- a/clang/test/OpenMP/tile_codegen_tile_for.cpp
+++ b/clang/test/OpenMP/tile_codegen_tile_for.cpp
@@ -1,3 +1,4 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
 // Check code generation
 // RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fopenmp -emit-llvm %s -o - | FileCheck %s --check-prefix=IR
 
@@ -16,229 +17,197 @@
 extern "C" void body(...) {}
 
 
-// IR-LABEL: define {{.*}}@func(
-// IR-NEXT:  [[ENTRY:.*]]:
-// IR-NEXT:    %[[START_ADDR:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[END_ADDR:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[STEP_ADDR:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_IV:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[TMP:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[I:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_1:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTNEW_STEP:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_2:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTFLOOR_0_IV_I:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_5:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_7:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_11:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_13:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTFLOOR_0_IV__FLOOR_0_IV_I:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_LB:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_UB:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_STRIDE:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_IS_LAST:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTFLOOR_0_IV__FLOOR_0_IV_I17:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTTILE_0_IV__FLOOR_0_IV_I:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTTILE_0_IV_I:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[TMP0:.+]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:.+]])
-// IR-NEXT:    store i32 %[[START:.+]], ptr %[[START_ADDR]], align 4
-// IR-NEXT:    store i32 %[[END:.+]], ptr %[[END_ADDR]], align 4
-// IR-NEXT:    store i32 %[[STEP:.+]], ptr %[[STEP_ADDR]], align 4
-// IR-NEXT:    %[[TMP1:.+]] = load i32, ptr %[[START_ADDR]], align 4
-// IR-NEXT:    store i32 %[[TMP1]], ptr %[[I]], align 4
-// IR-NEXT:    %[[TMP2:.+]] = load i32, ptr %[[START_ADDR]], align 4
-// IR-NEXT:    store i32 %[[TMP2]], ptr %[[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    %[[TMP3:.+]] = load i32, ptr %[[END_ADDR]], align 4
-// IR-NEXT:    store i32 %[[TMP3]], ptr %[[DOTCAPTURE_EXPR_1]], align 4
-// IR-NEXT:    %[[TMP4:.+]] = load i32, ptr %[[STEP_ADDR]], align 4
-// IR-NEXT:    store i32 %[[TMP4]], ptr %[[DOTNEW_STEP]], align 4
-// IR-NEXT:    %[[TMP5:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_1]], align 4
-// IR-NEXT:    %[[TMP6:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    %[[SUB:.+]] = sub i32 %[[TMP5]], %[[TMP6]]
-// IR-NEXT:    %[[SUB3:.+]] = sub i32 %[[SUB]], 1
-// IR-NEXT:    %[[TMP7:.+]] = load i32, ptr %[[DOTNEW_STEP]], align 4
-// IR-NEXT:    %[[ADD:.+]] = add i32 %[[SUB3]], %[[TMP7]]
-// IR-NEXT:    %[[TMP8:.+]] = load i32, ptr %[[DOTNEW_STEP]], align 4
-// IR-NEXT:    %[[DIV:.+]] = udiv i32 %[[ADD]], %[[TMP8]]
-// IR-NEXT:    %[[SUB4:.+]] = sub i32 %[[DIV]], 1
-// IR-NEXT:    store i32 %[[SUB4]], ptr %[[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    store i32 0, ptr %[[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    %[[TMP9:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    %[[ADD6:.+]] = add i32 %[[TMP9]], 1
-// IR-NEXT:    store i32 %[[ADD6]], ptr %[[DOTCAPTURE_EXPR_5]], align 4
-// IR-NEXT:    %[[TMP10:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_5]], align 4
-// IR-NEXT:    %[[SUB8:.+]] = sub i32 %[[TMP10]], -3
-// IR-NEXT:    %[[DIV9:.+]] = udiv i32 %[[SUB8]], 4
-// IR-NEXT:    %[[SUB10:.+]] = sub i32 %[[DIV9]], 1
-// IR-NEXT:    store i32 %[[SUB10]], ptr %[[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    %[[TMP11:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    %[[ADD12:.+]] = add i32 %[[TMP11]], 1
-// IR-NEXT:    store i32 %[[ADD12]], ptr %[[DOTCAPTURE_EXPR_11]], align 4
-// IR-NEXT:    %[[TMP12:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_11]], align 4
-// IR-NEXT:    %[[SUB14:.+]] = sub i32 %[[TMP12]], -2
-// IR-NEXT:    %[[DIV15:.+]] = udiv i32 %[[SUB14]], 3
-// IR-NEXT:    %[[SUB16:.+]] = sub i32 %[[DIV15]], 1
-// IR-NEXT:    store i32 %[[SUB16]], ptr %[[DOTCAPTURE_EXPR_13]], align 4
-// IR-NEXT:    store i32 0, ptr %[[DOTFLOOR_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    %[[TMP13:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_11]], align 4
-// IR-NEXT:    %[[CMP:.+]] = icmp ult i32 0, %[[TMP13]]
-// IR-NEXT:    br i1 %[[CMP]], label %[[OMP_PRECOND_THEN:.+]], label %[[OMP_PRECOND_END:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_PRECOND_THEN]]:
-// IR-NEXT:    store i32 0, ptr %[[DOTOMP_LB]], align 4
-// IR-NEXT:    %[[TMP14:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_13]], align 4
-// IR-NEXT:    store i32 %[[TMP14]], ptr %[[DOTOMP_UB]], align 4
-// IR-NEXT:    store i32 1, ptr %[[DOTOMP_STRIDE]], align 4
-// IR-NEXT:    store i32 0, ptr %[[DOTOMP_IS_LAST]], align 4
-// IR-NEXT:    call void @__kmpc_for_static_init_4u(ptr @[[GLOB1:.+]], i32 %[[TMP0]], i32 34, ptr %[[DOTOMP_IS_LAST]], ptr %[[DOTOMP_LB]], ptr %[[DOTOMP_UB]], ptr %[[DOTOMP_STRIDE]], i32 1, i32 1)
-// IR-NEXT:    %[[TMP15:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
-// IR-NEXT:    %[[TMP16:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_13]], align 4
-// IR-NEXT:    %[[CMP18:.+]] = icmp ugt i32 %[[TMP15]], %[[TMP16]]
-// IR-NEXT:    br i1 %[[CMP18]], label %[[COND_TRUE:.+]], label %[[COND_FALSE:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_TRUE]]:
-// IR-NEXT:    %[[TMP17:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_13]], align 4
-// IR-NEXT:    br label %[[COND_END:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_FALSE]]:
-// IR-NEXT:    %[[TMP18:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
+
+
+// IR-LABEL: define dso_local void @func(
+// IR-SAME: i32 noundef [[START:%.*]], i32 noundef [[END:%.*]], i32 noundef [[STEP:%.*]]) #[[ATTR0:[0-9]+]] {
+// IR-NEXT:  [[ENTRY:.*:]]
+// IR-NEXT:    [[START_ADDR:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[TMP:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_5:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV__FLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_13:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV__FLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV__FLOOR_0_IV_I17:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
+// IR-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
+// IR-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
+// IR-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// IR-NEXT:    store i32 [[TMP1]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// IR-NEXT:    store i32 [[TMP2]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// IR-NEXT:    store i32 [[TMP3]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// IR-NEXT:    store i32 [[TMP4]], ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    [[SUB:%.*]] = sub i32 [[TMP5]], [[TMP6]]
+// IR-NEXT:    [[SUB3:%.*]] = sub i32 [[SUB]], 1
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add i32 [[SUB3]], [[TMP7]]
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP8]]
+// IR-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
+// IR-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    [[ADD6:%.*]] = add i32 [[TMP9]], 1
+// IR-NEXT:    store i32 [[ADD6]], ptr [[DOTCAPTURE_EXPR_5]], align 4
+// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// IR-NEXT:    [[SUB8:%.*]] = sub i32 [[TMP10]], -3
+// IR-NEXT:    [[DIV9:%.*]] = udiv i32 [[SUB8]], 4
+// IR-NEXT:    [[SUB10:%.*]] = sub i32 [[DIV9]], 1
+// IR-NEXT:    store i32 [[SUB10]], ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    [[ADD12:%.*]] = add i32 [[TMP11]], 1
+// IR-NEXT:    store i32 [[ADD12]], ptr [[DOTCAPTURE_EXPR_11]], align 4
+// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// IR-NEXT:    [[SUB14:%.*]] = sub i32 [[TMP12]], -2
+// IR-NEXT:    [[DIV15:%.*]] = udiv i32 [[SUB14]], 3
+// IR-NEXT:    [[SUB16:%.*]] = sub i32 [[DIV15]], 1
+// IR-NEXT:    store i32 [[SUB16]], ptr [[DOTCAPTURE_EXPR_13]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// IR-NEXT:    [[CMP:%.*]] = icmp ult i32 0, [[TMP13]]
+// IR-NEXT:    br i1 [[CMP]], label %[[OMP_PRECOND_THEN:.*]], label %[[OMP_PRECOND_END:.*]]
+// IR:       [[OMP_PRECOND_THEN]]:
+// IR-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
+// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_13]], align 4
+// IR-NEXT:    store i32 [[TMP14]], ptr [[DOTOMP_UB]], align 4
+// IR-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
+// IR-NEXT:    call void @__kmpc_for_static_init_4u(ptr @[[GLOB1:[0-9]+]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
+// IR-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// IR-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_13]], align 4
+// IR-NEXT:    [[CMP18:%.*]] = icmp ugt i32 [[TMP15]], [[TMP16]]
+// IR-NEXT:    br i1 [[CMP18]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_13]], align 4
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
 // IR-NEXT:    br label %[[COND_END]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_END]]:
-// IR-NEXT:    %[[COND:.+]] = phi i32 [ %[[TMP17]], %[[COND_TRUE]] ], [ %[[TMP18]], %[[COND_FALSE]] ]
-// IR-NEXT:    store i32 %[[COND]], ptr %[[DOTOMP_UB]], align 4
-// IR-NEXT:    %[[TMP19:.+]] = load i32, ptr %[[DOTOMP_LB]], align 4
-// IR-NEXT:    store i32 %[[TMP19]], ptr %[[DOTOMP_IV]], align 4
-// IR-NEXT:    br label %[[OMP_INNER_FOR_COND:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_INNER_FOR_COND]]:
-// IR-NEXT:    %[[TMP20:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
-// IR-NEXT:    %[[TMP21:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
-// IR-NEXT:    %[[ADD19:.+]] = add i32 %[[TMP21]], 1
-// IR-NEXT:    %[[CMP20:.+]] = icmp ult i32 %[[TMP20]], %[[ADD19]]
-// IR-NEXT:    br i1 %[[CMP20]], label %[[OMP_INNER_FOR_BODY:.+]], label %[[OMP_INNER_FOR_END:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_INNER_FOR_BODY]]:
-// IR-NEXT:    %[[TMP22:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
-// IR-NEXT:    %[[MUL:.+]] = mul i32 %[[TMP22]], 3
-// IR-NEXT:    %[[ADD21:.+]] = add i32 0, %[[MUL]]
-// IR-NEXT:    store i32 %[[ADD21]], ptr %[[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
-// IR-NEXT:    %[[TMP23:.+]] = load i32, ptr %[[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
-// IR-NEXT:    store i32 %[[TMP23]], ptr %[[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_COND]]:
-// IR-NEXT:    %[[TMP24:.+]] = load i32, ptr %[[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    %[[TMP25:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    %[[ADD22:.+]] = add i32 %[[TMP25]], 1
-// IR-NEXT:    %[[TMP26:.+]] = load i32, ptr %[[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
-// IR-NEXT:    %[[ADD23:.+]] = add i32 %[[TMP26]], 3
-// IR-NEXT:    %[[CMP24:.+]] = icmp ult i32 %[[ADD22]], %[[ADD23]]
-// IR-NEXT:    br i1 %[[CMP24]], label %[[COND_TRUE25:.+]], label %[[COND_FALSE27:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_TRUE25]]:
-// IR-NEXT:    %[[TMP27:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    %[[ADD26:.+]] = add i32 %[[TMP27]], 1
-// IR-NEXT:    br label %[[COND_END29:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_FALSE27]]:
-// IR-NEXT:    %[[TMP28:.+]] = load i32, ptr %[[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
-// IR-NEXT:    %[[ADD28:.+]] = add i32 %[[TMP28]], 3
-// IR-NEXT:    br label %[[COND_END29]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_END29]]:
-// IR-NEXT:    %[[COND30:.+]] = phi i32 [ %[[ADD26]], %[[COND_TRUE25]] ], [ %[[ADD28]], %[[COND_FALSE27]] ]
-// IR-NEXT:    %[[CMP31:.+]] = icmp ult i32 %[[TMP24]], %[[COND30]]
-// IR-NEXT:    br i1 %[[CMP31]], label %[[FOR_BODY:.+]], label %[[FOR_END50:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_BODY]]:
-// IR-NEXT:    %[[TMP29:.+]] = load i32, ptr %[[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    %[[MUL32:.+]] = mul i32 %[[TMP29]], 4
-// IR-NEXT:    %[[ADD33:.+]] = add i32 0, %[[MUL32]]
-// IR-NEXT:    store i32 %[[ADD33]], ptr %[[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    %[[TMP30:.+]] = load i32, ptr %[[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    store i32 %[[TMP30]], ptr %[[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND34:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_COND34]]:
-// IR-NEXT:    %[[TMP31:.+]] = load i32, ptr %[[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    %[[TMP32:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    %[[ADD35:.+]] = add i32 %[[TMP32]], 1
-// IR-NEXT:    %[[TMP33:.+]] = load i32, ptr %[[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    %[[ADD36:.+]] = add i32 %[[TMP33]], 4
-// IR-NEXT:    %[[CMP37:.+]] = icmp ult i32 %[[ADD35]], %[[ADD36]]
-// IR-NEXT:    br i1 %[[CMP37]], label %[[COND_TRUE38:.+]], label %[[COND_FALSE40:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_TRUE38]]:
-// IR-NEXT:    %[[TMP34:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    %[[ADD39:.+]] = add i32 %[[TMP34]], 1
-// IR-NEXT:    br label %[[COND_END42:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_FALSE40]]:
-// IR-NEXT:    %[[TMP35:.+]] = load i32, ptr %[[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    %[[ADD41:.+]] = add i32 %[[TMP35]], 4
-// IR-NEXT:    br label %[[COND_END42]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_END42]]:
-// IR-NEXT:    %[[COND43:.+]] = phi i32 [ %[[ADD39]], %[[COND_TRUE38]] ], [ %[[ADD41]], %[[COND_FALSE40]] ]
-// IR-NEXT:    %[[CMP44:.+]] = icmp ult i32 %[[TMP31]], %[[COND43]]
-// IR-NEXT:    br i1 %[[CMP44]], label %[[FOR_BODY45:.+]], label %[[FOR_END:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_BODY45]]:
-// IR-NEXT:    %[[TMP36:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    %[[TMP37:.+]] = load i32, ptr %[[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    %[[TMP38:.+]] = load i32, ptr %[[DOTNEW_STEP]], align 4
-// IR-NEXT:    %[[MUL46:.+]] = mul i32 %[[TMP37]], %[[TMP38]]
-// IR-NEXT:    %[[ADD47:.+]] = add i32 %[[TMP36]], %[[MUL46]]
-// IR-NEXT:    store i32 %[[ADD47]], ptr %[[I]], align 4
-// IR-NEXT:    %[[TMP39:.+]] = load i32, ptr %[[START_ADDR]], align 4
-// IR-NEXT:    %[[TMP40:.+]] = load i32, ptr %[[END_ADDR]], align 4
-// IR-NEXT:    %[[TMP41:.+]] = load i32, ptr %[[STEP_ADDR]], align 4
-// IR-NEXT:    %[[TMP42:.+]] = load i32, ptr %[[I]], align 4
-// IR-NEXT:    call void (...) @body(i32 noundef %[[TMP39]], i32 noundef %[[TMP40]], i32 noundef %[[TMP41]], i32 noundef %[[TMP42]])
-// IR-NEXT:    br label %[[FOR_INC:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_INC]]:
-// IR-NEXT:    %[[TMP43:.+]] = load i32, ptr %[[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    %[[INC:.+]] = add i32 %[[TMP43]], 1
-// IR-NEXT:    store i32 %[[INC]], ptr %[[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND34]], !llvm.loop ![[LOOP2:[0-9]+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC48:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_INC48]]:
-// IR-NEXT:    %[[TMP44:.+]] = load i32, ptr %[[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    %[[INC49:.+]] = add i32 %[[TMP44]], 1
-// IR-NEXT:    store i32 %[[INC49]], ptr %[[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop ![[LOOP4:[0-9]+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_END50]]:
-// IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_BODY_CONTINUE]]:
-// IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_INNER_FOR_INC]]:
-// IR-NEXT:    %[[TMP45:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
-// IR-NEXT:    %[[ADD51:.+]] = add i32 %[[TMP45]], 1
-// IR-NEXT:    store i32 %[[ADD51]], ptr %[[DOTOMP_IV]], align 4
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP17]], %[[COND_TRUE]] ], [ [[TMP18]], %[[COND_FALSE]] ]
+// IR-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
+// IR-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
+// IR-NEXT:    store i32 [[TMP19]], ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    br label %[[OMP_INNER_FOR_COND:.*]]
+// IR:       [[OMP_INNER_FOR_COND]]:
+// IR-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// IR-NEXT:    [[ADD19:%.*]] = add i32 [[TMP21]], 1
+// IR-NEXT:    [[CMP20:%.*]] = icmp ult i32 [[TMP20]], [[ADD19]]
+// IR-NEXT:    br i1 [[CMP20]], label %[[OMP_INNER_FOR_BODY:.*]], label %[[OMP_INNER_FOR_END:.*]]
+// IR:       [[OMP_INNER_FOR_BODY]]:
+// IR-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul i32 [[TMP22]], 3
+// IR-NEXT:    [[ADD21:%.*]] = add i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD21]], ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND:.*]]
+// IR:       [[FOR_COND]]:
+// IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[CMP22:%.*]] = icmp ult i32 [[TMP23]], 3
+// IR-NEXT:    br i1 [[CMP22]], label %[[FOR_BODY:.*]], label %[[FOR_END40:.*]]
+// IR:       [[FOR_BODY]]:
+// IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
+// IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD23:%.*]] = add i32 [[TMP24]], [[TMP25]]
+// IR-NEXT:    store i32 [[ADD23]], ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[MUL24:%.*]] = mul i32 [[TMP26]], 4
+// IR-NEXT:    [[ADD25:%.*]] = add i32 0, [[MUL24]]
+// IR-NEXT:    store i32 [[ADD25]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    [[ADD26:%.*]] = add i32 [[TMP28]], 1
+// IR-NEXT:    [[CMP27:%.*]] = icmp ult i32 [[TMP27]], [[ADD26]]
+// IR-NEXT:    br i1 [[CMP27]], label %[[IF_THEN:.*]], label %[[IF_END37:.*]]
+// IR:       [[IF_THEN]]:
+// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND28:.*]]
+// IR:       [[FOR_COND28]]:
+// IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[CMP29:%.*]] = icmp ult i32 [[TMP29]], 4
+// IR-NEXT:    br i1 [[CMP29]], label %[[FOR_BODY30:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY30]]:
+// IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD31:%.*]] = add i32 [[TMP30]], [[TMP31]]
+// IR-NEXT:    store i32 [[ADD31]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[MUL32:%.*]] = mul i32 [[TMP33]], [[TMP34]]
+// IR-NEXT:    [[ADD33:%.*]] = add i32 [[TMP32]], [[MUL32]]
+// IR-NEXT:    store i32 [[ADD33]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    [[ADD34:%.*]] = add i32 [[TMP36]], 1
+// IR-NEXT:    [[CMP35:%.*]] = icmp ult i32 [[TMP35]], [[ADD34]]
+// IR-NEXT:    br i1 [[CMP35]], label %[[IF_THEN36:.*]], label %[[IF_END:.*]]
+// IR:       [[IF_THEN36]]:
+// IR-NEXT:    [[TMP37:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// IR-NEXT:    [[TMP38:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// IR-NEXT:    [[TMP39:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// IR-NEXT:    [[TMP40:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void (...) @body(i32 noundef [[TMP37]], i32 noundef [[TMP38]], i32 noundef [[TMP39]], i32 noundef [[TMP40]])
+// IR-NEXT:    br label %[[IF_END]]
+// IR:       [[IF_END]]:
+// IR-NEXT:    br label %[[FOR_INC:.*]]
+// IR:       [[FOR_INC]]:
+// IR-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add i32 [[TMP41]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND28]], !llvm.loop [[LOOP2:![0-9]+]]
+// IR:       [[FOR_END]]:
+// IR-NEXT:    br label %[[IF_END37]]
+// IR:       [[IF_END37]]:
+// IR-NEXT:    br label %[[FOR_INC38:.*]]
+// IR:       [[FOR_INC38]]:
+// IR-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[INC39:%.*]] = add i32 [[TMP42]], 1
+// IR-NEXT:    store i32 [[INC39]], ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP4:![0-9]+]]
+// IR:       [[FOR_END40]]:
+// IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.*]]
+// IR:       [[OMP_BODY_CONTINUE]]:
+// IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.*]]
+// IR:       [[OMP_INNER_FOR_INC]]:
+// IR-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[ADD41:%.*]] = add i32 [[TMP43]], 1
+// IR-NEXT:    store i32 [[ADD41]], ptr [[DOTOMP_IV]], align 4
 // IR-NEXT:    br label %[[OMP_INNER_FOR_COND]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_INNER_FOR_END]]:
-// IR-NEXT:    br label %[[OMP_LOOP_EXIT:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_LOOP_EXIT]]:
-// IR-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 %[[TMP0]])
+// IR:       [[OMP_INNER_FOR_END]]:
+// IR-NEXT:    br label %[[OMP_LOOP_EXIT:.*]]
+// IR:       [[OMP_LOOP_EXIT]]:
+// IR-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP0]])
 // IR-NEXT:    br label %[[OMP_PRECOND_END]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_PRECOND_END]]:
-// IR-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3:.+]], i32 %[[TMP0]])
+// IR:       [[OMP_PRECOND_END]]:
+// IR-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3:[0-9]+]], i32 [[TMP0]])
 // IR-NEXT:    ret void
-// IR-NEXT:  }
-
-
+//
 extern "C" void func(int start, int end, int step) {
 #pragma omp for
 #pragma omp tile sizes(3)
@@ -249,8 +218,8 @@ extern "C" void func(int start, int end, int step) {
 
 #endif /* HEADER */
 
-// IR: ![[META0:[0-9]+]] = !{i32 7, !"openmp", i32 51}
-// IR: ![[META1:[0-9]+]] =
-// IR: ![[LOOP2]] = distinct !{![[LOOP2]], ![[LOOPPROP3:[0-9]+]]}
-// IR: ![[LOOPPROP3]] = !{!"llvm.loop.mustprogress"}
-// IR: ![[LOOP4]] = distinct !{![[LOOP4]], ![[LOOPPROP3]]}
+//.
+// IR: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]]}
+// IR: [[META3]] = !{!"llvm.loop.mustprogress"}
+// IR: [[LOOP4]] = distinct !{[[LOOP4]], [[META3]]}
+//.
diff --git a/clang/test/OpenMP/tile_messages.cpp b/clang/test/OpenMP/tile_messages.cpp
index e1f5155c924e5..8915d1bb3c546 100644
--- a/clang/test/OpenMP/tile_messages.cpp
+++ b/clang/test/OpenMP/tile_messages.cpp
@@ -101,7 +101,7 @@ void func() {
     for (int j = 0; j < i; ++j)
       ;
 
-  // expected-error at +5 {{expected 3 for loops after '#pragma omp for', but found only 2}}
+  // expected-error@* {{expected 3 for loops after '#pragma omp for', but found only 2}}
   // expected-note at +1 {{as specified in 'collapse' clause}}
   #pragma omp for collapse(3)
   #pragma omp tile sizes(5)

>From 5fdb86f0bcdf5b36369d68db94619ec9d4107cf0 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Mon, 13 Apr 2026 03:19:54 -0400
Subject: [PATCH 06/19] ir-regenerate

---
 clang/test/OpenMP/interchange_codegen.cpp     | 4183 +++++++----------
 .../irbuilder_unroll_partial_factor_for.c     |   57 +-
 ...er_unroll_partial_heuristic_constant_for.c |   79 +-
 ...der_unroll_partial_heuristic_runtime_for.c |   79 +-
 .../irbuilder_unroll_unroll_partial_factor.c  |   57 +-
 ...rbuilder_unroll_unroll_partial_heuristic.c |   67 +-
 clang/test/OpenMP/unroll_codegen_tile_for.cpp |  404 +-
 7 files changed, 2096 insertions(+), 2830 deletions(-)

diff --git a/clang/test/OpenMP/interchange_codegen.cpp b/clang/test/OpenMP/interchange_codegen.cpp
index 8e833c9df324c..b062d42c9f162 100644
--- a/clang/test/OpenMP/interchange_codegen.cpp
+++ b/clang/test/OpenMP/interchange_codegen.cpp
@@ -123,6 +123,7 @@ extern "C" void foo10() {
 
 #endif /* HEADER */
 
+
 // CHECK1-LABEL: define {{[^@]+}}@body
 // CHECK1-SAME: (...) #[[ATTR0:[0-9]+]] {
 // CHECK1-NEXT:  entry:
@@ -156,7 +157,7 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP5]], [[TMP4]]
 // CHECK1-NEXT:    store i32 [[ADD]], ptr [[I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP3:![0-9]+]]
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP2:![0-9]+]]
 // CHECK1:       for.end:
 // CHECK1-NEXT:    ret void
 //
@@ -262,14 +263,14 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP28]], 1
 // CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTPERMUTED_1_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND16]], !llvm.loop [[LOOP5:![0-9]+]]
+// CHECK1-NEXT:    br label [[FOR_COND16]], !llvm.loop [[LOOP4:![0-9]+]]
 // CHECK1:       for.end:
 // CHECK1-NEXT:    br label [[FOR_INC22:%.*]]
 // CHECK1:       for.inc22:
 // CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_J]], align 4
 // CHECK1-NEXT:    [[INC23:%.*]] = add i32 [[TMP29]], 1
 // CHECK1-NEXT:    store i32 [[INC23]], ptr [[DOTPERMUTED_0_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP5:![0-9]+]]
 // CHECK1:       for.end24:
 // CHECK1-NEXT:    ret void
 //
@@ -342,7 +343,7 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP12]], 1
 // CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTPERMUTED_1_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP7:![0-9]+]]
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
 // CHECK1:       for.end:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
@@ -439,7 +440,7 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
 // CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTPERMUTED_1_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP7:![0-9]+]]
 // CHECK1:       for.end:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
@@ -754,28 +755,28 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTPERMUTED_3_IV_I]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP12]], 1
 // CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTPERMUTED_3_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND11]], !llvm.loop [[LOOP9:![0-9]+]]
+// CHECK1-NEXT:    br label [[FOR_COND11]], !llvm.loop [[LOOP8:![0-9]+]]
 // CHECK1:       for.end:
 // CHECK1-NEXT:    br label [[FOR_INC16:%.*]]
 // CHECK1:       for.inc16:
 // CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTPERMUTED_2_IV_L]], align 4
 // CHECK1-NEXT:    [[INC17:%.*]] = add nsw i32 [[TMP13]], 1
 // CHECK1-NEXT:    store i32 [[INC17]], ptr [[DOTPERMUTED_2_IV_L]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP10:![0-9]+]]
+// CHECK1-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP9:![0-9]+]]
 // CHECK1:       for.end18:
 // CHECK1-NEXT:    br label [[FOR_INC19:%.*]]
 // CHECK1:       for.inc19:
 // CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_K]], align 4
 // CHECK1-NEXT:    [[INC20:%.*]] = add nsw i32 [[TMP14]], 1
 // CHECK1-NEXT:    store i32 [[INC20]], ptr [[DOTPERMUTED_1_IV_K]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP11:![0-9]+]]
+// CHECK1-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP10:![0-9]+]]
 // CHECK1:       for.end21:
 // CHECK1-NEXT:    br label [[FOR_INC22:%.*]]
 // CHECK1:       for.inc22:
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_J]], align 4
 // CHECK1-NEXT:    [[INC23:%.*]] = add nsw i32 [[TMP15]], 1
 // CHECK1-NEXT:    store i32 [[INC23]], ptr [[DOTPERMUTED_0_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP12:![0-9]+]]
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP11:![0-9]+]]
 // CHECK1:       for.end24:
 // CHECK1-NEXT:    ret void
 //
@@ -810,22 +811,21 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTNEW_STEP10:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_14:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_17:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_22:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_K:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_K:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[I49:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[J50:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV_K51:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_K52:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[I35:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[J36:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV_K37:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_K38:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
@@ -863,630 +863,452 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
 // CHECK1-NEXT:    [[ADD15:%.*]] = add i32 [[TMP15]], 1
 // CHECK1-NEXT:    store i32 [[ADD15]], ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[_TMP2]], align 4
-// CHECK1-NEXT:    store i32 [[TMP16]], ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK1-NEXT:    [[ADD18:%.*]] = add i32 [[TMP17]], 1
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[_TMP2]], align 4
-// CHECK1-NEXT:    [[ADD19:%.*]] = add i32 [[TMP18]], 32
-// CHECK1-NEXT:    [[CMP:%.*]] = icmp ult i32 [[ADD18]], [[ADD19]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK1:       cond.true:
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK1-NEXT:    [[ADD20:%.*]] = add i32 [[TMP19]], 1
-// CHECK1-NEXT:    br label [[COND_END:%.*]]
-// CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[_TMP2]], align 4
-// CHECK1-NEXT:    [[ADD21:%.*]] = add i32 [[TMP20]], 32
-// CHECK1-NEXT:    br label [[COND_END]]
-// CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD20]], [[COND_TRUE]] ], [ [[ADD21]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[SUB23:%.*]] = sub i32 [[TMP21]], [[TMP22]]
-// CHECK1-NEXT:    [[SUB24:%.*]] = sub i32 [[SUB23]], 1
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[ADD25:%.*]] = add i32 [[SUB24]], [[TMP23]]
-// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[DIV26:%.*]] = udiv i32 [[ADD25]], [[TMP24]]
-// CHECK1-NEXT:    [[CONV:%.*]] = zext i32 [[DIV26]] to i64
-// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[SUB17:%.*]] = sub i32 [[TMP16]], [[TMP17]]
+// CHECK1-NEXT:    [[SUB18:%.*]] = sub i32 [[SUB17]], 1
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[ADD19:%.*]] = add i32 [[SUB18]], [[TMP18]]
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[DIV20:%.*]] = udiv i32 [[ADD19]], [[TMP19]]
+// CHECK1-NEXT:    [[CONV:%.*]] = zext i32 [[DIV20]] to i64
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB21:%.*]] = sub i32 [[TMP20]], [[TMP21]]
+// CHECK1-NEXT:    [[SUB22:%.*]] = sub i32 [[SUB21]], 1
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD23:%.*]] = add i32 [[SUB22]], [[TMP22]]
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV24:%.*]] = udiv i32 [[ADD23]], [[TMP23]]
+// CHECK1-NEXT:    [[CONV25:%.*]] = zext i32 [[DIV24]] to i64
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV25]]
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB26:%.*]] = sub i32 [[TMP24]], -31
+// CHECK1-NEXT:    [[DIV27:%.*]] = udiv i32 [[SUB26]], 32
+// CHECK1-NEXT:    [[CONV28:%.*]] = zext i32 [[DIV27]] to i64
+// CHECK1-NEXT:    [[MUL29:%.*]] = mul nsw i64 [[MUL]], [[CONV28]]
+// CHECK1-NEXT:    [[MUL30:%.*]] = mul nsw i64 [[MUL29]], 32
+// CHECK1-NEXT:    [[SUB31:%.*]] = sub nsw i64 [[MUL30]], 1
+// CHECK1-NEXT:    store i64 [[SUB31]], ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    store i32 [[TMP25]], ptr [[I]], align 4
 // CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB27:%.*]] = sub i32 [[TMP25]], [[TMP26]]
-// CHECK1-NEXT:    [[SUB28:%.*]] = sub i32 [[SUB27]], 1
-// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD29:%.*]] = add i32 [[SUB28]], [[TMP27]]
-// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV30:%.*]] = udiv i32 [[ADD29]], [[TMP28]]
-// CHECK1-NEXT:    [[CONV31:%.*]] = zext i32 [[DIV30]] to i64
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV31]]
-// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB32:%.*]] = sub i32 [[TMP29]], -31
-// CHECK1-NEXT:    [[DIV33:%.*]] = udiv i32 [[SUB32]], 32
-// CHECK1-NEXT:    [[CONV34:%.*]] = zext i32 [[DIV33]] to i64
-// CHECK1-NEXT:    [[MUL35:%.*]] = mul nsw i64 [[MUL]], [[CONV34]]
-// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB36:%.*]] = sub i32 [[TMP30]], [[TMP31]]
-// CHECK1-NEXT:    [[SUB37:%.*]] = sub i32 [[SUB36]], 1
-// CHECK1-NEXT:    [[ADD38:%.*]] = add i32 [[SUB37]], 1
-// CHECK1-NEXT:    [[DIV39:%.*]] = udiv i32 [[ADD38]], 1
-// CHECK1-NEXT:    [[CONV40:%.*]] = zext i32 [[DIV39]] to i64
-// CHECK1-NEXT:    [[MUL41:%.*]] = mul nsw i64 [[MUL35]], [[CONV40]]
-// CHECK1-NEXT:    [[SUB42:%.*]] = sub nsw i64 [[MUL41]], 1
-// CHECK1-NEXT:    store i64 [[SUB42]], ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    store i32 [[TMP32]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    store i32 [[TMP33]], ptr [[J]], align 4
+// CHECK1-NEXT:    store i32 [[TMP26]], ptr [[J]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_K]], align 4
-// CHECK1-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    store i32 [[TMP34]], ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK1-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK1-NEXT:    [[CMP43:%.*]] = icmp slt i32 [[TMP35]], [[TMP36]]
-// CHECK1-NEXT:    br i1 [[CMP43]], label [[LAND_LHS_TRUE:%.*]], label [[OMP_PRECOND_END:%.*]]
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_K]], align 4
+// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP27]], [[TMP28]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[LAND_LHS_TRUE:%.*]], label [[OMP_PRECOND_END:%.*]]
 // CHECK1:       land.lhs.true:
-// CHECK1-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[CMP44:%.*]] = icmp slt i32 [[TMP37]], [[TMP38]]
-// CHECK1-NEXT:    br i1 [[CMP44]], label [[LAND_LHS_TRUE45:%.*]], label [[OMP_PRECOND_END]]
-// CHECK1:       land.lhs.true45:
-// CHECK1-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[CMP46:%.*]] = icmp ult i32 0, [[TMP39]]
-// CHECK1-NEXT:    br i1 [[CMP46]], label [[LAND_LHS_TRUE47:%.*]], label [[OMP_PRECOND_END]]
-// CHECK1:       land.lhs.true47:
-// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[CMP48:%.*]] = icmp ult i32 [[TMP40]], [[TMP41]]
-// CHECK1-NEXT:    br i1 [[CMP48]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
+// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[CMP32:%.*]] = icmp slt i32 [[TMP29]], [[TMP30]]
+// CHECK1-NEXT:    br i1 [[CMP32]], label [[LAND_LHS_TRUE33:%.*]], label [[OMP_PRECOND_END]]
+// CHECK1:       land.lhs.true33:
+// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[CMP34:%.*]] = icmp ult i32 0, [[TMP31]]
+// CHECK1-NEXT:    br i1 [[CMP34]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
 // CHECK1:       omp.precond.then:
 // CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
-// CHECK1-NEXT:    [[TMP42:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK1-NEXT:    store i64 [[TMP42]], ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK1-NEXT:    store i64 [[TMP32]], ptr [[DOTOMP_UB]], align 8
 // CHECK1-NEXT:    store i64 1, ptr [[DOTOMP_STRIDE]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
 // CHECK1-NEXT:    call void @__kmpc_for_static_init_8(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i64 1, i64 1)
-// CHECK1-NEXT:    [[TMP43:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[TMP44:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK1-NEXT:    [[CMP53:%.*]] = icmp sgt i64 [[TMP43]], [[TMP44]]
-// CHECK1-NEXT:    br i1 [[CMP53]], label [[COND_TRUE54:%.*]], label [[COND_FALSE55:%.*]]
-// CHECK1:       cond.true54:
-// CHECK1-NEXT:    [[TMP45:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK1-NEXT:    br label [[COND_END56:%.*]]
-// CHECK1:       cond.false55:
-// CHECK1-NEXT:    [[TMP46:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    br label [[COND_END56]]
-// CHECK1:       cond.end56:
-// CHECK1-NEXT:    [[COND57:%.*]] = phi i64 [ [[TMP45]], [[COND_TRUE54]] ], [ [[TMP46]], [[COND_FALSE55]] ]
-// CHECK1-NEXT:    store i64 [[COND57]], ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[TMP47:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
-// CHECK1-NEXT:    store i64 [[TMP47]], ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK1-NEXT:    [[CMP39:%.*]] = icmp sgt i64 [[TMP33]], [[TMP34]]
+// CHECK1-NEXT:    br i1 [[CMP39]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i64 [ [[TMP35]], [[COND_TRUE]] ], [ [[TMP36]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    store i64 [[COND]], ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
+// CHECK1-NEXT:    store i64 [[TMP37]], ptr [[DOTOMP_IV]], align 8
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK1:       omp.inner.for.cond:
-// CHECK1-NEXT:    [[TMP48:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[CMP58:%.*]] = icmp sle i64 [[TMP48]], [[TMP49]]
-// CHECK1-NEXT:    br i1 [[CMP58]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK1-NEXT:    [[TMP38:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP39:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[CMP40:%.*]] = icmp sle i64 [[TMP38]], [[TMP39]]
+// CHECK1-NEXT:    br i1 [[CMP40]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
 // CHECK1:       omp.inner.for.body:
-// CHECK1-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[CONV59:%.*]] = sext i32 [[TMP50]] to i64
-// CHECK1-NEXT:    [[TMP51:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB60:%.*]] = sub i32 [[TMP52]], [[TMP53]]
-// CHECK1-NEXT:    [[SUB61:%.*]] = sub i32 [[SUB60]], 1
+// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[CONV41:%.*]] = sext i32 [[TMP40]] to i64
+// CHECK1-NEXT:    [[TMP41:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB42:%.*]] = sub i32 [[TMP42]], [[TMP43]]
+// CHECK1-NEXT:    [[SUB43:%.*]] = sub i32 [[SUB42]], 1
+// CHECK1-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD44:%.*]] = add i32 [[SUB43]], [[TMP44]]
+// CHECK1-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV45:%.*]] = udiv i32 [[ADD44]], [[TMP45]]
+// CHECK1-NEXT:    [[MUL46:%.*]] = mul i32 1, [[DIV45]]
+// CHECK1-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB47:%.*]] = sub i32 [[TMP46]], -31
+// CHECK1-NEXT:    [[DIV48:%.*]] = udiv i32 [[SUB47]], 32
+// CHECK1-NEXT:    [[MUL49:%.*]] = mul i32 [[MUL46]], [[DIV48]]
+// CHECK1-NEXT:    [[MUL50:%.*]] = mul i32 [[MUL49]], 32
+// CHECK1-NEXT:    [[CONV51:%.*]] = zext i32 [[MUL50]] to i64
+// CHECK1-NEXT:    [[DIV52:%.*]] = sdiv i64 [[TMP41]], [[CONV51]]
+// CHECK1-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[CONV53:%.*]] = sext i32 [[TMP47]] to i64
+// CHECK1-NEXT:    [[MUL54:%.*]] = mul nsw i64 [[DIV52]], [[CONV53]]
+// CHECK1-NEXT:    [[ADD55:%.*]] = add nsw i64 [[CONV41]], [[MUL54]]
+// CHECK1-NEXT:    [[CONV56:%.*]] = trunc i64 [[ADD55]] to i32
+// CHECK1-NEXT:    store i32 [[CONV56]], ptr [[I35]], align 4
+// CHECK1-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP48]] to i64
+// CHECK1-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP50:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB58:%.*]] = sub i32 [[TMP51]], [[TMP52]]
+// CHECK1-NEXT:    [[SUB59:%.*]] = sub i32 [[SUB58]], 1
+// CHECK1-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD60:%.*]] = add i32 [[SUB59]], [[TMP53]]
 // CHECK1-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD62:%.*]] = add i32 [[SUB61]], [[TMP54]]
-// CHECK1-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV63:%.*]] = udiv i32 [[ADD62]], [[TMP55]]
-// CHECK1-NEXT:    [[MUL64:%.*]] = mul i32 1, [[DIV63]]
-// CHECK1-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB65:%.*]] = sub i32 [[TMP56]], -31
-// CHECK1-NEXT:    [[DIV66:%.*]] = udiv i32 [[SUB65]], 32
-// CHECK1-NEXT:    [[MUL67:%.*]] = mul i32 [[MUL64]], [[DIV66]]
-// CHECK1-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB68:%.*]] = sub i32 [[TMP57]], [[TMP58]]
-// CHECK1-NEXT:    [[SUB69:%.*]] = sub i32 [[SUB68]], 1
-// CHECK1-NEXT:    [[ADD70:%.*]] = add i32 [[SUB69]], 1
-// CHECK1-NEXT:    [[DIV71:%.*]] = udiv i32 [[ADD70]], 1
-// CHECK1-NEXT:    [[MUL72:%.*]] = mul i32 [[MUL67]], [[DIV71]]
-// CHECK1-NEXT:    [[CONV73:%.*]] = zext i32 [[MUL72]] to i64
-// CHECK1-NEXT:    [[DIV74:%.*]] = sdiv i64 [[TMP51]], [[CONV73]]
-// CHECK1-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[CONV75:%.*]] = sext i32 [[TMP59]] to i64
-// CHECK1-NEXT:    [[MUL76:%.*]] = mul nsw i64 [[DIV74]], [[CONV75]]
-// CHECK1-NEXT:    [[ADD77:%.*]] = add nsw i64 [[CONV59]], [[MUL76]]
-// CHECK1-NEXT:    [[CONV78:%.*]] = trunc i64 [[ADD77]] to i32
-// CHECK1-NEXT:    store i32 [[CONV78]], ptr [[I49]], align 4
-// CHECK1-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[CONV79:%.*]] = sext i32 [[TMP60]] to i64
-// CHECK1-NEXT:    [[TMP61:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP62:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP63:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP64:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB80:%.*]] = sub i32 [[TMP63]], [[TMP64]]
-// CHECK1-NEXT:    [[SUB81:%.*]] = sub i32 [[SUB80]], 1
-// CHECK1-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD82:%.*]] = add i32 [[SUB81]], [[TMP65]]
-// CHECK1-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV83:%.*]] = udiv i32 [[ADD82]], [[TMP66]]
-// CHECK1-NEXT:    [[MUL84:%.*]] = mul i32 1, [[DIV83]]
-// CHECK1-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB85:%.*]] = sub i32 [[TMP67]], -31
-// CHECK1-NEXT:    [[DIV86:%.*]] = udiv i32 [[SUB85]], 32
-// CHECK1-NEXT:    [[MUL87:%.*]] = mul i32 [[MUL84]], [[DIV86]]
-// CHECK1-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB88:%.*]] = sub i32 [[TMP68]], [[TMP69]]
-// CHECK1-NEXT:    [[SUB89:%.*]] = sub i32 [[SUB88]], 1
-// CHECK1-NEXT:    [[ADD90:%.*]] = add i32 [[SUB89]], 1
-// CHECK1-NEXT:    [[DIV91:%.*]] = udiv i32 [[ADD90]], 1
-// CHECK1-NEXT:    [[MUL92:%.*]] = mul i32 [[MUL87]], [[DIV91]]
-// CHECK1-NEXT:    [[CONV93:%.*]] = zext i32 [[MUL92]] to i64
-// CHECK1-NEXT:    [[DIV94:%.*]] = sdiv i64 [[TMP62]], [[CONV93]]
+// CHECK1-NEXT:    [[DIV61:%.*]] = udiv i32 [[ADD60]], [[TMP54]]
+// CHECK1-NEXT:    [[MUL62:%.*]] = mul i32 1, [[DIV61]]
+// CHECK1-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB63:%.*]] = sub i32 [[TMP55]], -31
+// CHECK1-NEXT:    [[DIV64:%.*]] = udiv i32 [[SUB63]], 32
+// CHECK1-NEXT:    [[MUL65:%.*]] = mul i32 [[MUL62]], [[DIV64]]
+// CHECK1-NEXT:    [[MUL66:%.*]] = mul i32 [[MUL65]], 32
+// CHECK1-NEXT:    [[CONV67:%.*]] = zext i32 [[MUL66]] to i64
+// CHECK1-NEXT:    [[DIV68:%.*]] = sdiv i64 [[TMP50]], [[CONV67]]
+// CHECK1-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB69:%.*]] = sub i32 [[TMP56]], [[TMP57]]
+// CHECK1-NEXT:    [[SUB70:%.*]] = sub i32 [[SUB69]], 1
+// CHECK1-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD71:%.*]] = add i32 [[SUB70]], [[TMP58]]
+// CHECK1-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV72:%.*]] = udiv i32 [[ADD71]], [[TMP59]]
+// CHECK1-NEXT:    [[MUL73:%.*]] = mul i32 1, [[DIV72]]
+// CHECK1-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB74:%.*]] = sub i32 [[TMP60]], -31
+// CHECK1-NEXT:    [[DIV75:%.*]] = udiv i32 [[SUB74]], 32
+// CHECK1-NEXT:    [[MUL76:%.*]] = mul i32 [[MUL73]], [[DIV75]]
+// CHECK1-NEXT:    [[MUL77:%.*]] = mul i32 [[MUL76]], 32
+// CHECK1-NEXT:    [[CONV78:%.*]] = zext i32 [[MUL77]] to i64
+// CHECK1-NEXT:    [[MUL79:%.*]] = mul nsw i64 [[DIV68]], [[CONV78]]
+// CHECK1-NEXT:    [[SUB80:%.*]] = sub nsw i64 [[TMP49]], [[MUL79]]
+// CHECK1-NEXT:    [[TMP61:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB81:%.*]] = sub i32 [[TMP61]], -31
+// CHECK1-NEXT:    [[DIV82:%.*]] = udiv i32 [[SUB81]], 32
+// CHECK1-NEXT:    [[MUL83:%.*]] = mul i32 1, [[DIV82]]
+// CHECK1-NEXT:    [[MUL84:%.*]] = mul i32 [[MUL83]], 32
+// CHECK1-NEXT:    [[CONV85:%.*]] = zext i32 [[MUL84]] to i64
+// CHECK1-NEXT:    [[DIV86:%.*]] = sdiv i64 [[SUB80]], [[CONV85]]
+// CHECK1-NEXT:    [[TMP62:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[CONV87:%.*]] = sext i32 [[TMP62]] to i64
+// CHECK1-NEXT:    [[MUL88:%.*]] = mul nsw i64 [[DIV86]], [[CONV87]]
+// CHECK1-NEXT:    [[ADD89:%.*]] = add nsw i64 [[CONV57]], [[MUL88]]
+// CHECK1-NEXT:    [[CONV90:%.*]] = trunc i64 [[ADD89]] to i32
+// CHECK1-NEXT:    store i32 [[CONV90]], ptr [[J36]], align 4
+// CHECK1-NEXT:    [[TMP63:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP64:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB91:%.*]] = sub i32 [[TMP65]], [[TMP66]]
+// CHECK1-NEXT:    [[SUB92:%.*]] = sub i32 [[SUB91]], 1
+// CHECK1-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD93:%.*]] = add i32 [[SUB92]], [[TMP67]]
+// CHECK1-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV94:%.*]] = udiv i32 [[ADD93]], [[TMP68]]
+// CHECK1-NEXT:    [[MUL95:%.*]] = mul i32 1, [[DIV94]]
+// CHECK1-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB96:%.*]] = sub i32 [[TMP69]], -31
+// CHECK1-NEXT:    [[DIV97:%.*]] = udiv i32 [[SUB96]], 32
+// CHECK1-NEXT:    [[MUL98:%.*]] = mul i32 [[MUL95]], [[DIV97]]
+// CHECK1-NEXT:    [[MUL99:%.*]] = mul i32 [[MUL98]], 32
+// CHECK1-NEXT:    [[CONV100:%.*]] = zext i32 [[MUL99]] to i64
+// CHECK1-NEXT:    [[DIV101:%.*]] = sdiv i64 [[TMP64]], [[CONV100]]
 // CHECK1-NEXT:    [[TMP70:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
 // CHECK1-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB95:%.*]] = sub i32 [[TMP70]], [[TMP71]]
-// CHECK1-NEXT:    [[SUB96:%.*]] = sub i32 [[SUB95]], 1
+// CHECK1-NEXT:    [[SUB102:%.*]] = sub i32 [[TMP70]], [[TMP71]]
+// CHECK1-NEXT:    [[SUB103:%.*]] = sub i32 [[SUB102]], 1
 // CHECK1-NEXT:    [[TMP72:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD97:%.*]] = add i32 [[SUB96]], [[TMP72]]
+// CHECK1-NEXT:    [[ADD104:%.*]] = add i32 [[SUB103]], [[TMP72]]
 // CHECK1-NEXT:    [[TMP73:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV98:%.*]] = udiv i32 [[ADD97]], [[TMP73]]
-// CHECK1-NEXT:    [[MUL99:%.*]] = mul i32 1, [[DIV98]]
+// CHECK1-NEXT:    [[DIV105:%.*]] = udiv i32 [[ADD104]], [[TMP73]]
+// CHECK1-NEXT:    [[MUL106:%.*]] = mul i32 1, [[DIV105]]
 // CHECK1-NEXT:    [[TMP74:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB100:%.*]] = sub i32 [[TMP74]], -31
-// CHECK1-NEXT:    [[DIV101:%.*]] = udiv i32 [[SUB100]], 32
-// CHECK1-NEXT:    [[MUL102:%.*]] = mul i32 [[MUL99]], [[DIV101]]
-// CHECK1-NEXT:    [[TMP75:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP76:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB103:%.*]] = sub i32 [[TMP75]], [[TMP76]]
-// CHECK1-NEXT:    [[SUB104:%.*]] = sub i32 [[SUB103]], 1
-// CHECK1-NEXT:    [[ADD105:%.*]] = add i32 [[SUB104]], 1
-// CHECK1-NEXT:    [[DIV106:%.*]] = udiv i32 [[ADD105]], 1
-// CHECK1-NEXT:    [[MUL107:%.*]] = mul i32 [[MUL102]], [[DIV106]]
-// CHECK1-NEXT:    [[CONV108:%.*]] = zext i32 [[MUL107]] to i64
-// CHECK1-NEXT:    [[MUL109:%.*]] = mul nsw i64 [[DIV94]], [[CONV108]]
-// CHECK1-NEXT:    [[SUB110:%.*]] = sub nsw i64 [[TMP61]], [[MUL109]]
-// CHECK1-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB111:%.*]] = sub i32 [[TMP77]], -31
-// CHECK1-NEXT:    [[DIV112:%.*]] = udiv i32 [[SUB111]], 32
-// CHECK1-NEXT:    [[MUL113:%.*]] = mul i32 1, [[DIV112]]
-// CHECK1-NEXT:    [[TMP78:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP79:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB114:%.*]] = sub i32 [[TMP78]], [[TMP79]]
+// CHECK1-NEXT:    [[SUB107:%.*]] = sub i32 [[TMP74]], -31
+// CHECK1-NEXT:    [[DIV108:%.*]] = udiv i32 [[SUB107]], 32
+// CHECK1-NEXT:    [[MUL109:%.*]] = mul i32 [[MUL106]], [[DIV108]]
+// CHECK1-NEXT:    [[MUL110:%.*]] = mul i32 [[MUL109]], 32
+// CHECK1-NEXT:    [[CONV111:%.*]] = zext i32 [[MUL110]] to i64
+// CHECK1-NEXT:    [[MUL112:%.*]] = mul nsw i64 [[DIV101]], [[CONV111]]
+// CHECK1-NEXT:    [[SUB113:%.*]] = sub nsw i64 [[TMP63]], [[MUL112]]
+// CHECK1-NEXT:    [[TMP75:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP76:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP78:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB114:%.*]] = sub i32 [[TMP77]], [[TMP78]]
 // CHECK1-NEXT:    [[SUB115:%.*]] = sub i32 [[SUB114]], 1
-// CHECK1-NEXT:    [[ADD116:%.*]] = add i32 [[SUB115]], 1
-// CHECK1-NEXT:    [[DIV117:%.*]] = udiv i32 [[ADD116]], 1
-// CHECK1-NEXT:    [[MUL118:%.*]] = mul i32 [[MUL113]], [[DIV117]]
-// CHECK1-NEXT:    [[CONV119:%.*]] = zext i32 [[MUL118]] to i64
-// CHECK1-NEXT:    [[DIV120:%.*]] = sdiv i64 [[SUB110]], [[CONV119]]
+// CHECK1-NEXT:    [[TMP79:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD116:%.*]] = add i32 [[SUB115]], [[TMP79]]
 // CHECK1-NEXT:    [[TMP80:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[CONV121:%.*]] = sext i32 [[TMP80]] to i64
-// CHECK1-NEXT:    [[MUL122:%.*]] = mul nsw i64 [[DIV120]], [[CONV121]]
-// CHECK1-NEXT:    [[ADD123:%.*]] = add nsw i64 [[CONV79]], [[MUL122]]
-// CHECK1-NEXT:    [[CONV124:%.*]] = trunc i64 [[ADD123]] to i32
-// CHECK1-NEXT:    store i32 [[CONV124]], ptr [[J50]], align 4
-// CHECK1-NEXT:    [[TMP81:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP82:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB125:%.*]] = sub i32 [[TMP83]], [[TMP84]]
+// CHECK1-NEXT:    [[DIV117:%.*]] = udiv i32 [[ADD116]], [[TMP80]]
+// CHECK1-NEXT:    [[MUL118:%.*]] = mul i32 1, [[DIV117]]
+// CHECK1-NEXT:    [[TMP81:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB119:%.*]] = sub i32 [[TMP81]], -31
+// CHECK1-NEXT:    [[DIV120:%.*]] = udiv i32 [[SUB119]], 32
+// CHECK1-NEXT:    [[MUL121:%.*]] = mul i32 [[MUL118]], [[DIV120]]
+// CHECK1-NEXT:    [[MUL122:%.*]] = mul i32 [[MUL121]], 32
+// CHECK1-NEXT:    [[CONV123:%.*]] = zext i32 [[MUL122]] to i64
+// CHECK1-NEXT:    [[DIV124:%.*]] = sdiv i64 [[TMP76]], [[CONV123]]
+// CHECK1-NEXT:    [[TMP82:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB125:%.*]] = sub i32 [[TMP82]], [[TMP83]]
 // CHECK1-NEXT:    [[SUB126:%.*]] = sub i32 [[SUB125]], 1
+// CHECK1-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD127:%.*]] = add i32 [[SUB126]], [[TMP84]]
 // CHECK1-NEXT:    [[TMP85:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD127:%.*]] = add i32 [[SUB126]], [[TMP85]]
-// CHECK1-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV128:%.*]] = udiv i32 [[ADD127]], [[TMP86]]
+// CHECK1-NEXT:    [[DIV128:%.*]] = udiv i32 [[ADD127]], [[TMP85]]
 // CHECK1-NEXT:    [[MUL129:%.*]] = mul i32 1, [[DIV128]]
-// CHECK1-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB130:%.*]] = sub i32 [[TMP87]], -31
+// CHECK1-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB130:%.*]] = sub i32 [[TMP86]], -31
 // CHECK1-NEXT:    [[DIV131:%.*]] = udiv i32 [[SUB130]], 32
 // CHECK1-NEXT:    [[MUL132:%.*]] = mul i32 [[MUL129]], [[DIV131]]
-// CHECK1-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP89:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB133:%.*]] = sub i32 [[TMP88]], [[TMP89]]
-// CHECK1-NEXT:    [[SUB134:%.*]] = sub i32 [[SUB133]], 1
-// CHECK1-NEXT:    [[ADD135:%.*]] = add i32 [[SUB134]], 1
-// CHECK1-NEXT:    [[DIV136:%.*]] = udiv i32 [[ADD135]], 1
-// CHECK1-NEXT:    [[MUL137:%.*]] = mul i32 [[MUL132]], [[DIV136]]
-// CHECK1-NEXT:    [[CONV138:%.*]] = zext i32 [[MUL137]] to i64
-// CHECK1-NEXT:    [[DIV139:%.*]] = sdiv i64 [[TMP82]], [[CONV138]]
-// CHECK1-NEXT:    [[TMP90:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB140:%.*]] = sub i32 [[TMP90]], [[TMP91]]
-// CHECK1-NEXT:    [[SUB141:%.*]] = sub i32 [[SUB140]], 1
-// CHECK1-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD142:%.*]] = add i32 [[SUB141]], [[TMP92]]
+// CHECK1-NEXT:    [[MUL133:%.*]] = mul i32 [[MUL132]], 32
+// CHECK1-NEXT:    [[CONV134:%.*]] = zext i32 [[MUL133]] to i64
+// CHECK1-NEXT:    [[MUL135:%.*]] = mul nsw i64 [[DIV124]], [[CONV134]]
+// CHECK1-NEXT:    [[SUB136:%.*]] = sub nsw i64 [[TMP75]], [[MUL135]]
+// CHECK1-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB137:%.*]] = sub i32 [[TMP87]], -31
+// CHECK1-NEXT:    [[DIV138:%.*]] = udiv i32 [[SUB137]], 32
+// CHECK1-NEXT:    [[MUL139:%.*]] = mul i32 1, [[DIV138]]
+// CHECK1-NEXT:    [[MUL140:%.*]] = mul i32 [[MUL139]], 32
+// CHECK1-NEXT:    [[CONV141:%.*]] = zext i32 [[MUL140]] to i64
+// CHECK1-NEXT:    [[DIV142:%.*]] = sdiv i64 [[SUB136]], [[CONV141]]
+// CHECK1-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB143:%.*]] = sub i32 [[TMP88]], -31
+// CHECK1-NEXT:    [[DIV144:%.*]] = udiv i32 [[SUB143]], 32
+// CHECK1-NEXT:    [[MUL145:%.*]] = mul i32 1, [[DIV144]]
+// CHECK1-NEXT:    [[MUL146:%.*]] = mul i32 [[MUL145]], 32
+// CHECK1-NEXT:    [[CONV147:%.*]] = zext i32 [[MUL146]] to i64
+// CHECK1-NEXT:    [[MUL148:%.*]] = mul nsw i64 [[DIV142]], [[CONV147]]
+// CHECK1-NEXT:    [[SUB149:%.*]] = sub nsw i64 [[SUB113]], [[MUL148]]
+// CHECK1-NEXT:    [[DIV150:%.*]] = sdiv i64 [[SUB149]], 32
+// CHECK1-NEXT:    [[MUL151:%.*]] = mul nsw i64 [[DIV150]], 32
+// CHECK1-NEXT:    [[ADD152:%.*]] = add nsw i64 0, [[MUL151]]
+// CHECK1-NEXT:    [[CONV153:%.*]] = trunc i64 [[ADD152]] to i32
+// CHECK1-NEXT:    store i32 [[CONV153]], ptr [[DOTFLOOR_0_IV_K37]], align 4
+// CHECK1-NEXT:    [[TMP89:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP90:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB154:%.*]] = sub i32 [[TMP91]], [[TMP92]]
+// CHECK1-NEXT:    [[SUB155:%.*]] = sub i32 [[SUB154]], 1
 // CHECK1-NEXT:    [[TMP93:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV143:%.*]] = udiv i32 [[ADD142]], [[TMP93]]
-// CHECK1-NEXT:    [[MUL144:%.*]] = mul i32 1, [[DIV143]]
-// CHECK1-NEXT:    [[TMP94:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB145:%.*]] = sub i32 [[TMP94]], -31
-// CHECK1-NEXT:    [[DIV146:%.*]] = udiv i32 [[SUB145]], 32
-// CHECK1-NEXT:    [[MUL147:%.*]] = mul i32 [[MUL144]], [[DIV146]]
-// CHECK1-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB148:%.*]] = sub i32 [[TMP95]], [[TMP96]]
-// CHECK1-NEXT:    [[SUB149:%.*]] = sub i32 [[SUB148]], 1
-// CHECK1-NEXT:    [[ADD150:%.*]] = add i32 [[SUB149]], 1
-// CHECK1-NEXT:    [[DIV151:%.*]] = udiv i32 [[ADD150]], 1
-// CHECK1-NEXT:    [[MUL152:%.*]] = mul i32 [[MUL147]], [[DIV151]]
-// CHECK1-NEXT:    [[CONV153:%.*]] = zext i32 [[MUL152]] to i64
-// CHECK1-NEXT:    [[MUL154:%.*]] = mul nsw i64 [[DIV139]], [[CONV153]]
-// CHECK1-NEXT:    [[SUB155:%.*]] = sub nsw i64 [[TMP81]], [[MUL154]]
-// CHECK1-NEXT:    [[TMP97:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP98:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB156:%.*]] = sub i32 [[TMP99]], [[TMP100]]
-// CHECK1-NEXT:    [[SUB157:%.*]] = sub i32 [[SUB156]], 1
-// CHECK1-NEXT:    [[TMP101:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD158:%.*]] = add i32 [[SUB157]], [[TMP101]]
-// CHECK1-NEXT:    [[TMP102:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV159:%.*]] = udiv i32 [[ADD158]], [[TMP102]]
-// CHECK1-NEXT:    [[MUL160:%.*]] = mul i32 1, [[DIV159]]
-// CHECK1-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB161:%.*]] = sub i32 [[TMP103]], -31
-// CHECK1-NEXT:    [[DIV162:%.*]] = udiv i32 [[SUB161]], 32
-// CHECK1-NEXT:    [[MUL163:%.*]] = mul i32 [[MUL160]], [[DIV162]]
-// CHECK1-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP105:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB164:%.*]] = sub i32 [[TMP104]], [[TMP105]]
-// CHECK1-NEXT:    [[SUB165:%.*]] = sub i32 [[SUB164]], 1
-// CHECK1-NEXT:    [[ADD166:%.*]] = add i32 [[SUB165]], 1
-// CHECK1-NEXT:    [[DIV167:%.*]] = udiv i32 [[ADD166]], 1
-// CHECK1-NEXT:    [[MUL168:%.*]] = mul i32 [[MUL163]], [[DIV167]]
-// CHECK1-NEXT:    [[CONV169:%.*]] = zext i32 [[MUL168]] to i64
-// CHECK1-NEXT:    [[DIV170:%.*]] = sdiv i64 [[TMP98]], [[CONV169]]
-// CHECK1-NEXT:    [[TMP106:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB171:%.*]] = sub i32 [[TMP106]], [[TMP107]]
-// CHECK1-NEXT:    [[SUB172:%.*]] = sub i32 [[SUB171]], 1
-// CHECK1-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD173:%.*]] = add i32 [[SUB172]], [[TMP108]]
-// CHECK1-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV174:%.*]] = udiv i32 [[ADD173]], [[TMP109]]
-// CHECK1-NEXT:    [[MUL175:%.*]] = mul i32 1, [[DIV174]]
-// CHECK1-NEXT:    [[TMP110:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB176:%.*]] = sub i32 [[TMP110]], -31
-// CHECK1-NEXT:    [[DIV177:%.*]] = udiv i32 [[SUB176]], 32
-// CHECK1-NEXT:    [[MUL178:%.*]] = mul i32 [[MUL175]], [[DIV177]]
-// CHECK1-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB179:%.*]] = sub i32 [[TMP111]], [[TMP112]]
-// CHECK1-NEXT:    [[SUB180:%.*]] = sub i32 [[SUB179]], 1
-// CHECK1-NEXT:    [[ADD181:%.*]] = add i32 [[SUB180]], 1
-// CHECK1-NEXT:    [[DIV182:%.*]] = udiv i32 [[ADD181]], 1
-// CHECK1-NEXT:    [[MUL183:%.*]] = mul i32 [[MUL178]], [[DIV182]]
-// CHECK1-NEXT:    [[CONV184:%.*]] = zext i32 [[MUL183]] to i64
-// CHECK1-NEXT:    [[MUL185:%.*]] = mul nsw i64 [[DIV170]], [[CONV184]]
-// CHECK1-NEXT:    [[SUB186:%.*]] = sub nsw i64 [[TMP97]], [[MUL185]]
+// CHECK1-NEXT:    [[ADD156:%.*]] = add i32 [[SUB155]], [[TMP93]]
+// CHECK1-NEXT:    [[TMP94:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV157:%.*]] = udiv i32 [[ADD156]], [[TMP94]]
+// CHECK1-NEXT:    [[MUL158:%.*]] = mul i32 1, [[DIV157]]
+// CHECK1-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB159:%.*]] = sub i32 [[TMP95]], -31
+// CHECK1-NEXT:    [[DIV160:%.*]] = udiv i32 [[SUB159]], 32
+// CHECK1-NEXT:    [[MUL161:%.*]] = mul i32 [[MUL158]], [[DIV160]]
+// CHECK1-NEXT:    [[MUL162:%.*]] = mul i32 [[MUL161]], 32
+// CHECK1-NEXT:    [[CONV163:%.*]] = zext i32 [[MUL162]] to i64
+// CHECK1-NEXT:    [[DIV164:%.*]] = sdiv i64 [[TMP90]], [[CONV163]]
+// CHECK1-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP97:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB165:%.*]] = sub i32 [[TMP96]], [[TMP97]]
+// CHECK1-NEXT:    [[SUB166:%.*]] = sub i32 [[SUB165]], 1
+// CHECK1-NEXT:    [[TMP98:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD167:%.*]] = add i32 [[SUB166]], [[TMP98]]
+// CHECK1-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV168:%.*]] = udiv i32 [[ADD167]], [[TMP99]]
+// CHECK1-NEXT:    [[MUL169:%.*]] = mul i32 1, [[DIV168]]
+// CHECK1-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB170:%.*]] = sub i32 [[TMP100]], -31
+// CHECK1-NEXT:    [[DIV171:%.*]] = udiv i32 [[SUB170]], 32
+// CHECK1-NEXT:    [[MUL172:%.*]] = mul i32 [[MUL169]], [[DIV171]]
+// CHECK1-NEXT:    [[MUL173:%.*]] = mul i32 [[MUL172]], 32
+// CHECK1-NEXT:    [[CONV174:%.*]] = zext i32 [[MUL173]] to i64
+// CHECK1-NEXT:    [[MUL175:%.*]] = mul nsw i64 [[DIV164]], [[CONV174]]
+// CHECK1-NEXT:    [[SUB176:%.*]] = sub nsw i64 [[TMP89]], [[MUL175]]
+// CHECK1-NEXT:    [[TMP101:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP102:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB177:%.*]] = sub i32 [[TMP103]], [[TMP104]]
+// CHECK1-NEXT:    [[SUB178:%.*]] = sub i32 [[SUB177]], 1
+// CHECK1-NEXT:    [[TMP105:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD179:%.*]] = add i32 [[SUB178]], [[TMP105]]
+// CHECK1-NEXT:    [[TMP106:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV180:%.*]] = udiv i32 [[ADD179]], [[TMP106]]
+// CHECK1-NEXT:    [[MUL181:%.*]] = mul i32 1, [[DIV180]]
+// CHECK1-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB182:%.*]] = sub i32 [[TMP107]], -31
+// CHECK1-NEXT:    [[DIV183:%.*]] = udiv i32 [[SUB182]], 32
+// CHECK1-NEXT:    [[MUL184:%.*]] = mul i32 [[MUL181]], [[DIV183]]
+// CHECK1-NEXT:    [[MUL185:%.*]] = mul i32 [[MUL184]], 32
+// CHECK1-NEXT:    [[CONV186:%.*]] = zext i32 [[MUL185]] to i64
+// CHECK1-NEXT:    [[DIV187:%.*]] = sdiv i64 [[TMP102]], [[CONV186]]
+// CHECK1-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB188:%.*]] = sub i32 [[TMP108]], [[TMP109]]
+// CHECK1-NEXT:    [[SUB189:%.*]] = sub i32 [[SUB188]], 1
+// CHECK1-NEXT:    [[TMP110:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD190:%.*]] = add i32 [[SUB189]], [[TMP110]]
+// CHECK1-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV191:%.*]] = udiv i32 [[ADD190]], [[TMP111]]
+// CHECK1-NEXT:    [[MUL192:%.*]] = mul i32 1, [[DIV191]]
+// CHECK1-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB193:%.*]] = sub i32 [[TMP112]], -31
+// CHECK1-NEXT:    [[DIV194:%.*]] = udiv i32 [[SUB193]], 32
+// CHECK1-NEXT:    [[MUL195:%.*]] = mul i32 [[MUL192]], [[DIV194]]
+// CHECK1-NEXT:    [[MUL196:%.*]] = mul i32 [[MUL195]], 32
+// CHECK1-NEXT:    [[CONV197:%.*]] = zext i32 [[MUL196]] to i64
+// CHECK1-NEXT:    [[MUL198:%.*]] = mul nsw i64 [[DIV187]], [[CONV197]]
+// CHECK1-NEXT:    [[SUB199:%.*]] = sub nsw i64 [[TMP101]], [[MUL198]]
 // CHECK1-NEXT:    [[TMP113:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB187:%.*]] = sub i32 [[TMP113]], -31
-// CHECK1-NEXT:    [[DIV188:%.*]] = udiv i32 [[SUB187]], 32
-// CHECK1-NEXT:    [[MUL189:%.*]] = mul i32 1, [[DIV188]]
-// CHECK1-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP115:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB190:%.*]] = sub i32 [[TMP114]], [[TMP115]]
-// CHECK1-NEXT:    [[SUB191:%.*]] = sub i32 [[SUB190]], 1
-// CHECK1-NEXT:    [[ADD192:%.*]] = add i32 [[SUB191]], 1
-// CHECK1-NEXT:    [[DIV193:%.*]] = udiv i32 [[ADD192]], 1
-// CHECK1-NEXT:    [[MUL194:%.*]] = mul i32 [[MUL189]], [[DIV193]]
-// CHECK1-NEXT:    [[CONV195:%.*]] = zext i32 [[MUL194]] to i64
-// CHECK1-NEXT:    [[DIV196:%.*]] = sdiv i64 [[SUB186]], [[CONV195]]
-// CHECK1-NEXT:    [[TMP116:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB197:%.*]] = sub i32 [[TMP116]], -31
-// CHECK1-NEXT:    [[DIV198:%.*]] = udiv i32 [[SUB197]], 32
-// CHECK1-NEXT:    [[MUL199:%.*]] = mul i32 1, [[DIV198]]
-// CHECK1-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB200:%.*]] = sub i32 [[TMP117]], [[TMP118]]
-// CHECK1-NEXT:    [[SUB201:%.*]] = sub i32 [[SUB200]], 1
-// CHECK1-NEXT:    [[ADD202:%.*]] = add i32 [[SUB201]], 1
-// CHECK1-NEXT:    [[DIV203:%.*]] = udiv i32 [[ADD202]], 1
-// CHECK1-NEXT:    [[MUL204:%.*]] = mul i32 [[MUL199]], [[DIV203]]
-// CHECK1-NEXT:    [[CONV205:%.*]] = zext i32 [[MUL204]] to i64
-// CHECK1-NEXT:    [[MUL206:%.*]] = mul nsw i64 [[DIV196]], [[CONV205]]
-// CHECK1-NEXT:    [[SUB207:%.*]] = sub nsw i64 [[SUB155]], [[MUL206]]
-// CHECK1-NEXT:    [[TMP119:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP120:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB208:%.*]] = sub i32 [[TMP119]], [[TMP120]]
-// CHECK1-NEXT:    [[SUB209:%.*]] = sub i32 [[SUB208]], 1
-// CHECK1-NEXT:    [[ADD210:%.*]] = add i32 [[SUB209]], 1
-// CHECK1-NEXT:    [[DIV211:%.*]] = udiv i32 [[ADD210]], 1
-// CHECK1-NEXT:    [[MUL212:%.*]] = mul i32 1, [[DIV211]]
-// CHECK1-NEXT:    [[CONV213:%.*]] = zext i32 [[MUL212]] to i64
-// CHECK1-NEXT:    [[DIV214:%.*]] = sdiv i64 [[SUB207]], [[CONV213]]
-// CHECK1-NEXT:    [[MUL215:%.*]] = mul nsw i64 [[DIV214]], 32
-// CHECK1-NEXT:    [[ADD216:%.*]] = add nsw i64 0, [[MUL215]]
-// CHECK1-NEXT:    [[CONV217:%.*]] = trunc i64 [[ADD216]] to i32
-// CHECK1-NEXT:    store i32 [[CONV217]], ptr [[DOTFLOOR_0_IV_K51]], align 4
-// CHECK1-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[CONV218:%.*]] = zext i32 [[TMP121]] to i64
-// CHECK1-NEXT:    [[TMP122:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP123:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP124:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB219:%.*]] = sub i32 [[TMP124]], [[TMP125]]
-// CHECK1-NEXT:    [[SUB220:%.*]] = sub i32 [[SUB219]], 1
-// CHECK1-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD221:%.*]] = add i32 [[SUB220]], [[TMP126]]
-// CHECK1-NEXT:    [[TMP127:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV222:%.*]] = udiv i32 [[ADD221]], [[TMP127]]
-// CHECK1-NEXT:    [[MUL223:%.*]] = mul i32 1, [[DIV222]]
-// CHECK1-NEXT:    [[TMP128:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB224:%.*]] = sub i32 [[TMP128]], -31
-// CHECK1-NEXT:    [[DIV225:%.*]] = udiv i32 [[SUB224]], 32
-// CHECK1-NEXT:    [[MUL226:%.*]] = mul i32 [[MUL223]], [[DIV225]]
-// CHECK1-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB227:%.*]] = sub i32 [[TMP129]], [[TMP130]]
-// CHECK1-NEXT:    [[SUB228:%.*]] = sub i32 [[SUB227]], 1
-// CHECK1-NEXT:    [[ADD229:%.*]] = add i32 [[SUB228]], 1
-// CHECK1-NEXT:    [[DIV230:%.*]] = udiv i32 [[ADD229]], 1
-// CHECK1-NEXT:    [[MUL231:%.*]] = mul i32 [[MUL226]], [[DIV230]]
-// CHECK1-NEXT:    [[CONV232:%.*]] = zext i32 [[MUL231]] to i64
-// CHECK1-NEXT:    [[DIV233:%.*]] = sdiv i64 [[TMP123]], [[CONV232]]
-// CHECK1-NEXT:    [[TMP131:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP132:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB234:%.*]] = sub i32 [[TMP131]], [[TMP132]]
-// CHECK1-NEXT:    [[SUB235:%.*]] = sub i32 [[SUB234]], 1
-// CHECK1-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD236:%.*]] = add i32 [[SUB235]], [[TMP133]]
-// CHECK1-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV237:%.*]] = udiv i32 [[ADD236]], [[TMP134]]
-// CHECK1-NEXT:    [[MUL238:%.*]] = mul i32 1, [[DIV237]]
-// CHECK1-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB239:%.*]] = sub i32 [[TMP135]], -31
-// CHECK1-NEXT:    [[DIV240:%.*]] = udiv i32 [[SUB239]], 32
-// CHECK1-NEXT:    [[MUL241:%.*]] = mul i32 [[MUL238]], [[DIV240]]
-// CHECK1-NEXT:    [[TMP136:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB242:%.*]] = sub i32 [[TMP136]], [[TMP137]]
-// CHECK1-NEXT:    [[SUB243:%.*]] = sub i32 [[SUB242]], 1
-// CHECK1-NEXT:    [[ADD244:%.*]] = add i32 [[SUB243]], 1
-// CHECK1-NEXT:    [[DIV245:%.*]] = udiv i32 [[ADD244]], 1
-// CHECK1-NEXT:    [[MUL246:%.*]] = mul i32 [[MUL241]], [[DIV245]]
-// CHECK1-NEXT:    [[CONV247:%.*]] = zext i32 [[MUL246]] to i64
-// CHECK1-NEXT:    [[MUL248:%.*]] = mul nsw i64 [[DIV233]], [[CONV247]]
-// CHECK1-NEXT:    [[SUB249:%.*]] = sub nsw i64 [[TMP122]], [[MUL248]]
-// CHECK1-NEXT:    [[TMP138:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP139:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB250:%.*]] = sub i32 [[TMP140]], [[TMP141]]
-// CHECK1-NEXT:    [[SUB251:%.*]] = sub i32 [[SUB250]], 1
-// CHECK1-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD252:%.*]] = add i32 [[SUB251]], [[TMP142]]
-// CHECK1-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV253:%.*]] = udiv i32 [[ADD252]], [[TMP143]]
-// CHECK1-NEXT:    [[MUL254:%.*]] = mul i32 1, [[DIV253]]
-// CHECK1-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB255:%.*]] = sub i32 [[TMP144]], -31
-// CHECK1-NEXT:    [[DIV256:%.*]] = udiv i32 [[SUB255]], 32
-// CHECK1-NEXT:    [[MUL257:%.*]] = mul i32 [[MUL254]], [[DIV256]]
-// CHECK1-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB258:%.*]] = sub i32 [[TMP145]], [[TMP146]]
-// CHECK1-NEXT:    [[SUB259:%.*]] = sub i32 [[SUB258]], 1
-// CHECK1-NEXT:    [[ADD260:%.*]] = add i32 [[SUB259]], 1
-// CHECK1-NEXT:    [[DIV261:%.*]] = udiv i32 [[ADD260]], 1
-// CHECK1-NEXT:    [[MUL262:%.*]] = mul i32 [[MUL257]], [[DIV261]]
+// CHECK1-NEXT:    [[SUB200:%.*]] = sub i32 [[TMP113]], -31
+// CHECK1-NEXT:    [[DIV201:%.*]] = udiv i32 [[SUB200]], 32
+// CHECK1-NEXT:    [[MUL202:%.*]] = mul i32 1, [[DIV201]]
+// CHECK1-NEXT:    [[MUL203:%.*]] = mul i32 [[MUL202]], 32
+// CHECK1-NEXT:    [[CONV204:%.*]] = zext i32 [[MUL203]] to i64
+// CHECK1-NEXT:    [[DIV205:%.*]] = sdiv i64 [[SUB199]], [[CONV204]]
+// CHECK1-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB206:%.*]] = sub i32 [[TMP114]], -31
+// CHECK1-NEXT:    [[DIV207:%.*]] = udiv i32 [[SUB206]], 32
+// CHECK1-NEXT:    [[MUL208:%.*]] = mul i32 1, [[DIV207]]
+// CHECK1-NEXT:    [[MUL209:%.*]] = mul i32 [[MUL208]], 32
+// CHECK1-NEXT:    [[CONV210:%.*]] = zext i32 [[MUL209]] to i64
+// CHECK1-NEXT:    [[MUL211:%.*]] = mul nsw i64 [[DIV205]], [[CONV210]]
+// CHECK1-NEXT:    [[SUB212:%.*]] = sub nsw i64 [[SUB176]], [[MUL211]]
+// CHECK1-NEXT:    [[TMP115:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP116:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB213:%.*]] = sub i32 [[TMP117]], [[TMP118]]
+// CHECK1-NEXT:    [[SUB214:%.*]] = sub i32 [[SUB213]], 1
+// CHECK1-NEXT:    [[TMP119:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD215:%.*]] = add i32 [[SUB214]], [[TMP119]]
+// CHECK1-NEXT:    [[TMP120:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV216:%.*]] = udiv i32 [[ADD215]], [[TMP120]]
+// CHECK1-NEXT:    [[MUL217:%.*]] = mul i32 1, [[DIV216]]
+// CHECK1-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB218:%.*]] = sub i32 [[TMP121]], -31
+// CHECK1-NEXT:    [[DIV219:%.*]] = udiv i32 [[SUB218]], 32
+// CHECK1-NEXT:    [[MUL220:%.*]] = mul i32 [[MUL217]], [[DIV219]]
+// CHECK1-NEXT:    [[MUL221:%.*]] = mul i32 [[MUL220]], 32
+// CHECK1-NEXT:    [[CONV222:%.*]] = zext i32 [[MUL221]] to i64
+// CHECK1-NEXT:    [[DIV223:%.*]] = sdiv i64 [[TMP116]], [[CONV222]]
+// CHECK1-NEXT:    [[TMP122:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP123:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB224:%.*]] = sub i32 [[TMP122]], [[TMP123]]
+// CHECK1-NEXT:    [[SUB225:%.*]] = sub i32 [[SUB224]], 1
+// CHECK1-NEXT:    [[TMP124:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD226:%.*]] = add i32 [[SUB225]], [[TMP124]]
+// CHECK1-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV227:%.*]] = udiv i32 [[ADD226]], [[TMP125]]
+// CHECK1-NEXT:    [[MUL228:%.*]] = mul i32 1, [[DIV227]]
+// CHECK1-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB229:%.*]] = sub i32 [[TMP126]], -31
+// CHECK1-NEXT:    [[DIV230:%.*]] = udiv i32 [[SUB229]], 32
+// CHECK1-NEXT:    [[MUL231:%.*]] = mul i32 [[MUL228]], [[DIV230]]
+// CHECK1-NEXT:    [[MUL232:%.*]] = mul i32 [[MUL231]], 32
+// CHECK1-NEXT:    [[CONV233:%.*]] = zext i32 [[MUL232]] to i64
+// CHECK1-NEXT:    [[MUL234:%.*]] = mul nsw i64 [[DIV223]], [[CONV233]]
+// CHECK1-NEXT:    [[SUB235:%.*]] = sub nsw i64 [[TMP115]], [[MUL234]]
+// CHECK1-NEXT:    [[TMP127:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP128:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB236:%.*]] = sub i32 [[TMP129]], [[TMP130]]
+// CHECK1-NEXT:    [[SUB237:%.*]] = sub i32 [[SUB236]], 1
+// CHECK1-NEXT:    [[TMP131:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD238:%.*]] = add i32 [[SUB237]], [[TMP131]]
+// CHECK1-NEXT:    [[TMP132:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV239:%.*]] = udiv i32 [[ADD238]], [[TMP132]]
+// CHECK1-NEXT:    [[MUL240:%.*]] = mul i32 1, [[DIV239]]
+// CHECK1-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB241:%.*]] = sub i32 [[TMP133]], -31
+// CHECK1-NEXT:    [[DIV242:%.*]] = udiv i32 [[SUB241]], 32
+// CHECK1-NEXT:    [[MUL243:%.*]] = mul i32 [[MUL240]], [[DIV242]]
+// CHECK1-NEXT:    [[MUL244:%.*]] = mul i32 [[MUL243]], 32
+// CHECK1-NEXT:    [[CONV245:%.*]] = zext i32 [[MUL244]] to i64
+// CHECK1-NEXT:    [[DIV246:%.*]] = sdiv i64 [[TMP128]], [[CONV245]]
+// CHECK1-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB247:%.*]] = sub i32 [[TMP134]], [[TMP135]]
+// CHECK1-NEXT:    [[SUB248:%.*]] = sub i32 [[SUB247]], 1
+// CHECK1-NEXT:    [[TMP136:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD249:%.*]] = add i32 [[SUB248]], [[TMP136]]
+// CHECK1-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV250:%.*]] = udiv i32 [[ADD249]], [[TMP137]]
+// CHECK1-NEXT:    [[MUL251:%.*]] = mul i32 1, [[DIV250]]
+// CHECK1-NEXT:    [[TMP138:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB252:%.*]] = sub i32 [[TMP138]], -31
+// CHECK1-NEXT:    [[DIV253:%.*]] = udiv i32 [[SUB252]], 32
+// CHECK1-NEXT:    [[MUL254:%.*]] = mul i32 [[MUL251]], [[DIV253]]
+// CHECK1-NEXT:    [[MUL255:%.*]] = mul i32 [[MUL254]], 32
+// CHECK1-NEXT:    [[CONV256:%.*]] = zext i32 [[MUL255]] to i64
+// CHECK1-NEXT:    [[MUL257:%.*]] = mul nsw i64 [[DIV246]], [[CONV256]]
+// CHECK1-NEXT:    [[SUB258:%.*]] = sub nsw i64 [[TMP127]], [[MUL257]]
+// CHECK1-NEXT:    [[TMP139:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB259:%.*]] = sub i32 [[TMP139]], -31
+// CHECK1-NEXT:    [[DIV260:%.*]] = udiv i32 [[SUB259]], 32
+// CHECK1-NEXT:    [[MUL261:%.*]] = mul i32 1, [[DIV260]]
+// CHECK1-NEXT:    [[MUL262:%.*]] = mul i32 [[MUL261]], 32
 // CHECK1-NEXT:    [[CONV263:%.*]] = zext i32 [[MUL262]] to i64
-// CHECK1-NEXT:    [[DIV264:%.*]] = sdiv i64 [[TMP139]], [[CONV263]]
-// CHECK1-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB265:%.*]] = sub i32 [[TMP147]], [[TMP148]]
-// CHECK1-NEXT:    [[SUB266:%.*]] = sub i32 [[SUB265]], 1
-// CHECK1-NEXT:    [[TMP149:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD267:%.*]] = add i32 [[SUB266]], [[TMP149]]
-// CHECK1-NEXT:    [[TMP150:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV268:%.*]] = udiv i32 [[ADD267]], [[TMP150]]
-// CHECK1-NEXT:    [[MUL269:%.*]] = mul i32 1, [[DIV268]]
-// CHECK1-NEXT:    [[TMP151:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB270:%.*]] = sub i32 [[TMP151]], -31
-// CHECK1-NEXT:    [[DIV271:%.*]] = udiv i32 [[SUB270]], 32
-// CHECK1-NEXT:    [[MUL272:%.*]] = mul i32 [[MUL269]], [[DIV271]]
-// CHECK1-NEXT:    [[TMP152:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP153:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB273:%.*]] = sub i32 [[TMP152]], [[TMP153]]
-// CHECK1-NEXT:    [[SUB274:%.*]] = sub i32 [[SUB273]], 1
-// CHECK1-NEXT:    [[ADD275:%.*]] = add i32 [[SUB274]], 1
-// CHECK1-NEXT:    [[DIV276:%.*]] = udiv i32 [[ADD275]], 1
-// CHECK1-NEXT:    [[MUL277:%.*]] = mul i32 [[MUL272]], [[DIV276]]
-// CHECK1-NEXT:    [[CONV278:%.*]] = zext i32 [[MUL277]] to i64
-// CHECK1-NEXT:    [[MUL279:%.*]] = mul nsw i64 [[DIV264]], [[CONV278]]
-// CHECK1-NEXT:    [[SUB280:%.*]] = sub nsw i64 [[TMP138]], [[MUL279]]
-// CHECK1-NEXT:    [[TMP154:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB281:%.*]] = sub i32 [[TMP154]], -31
-// CHECK1-NEXT:    [[DIV282:%.*]] = udiv i32 [[SUB281]], 32
-// CHECK1-NEXT:    [[MUL283:%.*]] = mul i32 1, [[DIV282]]
-// CHECK1-NEXT:    [[TMP155:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP156:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB284:%.*]] = sub i32 [[TMP155]], [[TMP156]]
-// CHECK1-NEXT:    [[SUB285:%.*]] = sub i32 [[SUB284]], 1
-// CHECK1-NEXT:    [[ADD286:%.*]] = add i32 [[SUB285]], 1
-// CHECK1-NEXT:    [[DIV287:%.*]] = udiv i32 [[ADD286]], 1
-// CHECK1-NEXT:    [[MUL288:%.*]] = mul i32 [[MUL283]], [[DIV287]]
-// CHECK1-NEXT:    [[CONV289:%.*]] = zext i32 [[MUL288]] to i64
-// CHECK1-NEXT:    [[DIV290:%.*]] = sdiv i64 [[SUB280]], [[CONV289]]
-// CHECK1-NEXT:    [[TMP157:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB291:%.*]] = sub i32 [[TMP157]], -31
-// CHECK1-NEXT:    [[DIV292:%.*]] = udiv i32 [[SUB291]], 32
-// CHECK1-NEXT:    [[MUL293:%.*]] = mul i32 1, [[DIV292]]
-// CHECK1-NEXT:    [[TMP158:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP159:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB294:%.*]] = sub i32 [[TMP158]], [[TMP159]]
-// CHECK1-NEXT:    [[SUB295:%.*]] = sub i32 [[SUB294]], 1
-// CHECK1-NEXT:    [[ADD296:%.*]] = add i32 [[SUB295]], 1
-// CHECK1-NEXT:    [[DIV297:%.*]] = udiv i32 [[ADD296]], 1
-// CHECK1-NEXT:    [[MUL298:%.*]] = mul i32 [[MUL293]], [[DIV297]]
-// CHECK1-NEXT:    [[CONV299:%.*]] = zext i32 [[MUL298]] to i64
-// CHECK1-NEXT:    [[MUL300:%.*]] = mul nsw i64 [[DIV290]], [[CONV299]]
-// CHECK1-NEXT:    [[SUB301:%.*]] = sub nsw i64 [[SUB249]], [[MUL300]]
-// CHECK1-NEXT:    [[TMP160:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP161:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP162:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP163:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB302:%.*]] = sub i32 [[TMP162]], [[TMP163]]
-// CHECK1-NEXT:    [[SUB303:%.*]] = sub i32 [[SUB302]], 1
-// CHECK1-NEXT:    [[TMP164:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD304:%.*]] = add i32 [[SUB303]], [[TMP164]]
-// CHECK1-NEXT:    [[TMP165:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV305:%.*]] = udiv i32 [[ADD304]], [[TMP165]]
-// CHECK1-NEXT:    [[MUL306:%.*]] = mul i32 1, [[DIV305]]
-// CHECK1-NEXT:    [[TMP166:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB307:%.*]] = sub i32 [[TMP166]], -31
-// CHECK1-NEXT:    [[DIV308:%.*]] = udiv i32 [[SUB307]], 32
-// CHECK1-NEXT:    [[MUL309:%.*]] = mul i32 [[MUL306]], [[DIV308]]
-// CHECK1-NEXT:    [[TMP167:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP168:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB310:%.*]] = sub i32 [[TMP167]], [[TMP168]]
-// CHECK1-NEXT:    [[SUB311:%.*]] = sub i32 [[SUB310]], 1
-// CHECK1-NEXT:    [[ADD312:%.*]] = add i32 [[SUB311]], 1
-// CHECK1-NEXT:    [[DIV313:%.*]] = udiv i32 [[ADD312]], 1
-// CHECK1-NEXT:    [[MUL314:%.*]] = mul i32 [[MUL309]], [[DIV313]]
-// CHECK1-NEXT:    [[CONV315:%.*]] = zext i32 [[MUL314]] to i64
-// CHECK1-NEXT:    [[DIV316:%.*]] = sdiv i64 [[TMP161]], [[CONV315]]
-// CHECK1-NEXT:    [[TMP169:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP170:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB317:%.*]] = sub i32 [[TMP169]], [[TMP170]]
-// CHECK1-NEXT:    [[SUB318:%.*]] = sub i32 [[SUB317]], 1
-// CHECK1-NEXT:    [[TMP171:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD319:%.*]] = add i32 [[SUB318]], [[TMP171]]
-// CHECK1-NEXT:    [[TMP172:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV320:%.*]] = udiv i32 [[ADD319]], [[TMP172]]
-// CHECK1-NEXT:    [[MUL321:%.*]] = mul i32 1, [[DIV320]]
-// CHECK1-NEXT:    [[TMP173:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB322:%.*]] = sub i32 [[TMP173]], -31
-// CHECK1-NEXT:    [[DIV323:%.*]] = udiv i32 [[SUB322]], 32
-// CHECK1-NEXT:    [[MUL324:%.*]] = mul i32 [[MUL321]], [[DIV323]]
-// CHECK1-NEXT:    [[TMP174:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP175:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB325:%.*]] = sub i32 [[TMP174]], [[TMP175]]
-// CHECK1-NEXT:    [[SUB326:%.*]] = sub i32 [[SUB325]], 1
-// CHECK1-NEXT:    [[ADD327:%.*]] = add i32 [[SUB326]], 1
-// CHECK1-NEXT:    [[DIV328:%.*]] = udiv i32 [[ADD327]], 1
-// CHECK1-NEXT:    [[MUL329:%.*]] = mul i32 [[MUL324]], [[DIV328]]
-// CHECK1-NEXT:    [[CONV330:%.*]] = zext i32 [[MUL329]] to i64
-// CHECK1-NEXT:    [[MUL331:%.*]] = mul nsw i64 [[DIV316]], [[CONV330]]
-// CHECK1-NEXT:    [[SUB332:%.*]] = sub nsw i64 [[TMP160]], [[MUL331]]
-// CHECK1-NEXT:    [[TMP176:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP177:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP178:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP179:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB333:%.*]] = sub i32 [[TMP178]], [[TMP179]]
-// CHECK1-NEXT:    [[SUB334:%.*]] = sub i32 [[SUB333]], 1
-// CHECK1-NEXT:    [[TMP180:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD335:%.*]] = add i32 [[SUB334]], [[TMP180]]
-// CHECK1-NEXT:    [[TMP181:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV336:%.*]] = udiv i32 [[ADD335]], [[TMP181]]
-// CHECK1-NEXT:    [[MUL337:%.*]] = mul i32 1, [[DIV336]]
-// CHECK1-NEXT:    [[TMP182:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB338:%.*]] = sub i32 [[TMP182]], -31
-// CHECK1-NEXT:    [[DIV339:%.*]] = udiv i32 [[SUB338]], 32
-// CHECK1-NEXT:    [[MUL340:%.*]] = mul i32 [[MUL337]], [[DIV339]]
-// CHECK1-NEXT:    [[TMP183:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP184:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB341:%.*]] = sub i32 [[TMP183]], [[TMP184]]
-// CHECK1-NEXT:    [[SUB342:%.*]] = sub i32 [[SUB341]], 1
-// CHECK1-NEXT:    [[ADD343:%.*]] = add i32 [[SUB342]], 1
-// CHECK1-NEXT:    [[DIV344:%.*]] = udiv i32 [[ADD343]], 1
-// CHECK1-NEXT:    [[MUL345:%.*]] = mul i32 [[MUL340]], [[DIV344]]
-// CHECK1-NEXT:    [[CONV346:%.*]] = zext i32 [[MUL345]] to i64
-// CHECK1-NEXT:    [[DIV347:%.*]] = sdiv i64 [[TMP177]], [[CONV346]]
-// CHECK1-NEXT:    [[TMP185:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP186:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB348:%.*]] = sub i32 [[TMP185]], [[TMP186]]
-// CHECK1-NEXT:    [[SUB349:%.*]] = sub i32 [[SUB348]], 1
-// CHECK1-NEXT:    [[TMP187:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD350:%.*]] = add i32 [[SUB349]], [[TMP187]]
-// CHECK1-NEXT:    [[TMP188:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV351:%.*]] = udiv i32 [[ADD350]], [[TMP188]]
-// CHECK1-NEXT:    [[MUL352:%.*]] = mul i32 1, [[DIV351]]
-// CHECK1-NEXT:    [[TMP189:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB353:%.*]] = sub i32 [[TMP189]], -31
-// CHECK1-NEXT:    [[DIV354:%.*]] = udiv i32 [[SUB353]], 32
-// CHECK1-NEXT:    [[MUL355:%.*]] = mul i32 [[MUL352]], [[DIV354]]
-// CHECK1-NEXT:    [[TMP190:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP191:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB356:%.*]] = sub i32 [[TMP190]], [[TMP191]]
-// CHECK1-NEXT:    [[SUB357:%.*]] = sub i32 [[SUB356]], 1
-// CHECK1-NEXT:    [[ADD358:%.*]] = add i32 [[SUB357]], 1
-// CHECK1-NEXT:    [[DIV359:%.*]] = udiv i32 [[ADD358]], 1
-// CHECK1-NEXT:    [[MUL360:%.*]] = mul i32 [[MUL355]], [[DIV359]]
-// CHECK1-NEXT:    [[CONV361:%.*]] = zext i32 [[MUL360]] to i64
-// CHECK1-NEXT:    [[MUL362:%.*]] = mul nsw i64 [[DIV347]], [[CONV361]]
-// CHECK1-NEXT:    [[SUB363:%.*]] = sub nsw i64 [[TMP176]], [[MUL362]]
-// CHECK1-NEXT:    [[TMP192:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB364:%.*]] = sub i32 [[TMP192]], -31
-// CHECK1-NEXT:    [[DIV365:%.*]] = udiv i32 [[SUB364]], 32
-// CHECK1-NEXT:    [[MUL366:%.*]] = mul i32 1, [[DIV365]]
-// CHECK1-NEXT:    [[TMP193:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP194:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB367:%.*]] = sub i32 [[TMP193]], [[TMP194]]
-// CHECK1-NEXT:    [[SUB368:%.*]] = sub i32 [[SUB367]], 1
-// CHECK1-NEXT:    [[ADD369:%.*]] = add i32 [[SUB368]], 1
-// CHECK1-NEXT:    [[DIV370:%.*]] = udiv i32 [[ADD369]], 1
-// CHECK1-NEXT:    [[MUL371:%.*]] = mul i32 [[MUL366]], [[DIV370]]
-// CHECK1-NEXT:    [[CONV372:%.*]] = zext i32 [[MUL371]] to i64
-// CHECK1-NEXT:    [[DIV373:%.*]] = sdiv i64 [[SUB363]], [[CONV372]]
-// CHECK1-NEXT:    [[TMP195:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB374:%.*]] = sub i32 [[TMP195]], -31
-// CHECK1-NEXT:    [[DIV375:%.*]] = udiv i32 [[SUB374]], 32
-// CHECK1-NEXT:    [[MUL376:%.*]] = mul i32 1, [[DIV375]]
-// CHECK1-NEXT:    [[TMP196:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP197:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB377:%.*]] = sub i32 [[TMP196]], [[TMP197]]
-// CHECK1-NEXT:    [[SUB378:%.*]] = sub i32 [[SUB377]], 1
-// CHECK1-NEXT:    [[ADD379:%.*]] = add i32 [[SUB378]], 1
-// CHECK1-NEXT:    [[DIV380:%.*]] = udiv i32 [[ADD379]], 1
-// CHECK1-NEXT:    [[MUL381:%.*]] = mul i32 [[MUL376]], [[DIV380]]
-// CHECK1-NEXT:    [[CONV382:%.*]] = zext i32 [[MUL381]] to i64
-// CHECK1-NEXT:    [[MUL383:%.*]] = mul nsw i64 [[DIV373]], [[CONV382]]
-// CHECK1-NEXT:    [[SUB384:%.*]] = sub nsw i64 [[SUB332]], [[MUL383]]
-// CHECK1-NEXT:    [[TMP198:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP199:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB385:%.*]] = sub i32 [[TMP198]], [[TMP199]]
-// CHECK1-NEXT:    [[SUB386:%.*]] = sub i32 [[SUB385]], 1
-// CHECK1-NEXT:    [[ADD387:%.*]] = add i32 [[SUB386]], 1
-// CHECK1-NEXT:    [[DIV388:%.*]] = udiv i32 [[ADD387]], 1
-// CHECK1-NEXT:    [[MUL389:%.*]] = mul i32 1, [[DIV388]]
-// CHECK1-NEXT:    [[CONV390:%.*]] = zext i32 [[MUL389]] to i64
-// CHECK1-NEXT:    [[DIV391:%.*]] = sdiv i64 [[SUB384]], [[CONV390]]
-// CHECK1-NEXT:    [[TMP200:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP201:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB392:%.*]] = sub i32 [[TMP200]], [[TMP201]]
-// CHECK1-NEXT:    [[SUB393:%.*]] = sub i32 [[SUB392]], 1
-// CHECK1-NEXT:    [[ADD394:%.*]] = add i32 [[SUB393]], 1
-// CHECK1-NEXT:    [[DIV395:%.*]] = udiv i32 [[ADD394]], 1
-// CHECK1-NEXT:    [[MUL396:%.*]] = mul i32 1, [[DIV395]]
-// CHECK1-NEXT:    [[CONV397:%.*]] = zext i32 [[MUL396]] to i64
-// CHECK1-NEXT:    [[MUL398:%.*]] = mul nsw i64 [[DIV391]], [[CONV397]]
-// CHECK1-NEXT:    [[SUB399:%.*]] = sub nsw i64 [[SUB301]], [[MUL398]]
-// CHECK1-NEXT:    [[MUL400:%.*]] = mul nsw i64 [[SUB399]], 1
-// CHECK1-NEXT:    [[ADD401:%.*]] = add nsw i64 [[CONV218]], [[MUL400]]
-// CHECK1-NEXT:    [[CONV402:%.*]] = trunc i64 [[ADD401]] to i32
-// CHECK1-NEXT:    store i32 [[CONV402]], ptr [[DOTTILE_0_IV_K52]], align 4
-// CHECK1-NEXT:    [[TMP202:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
-// CHECK1-NEXT:    [[TMP203:%.*]] = load i32, ptr [[DOTTILE_0_IV_K52]], align 4
-// CHECK1-NEXT:    [[TMP204:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
-// CHECK1-NEXT:    [[MUL403:%.*]] = mul i32 [[TMP203]], [[TMP204]]
-// CHECK1-NEXT:    [[ADD404:%.*]] = add i32 [[TMP202]], [[MUL403]]
-// CHECK1-NEXT:    store i32 [[ADD404]], ptr [[K]], align 4
-// CHECK1-NEXT:    [[TMP205:%.*]] = load i32, ptr [[I49]], align 4
-// CHECK1-NEXT:    [[TMP206:%.*]] = load i32, ptr [[J50]], align 4
-// CHECK1-NEXT:    [[TMP207:%.*]] = load i32, ptr [[K]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP205]], i32 noundef [[TMP206]], i32 noundef [[TMP207]])
+// CHECK1-NEXT:    [[DIV264:%.*]] = sdiv i64 [[SUB258]], [[CONV263]]
+// CHECK1-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB265:%.*]] = sub i32 [[TMP140]], -31
+// CHECK1-NEXT:    [[DIV266:%.*]] = udiv i32 [[SUB265]], 32
+// CHECK1-NEXT:    [[MUL267:%.*]] = mul i32 1, [[DIV266]]
+// CHECK1-NEXT:    [[MUL268:%.*]] = mul i32 [[MUL267]], 32
+// CHECK1-NEXT:    [[CONV269:%.*]] = zext i32 [[MUL268]] to i64
+// CHECK1-NEXT:    [[MUL270:%.*]] = mul nsw i64 [[DIV264]], [[CONV269]]
+// CHECK1-NEXT:    [[SUB271:%.*]] = sub nsw i64 [[SUB235]], [[MUL270]]
+// CHECK1-NEXT:    [[DIV272:%.*]] = sdiv i64 [[SUB271]], 32
+// CHECK1-NEXT:    [[MUL273:%.*]] = mul nsw i64 [[DIV272]], 32
+// CHECK1-NEXT:    [[SUB274:%.*]] = sub nsw i64 [[SUB212]], [[MUL273]]
+// CHECK1-NEXT:    [[MUL275:%.*]] = mul nsw i64 [[SUB274]], 1
+// CHECK1-NEXT:    [[ADD276:%.*]] = add nsw i64 0, [[MUL275]]
+// CHECK1-NEXT:    [[CONV277:%.*]] = trunc i64 [[ADD276]] to i32
+// CHECK1-NEXT:    store i32 [[CONV277]], ptr [[DOTTILE_CNT_0_IV_K38]], align 4
+// CHECK1-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
+// CHECK1-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
+// CHECK1-NEXT:    [[ADD278:%.*]] = add i32 [[TMP141]], [[TMP142]]
+// CHECK1-NEXT:    store i32 [[ADD278]], ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK1-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
+// CHECK1-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK1-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
+// CHECK1-NEXT:    [[MUL279:%.*]] = mul i32 [[TMP144]], [[TMP145]]
+// CHECK1-NEXT:    [[ADD280:%.*]] = add i32 [[TMP143]], [[MUL279]]
+// CHECK1-NEXT:    store i32 [[ADD280]], ptr [[K]], align 4
+// CHECK1-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK1-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// CHECK1-NEXT:    [[ADD281:%.*]] = add i32 [[TMP147]], 1
+// CHECK1-NEXT:    [[CMP282:%.*]] = icmp ult i32 [[TMP146]], [[ADD281]]
+// CHECK1-NEXT:    br i1 [[CMP282]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK1:       if.then:
+// CHECK1-NEXT:    [[TMP148:%.*]] = load i32, ptr [[I35]], align 4
+// CHECK1-NEXT:    [[TMP149:%.*]] = load i32, ptr [[J36]], align 4
+// CHECK1-NEXT:    [[TMP150:%.*]] = load i32, ptr [[K]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP148]], i32 noundef [[TMP149]], i32 noundef [[TMP150]])
+// CHECK1-NEXT:    br label [[IF_END]]
+// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP208:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[ADD405:%.*]] = add nsw i64 [[TMP208]], 1
-// CHECK1-NEXT:    store i64 [[ADD405]], ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP151:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[ADD283:%.*]] = add nsw i64 [[TMP151]], 1
+// CHECK1-NEXT:    store i64 [[ADD283]], ptr [[DOTOMP_IV]], align 8
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -1520,22 +1342,21 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTNEW_STEP10:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_14:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_17:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_22:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_K:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_K:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[I49:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[J50:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV_K51:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_K52:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[I35:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[J36:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV_K37:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_K38:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
@@ -1573,630 +1394,452 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
 // CHECK1-NEXT:    [[ADD15:%.*]] = add i32 [[TMP15]], 1
 // CHECK1-NEXT:    store i32 [[ADD15]], ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[_TMP2]], align 4
-// CHECK1-NEXT:    store i32 [[TMP16]], ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK1-NEXT:    [[ADD18:%.*]] = add i32 [[TMP17]], 1
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[_TMP2]], align 4
-// CHECK1-NEXT:    [[ADD19:%.*]] = add i32 [[TMP18]], 64
-// CHECK1-NEXT:    [[CMP:%.*]] = icmp ult i32 [[ADD18]], [[ADD19]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK1:       cond.true:
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK1-NEXT:    [[ADD20:%.*]] = add i32 [[TMP19]], 1
-// CHECK1-NEXT:    br label [[COND_END:%.*]]
-// CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[_TMP2]], align 4
-// CHECK1-NEXT:    [[ADD21:%.*]] = add i32 [[TMP20]], 64
-// CHECK1-NEXT:    br label [[COND_END]]
-// CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD20]], [[COND_TRUE]] ], [ [[ADD21]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[SUB23:%.*]] = sub i32 [[TMP21]], [[TMP22]]
-// CHECK1-NEXT:    [[SUB24:%.*]] = sub i32 [[SUB23]], 1
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[ADD25:%.*]] = add i32 [[SUB24]], [[TMP23]]
-// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[DIV26:%.*]] = udiv i32 [[ADD25]], [[TMP24]]
-// CHECK1-NEXT:    [[CONV:%.*]] = zext i32 [[DIV26]] to i64
-// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[SUB17:%.*]] = sub i32 [[TMP16]], [[TMP17]]
+// CHECK1-NEXT:    [[SUB18:%.*]] = sub i32 [[SUB17]], 1
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[ADD19:%.*]] = add i32 [[SUB18]], [[TMP18]]
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[DIV20:%.*]] = udiv i32 [[ADD19]], [[TMP19]]
+// CHECK1-NEXT:    [[CONV:%.*]] = zext i32 [[DIV20]] to i64
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB21:%.*]] = sub i32 [[TMP20]], [[TMP21]]
+// CHECK1-NEXT:    [[SUB22:%.*]] = sub i32 [[SUB21]], 1
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD23:%.*]] = add i32 [[SUB22]], [[TMP22]]
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV24:%.*]] = udiv i32 [[ADD23]], [[TMP23]]
+// CHECK1-NEXT:    [[CONV25:%.*]] = zext i32 [[DIV24]] to i64
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV25]]
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB26:%.*]] = sub i32 [[TMP24]], -63
+// CHECK1-NEXT:    [[DIV27:%.*]] = udiv i32 [[SUB26]], 64
+// CHECK1-NEXT:    [[CONV28:%.*]] = zext i32 [[DIV27]] to i64
+// CHECK1-NEXT:    [[MUL29:%.*]] = mul nsw i64 [[MUL]], [[CONV28]]
+// CHECK1-NEXT:    [[MUL30:%.*]] = mul nsw i64 [[MUL29]], 64
+// CHECK1-NEXT:    [[SUB31:%.*]] = sub nsw i64 [[MUL30]], 1
+// CHECK1-NEXT:    store i64 [[SUB31]], ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    store i32 [[TMP25]], ptr [[I]], align 4
 // CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB27:%.*]] = sub i32 [[TMP25]], [[TMP26]]
-// CHECK1-NEXT:    [[SUB28:%.*]] = sub i32 [[SUB27]], 1
-// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD29:%.*]] = add i32 [[SUB28]], [[TMP27]]
-// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV30:%.*]] = udiv i32 [[ADD29]], [[TMP28]]
-// CHECK1-NEXT:    [[CONV31:%.*]] = zext i32 [[DIV30]] to i64
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV31]]
-// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB32:%.*]] = sub i32 [[TMP29]], -63
-// CHECK1-NEXT:    [[DIV33:%.*]] = udiv i32 [[SUB32]], 64
-// CHECK1-NEXT:    [[CONV34:%.*]] = zext i32 [[DIV33]] to i64
-// CHECK1-NEXT:    [[MUL35:%.*]] = mul nsw i64 [[MUL]], [[CONV34]]
-// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB36:%.*]] = sub i32 [[TMP30]], [[TMP31]]
-// CHECK1-NEXT:    [[SUB37:%.*]] = sub i32 [[SUB36]], 1
-// CHECK1-NEXT:    [[ADD38:%.*]] = add i32 [[SUB37]], 1
-// CHECK1-NEXT:    [[DIV39:%.*]] = udiv i32 [[ADD38]], 1
-// CHECK1-NEXT:    [[CONV40:%.*]] = zext i32 [[DIV39]] to i64
-// CHECK1-NEXT:    [[MUL41:%.*]] = mul nsw i64 [[MUL35]], [[CONV40]]
-// CHECK1-NEXT:    [[SUB42:%.*]] = sub nsw i64 [[MUL41]], 1
-// CHECK1-NEXT:    store i64 [[SUB42]], ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    store i32 [[TMP32]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    store i32 [[TMP33]], ptr [[J]], align 4
+// CHECK1-NEXT:    store i32 [[TMP26]], ptr [[J]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_K]], align 4
-// CHECK1-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    store i32 [[TMP34]], ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK1-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK1-NEXT:    [[CMP43:%.*]] = icmp slt i32 [[TMP35]], [[TMP36]]
-// CHECK1-NEXT:    br i1 [[CMP43]], label [[LAND_LHS_TRUE:%.*]], label [[OMP_PRECOND_END:%.*]]
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_K]], align 4
+// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP27]], [[TMP28]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[LAND_LHS_TRUE:%.*]], label [[OMP_PRECOND_END:%.*]]
 // CHECK1:       land.lhs.true:
-// CHECK1-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[CMP44:%.*]] = icmp slt i32 [[TMP37]], [[TMP38]]
-// CHECK1-NEXT:    br i1 [[CMP44]], label [[LAND_LHS_TRUE45:%.*]], label [[OMP_PRECOND_END]]
-// CHECK1:       land.lhs.true45:
-// CHECK1-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[CMP46:%.*]] = icmp ult i32 0, [[TMP39]]
-// CHECK1-NEXT:    br i1 [[CMP46]], label [[LAND_LHS_TRUE47:%.*]], label [[OMP_PRECOND_END]]
-// CHECK1:       land.lhs.true47:
-// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[CMP48:%.*]] = icmp ult i32 [[TMP40]], [[TMP41]]
-// CHECK1-NEXT:    br i1 [[CMP48]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
+// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[CMP32:%.*]] = icmp slt i32 [[TMP29]], [[TMP30]]
+// CHECK1-NEXT:    br i1 [[CMP32]], label [[LAND_LHS_TRUE33:%.*]], label [[OMP_PRECOND_END]]
+// CHECK1:       land.lhs.true33:
+// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[CMP34:%.*]] = icmp ult i32 0, [[TMP31]]
+// CHECK1-NEXT:    br i1 [[CMP34]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
 // CHECK1:       omp.precond.then:
 // CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
-// CHECK1-NEXT:    [[TMP42:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK1-NEXT:    store i64 [[TMP42]], ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK1-NEXT:    store i64 [[TMP32]], ptr [[DOTOMP_UB]], align 8
 // CHECK1-NEXT:    store i64 1, ptr [[DOTOMP_STRIDE]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
 // CHECK1-NEXT:    call void @__kmpc_for_static_init_8(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i64 1, i64 1)
-// CHECK1-NEXT:    [[TMP43:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[TMP44:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK1-NEXT:    [[CMP53:%.*]] = icmp sgt i64 [[TMP43]], [[TMP44]]
-// CHECK1-NEXT:    br i1 [[CMP53]], label [[COND_TRUE54:%.*]], label [[COND_FALSE55:%.*]]
-// CHECK1:       cond.true54:
-// CHECK1-NEXT:    [[TMP45:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK1-NEXT:    br label [[COND_END56:%.*]]
-// CHECK1:       cond.false55:
-// CHECK1-NEXT:    [[TMP46:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    br label [[COND_END56]]
-// CHECK1:       cond.end56:
-// CHECK1-NEXT:    [[COND57:%.*]] = phi i64 [ [[TMP45]], [[COND_TRUE54]] ], [ [[TMP46]], [[COND_FALSE55]] ]
-// CHECK1-NEXT:    store i64 [[COND57]], ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[TMP47:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
-// CHECK1-NEXT:    store i64 [[TMP47]], ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK1-NEXT:    [[CMP39:%.*]] = icmp sgt i64 [[TMP33]], [[TMP34]]
+// CHECK1-NEXT:    br i1 [[CMP39]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i64 [ [[TMP35]], [[COND_TRUE]] ], [ [[TMP36]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    store i64 [[COND]], ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
+// CHECK1-NEXT:    store i64 [[TMP37]], ptr [[DOTOMP_IV]], align 8
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK1:       omp.inner.for.cond:
-// CHECK1-NEXT:    [[TMP48:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[CMP58:%.*]] = icmp sle i64 [[TMP48]], [[TMP49]]
-// CHECK1-NEXT:    br i1 [[CMP58]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK1-NEXT:    [[TMP38:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP39:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[CMP40:%.*]] = icmp sle i64 [[TMP38]], [[TMP39]]
+// CHECK1-NEXT:    br i1 [[CMP40]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
 // CHECK1:       omp.inner.for.body:
-// CHECK1-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[CONV59:%.*]] = sext i32 [[TMP50]] to i64
-// CHECK1-NEXT:    [[TMP51:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB60:%.*]] = sub i32 [[TMP52]], [[TMP53]]
-// CHECK1-NEXT:    [[SUB61:%.*]] = sub i32 [[SUB60]], 1
+// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[CONV41:%.*]] = sext i32 [[TMP40]] to i64
+// CHECK1-NEXT:    [[TMP41:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB42:%.*]] = sub i32 [[TMP42]], [[TMP43]]
+// CHECK1-NEXT:    [[SUB43:%.*]] = sub i32 [[SUB42]], 1
+// CHECK1-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD44:%.*]] = add i32 [[SUB43]], [[TMP44]]
+// CHECK1-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV45:%.*]] = udiv i32 [[ADD44]], [[TMP45]]
+// CHECK1-NEXT:    [[MUL46:%.*]] = mul i32 1, [[DIV45]]
+// CHECK1-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB47:%.*]] = sub i32 [[TMP46]], -63
+// CHECK1-NEXT:    [[DIV48:%.*]] = udiv i32 [[SUB47]], 64
+// CHECK1-NEXT:    [[MUL49:%.*]] = mul i32 [[MUL46]], [[DIV48]]
+// CHECK1-NEXT:    [[MUL50:%.*]] = mul i32 [[MUL49]], 64
+// CHECK1-NEXT:    [[CONV51:%.*]] = zext i32 [[MUL50]] to i64
+// CHECK1-NEXT:    [[DIV52:%.*]] = sdiv i64 [[TMP41]], [[CONV51]]
+// CHECK1-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[CONV53:%.*]] = sext i32 [[TMP47]] to i64
+// CHECK1-NEXT:    [[MUL54:%.*]] = mul nsw i64 [[DIV52]], [[CONV53]]
+// CHECK1-NEXT:    [[ADD55:%.*]] = add nsw i64 [[CONV41]], [[MUL54]]
+// CHECK1-NEXT:    [[CONV56:%.*]] = trunc i64 [[ADD55]] to i32
+// CHECK1-NEXT:    store i32 [[CONV56]], ptr [[I35]], align 4
+// CHECK1-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP48]] to i64
+// CHECK1-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP50:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB58:%.*]] = sub i32 [[TMP51]], [[TMP52]]
+// CHECK1-NEXT:    [[SUB59:%.*]] = sub i32 [[SUB58]], 1
+// CHECK1-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD60:%.*]] = add i32 [[SUB59]], [[TMP53]]
 // CHECK1-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD62:%.*]] = add i32 [[SUB61]], [[TMP54]]
-// CHECK1-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV63:%.*]] = udiv i32 [[ADD62]], [[TMP55]]
-// CHECK1-NEXT:    [[MUL64:%.*]] = mul i32 1, [[DIV63]]
-// CHECK1-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB65:%.*]] = sub i32 [[TMP56]], -63
-// CHECK1-NEXT:    [[DIV66:%.*]] = udiv i32 [[SUB65]], 64
-// CHECK1-NEXT:    [[MUL67:%.*]] = mul i32 [[MUL64]], [[DIV66]]
-// CHECK1-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB68:%.*]] = sub i32 [[TMP57]], [[TMP58]]
-// CHECK1-NEXT:    [[SUB69:%.*]] = sub i32 [[SUB68]], 1
-// CHECK1-NEXT:    [[ADD70:%.*]] = add i32 [[SUB69]], 1
-// CHECK1-NEXT:    [[DIV71:%.*]] = udiv i32 [[ADD70]], 1
-// CHECK1-NEXT:    [[MUL72:%.*]] = mul i32 [[MUL67]], [[DIV71]]
-// CHECK1-NEXT:    [[CONV73:%.*]] = zext i32 [[MUL72]] to i64
-// CHECK1-NEXT:    [[DIV74:%.*]] = sdiv i64 [[TMP51]], [[CONV73]]
-// CHECK1-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[CONV75:%.*]] = sext i32 [[TMP59]] to i64
-// CHECK1-NEXT:    [[MUL76:%.*]] = mul nsw i64 [[DIV74]], [[CONV75]]
-// CHECK1-NEXT:    [[ADD77:%.*]] = add nsw i64 [[CONV59]], [[MUL76]]
-// CHECK1-NEXT:    [[CONV78:%.*]] = trunc i64 [[ADD77]] to i32
-// CHECK1-NEXT:    store i32 [[CONV78]], ptr [[I49]], align 4
-// CHECK1-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[CONV79:%.*]] = sext i32 [[TMP60]] to i64
-// CHECK1-NEXT:    [[TMP61:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP62:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP63:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP64:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB80:%.*]] = sub i32 [[TMP63]], [[TMP64]]
-// CHECK1-NEXT:    [[SUB81:%.*]] = sub i32 [[SUB80]], 1
-// CHECK1-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD82:%.*]] = add i32 [[SUB81]], [[TMP65]]
-// CHECK1-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV83:%.*]] = udiv i32 [[ADD82]], [[TMP66]]
-// CHECK1-NEXT:    [[MUL84:%.*]] = mul i32 1, [[DIV83]]
-// CHECK1-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB85:%.*]] = sub i32 [[TMP67]], -63
-// CHECK1-NEXT:    [[DIV86:%.*]] = udiv i32 [[SUB85]], 64
-// CHECK1-NEXT:    [[MUL87:%.*]] = mul i32 [[MUL84]], [[DIV86]]
-// CHECK1-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB88:%.*]] = sub i32 [[TMP68]], [[TMP69]]
-// CHECK1-NEXT:    [[SUB89:%.*]] = sub i32 [[SUB88]], 1
-// CHECK1-NEXT:    [[ADD90:%.*]] = add i32 [[SUB89]], 1
-// CHECK1-NEXT:    [[DIV91:%.*]] = udiv i32 [[ADD90]], 1
-// CHECK1-NEXT:    [[MUL92:%.*]] = mul i32 [[MUL87]], [[DIV91]]
-// CHECK1-NEXT:    [[CONV93:%.*]] = zext i32 [[MUL92]] to i64
-// CHECK1-NEXT:    [[DIV94:%.*]] = sdiv i64 [[TMP62]], [[CONV93]]
+// CHECK1-NEXT:    [[DIV61:%.*]] = udiv i32 [[ADD60]], [[TMP54]]
+// CHECK1-NEXT:    [[MUL62:%.*]] = mul i32 1, [[DIV61]]
+// CHECK1-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB63:%.*]] = sub i32 [[TMP55]], -63
+// CHECK1-NEXT:    [[DIV64:%.*]] = udiv i32 [[SUB63]], 64
+// CHECK1-NEXT:    [[MUL65:%.*]] = mul i32 [[MUL62]], [[DIV64]]
+// CHECK1-NEXT:    [[MUL66:%.*]] = mul i32 [[MUL65]], 64
+// CHECK1-NEXT:    [[CONV67:%.*]] = zext i32 [[MUL66]] to i64
+// CHECK1-NEXT:    [[DIV68:%.*]] = sdiv i64 [[TMP50]], [[CONV67]]
+// CHECK1-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB69:%.*]] = sub i32 [[TMP56]], [[TMP57]]
+// CHECK1-NEXT:    [[SUB70:%.*]] = sub i32 [[SUB69]], 1
+// CHECK1-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD71:%.*]] = add i32 [[SUB70]], [[TMP58]]
+// CHECK1-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV72:%.*]] = udiv i32 [[ADD71]], [[TMP59]]
+// CHECK1-NEXT:    [[MUL73:%.*]] = mul i32 1, [[DIV72]]
+// CHECK1-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB74:%.*]] = sub i32 [[TMP60]], -63
+// CHECK1-NEXT:    [[DIV75:%.*]] = udiv i32 [[SUB74]], 64
+// CHECK1-NEXT:    [[MUL76:%.*]] = mul i32 [[MUL73]], [[DIV75]]
+// CHECK1-NEXT:    [[MUL77:%.*]] = mul i32 [[MUL76]], 64
+// CHECK1-NEXT:    [[CONV78:%.*]] = zext i32 [[MUL77]] to i64
+// CHECK1-NEXT:    [[MUL79:%.*]] = mul nsw i64 [[DIV68]], [[CONV78]]
+// CHECK1-NEXT:    [[SUB80:%.*]] = sub nsw i64 [[TMP49]], [[MUL79]]
+// CHECK1-NEXT:    [[TMP61:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB81:%.*]] = sub i32 [[TMP61]], -63
+// CHECK1-NEXT:    [[DIV82:%.*]] = udiv i32 [[SUB81]], 64
+// CHECK1-NEXT:    [[MUL83:%.*]] = mul i32 1, [[DIV82]]
+// CHECK1-NEXT:    [[MUL84:%.*]] = mul i32 [[MUL83]], 64
+// CHECK1-NEXT:    [[CONV85:%.*]] = zext i32 [[MUL84]] to i64
+// CHECK1-NEXT:    [[DIV86:%.*]] = sdiv i64 [[SUB80]], [[CONV85]]
+// CHECK1-NEXT:    [[TMP62:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[CONV87:%.*]] = sext i32 [[TMP62]] to i64
+// CHECK1-NEXT:    [[MUL88:%.*]] = mul nsw i64 [[DIV86]], [[CONV87]]
+// CHECK1-NEXT:    [[ADD89:%.*]] = add nsw i64 [[CONV57]], [[MUL88]]
+// CHECK1-NEXT:    [[CONV90:%.*]] = trunc i64 [[ADD89]] to i32
+// CHECK1-NEXT:    store i32 [[CONV90]], ptr [[J36]], align 4
+// CHECK1-NEXT:    [[TMP63:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP64:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB91:%.*]] = sub i32 [[TMP65]], [[TMP66]]
+// CHECK1-NEXT:    [[SUB92:%.*]] = sub i32 [[SUB91]], 1
+// CHECK1-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD93:%.*]] = add i32 [[SUB92]], [[TMP67]]
+// CHECK1-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV94:%.*]] = udiv i32 [[ADD93]], [[TMP68]]
+// CHECK1-NEXT:    [[MUL95:%.*]] = mul i32 1, [[DIV94]]
+// CHECK1-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB96:%.*]] = sub i32 [[TMP69]], -63
+// CHECK1-NEXT:    [[DIV97:%.*]] = udiv i32 [[SUB96]], 64
+// CHECK1-NEXT:    [[MUL98:%.*]] = mul i32 [[MUL95]], [[DIV97]]
+// CHECK1-NEXT:    [[MUL99:%.*]] = mul i32 [[MUL98]], 64
+// CHECK1-NEXT:    [[CONV100:%.*]] = zext i32 [[MUL99]] to i64
+// CHECK1-NEXT:    [[DIV101:%.*]] = sdiv i64 [[TMP64]], [[CONV100]]
 // CHECK1-NEXT:    [[TMP70:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
 // CHECK1-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB95:%.*]] = sub i32 [[TMP70]], [[TMP71]]
-// CHECK1-NEXT:    [[SUB96:%.*]] = sub i32 [[SUB95]], 1
+// CHECK1-NEXT:    [[SUB102:%.*]] = sub i32 [[TMP70]], [[TMP71]]
+// CHECK1-NEXT:    [[SUB103:%.*]] = sub i32 [[SUB102]], 1
 // CHECK1-NEXT:    [[TMP72:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD97:%.*]] = add i32 [[SUB96]], [[TMP72]]
+// CHECK1-NEXT:    [[ADD104:%.*]] = add i32 [[SUB103]], [[TMP72]]
 // CHECK1-NEXT:    [[TMP73:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV98:%.*]] = udiv i32 [[ADD97]], [[TMP73]]
-// CHECK1-NEXT:    [[MUL99:%.*]] = mul i32 1, [[DIV98]]
+// CHECK1-NEXT:    [[DIV105:%.*]] = udiv i32 [[ADD104]], [[TMP73]]
+// CHECK1-NEXT:    [[MUL106:%.*]] = mul i32 1, [[DIV105]]
 // CHECK1-NEXT:    [[TMP74:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB100:%.*]] = sub i32 [[TMP74]], -63
-// CHECK1-NEXT:    [[DIV101:%.*]] = udiv i32 [[SUB100]], 64
-// CHECK1-NEXT:    [[MUL102:%.*]] = mul i32 [[MUL99]], [[DIV101]]
-// CHECK1-NEXT:    [[TMP75:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP76:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB103:%.*]] = sub i32 [[TMP75]], [[TMP76]]
-// CHECK1-NEXT:    [[SUB104:%.*]] = sub i32 [[SUB103]], 1
-// CHECK1-NEXT:    [[ADD105:%.*]] = add i32 [[SUB104]], 1
-// CHECK1-NEXT:    [[DIV106:%.*]] = udiv i32 [[ADD105]], 1
-// CHECK1-NEXT:    [[MUL107:%.*]] = mul i32 [[MUL102]], [[DIV106]]
-// CHECK1-NEXT:    [[CONV108:%.*]] = zext i32 [[MUL107]] to i64
-// CHECK1-NEXT:    [[MUL109:%.*]] = mul nsw i64 [[DIV94]], [[CONV108]]
-// CHECK1-NEXT:    [[SUB110:%.*]] = sub nsw i64 [[TMP61]], [[MUL109]]
-// CHECK1-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB111:%.*]] = sub i32 [[TMP77]], -63
-// CHECK1-NEXT:    [[DIV112:%.*]] = udiv i32 [[SUB111]], 64
-// CHECK1-NEXT:    [[MUL113:%.*]] = mul i32 1, [[DIV112]]
-// CHECK1-NEXT:    [[TMP78:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP79:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB114:%.*]] = sub i32 [[TMP78]], [[TMP79]]
+// CHECK1-NEXT:    [[SUB107:%.*]] = sub i32 [[TMP74]], -63
+// CHECK1-NEXT:    [[DIV108:%.*]] = udiv i32 [[SUB107]], 64
+// CHECK1-NEXT:    [[MUL109:%.*]] = mul i32 [[MUL106]], [[DIV108]]
+// CHECK1-NEXT:    [[MUL110:%.*]] = mul i32 [[MUL109]], 64
+// CHECK1-NEXT:    [[CONV111:%.*]] = zext i32 [[MUL110]] to i64
+// CHECK1-NEXT:    [[MUL112:%.*]] = mul nsw i64 [[DIV101]], [[CONV111]]
+// CHECK1-NEXT:    [[SUB113:%.*]] = sub nsw i64 [[TMP63]], [[MUL112]]
+// CHECK1-NEXT:    [[TMP75:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP76:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP78:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB114:%.*]] = sub i32 [[TMP77]], [[TMP78]]
 // CHECK1-NEXT:    [[SUB115:%.*]] = sub i32 [[SUB114]], 1
-// CHECK1-NEXT:    [[ADD116:%.*]] = add i32 [[SUB115]], 1
-// CHECK1-NEXT:    [[DIV117:%.*]] = udiv i32 [[ADD116]], 1
-// CHECK1-NEXT:    [[MUL118:%.*]] = mul i32 [[MUL113]], [[DIV117]]
-// CHECK1-NEXT:    [[CONV119:%.*]] = zext i32 [[MUL118]] to i64
-// CHECK1-NEXT:    [[DIV120:%.*]] = sdiv i64 [[SUB110]], [[CONV119]]
+// CHECK1-NEXT:    [[TMP79:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD116:%.*]] = add i32 [[SUB115]], [[TMP79]]
 // CHECK1-NEXT:    [[TMP80:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[CONV121:%.*]] = sext i32 [[TMP80]] to i64
-// CHECK1-NEXT:    [[MUL122:%.*]] = mul nsw i64 [[DIV120]], [[CONV121]]
-// CHECK1-NEXT:    [[ADD123:%.*]] = add nsw i64 [[CONV79]], [[MUL122]]
-// CHECK1-NEXT:    [[CONV124:%.*]] = trunc i64 [[ADD123]] to i32
-// CHECK1-NEXT:    store i32 [[CONV124]], ptr [[J50]], align 4
-// CHECK1-NEXT:    [[TMP81:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP82:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB125:%.*]] = sub i32 [[TMP83]], [[TMP84]]
+// CHECK1-NEXT:    [[DIV117:%.*]] = udiv i32 [[ADD116]], [[TMP80]]
+// CHECK1-NEXT:    [[MUL118:%.*]] = mul i32 1, [[DIV117]]
+// CHECK1-NEXT:    [[TMP81:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB119:%.*]] = sub i32 [[TMP81]], -63
+// CHECK1-NEXT:    [[DIV120:%.*]] = udiv i32 [[SUB119]], 64
+// CHECK1-NEXT:    [[MUL121:%.*]] = mul i32 [[MUL118]], [[DIV120]]
+// CHECK1-NEXT:    [[MUL122:%.*]] = mul i32 [[MUL121]], 64
+// CHECK1-NEXT:    [[CONV123:%.*]] = zext i32 [[MUL122]] to i64
+// CHECK1-NEXT:    [[DIV124:%.*]] = sdiv i64 [[TMP76]], [[CONV123]]
+// CHECK1-NEXT:    [[TMP82:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB125:%.*]] = sub i32 [[TMP82]], [[TMP83]]
 // CHECK1-NEXT:    [[SUB126:%.*]] = sub i32 [[SUB125]], 1
+// CHECK1-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD127:%.*]] = add i32 [[SUB126]], [[TMP84]]
 // CHECK1-NEXT:    [[TMP85:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD127:%.*]] = add i32 [[SUB126]], [[TMP85]]
-// CHECK1-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV128:%.*]] = udiv i32 [[ADD127]], [[TMP86]]
+// CHECK1-NEXT:    [[DIV128:%.*]] = udiv i32 [[ADD127]], [[TMP85]]
 // CHECK1-NEXT:    [[MUL129:%.*]] = mul i32 1, [[DIV128]]
-// CHECK1-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB130:%.*]] = sub i32 [[TMP87]], -63
+// CHECK1-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB130:%.*]] = sub i32 [[TMP86]], -63
 // CHECK1-NEXT:    [[DIV131:%.*]] = udiv i32 [[SUB130]], 64
 // CHECK1-NEXT:    [[MUL132:%.*]] = mul i32 [[MUL129]], [[DIV131]]
-// CHECK1-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP89:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB133:%.*]] = sub i32 [[TMP88]], [[TMP89]]
-// CHECK1-NEXT:    [[SUB134:%.*]] = sub i32 [[SUB133]], 1
-// CHECK1-NEXT:    [[ADD135:%.*]] = add i32 [[SUB134]], 1
-// CHECK1-NEXT:    [[DIV136:%.*]] = udiv i32 [[ADD135]], 1
-// CHECK1-NEXT:    [[MUL137:%.*]] = mul i32 [[MUL132]], [[DIV136]]
-// CHECK1-NEXT:    [[CONV138:%.*]] = zext i32 [[MUL137]] to i64
-// CHECK1-NEXT:    [[DIV139:%.*]] = sdiv i64 [[TMP82]], [[CONV138]]
-// CHECK1-NEXT:    [[TMP90:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB140:%.*]] = sub i32 [[TMP90]], [[TMP91]]
-// CHECK1-NEXT:    [[SUB141:%.*]] = sub i32 [[SUB140]], 1
-// CHECK1-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD142:%.*]] = add i32 [[SUB141]], [[TMP92]]
+// CHECK1-NEXT:    [[MUL133:%.*]] = mul i32 [[MUL132]], 64
+// CHECK1-NEXT:    [[CONV134:%.*]] = zext i32 [[MUL133]] to i64
+// CHECK1-NEXT:    [[MUL135:%.*]] = mul nsw i64 [[DIV124]], [[CONV134]]
+// CHECK1-NEXT:    [[SUB136:%.*]] = sub nsw i64 [[TMP75]], [[MUL135]]
+// CHECK1-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB137:%.*]] = sub i32 [[TMP87]], -63
+// CHECK1-NEXT:    [[DIV138:%.*]] = udiv i32 [[SUB137]], 64
+// CHECK1-NEXT:    [[MUL139:%.*]] = mul i32 1, [[DIV138]]
+// CHECK1-NEXT:    [[MUL140:%.*]] = mul i32 [[MUL139]], 64
+// CHECK1-NEXT:    [[CONV141:%.*]] = zext i32 [[MUL140]] to i64
+// CHECK1-NEXT:    [[DIV142:%.*]] = sdiv i64 [[SUB136]], [[CONV141]]
+// CHECK1-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB143:%.*]] = sub i32 [[TMP88]], -63
+// CHECK1-NEXT:    [[DIV144:%.*]] = udiv i32 [[SUB143]], 64
+// CHECK1-NEXT:    [[MUL145:%.*]] = mul i32 1, [[DIV144]]
+// CHECK1-NEXT:    [[MUL146:%.*]] = mul i32 [[MUL145]], 64
+// CHECK1-NEXT:    [[CONV147:%.*]] = zext i32 [[MUL146]] to i64
+// CHECK1-NEXT:    [[MUL148:%.*]] = mul nsw i64 [[DIV142]], [[CONV147]]
+// CHECK1-NEXT:    [[SUB149:%.*]] = sub nsw i64 [[SUB113]], [[MUL148]]
+// CHECK1-NEXT:    [[DIV150:%.*]] = sdiv i64 [[SUB149]], 64
+// CHECK1-NEXT:    [[MUL151:%.*]] = mul nsw i64 [[DIV150]], 64
+// CHECK1-NEXT:    [[ADD152:%.*]] = add nsw i64 0, [[MUL151]]
+// CHECK1-NEXT:    [[CONV153:%.*]] = trunc i64 [[ADD152]] to i32
+// CHECK1-NEXT:    store i32 [[CONV153]], ptr [[DOTFLOOR_0_IV_K37]], align 4
+// CHECK1-NEXT:    [[TMP89:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP90:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB154:%.*]] = sub i32 [[TMP91]], [[TMP92]]
+// CHECK1-NEXT:    [[SUB155:%.*]] = sub i32 [[SUB154]], 1
 // CHECK1-NEXT:    [[TMP93:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV143:%.*]] = udiv i32 [[ADD142]], [[TMP93]]
-// CHECK1-NEXT:    [[MUL144:%.*]] = mul i32 1, [[DIV143]]
-// CHECK1-NEXT:    [[TMP94:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB145:%.*]] = sub i32 [[TMP94]], -63
-// CHECK1-NEXT:    [[DIV146:%.*]] = udiv i32 [[SUB145]], 64
-// CHECK1-NEXT:    [[MUL147:%.*]] = mul i32 [[MUL144]], [[DIV146]]
-// CHECK1-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB148:%.*]] = sub i32 [[TMP95]], [[TMP96]]
-// CHECK1-NEXT:    [[SUB149:%.*]] = sub i32 [[SUB148]], 1
-// CHECK1-NEXT:    [[ADD150:%.*]] = add i32 [[SUB149]], 1
-// CHECK1-NEXT:    [[DIV151:%.*]] = udiv i32 [[ADD150]], 1
-// CHECK1-NEXT:    [[MUL152:%.*]] = mul i32 [[MUL147]], [[DIV151]]
-// CHECK1-NEXT:    [[CONV153:%.*]] = zext i32 [[MUL152]] to i64
-// CHECK1-NEXT:    [[MUL154:%.*]] = mul nsw i64 [[DIV139]], [[CONV153]]
-// CHECK1-NEXT:    [[SUB155:%.*]] = sub nsw i64 [[TMP81]], [[MUL154]]
-// CHECK1-NEXT:    [[TMP97:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP98:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB156:%.*]] = sub i32 [[TMP99]], [[TMP100]]
-// CHECK1-NEXT:    [[SUB157:%.*]] = sub i32 [[SUB156]], 1
-// CHECK1-NEXT:    [[TMP101:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD158:%.*]] = add i32 [[SUB157]], [[TMP101]]
-// CHECK1-NEXT:    [[TMP102:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV159:%.*]] = udiv i32 [[ADD158]], [[TMP102]]
-// CHECK1-NEXT:    [[MUL160:%.*]] = mul i32 1, [[DIV159]]
-// CHECK1-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB161:%.*]] = sub i32 [[TMP103]], -63
-// CHECK1-NEXT:    [[DIV162:%.*]] = udiv i32 [[SUB161]], 64
-// CHECK1-NEXT:    [[MUL163:%.*]] = mul i32 [[MUL160]], [[DIV162]]
-// CHECK1-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP105:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB164:%.*]] = sub i32 [[TMP104]], [[TMP105]]
-// CHECK1-NEXT:    [[SUB165:%.*]] = sub i32 [[SUB164]], 1
-// CHECK1-NEXT:    [[ADD166:%.*]] = add i32 [[SUB165]], 1
-// CHECK1-NEXT:    [[DIV167:%.*]] = udiv i32 [[ADD166]], 1
-// CHECK1-NEXT:    [[MUL168:%.*]] = mul i32 [[MUL163]], [[DIV167]]
-// CHECK1-NEXT:    [[CONV169:%.*]] = zext i32 [[MUL168]] to i64
-// CHECK1-NEXT:    [[DIV170:%.*]] = sdiv i64 [[TMP98]], [[CONV169]]
-// CHECK1-NEXT:    [[TMP106:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB171:%.*]] = sub i32 [[TMP106]], [[TMP107]]
-// CHECK1-NEXT:    [[SUB172:%.*]] = sub i32 [[SUB171]], 1
-// CHECK1-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD173:%.*]] = add i32 [[SUB172]], [[TMP108]]
-// CHECK1-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV174:%.*]] = udiv i32 [[ADD173]], [[TMP109]]
-// CHECK1-NEXT:    [[MUL175:%.*]] = mul i32 1, [[DIV174]]
-// CHECK1-NEXT:    [[TMP110:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB176:%.*]] = sub i32 [[TMP110]], -63
-// CHECK1-NEXT:    [[DIV177:%.*]] = udiv i32 [[SUB176]], 64
-// CHECK1-NEXT:    [[MUL178:%.*]] = mul i32 [[MUL175]], [[DIV177]]
-// CHECK1-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB179:%.*]] = sub i32 [[TMP111]], [[TMP112]]
-// CHECK1-NEXT:    [[SUB180:%.*]] = sub i32 [[SUB179]], 1
-// CHECK1-NEXT:    [[ADD181:%.*]] = add i32 [[SUB180]], 1
-// CHECK1-NEXT:    [[DIV182:%.*]] = udiv i32 [[ADD181]], 1
-// CHECK1-NEXT:    [[MUL183:%.*]] = mul i32 [[MUL178]], [[DIV182]]
-// CHECK1-NEXT:    [[CONV184:%.*]] = zext i32 [[MUL183]] to i64
-// CHECK1-NEXT:    [[MUL185:%.*]] = mul nsw i64 [[DIV170]], [[CONV184]]
-// CHECK1-NEXT:    [[SUB186:%.*]] = sub nsw i64 [[TMP97]], [[MUL185]]
+// CHECK1-NEXT:    [[ADD156:%.*]] = add i32 [[SUB155]], [[TMP93]]
+// CHECK1-NEXT:    [[TMP94:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV157:%.*]] = udiv i32 [[ADD156]], [[TMP94]]
+// CHECK1-NEXT:    [[MUL158:%.*]] = mul i32 1, [[DIV157]]
+// CHECK1-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB159:%.*]] = sub i32 [[TMP95]], -63
+// CHECK1-NEXT:    [[DIV160:%.*]] = udiv i32 [[SUB159]], 64
+// CHECK1-NEXT:    [[MUL161:%.*]] = mul i32 [[MUL158]], [[DIV160]]
+// CHECK1-NEXT:    [[MUL162:%.*]] = mul i32 [[MUL161]], 64
+// CHECK1-NEXT:    [[CONV163:%.*]] = zext i32 [[MUL162]] to i64
+// CHECK1-NEXT:    [[DIV164:%.*]] = sdiv i64 [[TMP90]], [[CONV163]]
+// CHECK1-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP97:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB165:%.*]] = sub i32 [[TMP96]], [[TMP97]]
+// CHECK1-NEXT:    [[SUB166:%.*]] = sub i32 [[SUB165]], 1
+// CHECK1-NEXT:    [[TMP98:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD167:%.*]] = add i32 [[SUB166]], [[TMP98]]
+// CHECK1-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV168:%.*]] = udiv i32 [[ADD167]], [[TMP99]]
+// CHECK1-NEXT:    [[MUL169:%.*]] = mul i32 1, [[DIV168]]
+// CHECK1-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB170:%.*]] = sub i32 [[TMP100]], -63
+// CHECK1-NEXT:    [[DIV171:%.*]] = udiv i32 [[SUB170]], 64
+// CHECK1-NEXT:    [[MUL172:%.*]] = mul i32 [[MUL169]], [[DIV171]]
+// CHECK1-NEXT:    [[MUL173:%.*]] = mul i32 [[MUL172]], 64
+// CHECK1-NEXT:    [[CONV174:%.*]] = zext i32 [[MUL173]] to i64
+// CHECK1-NEXT:    [[MUL175:%.*]] = mul nsw i64 [[DIV164]], [[CONV174]]
+// CHECK1-NEXT:    [[SUB176:%.*]] = sub nsw i64 [[TMP89]], [[MUL175]]
+// CHECK1-NEXT:    [[TMP101:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP102:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB177:%.*]] = sub i32 [[TMP103]], [[TMP104]]
+// CHECK1-NEXT:    [[SUB178:%.*]] = sub i32 [[SUB177]], 1
+// CHECK1-NEXT:    [[TMP105:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD179:%.*]] = add i32 [[SUB178]], [[TMP105]]
+// CHECK1-NEXT:    [[TMP106:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV180:%.*]] = udiv i32 [[ADD179]], [[TMP106]]
+// CHECK1-NEXT:    [[MUL181:%.*]] = mul i32 1, [[DIV180]]
+// CHECK1-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB182:%.*]] = sub i32 [[TMP107]], -63
+// CHECK1-NEXT:    [[DIV183:%.*]] = udiv i32 [[SUB182]], 64
+// CHECK1-NEXT:    [[MUL184:%.*]] = mul i32 [[MUL181]], [[DIV183]]
+// CHECK1-NEXT:    [[MUL185:%.*]] = mul i32 [[MUL184]], 64
+// CHECK1-NEXT:    [[CONV186:%.*]] = zext i32 [[MUL185]] to i64
+// CHECK1-NEXT:    [[DIV187:%.*]] = sdiv i64 [[TMP102]], [[CONV186]]
+// CHECK1-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB188:%.*]] = sub i32 [[TMP108]], [[TMP109]]
+// CHECK1-NEXT:    [[SUB189:%.*]] = sub i32 [[SUB188]], 1
+// CHECK1-NEXT:    [[TMP110:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD190:%.*]] = add i32 [[SUB189]], [[TMP110]]
+// CHECK1-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV191:%.*]] = udiv i32 [[ADD190]], [[TMP111]]
+// CHECK1-NEXT:    [[MUL192:%.*]] = mul i32 1, [[DIV191]]
+// CHECK1-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB193:%.*]] = sub i32 [[TMP112]], -63
+// CHECK1-NEXT:    [[DIV194:%.*]] = udiv i32 [[SUB193]], 64
+// CHECK1-NEXT:    [[MUL195:%.*]] = mul i32 [[MUL192]], [[DIV194]]
+// CHECK1-NEXT:    [[MUL196:%.*]] = mul i32 [[MUL195]], 64
+// CHECK1-NEXT:    [[CONV197:%.*]] = zext i32 [[MUL196]] to i64
+// CHECK1-NEXT:    [[MUL198:%.*]] = mul nsw i64 [[DIV187]], [[CONV197]]
+// CHECK1-NEXT:    [[SUB199:%.*]] = sub nsw i64 [[TMP101]], [[MUL198]]
 // CHECK1-NEXT:    [[TMP113:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB187:%.*]] = sub i32 [[TMP113]], -63
-// CHECK1-NEXT:    [[DIV188:%.*]] = udiv i32 [[SUB187]], 64
-// CHECK1-NEXT:    [[MUL189:%.*]] = mul i32 1, [[DIV188]]
-// CHECK1-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP115:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB190:%.*]] = sub i32 [[TMP114]], [[TMP115]]
-// CHECK1-NEXT:    [[SUB191:%.*]] = sub i32 [[SUB190]], 1
-// CHECK1-NEXT:    [[ADD192:%.*]] = add i32 [[SUB191]], 1
-// CHECK1-NEXT:    [[DIV193:%.*]] = udiv i32 [[ADD192]], 1
-// CHECK1-NEXT:    [[MUL194:%.*]] = mul i32 [[MUL189]], [[DIV193]]
-// CHECK1-NEXT:    [[CONV195:%.*]] = zext i32 [[MUL194]] to i64
-// CHECK1-NEXT:    [[DIV196:%.*]] = sdiv i64 [[SUB186]], [[CONV195]]
-// CHECK1-NEXT:    [[TMP116:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB197:%.*]] = sub i32 [[TMP116]], -63
-// CHECK1-NEXT:    [[DIV198:%.*]] = udiv i32 [[SUB197]], 64
-// CHECK1-NEXT:    [[MUL199:%.*]] = mul i32 1, [[DIV198]]
-// CHECK1-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB200:%.*]] = sub i32 [[TMP117]], [[TMP118]]
-// CHECK1-NEXT:    [[SUB201:%.*]] = sub i32 [[SUB200]], 1
-// CHECK1-NEXT:    [[ADD202:%.*]] = add i32 [[SUB201]], 1
-// CHECK1-NEXT:    [[DIV203:%.*]] = udiv i32 [[ADD202]], 1
-// CHECK1-NEXT:    [[MUL204:%.*]] = mul i32 [[MUL199]], [[DIV203]]
-// CHECK1-NEXT:    [[CONV205:%.*]] = zext i32 [[MUL204]] to i64
-// CHECK1-NEXT:    [[MUL206:%.*]] = mul nsw i64 [[DIV196]], [[CONV205]]
-// CHECK1-NEXT:    [[SUB207:%.*]] = sub nsw i64 [[SUB155]], [[MUL206]]
-// CHECK1-NEXT:    [[TMP119:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP120:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB208:%.*]] = sub i32 [[TMP119]], [[TMP120]]
-// CHECK1-NEXT:    [[SUB209:%.*]] = sub i32 [[SUB208]], 1
-// CHECK1-NEXT:    [[ADD210:%.*]] = add i32 [[SUB209]], 1
-// CHECK1-NEXT:    [[DIV211:%.*]] = udiv i32 [[ADD210]], 1
-// CHECK1-NEXT:    [[MUL212:%.*]] = mul i32 1, [[DIV211]]
-// CHECK1-NEXT:    [[CONV213:%.*]] = zext i32 [[MUL212]] to i64
-// CHECK1-NEXT:    [[DIV214:%.*]] = sdiv i64 [[SUB207]], [[CONV213]]
-// CHECK1-NEXT:    [[MUL215:%.*]] = mul nsw i64 [[DIV214]], 64
-// CHECK1-NEXT:    [[ADD216:%.*]] = add nsw i64 0, [[MUL215]]
-// CHECK1-NEXT:    [[CONV217:%.*]] = trunc i64 [[ADD216]] to i32
-// CHECK1-NEXT:    store i32 [[CONV217]], ptr [[DOTFLOOR_0_IV_K51]], align 4
-// CHECK1-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[CONV218:%.*]] = zext i32 [[TMP121]] to i64
-// CHECK1-NEXT:    [[TMP122:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP123:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP124:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB219:%.*]] = sub i32 [[TMP124]], [[TMP125]]
-// CHECK1-NEXT:    [[SUB220:%.*]] = sub i32 [[SUB219]], 1
-// CHECK1-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD221:%.*]] = add i32 [[SUB220]], [[TMP126]]
-// CHECK1-NEXT:    [[TMP127:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV222:%.*]] = udiv i32 [[ADD221]], [[TMP127]]
-// CHECK1-NEXT:    [[MUL223:%.*]] = mul i32 1, [[DIV222]]
-// CHECK1-NEXT:    [[TMP128:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB224:%.*]] = sub i32 [[TMP128]], -63
-// CHECK1-NEXT:    [[DIV225:%.*]] = udiv i32 [[SUB224]], 64
-// CHECK1-NEXT:    [[MUL226:%.*]] = mul i32 [[MUL223]], [[DIV225]]
-// CHECK1-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB227:%.*]] = sub i32 [[TMP129]], [[TMP130]]
-// CHECK1-NEXT:    [[SUB228:%.*]] = sub i32 [[SUB227]], 1
-// CHECK1-NEXT:    [[ADD229:%.*]] = add i32 [[SUB228]], 1
-// CHECK1-NEXT:    [[DIV230:%.*]] = udiv i32 [[ADD229]], 1
-// CHECK1-NEXT:    [[MUL231:%.*]] = mul i32 [[MUL226]], [[DIV230]]
-// CHECK1-NEXT:    [[CONV232:%.*]] = zext i32 [[MUL231]] to i64
-// CHECK1-NEXT:    [[DIV233:%.*]] = sdiv i64 [[TMP123]], [[CONV232]]
-// CHECK1-NEXT:    [[TMP131:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP132:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB234:%.*]] = sub i32 [[TMP131]], [[TMP132]]
-// CHECK1-NEXT:    [[SUB235:%.*]] = sub i32 [[SUB234]], 1
-// CHECK1-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD236:%.*]] = add i32 [[SUB235]], [[TMP133]]
-// CHECK1-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV237:%.*]] = udiv i32 [[ADD236]], [[TMP134]]
-// CHECK1-NEXT:    [[MUL238:%.*]] = mul i32 1, [[DIV237]]
-// CHECK1-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB239:%.*]] = sub i32 [[TMP135]], -63
-// CHECK1-NEXT:    [[DIV240:%.*]] = udiv i32 [[SUB239]], 64
-// CHECK1-NEXT:    [[MUL241:%.*]] = mul i32 [[MUL238]], [[DIV240]]
-// CHECK1-NEXT:    [[TMP136:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB242:%.*]] = sub i32 [[TMP136]], [[TMP137]]
-// CHECK1-NEXT:    [[SUB243:%.*]] = sub i32 [[SUB242]], 1
-// CHECK1-NEXT:    [[ADD244:%.*]] = add i32 [[SUB243]], 1
-// CHECK1-NEXT:    [[DIV245:%.*]] = udiv i32 [[ADD244]], 1
-// CHECK1-NEXT:    [[MUL246:%.*]] = mul i32 [[MUL241]], [[DIV245]]
-// CHECK1-NEXT:    [[CONV247:%.*]] = zext i32 [[MUL246]] to i64
-// CHECK1-NEXT:    [[MUL248:%.*]] = mul nsw i64 [[DIV233]], [[CONV247]]
-// CHECK1-NEXT:    [[SUB249:%.*]] = sub nsw i64 [[TMP122]], [[MUL248]]
-// CHECK1-NEXT:    [[TMP138:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP139:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB250:%.*]] = sub i32 [[TMP140]], [[TMP141]]
-// CHECK1-NEXT:    [[SUB251:%.*]] = sub i32 [[SUB250]], 1
-// CHECK1-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD252:%.*]] = add i32 [[SUB251]], [[TMP142]]
-// CHECK1-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV253:%.*]] = udiv i32 [[ADD252]], [[TMP143]]
-// CHECK1-NEXT:    [[MUL254:%.*]] = mul i32 1, [[DIV253]]
-// CHECK1-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB255:%.*]] = sub i32 [[TMP144]], -63
-// CHECK1-NEXT:    [[DIV256:%.*]] = udiv i32 [[SUB255]], 64
-// CHECK1-NEXT:    [[MUL257:%.*]] = mul i32 [[MUL254]], [[DIV256]]
-// CHECK1-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB258:%.*]] = sub i32 [[TMP145]], [[TMP146]]
-// CHECK1-NEXT:    [[SUB259:%.*]] = sub i32 [[SUB258]], 1
-// CHECK1-NEXT:    [[ADD260:%.*]] = add i32 [[SUB259]], 1
-// CHECK1-NEXT:    [[DIV261:%.*]] = udiv i32 [[ADD260]], 1
-// CHECK1-NEXT:    [[MUL262:%.*]] = mul i32 [[MUL257]], [[DIV261]]
+// CHECK1-NEXT:    [[SUB200:%.*]] = sub i32 [[TMP113]], -63
+// CHECK1-NEXT:    [[DIV201:%.*]] = udiv i32 [[SUB200]], 64
+// CHECK1-NEXT:    [[MUL202:%.*]] = mul i32 1, [[DIV201]]
+// CHECK1-NEXT:    [[MUL203:%.*]] = mul i32 [[MUL202]], 64
+// CHECK1-NEXT:    [[CONV204:%.*]] = zext i32 [[MUL203]] to i64
+// CHECK1-NEXT:    [[DIV205:%.*]] = sdiv i64 [[SUB199]], [[CONV204]]
+// CHECK1-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB206:%.*]] = sub i32 [[TMP114]], -63
+// CHECK1-NEXT:    [[DIV207:%.*]] = udiv i32 [[SUB206]], 64
+// CHECK1-NEXT:    [[MUL208:%.*]] = mul i32 1, [[DIV207]]
+// CHECK1-NEXT:    [[MUL209:%.*]] = mul i32 [[MUL208]], 64
+// CHECK1-NEXT:    [[CONV210:%.*]] = zext i32 [[MUL209]] to i64
+// CHECK1-NEXT:    [[MUL211:%.*]] = mul nsw i64 [[DIV205]], [[CONV210]]
+// CHECK1-NEXT:    [[SUB212:%.*]] = sub nsw i64 [[SUB176]], [[MUL211]]
+// CHECK1-NEXT:    [[TMP115:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP116:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB213:%.*]] = sub i32 [[TMP117]], [[TMP118]]
+// CHECK1-NEXT:    [[SUB214:%.*]] = sub i32 [[SUB213]], 1
+// CHECK1-NEXT:    [[TMP119:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD215:%.*]] = add i32 [[SUB214]], [[TMP119]]
+// CHECK1-NEXT:    [[TMP120:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV216:%.*]] = udiv i32 [[ADD215]], [[TMP120]]
+// CHECK1-NEXT:    [[MUL217:%.*]] = mul i32 1, [[DIV216]]
+// CHECK1-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB218:%.*]] = sub i32 [[TMP121]], -63
+// CHECK1-NEXT:    [[DIV219:%.*]] = udiv i32 [[SUB218]], 64
+// CHECK1-NEXT:    [[MUL220:%.*]] = mul i32 [[MUL217]], [[DIV219]]
+// CHECK1-NEXT:    [[MUL221:%.*]] = mul i32 [[MUL220]], 64
+// CHECK1-NEXT:    [[CONV222:%.*]] = zext i32 [[MUL221]] to i64
+// CHECK1-NEXT:    [[DIV223:%.*]] = sdiv i64 [[TMP116]], [[CONV222]]
+// CHECK1-NEXT:    [[TMP122:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP123:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB224:%.*]] = sub i32 [[TMP122]], [[TMP123]]
+// CHECK1-NEXT:    [[SUB225:%.*]] = sub i32 [[SUB224]], 1
+// CHECK1-NEXT:    [[TMP124:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD226:%.*]] = add i32 [[SUB225]], [[TMP124]]
+// CHECK1-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV227:%.*]] = udiv i32 [[ADD226]], [[TMP125]]
+// CHECK1-NEXT:    [[MUL228:%.*]] = mul i32 1, [[DIV227]]
+// CHECK1-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB229:%.*]] = sub i32 [[TMP126]], -63
+// CHECK1-NEXT:    [[DIV230:%.*]] = udiv i32 [[SUB229]], 64
+// CHECK1-NEXT:    [[MUL231:%.*]] = mul i32 [[MUL228]], [[DIV230]]
+// CHECK1-NEXT:    [[MUL232:%.*]] = mul i32 [[MUL231]], 64
+// CHECK1-NEXT:    [[CONV233:%.*]] = zext i32 [[MUL232]] to i64
+// CHECK1-NEXT:    [[MUL234:%.*]] = mul nsw i64 [[DIV223]], [[CONV233]]
+// CHECK1-NEXT:    [[SUB235:%.*]] = sub nsw i64 [[TMP115]], [[MUL234]]
+// CHECK1-NEXT:    [[TMP127:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP128:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB236:%.*]] = sub i32 [[TMP129]], [[TMP130]]
+// CHECK1-NEXT:    [[SUB237:%.*]] = sub i32 [[SUB236]], 1
+// CHECK1-NEXT:    [[TMP131:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD238:%.*]] = add i32 [[SUB237]], [[TMP131]]
+// CHECK1-NEXT:    [[TMP132:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV239:%.*]] = udiv i32 [[ADD238]], [[TMP132]]
+// CHECK1-NEXT:    [[MUL240:%.*]] = mul i32 1, [[DIV239]]
+// CHECK1-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB241:%.*]] = sub i32 [[TMP133]], -63
+// CHECK1-NEXT:    [[DIV242:%.*]] = udiv i32 [[SUB241]], 64
+// CHECK1-NEXT:    [[MUL243:%.*]] = mul i32 [[MUL240]], [[DIV242]]
+// CHECK1-NEXT:    [[MUL244:%.*]] = mul i32 [[MUL243]], 64
+// CHECK1-NEXT:    [[CONV245:%.*]] = zext i32 [[MUL244]] to i64
+// CHECK1-NEXT:    [[DIV246:%.*]] = sdiv i64 [[TMP128]], [[CONV245]]
+// CHECK1-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB247:%.*]] = sub i32 [[TMP134]], [[TMP135]]
+// CHECK1-NEXT:    [[SUB248:%.*]] = sub i32 [[SUB247]], 1
+// CHECK1-NEXT:    [[TMP136:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD249:%.*]] = add i32 [[SUB248]], [[TMP136]]
+// CHECK1-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV250:%.*]] = udiv i32 [[ADD249]], [[TMP137]]
+// CHECK1-NEXT:    [[MUL251:%.*]] = mul i32 1, [[DIV250]]
+// CHECK1-NEXT:    [[TMP138:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB252:%.*]] = sub i32 [[TMP138]], -63
+// CHECK1-NEXT:    [[DIV253:%.*]] = udiv i32 [[SUB252]], 64
+// CHECK1-NEXT:    [[MUL254:%.*]] = mul i32 [[MUL251]], [[DIV253]]
+// CHECK1-NEXT:    [[MUL255:%.*]] = mul i32 [[MUL254]], 64
+// CHECK1-NEXT:    [[CONV256:%.*]] = zext i32 [[MUL255]] to i64
+// CHECK1-NEXT:    [[MUL257:%.*]] = mul nsw i64 [[DIV246]], [[CONV256]]
+// CHECK1-NEXT:    [[SUB258:%.*]] = sub nsw i64 [[TMP127]], [[MUL257]]
+// CHECK1-NEXT:    [[TMP139:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB259:%.*]] = sub i32 [[TMP139]], -63
+// CHECK1-NEXT:    [[DIV260:%.*]] = udiv i32 [[SUB259]], 64
+// CHECK1-NEXT:    [[MUL261:%.*]] = mul i32 1, [[DIV260]]
+// CHECK1-NEXT:    [[MUL262:%.*]] = mul i32 [[MUL261]], 64
 // CHECK1-NEXT:    [[CONV263:%.*]] = zext i32 [[MUL262]] to i64
-// CHECK1-NEXT:    [[DIV264:%.*]] = sdiv i64 [[TMP139]], [[CONV263]]
-// CHECK1-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB265:%.*]] = sub i32 [[TMP147]], [[TMP148]]
-// CHECK1-NEXT:    [[SUB266:%.*]] = sub i32 [[SUB265]], 1
-// CHECK1-NEXT:    [[TMP149:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD267:%.*]] = add i32 [[SUB266]], [[TMP149]]
-// CHECK1-NEXT:    [[TMP150:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV268:%.*]] = udiv i32 [[ADD267]], [[TMP150]]
-// CHECK1-NEXT:    [[MUL269:%.*]] = mul i32 1, [[DIV268]]
-// CHECK1-NEXT:    [[TMP151:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB270:%.*]] = sub i32 [[TMP151]], -63
-// CHECK1-NEXT:    [[DIV271:%.*]] = udiv i32 [[SUB270]], 64
-// CHECK1-NEXT:    [[MUL272:%.*]] = mul i32 [[MUL269]], [[DIV271]]
-// CHECK1-NEXT:    [[TMP152:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP153:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB273:%.*]] = sub i32 [[TMP152]], [[TMP153]]
-// CHECK1-NEXT:    [[SUB274:%.*]] = sub i32 [[SUB273]], 1
-// CHECK1-NEXT:    [[ADD275:%.*]] = add i32 [[SUB274]], 1
-// CHECK1-NEXT:    [[DIV276:%.*]] = udiv i32 [[ADD275]], 1
-// CHECK1-NEXT:    [[MUL277:%.*]] = mul i32 [[MUL272]], [[DIV276]]
-// CHECK1-NEXT:    [[CONV278:%.*]] = zext i32 [[MUL277]] to i64
-// CHECK1-NEXT:    [[MUL279:%.*]] = mul nsw i64 [[DIV264]], [[CONV278]]
-// CHECK1-NEXT:    [[SUB280:%.*]] = sub nsw i64 [[TMP138]], [[MUL279]]
-// CHECK1-NEXT:    [[TMP154:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB281:%.*]] = sub i32 [[TMP154]], -63
-// CHECK1-NEXT:    [[DIV282:%.*]] = udiv i32 [[SUB281]], 64
-// CHECK1-NEXT:    [[MUL283:%.*]] = mul i32 1, [[DIV282]]
-// CHECK1-NEXT:    [[TMP155:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP156:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB284:%.*]] = sub i32 [[TMP155]], [[TMP156]]
-// CHECK1-NEXT:    [[SUB285:%.*]] = sub i32 [[SUB284]], 1
-// CHECK1-NEXT:    [[ADD286:%.*]] = add i32 [[SUB285]], 1
-// CHECK1-NEXT:    [[DIV287:%.*]] = udiv i32 [[ADD286]], 1
-// CHECK1-NEXT:    [[MUL288:%.*]] = mul i32 [[MUL283]], [[DIV287]]
-// CHECK1-NEXT:    [[CONV289:%.*]] = zext i32 [[MUL288]] to i64
-// CHECK1-NEXT:    [[DIV290:%.*]] = sdiv i64 [[SUB280]], [[CONV289]]
-// CHECK1-NEXT:    [[TMP157:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB291:%.*]] = sub i32 [[TMP157]], -63
-// CHECK1-NEXT:    [[DIV292:%.*]] = udiv i32 [[SUB291]], 64
-// CHECK1-NEXT:    [[MUL293:%.*]] = mul i32 1, [[DIV292]]
-// CHECK1-NEXT:    [[TMP158:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP159:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB294:%.*]] = sub i32 [[TMP158]], [[TMP159]]
-// CHECK1-NEXT:    [[SUB295:%.*]] = sub i32 [[SUB294]], 1
-// CHECK1-NEXT:    [[ADD296:%.*]] = add i32 [[SUB295]], 1
-// CHECK1-NEXT:    [[DIV297:%.*]] = udiv i32 [[ADD296]], 1
-// CHECK1-NEXT:    [[MUL298:%.*]] = mul i32 [[MUL293]], [[DIV297]]
-// CHECK1-NEXT:    [[CONV299:%.*]] = zext i32 [[MUL298]] to i64
-// CHECK1-NEXT:    [[MUL300:%.*]] = mul nsw i64 [[DIV290]], [[CONV299]]
-// CHECK1-NEXT:    [[SUB301:%.*]] = sub nsw i64 [[SUB249]], [[MUL300]]
-// CHECK1-NEXT:    [[TMP160:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP161:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP162:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP163:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB302:%.*]] = sub i32 [[TMP162]], [[TMP163]]
-// CHECK1-NEXT:    [[SUB303:%.*]] = sub i32 [[SUB302]], 1
-// CHECK1-NEXT:    [[TMP164:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD304:%.*]] = add i32 [[SUB303]], [[TMP164]]
-// CHECK1-NEXT:    [[TMP165:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV305:%.*]] = udiv i32 [[ADD304]], [[TMP165]]
-// CHECK1-NEXT:    [[MUL306:%.*]] = mul i32 1, [[DIV305]]
-// CHECK1-NEXT:    [[TMP166:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB307:%.*]] = sub i32 [[TMP166]], -63
-// CHECK1-NEXT:    [[DIV308:%.*]] = udiv i32 [[SUB307]], 64
-// CHECK1-NEXT:    [[MUL309:%.*]] = mul i32 [[MUL306]], [[DIV308]]
-// CHECK1-NEXT:    [[TMP167:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP168:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB310:%.*]] = sub i32 [[TMP167]], [[TMP168]]
-// CHECK1-NEXT:    [[SUB311:%.*]] = sub i32 [[SUB310]], 1
-// CHECK1-NEXT:    [[ADD312:%.*]] = add i32 [[SUB311]], 1
-// CHECK1-NEXT:    [[DIV313:%.*]] = udiv i32 [[ADD312]], 1
-// CHECK1-NEXT:    [[MUL314:%.*]] = mul i32 [[MUL309]], [[DIV313]]
-// CHECK1-NEXT:    [[CONV315:%.*]] = zext i32 [[MUL314]] to i64
-// CHECK1-NEXT:    [[DIV316:%.*]] = sdiv i64 [[TMP161]], [[CONV315]]
-// CHECK1-NEXT:    [[TMP169:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP170:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB317:%.*]] = sub i32 [[TMP169]], [[TMP170]]
-// CHECK1-NEXT:    [[SUB318:%.*]] = sub i32 [[SUB317]], 1
-// CHECK1-NEXT:    [[TMP171:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD319:%.*]] = add i32 [[SUB318]], [[TMP171]]
-// CHECK1-NEXT:    [[TMP172:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV320:%.*]] = udiv i32 [[ADD319]], [[TMP172]]
-// CHECK1-NEXT:    [[MUL321:%.*]] = mul i32 1, [[DIV320]]
-// CHECK1-NEXT:    [[TMP173:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB322:%.*]] = sub i32 [[TMP173]], -63
-// CHECK1-NEXT:    [[DIV323:%.*]] = udiv i32 [[SUB322]], 64
-// CHECK1-NEXT:    [[MUL324:%.*]] = mul i32 [[MUL321]], [[DIV323]]
-// CHECK1-NEXT:    [[TMP174:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP175:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB325:%.*]] = sub i32 [[TMP174]], [[TMP175]]
-// CHECK1-NEXT:    [[SUB326:%.*]] = sub i32 [[SUB325]], 1
-// CHECK1-NEXT:    [[ADD327:%.*]] = add i32 [[SUB326]], 1
-// CHECK1-NEXT:    [[DIV328:%.*]] = udiv i32 [[ADD327]], 1
-// CHECK1-NEXT:    [[MUL329:%.*]] = mul i32 [[MUL324]], [[DIV328]]
-// CHECK1-NEXT:    [[CONV330:%.*]] = zext i32 [[MUL329]] to i64
-// CHECK1-NEXT:    [[MUL331:%.*]] = mul nsw i64 [[DIV316]], [[CONV330]]
-// CHECK1-NEXT:    [[SUB332:%.*]] = sub nsw i64 [[TMP160]], [[MUL331]]
-// CHECK1-NEXT:    [[TMP176:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP177:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP178:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP179:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB333:%.*]] = sub i32 [[TMP178]], [[TMP179]]
-// CHECK1-NEXT:    [[SUB334:%.*]] = sub i32 [[SUB333]], 1
-// CHECK1-NEXT:    [[TMP180:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD335:%.*]] = add i32 [[SUB334]], [[TMP180]]
-// CHECK1-NEXT:    [[TMP181:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV336:%.*]] = udiv i32 [[ADD335]], [[TMP181]]
-// CHECK1-NEXT:    [[MUL337:%.*]] = mul i32 1, [[DIV336]]
-// CHECK1-NEXT:    [[TMP182:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB338:%.*]] = sub i32 [[TMP182]], -63
-// CHECK1-NEXT:    [[DIV339:%.*]] = udiv i32 [[SUB338]], 64
-// CHECK1-NEXT:    [[MUL340:%.*]] = mul i32 [[MUL337]], [[DIV339]]
-// CHECK1-NEXT:    [[TMP183:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP184:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB341:%.*]] = sub i32 [[TMP183]], [[TMP184]]
-// CHECK1-NEXT:    [[SUB342:%.*]] = sub i32 [[SUB341]], 1
-// CHECK1-NEXT:    [[ADD343:%.*]] = add i32 [[SUB342]], 1
-// CHECK1-NEXT:    [[DIV344:%.*]] = udiv i32 [[ADD343]], 1
-// CHECK1-NEXT:    [[MUL345:%.*]] = mul i32 [[MUL340]], [[DIV344]]
-// CHECK1-NEXT:    [[CONV346:%.*]] = zext i32 [[MUL345]] to i64
-// CHECK1-NEXT:    [[DIV347:%.*]] = sdiv i64 [[TMP177]], [[CONV346]]
-// CHECK1-NEXT:    [[TMP185:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP186:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB348:%.*]] = sub i32 [[TMP185]], [[TMP186]]
-// CHECK1-NEXT:    [[SUB349:%.*]] = sub i32 [[SUB348]], 1
-// CHECK1-NEXT:    [[TMP187:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD350:%.*]] = add i32 [[SUB349]], [[TMP187]]
-// CHECK1-NEXT:    [[TMP188:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV351:%.*]] = udiv i32 [[ADD350]], [[TMP188]]
-// CHECK1-NEXT:    [[MUL352:%.*]] = mul i32 1, [[DIV351]]
-// CHECK1-NEXT:    [[TMP189:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB353:%.*]] = sub i32 [[TMP189]], -63
-// CHECK1-NEXT:    [[DIV354:%.*]] = udiv i32 [[SUB353]], 64
-// CHECK1-NEXT:    [[MUL355:%.*]] = mul i32 [[MUL352]], [[DIV354]]
-// CHECK1-NEXT:    [[TMP190:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP191:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB356:%.*]] = sub i32 [[TMP190]], [[TMP191]]
-// CHECK1-NEXT:    [[SUB357:%.*]] = sub i32 [[SUB356]], 1
-// CHECK1-NEXT:    [[ADD358:%.*]] = add i32 [[SUB357]], 1
-// CHECK1-NEXT:    [[DIV359:%.*]] = udiv i32 [[ADD358]], 1
-// CHECK1-NEXT:    [[MUL360:%.*]] = mul i32 [[MUL355]], [[DIV359]]
-// CHECK1-NEXT:    [[CONV361:%.*]] = zext i32 [[MUL360]] to i64
-// CHECK1-NEXT:    [[MUL362:%.*]] = mul nsw i64 [[DIV347]], [[CONV361]]
-// CHECK1-NEXT:    [[SUB363:%.*]] = sub nsw i64 [[TMP176]], [[MUL362]]
-// CHECK1-NEXT:    [[TMP192:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB364:%.*]] = sub i32 [[TMP192]], -63
-// CHECK1-NEXT:    [[DIV365:%.*]] = udiv i32 [[SUB364]], 64
-// CHECK1-NEXT:    [[MUL366:%.*]] = mul i32 1, [[DIV365]]
-// CHECK1-NEXT:    [[TMP193:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP194:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB367:%.*]] = sub i32 [[TMP193]], [[TMP194]]
-// CHECK1-NEXT:    [[SUB368:%.*]] = sub i32 [[SUB367]], 1
-// CHECK1-NEXT:    [[ADD369:%.*]] = add i32 [[SUB368]], 1
-// CHECK1-NEXT:    [[DIV370:%.*]] = udiv i32 [[ADD369]], 1
-// CHECK1-NEXT:    [[MUL371:%.*]] = mul i32 [[MUL366]], [[DIV370]]
-// CHECK1-NEXT:    [[CONV372:%.*]] = zext i32 [[MUL371]] to i64
-// CHECK1-NEXT:    [[DIV373:%.*]] = sdiv i64 [[SUB363]], [[CONV372]]
-// CHECK1-NEXT:    [[TMP195:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB374:%.*]] = sub i32 [[TMP195]], -63
-// CHECK1-NEXT:    [[DIV375:%.*]] = udiv i32 [[SUB374]], 64
-// CHECK1-NEXT:    [[MUL376:%.*]] = mul i32 1, [[DIV375]]
-// CHECK1-NEXT:    [[TMP196:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP197:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB377:%.*]] = sub i32 [[TMP196]], [[TMP197]]
-// CHECK1-NEXT:    [[SUB378:%.*]] = sub i32 [[SUB377]], 1
-// CHECK1-NEXT:    [[ADD379:%.*]] = add i32 [[SUB378]], 1
-// CHECK1-NEXT:    [[DIV380:%.*]] = udiv i32 [[ADD379]], 1
-// CHECK1-NEXT:    [[MUL381:%.*]] = mul i32 [[MUL376]], [[DIV380]]
-// CHECK1-NEXT:    [[CONV382:%.*]] = zext i32 [[MUL381]] to i64
-// CHECK1-NEXT:    [[MUL383:%.*]] = mul nsw i64 [[DIV373]], [[CONV382]]
-// CHECK1-NEXT:    [[SUB384:%.*]] = sub nsw i64 [[SUB332]], [[MUL383]]
-// CHECK1-NEXT:    [[TMP198:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP199:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB385:%.*]] = sub i32 [[TMP198]], [[TMP199]]
-// CHECK1-NEXT:    [[SUB386:%.*]] = sub i32 [[SUB385]], 1
-// CHECK1-NEXT:    [[ADD387:%.*]] = add i32 [[SUB386]], 1
-// CHECK1-NEXT:    [[DIV388:%.*]] = udiv i32 [[ADD387]], 1
-// CHECK1-NEXT:    [[MUL389:%.*]] = mul i32 1, [[DIV388]]
-// CHECK1-NEXT:    [[CONV390:%.*]] = zext i32 [[MUL389]] to i64
-// CHECK1-NEXT:    [[DIV391:%.*]] = sdiv i64 [[SUB384]], [[CONV390]]
-// CHECK1-NEXT:    [[TMP200:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK1-NEXT:    [[TMP201:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK1-NEXT:    [[SUB392:%.*]] = sub i32 [[TMP200]], [[TMP201]]
-// CHECK1-NEXT:    [[SUB393:%.*]] = sub i32 [[SUB392]], 1
-// CHECK1-NEXT:    [[ADD394:%.*]] = add i32 [[SUB393]], 1
-// CHECK1-NEXT:    [[DIV395:%.*]] = udiv i32 [[ADD394]], 1
-// CHECK1-NEXT:    [[MUL396:%.*]] = mul i32 1, [[DIV395]]
-// CHECK1-NEXT:    [[CONV397:%.*]] = zext i32 [[MUL396]] to i64
-// CHECK1-NEXT:    [[MUL398:%.*]] = mul nsw i64 [[DIV391]], [[CONV397]]
-// CHECK1-NEXT:    [[SUB399:%.*]] = sub nsw i64 [[SUB301]], [[MUL398]]
-// CHECK1-NEXT:    [[MUL400:%.*]] = mul nsw i64 [[SUB399]], 1
-// CHECK1-NEXT:    [[ADD401:%.*]] = add nsw i64 [[CONV218]], [[MUL400]]
-// CHECK1-NEXT:    [[CONV402:%.*]] = trunc i64 [[ADD401]] to i32
-// CHECK1-NEXT:    store i32 [[CONV402]], ptr [[DOTTILE_0_IV_K52]], align 4
-// CHECK1-NEXT:    [[TMP202:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
-// CHECK1-NEXT:    [[TMP203:%.*]] = load i32, ptr [[DOTTILE_0_IV_K52]], align 4
-// CHECK1-NEXT:    [[TMP204:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
-// CHECK1-NEXT:    [[MUL403:%.*]] = mul i32 [[TMP203]], [[TMP204]]
-// CHECK1-NEXT:    [[ADD404:%.*]] = add i32 [[TMP202]], [[MUL403]]
-// CHECK1-NEXT:    store i32 [[ADD404]], ptr [[K]], align 4
-// CHECK1-NEXT:    [[TMP205:%.*]] = load i32, ptr [[I49]], align 4
-// CHECK1-NEXT:    [[TMP206:%.*]] = load i32, ptr [[J50]], align 4
-// CHECK1-NEXT:    [[TMP207:%.*]] = load i32, ptr [[K]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP205]], i32 noundef [[TMP206]], i32 noundef [[TMP207]])
+// CHECK1-NEXT:    [[DIV264:%.*]] = sdiv i64 [[SUB258]], [[CONV263]]
+// CHECK1-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB265:%.*]] = sub i32 [[TMP140]], -63
+// CHECK1-NEXT:    [[DIV266:%.*]] = udiv i32 [[SUB265]], 64
+// CHECK1-NEXT:    [[MUL267:%.*]] = mul i32 1, [[DIV266]]
+// CHECK1-NEXT:    [[MUL268:%.*]] = mul i32 [[MUL267]], 64
+// CHECK1-NEXT:    [[CONV269:%.*]] = zext i32 [[MUL268]] to i64
+// CHECK1-NEXT:    [[MUL270:%.*]] = mul nsw i64 [[DIV264]], [[CONV269]]
+// CHECK1-NEXT:    [[SUB271:%.*]] = sub nsw i64 [[SUB235]], [[MUL270]]
+// CHECK1-NEXT:    [[DIV272:%.*]] = sdiv i64 [[SUB271]], 64
+// CHECK1-NEXT:    [[MUL273:%.*]] = mul nsw i64 [[DIV272]], 64
+// CHECK1-NEXT:    [[SUB274:%.*]] = sub nsw i64 [[SUB212]], [[MUL273]]
+// CHECK1-NEXT:    [[MUL275:%.*]] = mul nsw i64 [[SUB274]], 1
+// CHECK1-NEXT:    [[ADD276:%.*]] = add nsw i64 0, [[MUL275]]
+// CHECK1-NEXT:    [[CONV277:%.*]] = trunc i64 [[ADD276]] to i32
+// CHECK1-NEXT:    store i32 [[CONV277]], ptr [[DOTTILE_CNT_0_IV_K38]], align 4
+// CHECK1-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
+// CHECK1-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
+// CHECK1-NEXT:    [[ADD278:%.*]] = add i32 [[TMP141]], [[TMP142]]
+// CHECK1-NEXT:    store i32 [[ADD278]], ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK1-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
+// CHECK1-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK1-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
+// CHECK1-NEXT:    [[MUL279:%.*]] = mul i32 [[TMP144]], [[TMP145]]
+// CHECK1-NEXT:    [[ADD280:%.*]] = add i32 [[TMP143]], [[MUL279]]
+// CHECK1-NEXT:    store i32 [[ADD280]], ptr [[K]], align 4
+// CHECK1-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK1-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// CHECK1-NEXT:    [[ADD281:%.*]] = add i32 [[TMP147]], 1
+// CHECK1-NEXT:    [[CMP282:%.*]] = icmp ult i32 [[TMP146]], [[ADD281]]
+// CHECK1-NEXT:    br i1 [[CMP282]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK1:       if.then:
+// CHECK1-NEXT:    [[TMP148:%.*]] = load i32, ptr [[I35]], align 4
+// CHECK1-NEXT:    [[TMP149:%.*]] = load i32, ptr [[J36]], align 4
+// CHECK1-NEXT:    [[TMP150:%.*]] = load i32, ptr [[K]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP148]], i32 noundef [[TMP149]], i32 noundef [[TMP150]])
+// CHECK1-NEXT:    br label [[IF_END]]
+// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP208:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[ADD405:%.*]] = add nsw i64 [[TMP208]], 1
-// CHECK1-NEXT:    store i64 [[ADD405]], ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP151:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[ADD283:%.*]] = add nsw i64 [[TMP151]], 1
+// CHECK1-NEXT:    store i64 [[ADD283]], ptr [[DOTOMP_IV]], align 8
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -2225,14 +1868,14 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[V:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    store double 4.200000e+01, ptr [[C]], align 8
 // CHECK1-NEXT:    store ptr [[ARR]], ptr [[__RANGE2]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META12:![0-9]+]], !align [[META13:![0-9]+]]
 // CHECK1-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP0]], i64 0, i64 0
 // CHECK1-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 128
 // CHECK1-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META12]], !align [[META13]]
 // CHECK1-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP1]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[ARRAYDECAY1]], ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META12]], !align [[META13]]
 // CHECK1-NEXT:    [[ARRAYDECAY2:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP2]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[ARRAYDECAY2]], ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__END2]], align 8
@@ -2277,7 +1920,7 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
 // CHECK1-NEXT:    store ptr [[TMP12]], ptr [[V]], align 8
 // CHECK1-NEXT:    [[TMP13:%.*]] = load double, ptr [[C]], align 8
-// CHECK1-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[V]], align 8
+// CHECK1-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[V]], align 8, !nonnull [[META12]], !align [[META13]]
 // CHECK1-NEXT:    [[TMP15:%.*]] = load double, ptr [[TMP14]], align 8
 // CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    call void (...) @body(double noundef [[TMP13]], double noundef [[TMP15]], i32 noundef [[TMP16]])
@@ -2286,14 +1929,14 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[TMP17:%.*]] = load i64, ptr [[DOTPERMUTED_1_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP17]], 1
 // CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTPERMUTED_1_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP13:![0-9]+]]
+// CHECK1-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP14:![0-9]+]]
 // CHECK1:       for.end:
 // CHECK1-NEXT:    br label [[FOR_INC13:%.*]]
 // CHECK1:       for.inc13:
 // CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_I]], align 4
 // CHECK1-NEXT:    [[INC14:%.*]] = add nsw i32 [[TMP18]], 1
 // CHECK1-NEXT:    store i32 [[INC14]], ptr [[DOTPERMUTED_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP14:![0-9]+]]
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP15:![0-9]+]]
 // CHECK1:       for.end15:
 // CHECK1-NEXT:    ret void
 //
@@ -2342,14 +1985,14 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK1-NEXT:    store double 4.200000e+01, ptr [[C]], align 8
 // CHECK1-NEXT:    store ptr [[A]], ptr [[__RANGE3]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE3]], align 8
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE3]], align 8, !nonnull [[META12]], !align [[META13]]
 // CHECK1-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP1]], i64 0, i64 0
 // CHECK1-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 128
 // CHECK1-NEXT:    store ptr [[ADD_PTR]], ptr [[__END3]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE3]], align 8
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE3]], align 8, !nonnull [[META12]], !align [[META13]]
 // CHECK1-NEXT:    [[ARRAYDECAY4:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP2]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[ARRAYDECAY4]], ptr [[__BEGIN3]], align 8
-// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__RANGE3]], align 8
+// CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__RANGE3]], align 8, !nonnull [[META12]], !align [[META13]]
 // CHECK1-NEXT:    [[ARRAYDECAY5:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP3]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[ARRAYDECAY5]], ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK1-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[__END3]], align 8
@@ -2367,14 +2010,14 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    store i64 [[SUB8]], ptr [[DOTCAPTURE_EXPR_7]], align 8
 // CHECK1-NEXT:    store double 4.200000e+01, ptr [[D]], align 8
 // CHECK1-NEXT:    store ptr [[B]], ptr [[__RANGE4]], align 8
-// CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[__RANGE4]], align 8
+// CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[__RANGE4]], align 8, !nonnull [[META12]], !align [[META13]]
 // CHECK1-NEXT:    [[ARRAYDECAY9:%.*]] = getelementptr inbounds [16 x double], ptr [[TMP7]], i64 0, i64 0
 // CHECK1-NEXT:    [[ADD_PTR10:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY9]], i64 16
 // CHECK1-NEXT:    store ptr [[ADD_PTR10]], ptr [[__END4]], align 8
-// CHECK1-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[__RANGE4]], align 8
+// CHECK1-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[__RANGE4]], align 8, !nonnull [[META12]], !align [[META13]]
 // CHECK1-NEXT:    [[ARRAYDECAY11:%.*]] = getelementptr inbounds [16 x double], ptr [[TMP8]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[ARRAYDECAY11]], ptr [[__BEGIN4]], align 8
-// CHECK1-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[__RANGE4]], align 8
+// CHECK1-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[__RANGE4]], align 8, !nonnull [[META12]], !align [[META13]]
 // CHECK1-NEXT:    [[ARRAYDECAY13:%.*]] = getelementptr inbounds [16 x double], ptr [[TMP9]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[ARRAYDECAY13]], ptr [[DOTCAPTURE_EXPR_12]], align 8
 // CHECK1-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[__END4]], align 8
@@ -2701,7 +2344,7 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[TMP87:%.*]] = load double, ptr [[C]], align 8
 // CHECK1-NEXT:    [[TMP88:%.*]] = load double, ptr [[AA]], align 8
 // CHECK1-NEXT:    [[TMP89:%.*]] = load double, ptr [[D]], align 8
-// CHECK1-NEXT:    [[TMP90:%.*]] = load ptr, ptr [[BB]], align 8
+// CHECK1-NEXT:    [[TMP90:%.*]] = load ptr, ptr [[BB]], align 8, !nonnull [[META12]], !align [[META13]]
 // CHECK1-NEXT:    [[TMP91:%.*]] = load double, ptr [[TMP90]], align 8
 // CHECK1-NEXT:    [[TMP92:%.*]] = load i32, ptr [[J40]], align 4
 // CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP86]], double noundef [[TMP87]], double noundef [[TMP88]], double noundef [[TMP89]], double noundef [[TMP91]], i32 noundef [[TMP92]])
@@ -2756,7 +2399,7 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[I]], align 4
 // CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP5]], [[TMP4]]
 // CHECK2-NEXT:    store i32 [[ADD]], ptr [[I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP3:![0-9]+]]
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP2:![0-9]+]]
 // CHECK2:       for.end:
 // CHECK2-NEXT:    ret void
 //
@@ -2805,14 +2448,14 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
 // CHECK2-NEXT:    store double 4.200000e+01, ptr [[C]], align 8
 // CHECK2-NEXT:    store ptr [[A]], ptr [[__RANGE3]], align 8
-// CHECK2-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE3]], align 8
+// CHECK2-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE3]], align 8, !nonnull [[META4:![0-9]+]], !align [[META5:![0-9]+]]
 // CHECK2-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP1]], i64 0, i64 0
 // CHECK2-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 128
 // CHECK2-NEXT:    store ptr [[ADD_PTR]], ptr [[__END3]], align 8
-// CHECK2-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE3]], align 8
+// CHECK2-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE3]], align 8, !nonnull [[META4]], !align [[META5]]
 // CHECK2-NEXT:    [[ARRAYDECAY4:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP2]], i64 0, i64 0
 // CHECK2-NEXT:    store ptr [[ARRAYDECAY4]], ptr [[__BEGIN3]], align 8
-// CHECK2-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__RANGE3]], align 8
+// CHECK2-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__RANGE3]], align 8, !nonnull [[META4]], !align [[META5]]
 // CHECK2-NEXT:    [[ARRAYDECAY5:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP3]], i64 0, i64 0
 // CHECK2-NEXT:    store ptr [[ARRAYDECAY5]], ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK2-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[__END3]], align 8
@@ -2830,14 +2473,14 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    store i64 [[SUB8]], ptr [[DOTCAPTURE_EXPR_7]], align 8
 // CHECK2-NEXT:    store double 4.200000e+01, ptr [[D]], align 8
 // CHECK2-NEXT:    store ptr [[B]], ptr [[__RANGE4]], align 8
-// CHECK2-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[__RANGE4]], align 8
+// CHECK2-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[__RANGE4]], align 8, !nonnull [[META4]], !align [[META5]]
 // CHECK2-NEXT:    [[ARRAYDECAY9:%.*]] = getelementptr inbounds [16 x double], ptr [[TMP7]], i64 0, i64 0
 // CHECK2-NEXT:    [[ADD_PTR10:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY9]], i64 16
 // CHECK2-NEXT:    store ptr [[ADD_PTR10]], ptr [[__END4]], align 8
-// CHECK2-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[__RANGE4]], align 8
+// CHECK2-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[__RANGE4]], align 8, !nonnull [[META4]], !align [[META5]]
 // CHECK2-NEXT:    [[ARRAYDECAY11:%.*]] = getelementptr inbounds [16 x double], ptr [[TMP8]], i64 0, i64 0
 // CHECK2-NEXT:    store ptr [[ARRAYDECAY11]], ptr [[__BEGIN4]], align 8
-// CHECK2-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[__RANGE4]], align 8
+// CHECK2-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[__RANGE4]], align 8, !nonnull [[META4]], !align [[META5]]
 // CHECK2-NEXT:    [[ARRAYDECAY13:%.*]] = getelementptr inbounds [16 x double], ptr [[TMP9]], i64 0, i64 0
 // CHECK2-NEXT:    store ptr [[ARRAYDECAY13]], ptr [[DOTCAPTURE_EXPR_12]], align 8
 // CHECK2-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[__END4]], align 8
@@ -3164,7 +2807,7 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[TMP87:%.*]] = load double, ptr [[C]], align 8
 // CHECK2-NEXT:    [[TMP88:%.*]] = load double, ptr [[AA]], align 8
 // CHECK2-NEXT:    [[TMP89:%.*]] = load double, ptr [[D]], align 8
-// CHECK2-NEXT:    [[TMP90:%.*]] = load ptr, ptr [[BB]], align 8
+// CHECK2-NEXT:    [[TMP90:%.*]] = load ptr, ptr [[BB]], align 8, !nonnull [[META4]], !align [[META5]]
 // CHECK2-NEXT:    [[TMP91:%.*]] = load double, ptr [[TMP90]], align 8
 // CHECK2-NEXT:    [[TMP92:%.*]] = load i32, ptr [[J40]], align 4
 // CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP86]], double noundef [[TMP87]], double noundef [[TMP88]], double noundef [[TMP89]], double noundef [[TMP91]], i32 noundef [[TMP92]])
@@ -3287,14 +2930,14 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP28]], 1
 // CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTPERMUTED_1_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND16]], !llvm.loop [[LOOP5:![0-9]+]]
+// CHECK2-NEXT:    br label [[FOR_COND16]], !llvm.loop [[LOOP6:![0-9]+]]
 // CHECK2:       for.end:
 // CHECK2-NEXT:    br label [[FOR_INC22:%.*]]
 // CHECK2:       for.inc22:
 // CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_J]], align 4
 // CHECK2-NEXT:    [[INC23:%.*]] = add i32 [[TMP29]], 1
 // CHECK2-NEXT:    store i32 [[INC23]], ptr [[DOTPERMUTED_0_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP7:![0-9]+]]
 // CHECK2:       for.end24:
 // CHECK2-NEXT:    ret void
 //
@@ -3367,7 +3010,7 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP12]], 1
 // CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTPERMUTED_1_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP7:![0-9]+]]
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
 // CHECK2:       for.end:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
@@ -3464,7 +3107,7 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_I]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
 // CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTPERMUTED_1_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP9:![0-9]+]]
 // CHECK2:       for.end:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
@@ -3779,28 +3422,28 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTPERMUTED_3_IV_I]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP12]], 1
 // CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTPERMUTED_3_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND11]], !llvm.loop [[LOOP9:![0-9]+]]
+// CHECK2-NEXT:    br label [[FOR_COND11]], !llvm.loop [[LOOP10:![0-9]+]]
 // CHECK2:       for.end:
 // CHECK2-NEXT:    br label [[FOR_INC16:%.*]]
 // CHECK2:       for.inc16:
 // CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTPERMUTED_2_IV_L]], align 4
 // CHECK2-NEXT:    [[INC17:%.*]] = add nsw i32 [[TMP13]], 1
 // CHECK2-NEXT:    store i32 [[INC17]], ptr [[DOTPERMUTED_2_IV_L]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP10:![0-9]+]]
+// CHECK2-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP11:![0-9]+]]
 // CHECK2:       for.end18:
 // CHECK2-NEXT:    br label [[FOR_INC19:%.*]]
 // CHECK2:       for.inc19:
 // CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTPERMUTED_1_IV_K]], align 4
 // CHECK2-NEXT:    [[INC20:%.*]] = add nsw i32 [[TMP14]], 1
 // CHECK2-NEXT:    store i32 [[INC20]], ptr [[DOTPERMUTED_1_IV_K]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP11:![0-9]+]]
+// CHECK2-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP12:![0-9]+]]
 // CHECK2:       for.end21:
 // CHECK2-NEXT:    br label [[FOR_INC22:%.*]]
 // CHECK2:       for.inc22:
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_J]], align 4
 // CHECK2-NEXT:    [[INC23:%.*]] = add nsw i32 [[TMP15]], 1
 // CHECK2-NEXT:    store i32 [[INC23]], ptr [[DOTPERMUTED_0_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP12:![0-9]+]]
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP13:![0-9]+]]
 // CHECK2:       for.end24:
 // CHECK2-NEXT:    ret void
 //
@@ -3822,14 +3465,14 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[V:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    store double 4.200000e+01, ptr [[C]], align 8
 // CHECK2-NEXT:    store ptr [[ARR]], ptr [[__RANGE2]], align 8
-// CHECK2-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META4]], !align [[META5]]
 // CHECK2-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP0]], i64 0, i64 0
 // CHECK2-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 128
 // CHECK2-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
-// CHECK2-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META4]], !align [[META5]]
 // CHECK2-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP1]], i64 0, i64 0
 // CHECK2-NEXT:    store ptr [[ARRAYDECAY1]], ptr [[__BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8
+// CHECK2-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META4]], !align [[META5]]
 // CHECK2-NEXT:    [[ARRAYDECAY2:%.*]] = getelementptr inbounds [128 x double], ptr [[TMP2]], i64 0, i64 0
 // CHECK2-NEXT:    store ptr [[ARRAYDECAY2]], ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK2-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__END2]], align 8
@@ -3874,7 +3517,7 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
 // CHECK2-NEXT:    store ptr [[TMP12]], ptr [[V]], align 8
 // CHECK2-NEXT:    [[TMP13:%.*]] = load double, ptr [[C]], align 8
-// CHECK2-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[V]], align 8
+// CHECK2-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[V]], align 8, !nonnull [[META4]], !align [[META5]]
 // CHECK2-NEXT:    [[TMP15:%.*]] = load double, ptr [[TMP14]], align 8
 // CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
 // CHECK2-NEXT:    call void (...) @body(double noundef [[TMP13]], double noundef [[TMP15]], i32 noundef [[TMP16]])
@@ -3883,14 +3526,14 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[TMP17:%.*]] = load i64, ptr [[DOTPERMUTED_1_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP17]], 1
 // CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTPERMUTED_1_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP13:![0-9]+]]
+// CHECK2-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP14:![0-9]+]]
 // CHECK2:       for.end:
 // CHECK2-NEXT:    br label [[FOR_INC13:%.*]]
 // CHECK2:       for.inc13:
 // CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTPERMUTED_0_IV_I]], align 4
 // CHECK2-NEXT:    [[INC14:%.*]] = add nsw i32 [[TMP18]], 1
 // CHECK2-NEXT:    store i32 [[INC14]], ptr [[DOTPERMUTED_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP14:![0-9]+]]
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP15:![0-9]+]]
 // CHECK2:       for.end15:
 // CHECK2-NEXT:    ret void
 //
@@ -3925,22 +3568,21 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTNEW_STEP10:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_14:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_17:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_22:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_K:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_K:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[I49:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[J50:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV_K51:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_K52:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[I35:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[J36:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV_K37:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_K38:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
@@ -3978,630 +3620,452 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
 // CHECK2-NEXT:    [[ADD15:%.*]] = add i32 [[TMP15]], 1
 // CHECK2-NEXT:    store i32 [[ADD15]], ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[_TMP2]], align 4
-// CHECK2-NEXT:    store i32 [[TMP16]], ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK2-NEXT:    [[ADD18:%.*]] = add i32 [[TMP17]], 1
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[_TMP2]], align 4
-// CHECK2-NEXT:    [[ADD19:%.*]] = add i32 [[TMP18]], 32
-// CHECK2-NEXT:    [[CMP:%.*]] = icmp ult i32 [[ADD18]], [[ADD19]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK2:       cond.true:
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK2-NEXT:    [[ADD20:%.*]] = add i32 [[TMP19]], 1
-// CHECK2-NEXT:    br label [[COND_END:%.*]]
-// CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[_TMP2]], align 4
-// CHECK2-NEXT:    [[ADD21:%.*]] = add i32 [[TMP20]], 32
-// CHECK2-NEXT:    br label [[COND_END]]
-// CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD20]], [[COND_TRUE]] ], [ [[ADD21]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    store i32 [[COND]], ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[SUB23:%.*]] = sub i32 [[TMP21]], [[TMP22]]
-// CHECK2-NEXT:    [[SUB24:%.*]] = sub i32 [[SUB23]], 1
-// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[ADD25:%.*]] = add i32 [[SUB24]], [[TMP23]]
-// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[DIV26:%.*]] = udiv i32 [[ADD25]], [[TMP24]]
-// CHECK2-NEXT:    [[CONV:%.*]] = zext i32 [[DIV26]] to i64
-// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[SUB17:%.*]] = sub i32 [[TMP16]], [[TMP17]]
+// CHECK2-NEXT:    [[SUB18:%.*]] = sub i32 [[SUB17]], 1
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[ADD19:%.*]] = add i32 [[SUB18]], [[TMP18]]
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[DIV20:%.*]] = udiv i32 [[ADD19]], [[TMP19]]
+// CHECK2-NEXT:    [[CONV:%.*]] = zext i32 [[DIV20]] to i64
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB21:%.*]] = sub i32 [[TMP20]], [[TMP21]]
+// CHECK2-NEXT:    [[SUB22:%.*]] = sub i32 [[SUB21]], 1
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD23:%.*]] = add i32 [[SUB22]], [[TMP22]]
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV24:%.*]] = udiv i32 [[ADD23]], [[TMP23]]
+// CHECK2-NEXT:    [[CONV25:%.*]] = zext i32 [[DIV24]] to i64
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV25]]
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB26:%.*]] = sub i32 [[TMP24]], -31
+// CHECK2-NEXT:    [[DIV27:%.*]] = udiv i32 [[SUB26]], 32
+// CHECK2-NEXT:    [[CONV28:%.*]] = zext i32 [[DIV27]] to i64
+// CHECK2-NEXT:    [[MUL29:%.*]] = mul nsw i64 [[MUL]], [[CONV28]]
+// CHECK2-NEXT:    [[MUL30:%.*]] = mul nsw i64 [[MUL29]], 32
+// CHECK2-NEXT:    [[SUB31:%.*]] = sub nsw i64 [[MUL30]], 1
+// CHECK2-NEXT:    store i64 [[SUB31]], ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    store i32 [[TMP25]], ptr [[I]], align 4
 // CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB27:%.*]] = sub i32 [[TMP25]], [[TMP26]]
-// CHECK2-NEXT:    [[SUB28:%.*]] = sub i32 [[SUB27]], 1
-// CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD29:%.*]] = add i32 [[SUB28]], [[TMP27]]
-// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV30:%.*]] = udiv i32 [[ADD29]], [[TMP28]]
-// CHECK2-NEXT:    [[CONV31:%.*]] = zext i32 [[DIV30]] to i64
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV31]]
-// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB32:%.*]] = sub i32 [[TMP29]], -31
-// CHECK2-NEXT:    [[DIV33:%.*]] = udiv i32 [[SUB32]], 32
-// CHECK2-NEXT:    [[CONV34:%.*]] = zext i32 [[DIV33]] to i64
-// CHECK2-NEXT:    [[MUL35:%.*]] = mul nsw i64 [[MUL]], [[CONV34]]
-// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB36:%.*]] = sub i32 [[TMP30]], [[TMP31]]
-// CHECK2-NEXT:    [[SUB37:%.*]] = sub i32 [[SUB36]], 1
-// CHECK2-NEXT:    [[ADD38:%.*]] = add i32 [[SUB37]], 1
-// CHECK2-NEXT:    [[DIV39:%.*]] = udiv i32 [[ADD38]], 1
-// CHECK2-NEXT:    [[CONV40:%.*]] = zext i32 [[DIV39]] to i64
-// CHECK2-NEXT:    [[MUL41:%.*]] = mul nsw i64 [[MUL35]], [[CONV40]]
-// CHECK2-NEXT:    [[SUB42:%.*]] = sub nsw i64 [[MUL41]], 1
-// CHECK2-NEXT:    store i64 [[SUB42]], ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK2-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    store i32 [[TMP32]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    store i32 [[TMP33]], ptr [[J]], align 4
+// CHECK2-NEXT:    store i32 [[TMP26]], ptr [[J]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_K]], align 4
-// CHECK2-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    store i32 [[TMP34]], ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK2-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK2-NEXT:    [[CMP43:%.*]] = icmp slt i32 [[TMP35]], [[TMP36]]
-// CHECK2-NEXT:    br i1 [[CMP43]], label [[LAND_LHS_TRUE:%.*]], label [[OMP_PRECOND_END:%.*]]
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_K]], align 4
+// CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP27]], [[TMP28]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[LAND_LHS_TRUE:%.*]], label [[OMP_PRECOND_END:%.*]]
 // CHECK2:       land.lhs.true:
-// CHECK2-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[CMP44:%.*]] = icmp slt i32 [[TMP37]], [[TMP38]]
-// CHECK2-NEXT:    br i1 [[CMP44]], label [[LAND_LHS_TRUE45:%.*]], label [[OMP_PRECOND_END]]
-// CHECK2:       land.lhs.true45:
-// CHECK2-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[CMP46:%.*]] = icmp ult i32 0, [[TMP39]]
-// CHECK2-NEXT:    br i1 [[CMP46]], label [[LAND_LHS_TRUE47:%.*]], label [[OMP_PRECOND_END]]
-// CHECK2:       land.lhs.true47:
-// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[CMP48:%.*]] = icmp ult i32 [[TMP40]], [[TMP41]]
-// CHECK2-NEXT:    br i1 [[CMP48]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
+// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[CMP32:%.*]] = icmp slt i32 [[TMP29]], [[TMP30]]
+// CHECK2-NEXT:    br i1 [[CMP32]], label [[LAND_LHS_TRUE33:%.*]], label [[OMP_PRECOND_END]]
+// CHECK2:       land.lhs.true33:
+// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[CMP34:%.*]] = icmp ult i32 0, [[TMP31]]
+// CHECK2-NEXT:    br i1 [[CMP34]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
 // CHECK2:       omp.precond.then:
 // CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
-// CHECK2-NEXT:    [[TMP42:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK2-NEXT:    store i64 [[TMP42]], ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK2-NEXT:    store i64 [[TMP32]], ptr [[DOTOMP_UB]], align 8
 // CHECK2-NEXT:    store i64 1, ptr [[DOTOMP_STRIDE]], align 8
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
 // CHECK2-NEXT:    call void @__kmpc_for_static_init_8(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i64 1, i64 1)
-// CHECK2-NEXT:    [[TMP43:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[TMP44:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK2-NEXT:    [[CMP53:%.*]] = icmp sgt i64 [[TMP43]], [[TMP44]]
-// CHECK2-NEXT:    br i1 [[CMP53]], label [[COND_TRUE54:%.*]], label [[COND_FALSE55:%.*]]
-// CHECK2:       cond.true54:
-// CHECK2-NEXT:    [[TMP45:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK2-NEXT:    br label [[COND_END56:%.*]]
-// CHECK2:       cond.false55:
-// CHECK2-NEXT:    [[TMP46:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    br label [[COND_END56]]
-// CHECK2:       cond.end56:
-// CHECK2-NEXT:    [[COND57:%.*]] = phi i64 [ [[TMP45]], [[COND_TRUE54]] ], [ [[TMP46]], [[COND_FALSE55]] ]
-// CHECK2-NEXT:    store i64 [[COND57]], ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[TMP47:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
-// CHECK2-NEXT:    store i64 [[TMP47]], ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK2-NEXT:    [[CMP39:%.*]] = icmp sgt i64 [[TMP33]], [[TMP34]]
+// CHECK2-NEXT:    br i1 [[CMP39]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i64 [ [[TMP35]], [[COND_TRUE]] ], [ [[TMP36]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    store i64 [[COND]], ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
+// CHECK2-NEXT:    store i64 [[TMP37]], ptr [[DOTOMP_IV]], align 8
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK2:       omp.inner.for.cond:
-// CHECK2-NEXT:    [[TMP48:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[CMP58:%.*]] = icmp sle i64 [[TMP48]], [[TMP49]]
-// CHECK2-NEXT:    br i1 [[CMP58]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK2-NEXT:    [[TMP38:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP39:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[CMP40:%.*]] = icmp sle i64 [[TMP38]], [[TMP39]]
+// CHECK2-NEXT:    br i1 [[CMP40]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
 // CHECK2:       omp.inner.for.body:
-// CHECK2-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[CONV59:%.*]] = sext i32 [[TMP50]] to i64
-// CHECK2-NEXT:    [[TMP51:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB60:%.*]] = sub i32 [[TMP52]], [[TMP53]]
-// CHECK2-NEXT:    [[SUB61:%.*]] = sub i32 [[SUB60]], 1
+// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[CONV41:%.*]] = sext i32 [[TMP40]] to i64
+// CHECK2-NEXT:    [[TMP41:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB42:%.*]] = sub i32 [[TMP42]], [[TMP43]]
+// CHECK2-NEXT:    [[SUB43:%.*]] = sub i32 [[SUB42]], 1
+// CHECK2-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD44:%.*]] = add i32 [[SUB43]], [[TMP44]]
+// CHECK2-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV45:%.*]] = udiv i32 [[ADD44]], [[TMP45]]
+// CHECK2-NEXT:    [[MUL46:%.*]] = mul i32 1, [[DIV45]]
+// CHECK2-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB47:%.*]] = sub i32 [[TMP46]], -31
+// CHECK2-NEXT:    [[DIV48:%.*]] = udiv i32 [[SUB47]], 32
+// CHECK2-NEXT:    [[MUL49:%.*]] = mul i32 [[MUL46]], [[DIV48]]
+// CHECK2-NEXT:    [[MUL50:%.*]] = mul i32 [[MUL49]], 32
+// CHECK2-NEXT:    [[CONV51:%.*]] = zext i32 [[MUL50]] to i64
+// CHECK2-NEXT:    [[DIV52:%.*]] = sdiv i64 [[TMP41]], [[CONV51]]
+// CHECK2-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[CONV53:%.*]] = sext i32 [[TMP47]] to i64
+// CHECK2-NEXT:    [[MUL54:%.*]] = mul nsw i64 [[DIV52]], [[CONV53]]
+// CHECK2-NEXT:    [[ADD55:%.*]] = add nsw i64 [[CONV41]], [[MUL54]]
+// CHECK2-NEXT:    [[CONV56:%.*]] = trunc i64 [[ADD55]] to i32
+// CHECK2-NEXT:    store i32 [[CONV56]], ptr [[I35]], align 4
+// CHECK2-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP48]] to i64
+// CHECK2-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP50:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB58:%.*]] = sub i32 [[TMP51]], [[TMP52]]
+// CHECK2-NEXT:    [[SUB59:%.*]] = sub i32 [[SUB58]], 1
+// CHECK2-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD60:%.*]] = add i32 [[SUB59]], [[TMP53]]
 // CHECK2-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD62:%.*]] = add i32 [[SUB61]], [[TMP54]]
-// CHECK2-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV63:%.*]] = udiv i32 [[ADD62]], [[TMP55]]
-// CHECK2-NEXT:    [[MUL64:%.*]] = mul i32 1, [[DIV63]]
-// CHECK2-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB65:%.*]] = sub i32 [[TMP56]], -31
-// CHECK2-NEXT:    [[DIV66:%.*]] = udiv i32 [[SUB65]], 32
-// CHECK2-NEXT:    [[MUL67:%.*]] = mul i32 [[MUL64]], [[DIV66]]
-// CHECK2-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB68:%.*]] = sub i32 [[TMP57]], [[TMP58]]
-// CHECK2-NEXT:    [[SUB69:%.*]] = sub i32 [[SUB68]], 1
-// CHECK2-NEXT:    [[ADD70:%.*]] = add i32 [[SUB69]], 1
-// CHECK2-NEXT:    [[DIV71:%.*]] = udiv i32 [[ADD70]], 1
-// CHECK2-NEXT:    [[MUL72:%.*]] = mul i32 [[MUL67]], [[DIV71]]
-// CHECK2-NEXT:    [[CONV73:%.*]] = zext i32 [[MUL72]] to i64
-// CHECK2-NEXT:    [[DIV74:%.*]] = sdiv i64 [[TMP51]], [[CONV73]]
-// CHECK2-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[CONV75:%.*]] = sext i32 [[TMP59]] to i64
-// CHECK2-NEXT:    [[MUL76:%.*]] = mul nsw i64 [[DIV74]], [[CONV75]]
-// CHECK2-NEXT:    [[ADD77:%.*]] = add nsw i64 [[CONV59]], [[MUL76]]
-// CHECK2-NEXT:    [[CONV78:%.*]] = trunc i64 [[ADD77]] to i32
-// CHECK2-NEXT:    store i32 [[CONV78]], ptr [[I49]], align 4
-// CHECK2-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[CONV79:%.*]] = sext i32 [[TMP60]] to i64
-// CHECK2-NEXT:    [[TMP61:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP62:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP63:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP64:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB80:%.*]] = sub i32 [[TMP63]], [[TMP64]]
-// CHECK2-NEXT:    [[SUB81:%.*]] = sub i32 [[SUB80]], 1
-// CHECK2-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD82:%.*]] = add i32 [[SUB81]], [[TMP65]]
-// CHECK2-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV83:%.*]] = udiv i32 [[ADD82]], [[TMP66]]
-// CHECK2-NEXT:    [[MUL84:%.*]] = mul i32 1, [[DIV83]]
-// CHECK2-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB85:%.*]] = sub i32 [[TMP67]], -31
-// CHECK2-NEXT:    [[DIV86:%.*]] = udiv i32 [[SUB85]], 32
-// CHECK2-NEXT:    [[MUL87:%.*]] = mul i32 [[MUL84]], [[DIV86]]
-// CHECK2-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB88:%.*]] = sub i32 [[TMP68]], [[TMP69]]
-// CHECK2-NEXT:    [[SUB89:%.*]] = sub i32 [[SUB88]], 1
-// CHECK2-NEXT:    [[ADD90:%.*]] = add i32 [[SUB89]], 1
-// CHECK2-NEXT:    [[DIV91:%.*]] = udiv i32 [[ADD90]], 1
-// CHECK2-NEXT:    [[MUL92:%.*]] = mul i32 [[MUL87]], [[DIV91]]
-// CHECK2-NEXT:    [[CONV93:%.*]] = zext i32 [[MUL92]] to i64
-// CHECK2-NEXT:    [[DIV94:%.*]] = sdiv i64 [[TMP62]], [[CONV93]]
+// CHECK2-NEXT:    [[DIV61:%.*]] = udiv i32 [[ADD60]], [[TMP54]]
+// CHECK2-NEXT:    [[MUL62:%.*]] = mul i32 1, [[DIV61]]
+// CHECK2-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB63:%.*]] = sub i32 [[TMP55]], -31
+// CHECK2-NEXT:    [[DIV64:%.*]] = udiv i32 [[SUB63]], 32
+// CHECK2-NEXT:    [[MUL65:%.*]] = mul i32 [[MUL62]], [[DIV64]]
+// CHECK2-NEXT:    [[MUL66:%.*]] = mul i32 [[MUL65]], 32
+// CHECK2-NEXT:    [[CONV67:%.*]] = zext i32 [[MUL66]] to i64
+// CHECK2-NEXT:    [[DIV68:%.*]] = sdiv i64 [[TMP50]], [[CONV67]]
+// CHECK2-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB69:%.*]] = sub i32 [[TMP56]], [[TMP57]]
+// CHECK2-NEXT:    [[SUB70:%.*]] = sub i32 [[SUB69]], 1
+// CHECK2-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD71:%.*]] = add i32 [[SUB70]], [[TMP58]]
+// CHECK2-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV72:%.*]] = udiv i32 [[ADD71]], [[TMP59]]
+// CHECK2-NEXT:    [[MUL73:%.*]] = mul i32 1, [[DIV72]]
+// CHECK2-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB74:%.*]] = sub i32 [[TMP60]], -31
+// CHECK2-NEXT:    [[DIV75:%.*]] = udiv i32 [[SUB74]], 32
+// CHECK2-NEXT:    [[MUL76:%.*]] = mul i32 [[MUL73]], [[DIV75]]
+// CHECK2-NEXT:    [[MUL77:%.*]] = mul i32 [[MUL76]], 32
+// CHECK2-NEXT:    [[CONV78:%.*]] = zext i32 [[MUL77]] to i64
+// CHECK2-NEXT:    [[MUL79:%.*]] = mul nsw i64 [[DIV68]], [[CONV78]]
+// CHECK2-NEXT:    [[SUB80:%.*]] = sub nsw i64 [[TMP49]], [[MUL79]]
+// CHECK2-NEXT:    [[TMP61:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB81:%.*]] = sub i32 [[TMP61]], -31
+// CHECK2-NEXT:    [[DIV82:%.*]] = udiv i32 [[SUB81]], 32
+// CHECK2-NEXT:    [[MUL83:%.*]] = mul i32 1, [[DIV82]]
+// CHECK2-NEXT:    [[MUL84:%.*]] = mul i32 [[MUL83]], 32
+// CHECK2-NEXT:    [[CONV85:%.*]] = zext i32 [[MUL84]] to i64
+// CHECK2-NEXT:    [[DIV86:%.*]] = sdiv i64 [[SUB80]], [[CONV85]]
+// CHECK2-NEXT:    [[TMP62:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[CONV87:%.*]] = sext i32 [[TMP62]] to i64
+// CHECK2-NEXT:    [[MUL88:%.*]] = mul nsw i64 [[DIV86]], [[CONV87]]
+// CHECK2-NEXT:    [[ADD89:%.*]] = add nsw i64 [[CONV57]], [[MUL88]]
+// CHECK2-NEXT:    [[CONV90:%.*]] = trunc i64 [[ADD89]] to i32
+// CHECK2-NEXT:    store i32 [[CONV90]], ptr [[J36]], align 4
+// CHECK2-NEXT:    [[TMP63:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP64:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB91:%.*]] = sub i32 [[TMP65]], [[TMP66]]
+// CHECK2-NEXT:    [[SUB92:%.*]] = sub i32 [[SUB91]], 1
+// CHECK2-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD93:%.*]] = add i32 [[SUB92]], [[TMP67]]
+// CHECK2-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV94:%.*]] = udiv i32 [[ADD93]], [[TMP68]]
+// CHECK2-NEXT:    [[MUL95:%.*]] = mul i32 1, [[DIV94]]
+// CHECK2-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB96:%.*]] = sub i32 [[TMP69]], -31
+// CHECK2-NEXT:    [[DIV97:%.*]] = udiv i32 [[SUB96]], 32
+// CHECK2-NEXT:    [[MUL98:%.*]] = mul i32 [[MUL95]], [[DIV97]]
+// CHECK2-NEXT:    [[MUL99:%.*]] = mul i32 [[MUL98]], 32
+// CHECK2-NEXT:    [[CONV100:%.*]] = zext i32 [[MUL99]] to i64
+// CHECK2-NEXT:    [[DIV101:%.*]] = sdiv i64 [[TMP64]], [[CONV100]]
 // CHECK2-NEXT:    [[TMP70:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
 // CHECK2-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB95:%.*]] = sub i32 [[TMP70]], [[TMP71]]
-// CHECK2-NEXT:    [[SUB96:%.*]] = sub i32 [[SUB95]], 1
+// CHECK2-NEXT:    [[SUB102:%.*]] = sub i32 [[TMP70]], [[TMP71]]
+// CHECK2-NEXT:    [[SUB103:%.*]] = sub i32 [[SUB102]], 1
 // CHECK2-NEXT:    [[TMP72:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD97:%.*]] = add i32 [[SUB96]], [[TMP72]]
+// CHECK2-NEXT:    [[ADD104:%.*]] = add i32 [[SUB103]], [[TMP72]]
 // CHECK2-NEXT:    [[TMP73:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV98:%.*]] = udiv i32 [[ADD97]], [[TMP73]]
-// CHECK2-NEXT:    [[MUL99:%.*]] = mul i32 1, [[DIV98]]
+// CHECK2-NEXT:    [[DIV105:%.*]] = udiv i32 [[ADD104]], [[TMP73]]
+// CHECK2-NEXT:    [[MUL106:%.*]] = mul i32 1, [[DIV105]]
 // CHECK2-NEXT:    [[TMP74:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB100:%.*]] = sub i32 [[TMP74]], -31
-// CHECK2-NEXT:    [[DIV101:%.*]] = udiv i32 [[SUB100]], 32
-// CHECK2-NEXT:    [[MUL102:%.*]] = mul i32 [[MUL99]], [[DIV101]]
-// CHECK2-NEXT:    [[TMP75:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP76:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB103:%.*]] = sub i32 [[TMP75]], [[TMP76]]
-// CHECK2-NEXT:    [[SUB104:%.*]] = sub i32 [[SUB103]], 1
-// CHECK2-NEXT:    [[ADD105:%.*]] = add i32 [[SUB104]], 1
-// CHECK2-NEXT:    [[DIV106:%.*]] = udiv i32 [[ADD105]], 1
-// CHECK2-NEXT:    [[MUL107:%.*]] = mul i32 [[MUL102]], [[DIV106]]
-// CHECK2-NEXT:    [[CONV108:%.*]] = zext i32 [[MUL107]] to i64
-// CHECK2-NEXT:    [[MUL109:%.*]] = mul nsw i64 [[DIV94]], [[CONV108]]
-// CHECK2-NEXT:    [[SUB110:%.*]] = sub nsw i64 [[TMP61]], [[MUL109]]
-// CHECK2-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB111:%.*]] = sub i32 [[TMP77]], -31
-// CHECK2-NEXT:    [[DIV112:%.*]] = udiv i32 [[SUB111]], 32
-// CHECK2-NEXT:    [[MUL113:%.*]] = mul i32 1, [[DIV112]]
-// CHECK2-NEXT:    [[TMP78:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP79:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB114:%.*]] = sub i32 [[TMP78]], [[TMP79]]
+// CHECK2-NEXT:    [[SUB107:%.*]] = sub i32 [[TMP74]], -31
+// CHECK2-NEXT:    [[DIV108:%.*]] = udiv i32 [[SUB107]], 32
+// CHECK2-NEXT:    [[MUL109:%.*]] = mul i32 [[MUL106]], [[DIV108]]
+// CHECK2-NEXT:    [[MUL110:%.*]] = mul i32 [[MUL109]], 32
+// CHECK2-NEXT:    [[CONV111:%.*]] = zext i32 [[MUL110]] to i64
+// CHECK2-NEXT:    [[MUL112:%.*]] = mul nsw i64 [[DIV101]], [[CONV111]]
+// CHECK2-NEXT:    [[SUB113:%.*]] = sub nsw i64 [[TMP63]], [[MUL112]]
+// CHECK2-NEXT:    [[TMP75:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP76:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP78:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB114:%.*]] = sub i32 [[TMP77]], [[TMP78]]
 // CHECK2-NEXT:    [[SUB115:%.*]] = sub i32 [[SUB114]], 1
-// CHECK2-NEXT:    [[ADD116:%.*]] = add i32 [[SUB115]], 1
-// CHECK2-NEXT:    [[DIV117:%.*]] = udiv i32 [[ADD116]], 1
-// CHECK2-NEXT:    [[MUL118:%.*]] = mul i32 [[MUL113]], [[DIV117]]
-// CHECK2-NEXT:    [[CONV119:%.*]] = zext i32 [[MUL118]] to i64
-// CHECK2-NEXT:    [[DIV120:%.*]] = sdiv i64 [[SUB110]], [[CONV119]]
+// CHECK2-NEXT:    [[TMP79:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD116:%.*]] = add i32 [[SUB115]], [[TMP79]]
 // CHECK2-NEXT:    [[TMP80:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[CONV121:%.*]] = sext i32 [[TMP80]] to i64
-// CHECK2-NEXT:    [[MUL122:%.*]] = mul nsw i64 [[DIV120]], [[CONV121]]
-// CHECK2-NEXT:    [[ADD123:%.*]] = add nsw i64 [[CONV79]], [[MUL122]]
-// CHECK2-NEXT:    [[CONV124:%.*]] = trunc i64 [[ADD123]] to i32
-// CHECK2-NEXT:    store i32 [[CONV124]], ptr [[J50]], align 4
-// CHECK2-NEXT:    [[TMP81:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP82:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB125:%.*]] = sub i32 [[TMP83]], [[TMP84]]
+// CHECK2-NEXT:    [[DIV117:%.*]] = udiv i32 [[ADD116]], [[TMP80]]
+// CHECK2-NEXT:    [[MUL118:%.*]] = mul i32 1, [[DIV117]]
+// CHECK2-NEXT:    [[TMP81:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB119:%.*]] = sub i32 [[TMP81]], -31
+// CHECK2-NEXT:    [[DIV120:%.*]] = udiv i32 [[SUB119]], 32
+// CHECK2-NEXT:    [[MUL121:%.*]] = mul i32 [[MUL118]], [[DIV120]]
+// CHECK2-NEXT:    [[MUL122:%.*]] = mul i32 [[MUL121]], 32
+// CHECK2-NEXT:    [[CONV123:%.*]] = zext i32 [[MUL122]] to i64
+// CHECK2-NEXT:    [[DIV124:%.*]] = sdiv i64 [[TMP76]], [[CONV123]]
+// CHECK2-NEXT:    [[TMP82:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB125:%.*]] = sub i32 [[TMP82]], [[TMP83]]
 // CHECK2-NEXT:    [[SUB126:%.*]] = sub i32 [[SUB125]], 1
+// CHECK2-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD127:%.*]] = add i32 [[SUB126]], [[TMP84]]
 // CHECK2-NEXT:    [[TMP85:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD127:%.*]] = add i32 [[SUB126]], [[TMP85]]
-// CHECK2-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV128:%.*]] = udiv i32 [[ADD127]], [[TMP86]]
+// CHECK2-NEXT:    [[DIV128:%.*]] = udiv i32 [[ADD127]], [[TMP85]]
 // CHECK2-NEXT:    [[MUL129:%.*]] = mul i32 1, [[DIV128]]
-// CHECK2-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB130:%.*]] = sub i32 [[TMP87]], -31
+// CHECK2-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB130:%.*]] = sub i32 [[TMP86]], -31
 // CHECK2-NEXT:    [[DIV131:%.*]] = udiv i32 [[SUB130]], 32
 // CHECK2-NEXT:    [[MUL132:%.*]] = mul i32 [[MUL129]], [[DIV131]]
-// CHECK2-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP89:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB133:%.*]] = sub i32 [[TMP88]], [[TMP89]]
-// CHECK2-NEXT:    [[SUB134:%.*]] = sub i32 [[SUB133]], 1
-// CHECK2-NEXT:    [[ADD135:%.*]] = add i32 [[SUB134]], 1
-// CHECK2-NEXT:    [[DIV136:%.*]] = udiv i32 [[ADD135]], 1
-// CHECK2-NEXT:    [[MUL137:%.*]] = mul i32 [[MUL132]], [[DIV136]]
-// CHECK2-NEXT:    [[CONV138:%.*]] = zext i32 [[MUL137]] to i64
-// CHECK2-NEXT:    [[DIV139:%.*]] = sdiv i64 [[TMP82]], [[CONV138]]
-// CHECK2-NEXT:    [[TMP90:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB140:%.*]] = sub i32 [[TMP90]], [[TMP91]]
-// CHECK2-NEXT:    [[SUB141:%.*]] = sub i32 [[SUB140]], 1
-// CHECK2-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD142:%.*]] = add i32 [[SUB141]], [[TMP92]]
+// CHECK2-NEXT:    [[MUL133:%.*]] = mul i32 [[MUL132]], 32
+// CHECK2-NEXT:    [[CONV134:%.*]] = zext i32 [[MUL133]] to i64
+// CHECK2-NEXT:    [[MUL135:%.*]] = mul nsw i64 [[DIV124]], [[CONV134]]
+// CHECK2-NEXT:    [[SUB136:%.*]] = sub nsw i64 [[TMP75]], [[MUL135]]
+// CHECK2-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB137:%.*]] = sub i32 [[TMP87]], -31
+// CHECK2-NEXT:    [[DIV138:%.*]] = udiv i32 [[SUB137]], 32
+// CHECK2-NEXT:    [[MUL139:%.*]] = mul i32 1, [[DIV138]]
+// CHECK2-NEXT:    [[MUL140:%.*]] = mul i32 [[MUL139]], 32
+// CHECK2-NEXT:    [[CONV141:%.*]] = zext i32 [[MUL140]] to i64
+// CHECK2-NEXT:    [[DIV142:%.*]] = sdiv i64 [[SUB136]], [[CONV141]]
+// CHECK2-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB143:%.*]] = sub i32 [[TMP88]], -31
+// CHECK2-NEXT:    [[DIV144:%.*]] = udiv i32 [[SUB143]], 32
+// CHECK2-NEXT:    [[MUL145:%.*]] = mul i32 1, [[DIV144]]
+// CHECK2-NEXT:    [[MUL146:%.*]] = mul i32 [[MUL145]], 32
+// CHECK2-NEXT:    [[CONV147:%.*]] = zext i32 [[MUL146]] to i64
+// CHECK2-NEXT:    [[MUL148:%.*]] = mul nsw i64 [[DIV142]], [[CONV147]]
+// CHECK2-NEXT:    [[SUB149:%.*]] = sub nsw i64 [[SUB113]], [[MUL148]]
+// CHECK2-NEXT:    [[DIV150:%.*]] = sdiv i64 [[SUB149]], 32
+// CHECK2-NEXT:    [[MUL151:%.*]] = mul nsw i64 [[DIV150]], 32
+// CHECK2-NEXT:    [[ADD152:%.*]] = add nsw i64 0, [[MUL151]]
+// CHECK2-NEXT:    [[CONV153:%.*]] = trunc i64 [[ADD152]] to i32
+// CHECK2-NEXT:    store i32 [[CONV153]], ptr [[DOTFLOOR_0_IV_K37]], align 4
+// CHECK2-NEXT:    [[TMP89:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP90:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB154:%.*]] = sub i32 [[TMP91]], [[TMP92]]
+// CHECK2-NEXT:    [[SUB155:%.*]] = sub i32 [[SUB154]], 1
 // CHECK2-NEXT:    [[TMP93:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV143:%.*]] = udiv i32 [[ADD142]], [[TMP93]]
-// CHECK2-NEXT:    [[MUL144:%.*]] = mul i32 1, [[DIV143]]
-// CHECK2-NEXT:    [[TMP94:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB145:%.*]] = sub i32 [[TMP94]], -31
-// CHECK2-NEXT:    [[DIV146:%.*]] = udiv i32 [[SUB145]], 32
-// CHECK2-NEXT:    [[MUL147:%.*]] = mul i32 [[MUL144]], [[DIV146]]
-// CHECK2-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB148:%.*]] = sub i32 [[TMP95]], [[TMP96]]
-// CHECK2-NEXT:    [[SUB149:%.*]] = sub i32 [[SUB148]], 1
-// CHECK2-NEXT:    [[ADD150:%.*]] = add i32 [[SUB149]], 1
-// CHECK2-NEXT:    [[DIV151:%.*]] = udiv i32 [[ADD150]], 1
-// CHECK2-NEXT:    [[MUL152:%.*]] = mul i32 [[MUL147]], [[DIV151]]
-// CHECK2-NEXT:    [[CONV153:%.*]] = zext i32 [[MUL152]] to i64
-// CHECK2-NEXT:    [[MUL154:%.*]] = mul nsw i64 [[DIV139]], [[CONV153]]
-// CHECK2-NEXT:    [[SUB155:%.*]] = sub nsw i64 [[TMP81]], [[MUL154]]
-// CHECK2-NEXT:    [[TMP97:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP98:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB156:%.*]] = sub i32 [[TMP99]], [[TMP100]]
-// CHECK2-NEXT:    [[SUB157:%.*]] = sub i32 [[SUB156]], 1
-// CHECK2-NEXT:    [[TMP101:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD158:%.*]] = add i32 [[SUB157]], [[TMP101]]
-// CHECK2-NEXT:    [[TMP102:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV159:%.*]] = udiv i32 [[ADD158]], [[TMP102]]
-// CHECK2-NEXT:    [[MUL160:%.*]] = mul i32 1, [[DIV159]]
-// CHECK2-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB161:%.*]] = sub i32 [[TMP103]], -31
-// CHECK2-NEXT:    [[DIV162:%.*]] = udiv i32 [[SUB161]], 32
-// CHECK2-NEXT:    [[MUL163:%.*]] = mul i32 [[MUL160]], [[DIV162]]
-// CHECK2-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP105:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB164:%.*]] = sub i32 [[TMP104]], [[TMP105]]
-// CHECK2-NEXT:    [[SUB165:%.*]] = sub i32 [[SUB164]], 1
-// CHECK2-NEXT:    [[ADD166:%.*]] = add i32 [[SUB165]], 1
-// CHECK2-NEXT:    [[DIV167:%.*]] = udiv i32 [[ADD166]], 1
-// CHECK2-NEXT:    [[MUL168:%.*]] = mul i32 [[MUL163]], [[DIV167]]
-// CHECK2-NEXT:    [[CONV169:%.*]] = zext i32 [[MUL168]] to i64
-// CHECK2-NEXT:    [[DIV170:%.*]] = sdiv i64 [[TMP98]], [[CONV169]]
-// CHECK2-NEXT:    [[TMP106:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB171:%.*]] = sub i32 [[TMP106]], [[TMP107]]
-// CHECK2-NEXT:    [[SUB172:%.*]] = sub i32 [[SUB171]], 1
-// CHECK2-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD173:%.*]] = add i32 [[SUB172]], [[TMP108]]
-// CHECK2-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV174:%.*]] = udiv i32 [[ADD173]], [[TMP109]]
-// CHECK2-NEXT:    [[MUL175:%.*]] = mul i32 1, [[DIV174]]
-// CHECK2-NEXT:    [[TMP110:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB176:%.*]] = sub i32 [[TMP110]], -31
-// CHECK2-NEXT:    [[DIV177:%.*]] = udiv i32 [[SUB176]], 32
-// CHECK2-NEXT:    [[MUL178:%.*]] = mul i32 [[MUL175]], [[DIV177]]
-// CHECK2-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB179:%.*]] = sub i32 [[TMP111]], [[TMP112]]
-// CHECK2-NEXT:    [[SUB180:%.*]] = sub i32 [[SUB179]], 1
-// CHECK2-NEXT:    [[ADD181:%.*]] = add i32 [[SUB180]], 1
-// CHECK2-NEXT:    [[DIV182:%.*]] = udiv i32 [[ADD181]], 1
-// CHECK2-NEXT:    [[MUL183:%.*]] = mul i32 [[MUL178]], [[DIV182]]
-// CHECK2-NEXT:    [[CONV184:%.*]] = zext i32 [[MUL183]] to i64
-// CHECK2-NEXT:    [[MUL185:%.*]] = mul nsw i64 [[DIV170]], [[CONV184]]
-// CHECK2-NEXT:    [[SUB186:%.*]] = sub nsw i64 [[TMP97]], [[MUL185]]
+// CHECK2-NEXT:    [[ADD156:%.*]] = add i32 [[SUB155]], [[TMP93]]
+// CHECK2-NEXT:    [[TMP94:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV157:%.*]] = udiv i32 [[ADD156]], [[TMP94]]
+// CHECK2-NEXT:    [[MUL158:%.*]] = mul i32 1, [[DIV157]]
+// CHECK2-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB159:%.*]] = sub i32 [[TMP95]], -31
+// CHECK2-NEXT:    [[DIV160:%.*]] = udiv i32 [[SUB159]], 32
+// CHECK2-NEXT:    [[MUL161:%.*]] = mul i32 [[MUL158]], [[DIV160]]
+// CHECK2-NEXT:    [[MUL162:%.*]] = mul i32 [[MUL161]], 32
+// CHECK2-NEXT:    [[CONV163:%.*]] = zext i32 [[MUL162]] to i64
+// CHECK2-NEXT:    [[DIV164:%.*]] = sdiv i64 [[TMP90]], [[CONV163]]
+// CHECK2-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP97:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB165:%.*]] = sub i32 [[TMP96]], [[TMP97]]
+// CHECK2-NEXT:    [[SUB166:%.*]] = sub i32 [[SUB165]], 1
+// CHECK2-NEXT:    [[TMP98:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD167:%.*]] = add i32 [[SUB166]], [[TMP98]]
+// CHECK2-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV168:%.*]] = udiv i32 [[ADD167]], [[TMP99]]
+// CHECK2-NEXT:    [[MUL169:%.*]] = mul i32 1, [[DIV168]]
+// CHECK2-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB170:%.*]] = sub i32 [[TMP100]], -31
+// CHECK2-NEXT:    [[DIV171:%.*]] = udiv i32 [[SUB170]], 32
+// CHECK2-NEXT:    [[MUL172:%.*]] = mul i32 [[MUL169]], [[DIV171]]
+// CHECK2-NEXT:    [[MUL173:%.*]] = mul i32 [[MUL172]], 32
+// CHECK2-NEXT:    [[CONV174:%.*]] = zext i32 [[MUL173]] to i64
+// CHECK2-NEXT:    [[MUL175:%.*]] = mul nsw i64 [[DIV164]], [[CONV174]]
+// CHECK2-NEXT:    [[SUB176:%.*]] = sub nsw i64 [[TMP89]], [[MUL175]]
+// CHECK2-NEXT:    [[TMP101:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP102:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB177:%.*]] = sub i32 [[TMP103]], [[TMP104]]
+// CHECK2-NEXT:    [[SUB178:%.*]] = sub i32 [[SUB177]], 1
+// CHECK2-NEXT:    [[TMP105:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD179:%.*]] = add i32 [[SUB178]], [[TMP105]]
+// CHECK2-NEXT:    [[TMP106:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV180:%.*]] = udiv i32 [[ADD179]], [[TMP106]]
+// CHECK2-NEXT:    [[MUL181:%.*]] = mul i32 1, [[DIV180]]
+// CHECK2-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB182:%.*]] = sub i32 [[TMP107]], -31
+// CHECK2-NEXT:    [[DIV183:%.*]] = udiv i32 [[SUB182]], 32
+// CHECK2-NEXT:    [[MUL184:%.*]] = mul i32 [[MUL181]], [[DIV183]]
+// CHECK2-NEXT:    [[MUL185:%.*]] = mul i32 [[MUL184]], 32
+// CHECK2-NEXT:    [[CONV186:%.*]] = zext i32 [[MUL185]] to i64
+// CHECK2-NEXT:    [[DIV187:%.*]] = sdiv i64 [[TMP102]], [[CONV186]]
+// CHECK2-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB188:%.*]] = sub i32 [[TMP108]], [[TMP109]]
+// CHECK2-NEXT:    [[SUB189:%.*]] = sub i32 [[SUB188]], 1
+// CHECK2-NEXT:    [[TMP110:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD190:%.*]] = add i32 [[SUB189]], [[TMP110]]
+// CHECK2-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV191:%.*]] = udiv i32 [[ADD190]], [[TMP111]]
+// CHECK2-NEXT:    [[MUL192:%.*]] = mul i32 1, [[DIV191]]
+// CHECK2-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB193:%.*]] = sub i32 [[TMP112]], -31
+// CHECK2-NEXT:    [[DIV194:%.*]] = udiv i32 [[SUB193]], 32
+// CHECK2-NEXT:    [[MUL195:%.*]] = mul i32 [[MUL192]], [[DIV194]]
+// CHECK2-NEXT:    [[MUL196:%.*]] = mul i32 [[MUL195]], 32
+// CHECK2-NEXT:    [[CONV197:%.*]] = zext i32 [[MUL196]] to i64
+// CHECK2-NEXT:    [[MUL198:%.*]] = mul nsw i64 [[DIV187]], [[CONV197]]
+// CHECK2-NEXT:    [[SUB199:%.*]] = sub nsw i64 [[TMP101]], [[MUL198]]
 // CHECK2-NEXT:    [[TMP113:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB187:%.*]] = sub i32 [[TMP113]], -31
-// CHECK2-NEXT:    [[DIV188:%.*]] = udiv i32 [[SUB187]], 32
-// CHECK2-NEXT:    [[MUL189:%.*]] = mul i32 1, [[DIV188]]
-// CHECK2-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP115:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB190:%.*]] = sub i32 [[TMP114]], [[TMP115]]
-// CHECK2-NEXT:    [[SUB191:%.*]] = sub i32 [[SUB190]], 1
-// CHECK2-NEXT:    [[ADD192:%.*]] = add i32 [[SUB191]], 1
-// CHECK2-NEXT:    [[DIV193:%.*]] = udiv i32 [[ADD192]], 1
-// CHECK2-NEXT:    [[MUL194:%.*]] = mul i32 [[MUL189]], [[DIV193]]
-// CHECK2-NEXT:    [[CONV195:%.*]] = zext i32 [[MUL194]] to i64
-// CHECK2-NEXT:    [[DIV196:%.*]] = sdiv i64 [[SUB186]], [[CONV195]]
-// CHECK2-NEXT:    [[TMP116:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB197:%.*]] = sub i32 [[TMP116]], -31
-// CHECK2-NEXT:    [[DIV198:%.*]] = udiv i32 [[SUB197]], 32
-// CHECK2-NEXT:    [[MUL199:%.*]] = mul i32 1, [[DIV198]]
-// CHECK2-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB200:%.*]] = sub i32 [[TMP117]], [[TMP118]]
-// CHECK2-NEXT:    [[SUB201:%.*]] = sub i32 [[SUB200]], 1
-// CHECK2-NEXT:    [[ADD202:%.*]] = add i32 [[SUB201]], 1
-// CHECK2-NEXT:    [[DIV203:%.*]] = udiv i32 [[ADD202]], 1
-// CHECK2-NEXT:    [[MUL204:%.*]] = mul i32 [[MUL199]], [[DIV203]]
-// CHECK2-NEXT:    [[CONV205:%.*]] = zext i32 [[MUL204]] to i64
-// CHECK2-NEXT:    [[MUL206:%.*]] = mul nsw i64 [[DIV196]], [[CONV205]]
-// CHECK2-NEXT:    [[SUB207:%.*]] = sub nsw i64 [[SUB155]], [[MUL206]]
-// CHECK2-NEXT:    [[TMP119:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP120:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB208:%.*]] = sub i32 [[TMP119]], [[TMP120]]
-// CHECK2-NEXT:    [[SUB209:%.*]] = sub i32 [[SUB208]], 1
-// CHECK2-NEXT:    [[ADD210:%.*]] = add i32 [[SUB209]], 1
-// CHECK2-NEXT:    [[DIV211:%.*]] = udiv i32 [[ADD210]], 1
-// CHECK2-NEXT:    [[MUL212:%.*]] = mul i32 1, [[DIV211]]
-// CHECK2-NEXT:    [[CONV213:%.*]] = zext i32 [[MUL212]] to i64
-// CHECK2-NEXT:    [[DIV214:%.*]] = sdiv i64 [[SUB207]], [[CONV213]]
-// CHECK2-NEXT:    [[MUL215:%.*]] = mul nsw i64 [[DIV214]], 32
-// CHECK2-NEXT:    [[ADD216:%.*]] = add nsw i64 0, [[MUL215]]
-// CHECK2-NEXT:    [[CONV217:%.*]] = trunc i64 [[ADD216]] to i32
-// CHECK2-NEXT:    store i32 [[CONV217]], ptr [[DOTFLOOR_0_IV_K51]], align 4
-// CHECK2-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[CONV218:%.*]] = zext i32 [[TMP121]] to i64
-// CHECK2-NEXT:    [[TMP122:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP123:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP124:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB219:%.*]] = sub i32 [[TMP124]], [[TMP125]]
-// CHECK2-NEXT:    [[SUB220:%.*]] = sub i32 [[SUB219]], 1
-// CHECK2-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD221:%.*]] = add i32 [[SUB220]], [[TMP126]]
-// CHECK2-NEXT:    [[TMP127:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV222:%.*]] = udiv i32 [[ADD221]], [[TMP127]]
-// CHECK2-NEXT:    [[MUL223:%.*]] = mul i32 1, [[DIV222]]
-// CHECK2-NEXT:    [[TMP128:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB224:%.*]] = sub i32 [[TMP128]], -31
-// CHECK2-NEXT:    [[DIV225:%.*]] = udiv i32 [[SUB224]], 32
-// CHECK2-NEXT:    [[MUL226:%.*]] = mul i32 [[MUL223]], [[DIV225]]
-// CHECK2-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB227:%.*]] = sub i32 [[TMP129]], [[TMP130]]
-// CHECK2-NEXT:    [[SUB228:%.*]] = sub i32 [[SUB227]], 1
-// CHECK2-NEXT:    [[ADD229:%.*]] = add i32 [[SUB228]], 1
-// CHECK2-NEXT:    [[DIV230:%.*]] = udiv i32 [[ADD229]], 1
-// CHECK2-NEXT:    [[MUL231:%.*]] = mul i32 [[MUL226]], [[DIV230]]
-// CHECK2-NEXT:    [[CONV232:%.*]] = zext i32 [[MUL231]] to i64
-// CHECK2-NEXT:    [[DIV233:%.*]] = sdiv i64 [[TMP123]], [[CONV232]]
-// CHECK2-NEXT:    [[TMP131:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP132:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB234:%.*]] = sub i32 [[TMP131]], [[TMP132]]
-// CHECK2-NEXT:    [[SUB235:%.*]] = sub i32 [[SUB234]], 1
-// CHECK2-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD236:%.*]] = add i32 [[SUB235]], [[TMP133]]
-// CHECK2-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV237:%.*]] = udiv i32 [[ADD236]], [[TMP134]]
-// CHECK2-NEXT:    [[MUL238:%.*]] = mul i32 1, [[DIV237]]
-// CHECK2-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB239:%.*]] = sub i32 [[TMP135]], -31
-// CHECK2-NEXT:    [[DIV240:%.*]] = udiv i32 [[SUB239]], 32
-// CHECK2-NEXT:    [[MUL241:%.*]] = mul i32 [[MUL238]], [[DIV240]]
-// CHECK2-NEXT:    [[TMP136:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB242:%.*]] = sub i32 [[TMP136]], [[TMP137]]
-// CHECK2-NEXT:    [[SUB243:%.*]] = sub i32 [[SUB242]], 1
-// CHECK2-NEXT:    [[ADD244:%.*]] = add i32 [[SUB243]], 1
-// CHECK2-NEXT:    [[DIV245:%.*]] = udiv i32 [[ADD244]], 1
-// CHECK2-NEXT:    [[MUL246:%.*]] = mul i32 [[MUL241]], [[DIV245]]
-// CHECK2-NEXT:    [[CONV247:%.*]] = zext i32 [[MUL246]] to i64
-// CHECK2-NEXT:    [[MUL248:%.*]] = mul nsw i64 [[DIV233]], [[CONV247]]
-// CHECK2-NEXT:    [[SUB249:%.*]] = sub nsw i64 [[TMP122]], [[MUL248]]
-// CHECK2-NEXT:    [[TMP138:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP139:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB250:%.*]] = sub i32 [[TMP140]], [[TMP141]]
-// CHECK2-NEXT:    [[SUB251:%.*]] = sub i32 [[SUB250]], 1
-// CHECK2-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD252:%.*]] = add i32 [[SUB251]], [[TMP142]]
-// CHECK2-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV253:%.*]] = udiv i32 [[ADD252]], [[TMP143]]
-// CHECK2-NEXT:    [[MUL254:%.*]] = mul i32 1, [[DIV253]]
-// CHECK2-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB255:%.*]] = sub i32 [[TMP144]], -31
-// CHECK2-NEXT:    [[DIV256:%.*]] = udiv i32 [[SUB255]], 32
-// CHECK2-NEXT:    [[MUL257:%.*]] = mul i32 [[MUL254]], [[DIV256]]
-// CHECK2-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB258:%.*]] = sub i32 [[TMP145]], [[TMP146]]
-// CHECK2-NEXT:    [[SUB259:%.*]] = sub i32 [[SUB258]], 1
-// CHECK2-NEXT:    [[ADD260:%.*]] = add i32 [[SUB259]], 1
-// CHECK2-NEXT:    [[DIV261:%.*]] = udiv i32 [[ADD260]], 1
-// CHECK2-NEXT:    [[MUL262:%.*]] = mul i32 [[MUL257]], [[DIV261]]
+// CHECK2-NEXT:    [[SUB200:%.*]] = sub i32 [[TMP113]], -31
+// CHECK2-NEXT:    [[DIV201:%.*]] = udiv i32 [[SUB200]], 32
+// CHECK2-NEXT:    [[MUL202:%.*]] = mul i32 1, [[DIV201]]
+// CHECK2-NEXT:    [[MUL203:%.*]] = mul i32 [[MUL202]], 32
+// CHECK2-NEXT:    [[CONV204:%.*]] = zext i32 [[MUL203]] to i64
+// CHECK2-NEXT:    [[DIV205:%.*]] = sdiv i64 [[SUB199]], [[CONV204]]
+// CHECK2-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB206:%.*]] = sub i32 [[TMP114]], -31
+// CHECK2-NEXT:    [[DIV207:%.*]] = udiv i32 [[SUB206]], 32
+// CHECK2-NEXT:    [[MUL208:%.*]] = mul i32 1, [[DIV207]]
+// CHECK2-NEXT:    [[MUL209:%.*]] = mul i32 [[MUL208]], 32
+// CHECK2-NEXT:    [[CONV210:%.*]] = zext i32 [[MUL209]] to i64
+// CHECK2-NEXT:    [[MUL211:%.*]] = mul nsw i64 [[DIV205]], [[CONV210]]
+// CHECK2-NEXT:    [[SUB212:%.*]] = sub nsw i64 [[SUB176]], [[MUL211]]
+// CHECK2-NEXT:    [[TMP115:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP116:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB213:%.*]] = sub i32 [[TMP117]], [[TMP118]]
+// CHECK2-NEXT:    [[SUB214:%.*]] = sub i32 [[SUB213]], 1
+// CHECK2-NEXT:    [[TMP119:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD215:%.*]] = add i32 [[SUB214]], [[TMP119]]
+// CHECK2-NEXT:    [[TMP120:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV216:%.*]] = udiv i32 [[ADD215]], [[TMP120]]
+// CHECK2-NEXT:    [[MUL217:%.*]] = mul i32 1, [[DIV216]]
+// CHECK2-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB218:%.*]] = sub i32 [[TMP121]], -31
+// CHECK2-NEXT:    [[DIV219:%.*]] = udiv i32 [[SUB218]], 32
+// CHECK2-NEXT:    [[MUL220:%.*]] = mul i32 [[MUL217]], [[DIV219]]
+// CHECK2-NEXT:    [[MUL221:%.*]] = mul i32 [[MUL220]], 32
+// CHECK2-NEXT:    [[CONV222:%.*]] = zext i32 [[MUL221]] to i64
+// CHECK2-NEXT:    [[DIV223:%.*]] = sdiv i64 [[TMP116]], [[CONV222]]
+// CHECK2-NEXT:    [[TMP122:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP123:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB224:%.*]] = sub i32 [[TMP122]], [[TMP123]]
+// CHECK2-NEXT:    [[SUB225:%.*]] = sub i32 [[SUB224]], 1
+// CHECK2-NEXT:    [[TMP124:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD226:%.*]] = add i32 [[SUB225]], [[TMP124]]
+// CHECK2-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV227:%.*]] = udiv i32 [[ADD226]], [[TMP125]]
+// CHECK2-NEXT:    [[MUL228:%.*]] = mul i32 1, [[DIV227]]
+// CHECK2-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB229:%.*]] = sub i32 [[TMP126]], -31
+// CHECK2-NEXT:    [[DIV230:%.*]] = udiv i32 [[SUB229]], 32
+// CHECK2-NEXT:    [[MUL231:%.*]] = mul i32 [[MUL228]], [[DIV230]]
+// CHECK2-NEXT:    [[MUL232:%.*]] = mul i32 [[MUL231]], 32
+// CHECK2-NEXT:    [[CONV233:%.*]] = zext i32 [[MUL232]] to i64
+// CHECK2-NEXT:    [[MUL234:%.*]] = mul nsw i64 [[DIV223]], [[CONV233]]
+// CHECK2-NEXT:    [[SUB235:%.*]] = sub nsw i64 [[TMP115]], [[MUL234]]
+// CHECK2-NEXT:    [[TMP127:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP128:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB236:%.*]] = sub i32 [[TMP129]], [[TMP130]]
+// CHECK2-NEXT:    [[SUB237:%.*]] = sub i32 [[SUB236]], 1
+// CHECK2-NEXT:    [[TMP131:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD238:%.*]] = add i32 [[SUB237]], [[TMP131]]
+// CHECK2-NEXT:    [[TMP132:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV239:%.*]] = udiv i32 [[ADD238]], [[TMP132]]
+// CHECK2-NEXT:    [[MUL240:%.*]] = mul i32 1, [[DIV239]]
+// CHECK2-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB241:%.*]] = sub i32 [[TMP133]], -31
+// CHECK2-NEXT:    [[DIV242:%.*]] = udiv i32 [[SUB241]], 32
+// CHECK2-NEXT:    [[MUL243:%.*]] = mul i32 [[MUL240]], [[DIV242]]
+// CHECK2-NEXT:    [[MUL244:%.*]] = mul i32 [[MUL243]], 32
+// CHECK2-NEXT:    [[CONV245:%.*]] = zext i32 [[MUL244]] to i64
+// CHECK2-NEXT:    [[DIV246:%.*]] = sdiv i64 [[TMP128]], [[CONV245]]
+// CHECK2-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB247:%.*]] = sub i32 [[TMP134]], [[TMP135]]
+// CHECK2-NEXT:    [[SUB248:%.*]] = sub i32 [[SUB247]], 1
+// CHECK2-NEXT:    [[TMP136:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD249:%.*]] = add i32 [[SUB248]], [[TMP136]]
+// CHECK2-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV250:%.*]] = udiv i32 [[ADD249]], [[TMP137]]
+// CHECK2-NEXT:    [[MUL251:%.*]] = mul i32 1, [[DIV250]]
+// CHECK2-NEXT:    [[TMP138:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB252:%.*]] = sub i32 [[TMP138]], -31
+// CHECK2-NEXT:    [[DIV253:%.*]] = udiv i32 [[SUB252]], 32
+// CHECK2-NEXT:    [[MUL254:%.*]] = mul i32 [[MUL251]], [[DIV253]]
+// CHECK2-NEXT:    [[MUL255:%.*]] = mul i32 [[MUL254]], 32
+// CHECK2-NEXT:    [[CONV256:%.*]] = zext i32 [[MUL255]] to i64
+// CHECK2-NEXT:    [[MUL257:%.*]] = mul nsw i64 [[DIV246]], [[CONV256]]
+// CHECK2-NEXT:    [[SUB258:%.*]] = sub nsw i64 [[TMP127]], [[MUL257]]
+// CHECK2-NEXT:    [[TMP139:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB259:%.*]] = sub i32 [[TMP139]], -31
+// CHECK2-NEXT:    [[DIV260:%.*]] = udiv i32 [[SUB259]], 32
+// CHECK2-NEXT:    [[MUL261:%.*]] = mul i32 1, [[DIV260]]
+// CHECK2-NEXT:    [[MUL262:%.*]] = mul i32 [[MUL261]], 32
 // CHECK2-NEXT:    [[CONV263:%.*]] = zext i32 [[MUL262]] to i64
-// CHECK2-NEXT:    [[DIV264:%.*]] = sdiv i64 [[TMP139]], [[CONV263]]
-// CHECK2-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB265:%.*]] = sub i32 [[TMP147]], [[TMP148]]
-// CHECK2-NEXT:    [[SUB266:%.*]] = sub i32 [[SUB265]], 1
-// CHECK2-NEXT:    [[TMP149:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD267:%.*]] = add i32 [[SUB266]], [[TMP149]]
-// CHECK2-NEXT:    [[TMP150:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV268:%.*]] = udiv i32 [[ADD267]], [[TMP150]]
-// CHECK2-NEXT:    [[MUL269:%.*]] = mul i32 1, [[DIV268]]
-// CHECK2-NEXT:    [[TMP151:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB270:%.*]] = sub i32 [[TMP151]], -31
-// CHECK2-NEXT:    [[DIV271:%.*]] = udiv i32 [[SUB270]], 32
-// CHECK2-NEXT:    [[MUL272:%.*]] = mul i32 [[MUL269]], [[DIV271]]
-// CHECK2-NEXT:    [[TMP152:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP153:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB273:%.*]] = sub i32 [[TMP152]], [[TMP153]]
-// CHECK2-NEXT:    [[SUB274:%.*]] = sub i32 [[SUB273]], 1
-// CHECK2-NEXT:    [[ADD275:%.*]] = add i32 [[SUB274]], 1
-// CHECK2-NEXT:    [[DIV276:%.*]] = udiv i32 [[ADD275]], 1
-// CHECK2-NEXT:    [[MUL277:%.*]] = mul i32 [[MUL272]], [[DIV276]]
-// CHECK2-NEXT:    [[CONV278:%.*]] = zext i32 [[MUL277]] to i64
-// CHECK2-NEXT:    [[MUL279:%.*]] = mul nsw i64 [[DIV264]], [[CONV278]]
-// CHECK2-NEXT:    [[SUB280:%.*]] = sub nsw i64 [[TMP138]], [[MUL279]]
-// CHECK2-NEXT:    [[TMP154:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB281:%.*]] = sub i32 [[TMP154]], -31
-// CHECK2-NEXT:    [[DIV282:%.*]] = udiv i32 [[SUB281]], 32
-// CHECK2-NEXT:    [[MUL283:%.*]] = mul i32 1, [[DIV282]]
-// CHECK2-NEXT:    [[TMP155:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP156:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB284:%.*]] = sub i32 [[TMP155]], [[TMP156]]
-// CHECK2-NEXT:    [[SUB285:%.*]] = sub i32 [[SUB284]], 1
-// CHECK2-NEXT:    [[ADD286:%.*]] = add i32 [[SUB285]], 1
-// CHECK2-NEXT:    [[DIV287:%.*]] = udiv i32 [[ADD286]], 1
-// CHECK2-NEXT:    [[MUL288:%.*]] = mul i32 [[MUL283]], [[DIV287]]
-// CHECK2-NEXT:    [[CONV289:%.*]] = zext i32 [[MUL288]] to i64
-// CHECK2-NEXT:    [[DIV290:%.*]] = sdiv i64 [[SUB280]], [[CONV289]]
-// CHECK2-NEXT:    [[TMP157:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB291:%.*]] = sub i32 [[TMP157]], -31
-// CHECK2-NEXT:    [[DIV292:%.*]] = udiv i32 [[SUB291]], 32
-// CHECK2-NEXT:    [[MUL293:%.*]] = mul i32 1, [[DIV292]]
-// CHECK2-NEXT:    [[TMP158:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP159:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB294:%.*]] = sub i32 [[TMP158]], [[TMP159]]
-// CHECK2-NEXT:    [[SUB295:%.*]] = sub i32 [[SUB294]], 1
-// CHECK2-NEXT:    [[ADD296:%.*]] = add i32 [[SUB295]], 1
-// CHECK2-NEXT:    [[DIV297:%.*]] = udiv i32 [[ADD296]], 1
-// CHECK2-NEXT:    [[MUL298:%.*]] = mul i32 [[MUL293]], [[DIV297]]
-// CHECK2-NEXT:    [[CONV299:%.*]] = zext i32 [[MUL298]] to i64
-// CHECK2-NEXT:    [[MUL300:%.*]] = mul nsw i64 [[DIV290]], [[CONV299]]
-// CHECK2-NEXT:    [[SUB301:%.*]] = sub nsw i64 [[SUB249]], [[MUL300]]
-// CHECK2-NEXT:    [[TMP160:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP161:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP162:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP163:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB302:%.*]] = sub i32 [[TMP162]], [[TMP163]]
-// CHECK2-NEXT:    [[SUB303:%.*]] = sub i32 [[SUB302]], 1
-// CHECK2-NEXT:    [[TMP164:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD304:%.*]] = add i32 [[SUB303]], [[TMP164]]
-// CHECK2-NEXT:    [[TMP165:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV305:%.*]] = udiv i32 [[ADD304]], [[TMP165]]
-// CHECK2-NEXT:    [[MUL306:%.*]] = mul i32 1, [[DIV305]]
-// CHECK2-NEXT:    [[TMP166:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB307:%.*]] = sub i32 [[TMP166]], -31
-// CHECK2-NEXT:    [[DIV308:%.*]] = udiv i32 [[SUB307]], 32
-// CHECK2-NEXT:    [[MUL309:%.*]] = mul i32 [[MUL306]], [[DIV308]]
-// CHECK2-NEXT:    [[TMP167:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP168:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB310:%.*]] = sub i32 [[TMP167]], [[TMP168]]
-// CHECK2-NEXT:    [[SUB311:%.*]] = sub i32 [[SUB310]], 1
-// CHECK2-NEXT:    [[ADD312:%.*]] = add i32 [[SUB311]], 1
-// CHECK2-NEXT:    [[DIV313:%.*]] = udiv i32 [[ADD312]], 1
-// CHECK2-NEXT:    [[MUL314:%.*]] = mul i32 [[MUL309]], [[DIV313]]
-// CHECK2-NEXT:    [[CONV315:%.*]] = zext i32 [[MUL314]] to i64
-// CHECK2-NEXT:    [[DIV316:%.*]] = sdiv i64 [[TMP161]], [[CONV315]]
-// CHECK2-NEXT:    [[TMP169:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP170:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB317:%.*]] = sub i32 [[TMP169]], [[TMP170]]
-// CHECK2-NEXT:    [[SUB318:%.*]] = sub i32 [[SUB317]], 1
-// CHECK2-NEXT:    [[TMP171:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD319:%.*]] = add i32 [[SUB318]], [[TMP171]]
-// CHECK2-NEXT:    [[TMP172:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV320:%.*]] = udiv i32 [[ADD319]], [[TMP172]]
-// CHECK2-NEXT:    [[MUL321:%.*]] = mul i32 1, [[DIV320]]
-// CHECK2-NEXT:    [[TMP173:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB322:%.*]] = sub i32 [[TMP173]], -31
-// CHECK2-NEXT:    [[DIV323:%.*]] = udiv i32 [[SUB322]], 32
-// CHECK2-NEXT:    [[MUL324:%.*]] = mul i32 [[MUL321]], [[DIV323]]
-// CHECK2-NEXT:    [[TMP174:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP175:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB325:%.*]] = sub i32 [[TMP174]], [[TMP175]]
-// CHECK2-NEXT:    [[SUB326:%.*]] = sub i32 [[SUB325]], 1
-// CHECK2-NEXT:    [[ADD327:%.*]] = add i32 [[SUB326]], 1
-// CHECK2-NEXT:    [[DIV328:%.*]] = udiv i32 [[ADD327]], 1
-// CHECK2-NEXT:    [[MUL329:%.*]] = mul i32 [[MUL324]], [[DIV328]]
-// CHECK2-NEXT:    [[CONV330:%.*]] = zext i32 [[MUL329]] to i64
-// CHECK2-NEXT:    [[MUL331:%.*]] = mul nsw i64 [[DIV316]], [[CONV330]]
-// CHECK2-NEXT:    [[SUB332:%.*]] = sub nsw i64 [[TMP160]], [[MUL331]]
-// CHECK2-NEXT:    [[TMP176:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP177:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP178:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP179:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB333:%.*]] = sub i32 [[TMP178]], [[TMP179]]
-// CHECK2-NEXT:    [[SUB334:%.*]] = sub i32 [[SUB333]], 1
-// CHECK2-NEXT:    [[TMP180:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD335:%.*]] = add i32 [[SUB334]], [[TMP180]]
-// CHECK2-NEXT:    [[TMP181:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV336:%.*]] = udiv i32 [[ADD335]], [[TMP181]]
-// CHECK2-NEXT:    [[MUL337:%.*]] = mul i32 1, [[DIV336]]
-// CHECK2-NEXT:    [[TMP182:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB338:%.*]] = sub i32 [[TMP182]], -31
-// CHECK2-NEXT:    [[DIV339:%.*]] = udiv i32 [[SUB338]], 32
-// CHECK2-NEXT:    [[MUL340:%.*]] = mul i32 [[MUL337]], [[DIV339]]
-// CHECK2-NEXT:    [[TMP183:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP184:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB341:%.*]] = sub i32 [[TMP183]], [[TMP184]]
-// CHECK2-NEXT:    [[SUB342:%.*]] = sub i32 [[SUB341]], 1
-// CHECK2-NEXT:    [[ADD343:%.*]] = add i32 [[SUB342]], 1
-// CHECK2-NEXT:    [[DIV344:%.*]] = udiv i32 [[ADD343]], 1
-// CHECK2-NEXT:    [[MUL345:%.*]] = mul i32 [[MUL340]], [[DIV344]]
-// CHECK2-NEXT:    [[CONV346:%.*]] = zext i32 [[MUL345]] to i64
-// CHECK2-NEXT:    [[DIV347:%.*]] = sdiv i64 [[TMP177]], [[CONV346]]
-// CHECK2-NEXT:    [[TMP185:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP186:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB348:%.*]] = sub i32 [[TMP185]], [[TMP186]]
-// CHECK2-NEXT:    [[SUB349:%.*]] = sub i32 [[SUB348]], 1
-// CHECK2-NEXT:    [[TMP187:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD350:%.*]] = add i32 [[SUB349]], [[TMP187]]
-// CHECK2-NEXT:    [[TMP188:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV351:%.*]] = udiv i32 [[ADD350]], [[TMP188]]
-// CHECK2-NEXT:    [[MUL352:%.*]] = mul i32 1, [[DIV351]]
-// CHECK2-NEXT:    [[TMP189:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB353:%.*]] = sub i32 [[TMP189]], -31
-// CHECK2-NEXT:    [[DIV354:%.*]] = udiv i32 [[SUB353]], 32
-// CHECK2-NEXT:    [[MUL355:%.*]] = mul i32 [[MUL352]], [[DIV354]]
-// CHECK2-NEXT:    [[TMP190:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP191:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB356:%.*]] = sub i32 [[TMP190]], [[TMP191]]
-// CHECK2-NEXT:    [[SUB357:%.*]] = sub i32 [[SUB356]], 1
-// CHECK2-NEXT:    [[ADD358:%.*]] = add i32 [[SUB357]], 1
-// CHECK2-NEXT:    [[DIV359:%.*]] = udiv i32 [[ADD358]], 1
-// CHECK2-NEXT:    [[MUL360:%.*]] = mul i32 [[MUL355]], [[DIV359]]
-// CHECK2-NEXT:    [[CONV361:%.*]] = zext i32 [[MUL360]] to i64
-// CHECK2-NEXT:    [[MUL362:%.*]] = mul nsw i64 [[DIV347]], [[CONV361]]
-// CHECK2-NEXT:    [[SUB363:%.*]] = sub nsw i64 [[TMP176]], [[MUL362]]
-// CHECK2-NEXT:    [[TMP192:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB364:%.*]] = sub i32 [[TMP192]], -31
-// CHECK2-NEXT:    [[DIV365:%.*]] = udiv i32 [[SUB364]], 32
-// CHECK2-NEXT:    [[MUL366:%.*]] = mul i32 1, [[DIV365]]
-// CHECK2-NEXT:    [[TMP193:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP194:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB367:%.*]] = sub i32 [[TMP193]], [[TMP194]]
-// CHECK2-NEXT:    [[SUB368:%.*]] = sub i32 [[SUB367]], 1
-// CHECK2-NEXT:    [[ADD369:%.*]] = add i32 [[SUB368]], 1
-// CHECK2-NEXT:    [[DIV370:%.*]] = udiv i32 [[ADD369]], 1
-// CHECK2-NEXT:    [[MUL371:%.*]] = mul i32 [[MUL366]], [[DIV370]]
-// CHECK2-NEXT:    [[CONV372:%.*]] = zext i32 [[MUL371]] to i64
-// CHECK2-NEXT:    [[DIV373:%.*]] = sdiv i64 [[SUB363]], [[CONV372]]
-// CHECK2-NEXT:    [[TMP195:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB374:%.*]] = sub i32 [[TMP195]], -31
-// CHECK2-NEXT:    [[DIV375:%.*]] = udiv i32 [[SUB374]], 32
-// CHECK2-NEXT:    [[MUL376:%.*]] = mul i32 1, [[DIV375]]
-// CHECK2-NEXT:    [[TMP196:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP197:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB377:%.*]] = sub i32 [[TMP196]], [[TMP197]]
-// CHECK2-NEXT:    [[SUB378:%.*]] = sub i32 [[SUB377]], 1
-// CHECK2-NEXT:    [[ADD379:%.*]] = add i32 [[SUB378]], 1
-// CHECK2-NEXT:    [[DIV380:%.*]] = udiv i32 [[ADD379]], 1
-// CHECK2-NEXT:    [[MUL381:%.*]] = mul i32 [[MUL376]], [[DIV380]]
-// CHECK2-NEXT:    [[CONV382:%.*]] = zext i32 [[MUL381]] to i64
-// CHECK2-NEXT:    [[MUL383:%.*]] = mul nsw i64 [[DIV373]], [[CONV382]]
-// CHECK2-NEXT:    [[SUB384:%.*]] = sub nsw i64 [[SUB332]], [[MUL383]]
-// CHECK2-NEXT:    [[TMP198:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP199:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB385:%.*]] = sub i32 [[TMP198]], [[TMP199]]
-// CHECK2-NEXT:    [[SUB386:%.*]] = sub i32 [[SUB385]], 1
-// CHECK2-NEXT:    [[ADD387:%.*]] = add i32 [[SUB386]], 1
-// CHECK2-NEXT:    [[DIV388:%.*]] = udiv i32 [[ADD387]], 1
-// CHECK2-NEXT:    [[MUL389:%.*]] = mul i32 1, [[DIV388]]
-// CHECK2-NEXT:    [[CONV390:%.*]] = zext i32 [[MUL389]] to i64
-// CHECK2-NEXT:    [[DIV391:%.*]] = sdiv i64 [[SUB384]], [[CONV390]]
-// CHECK2-NEXT:    [[TMP200:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP201:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB392:%.*]] = sub i32 [[TMP200]], [[TMP201]]
-// CHECK2-NEXT:    [[SUB393:%.*]] = sub i32 [[SUB392]], 1
-// CHECK2-NEXT:    [[ADD394:%.*]] = add i32 [[SUB393]], 1
-// CHECK2-NEXT:    [[DIV395:%.*]] = udiv i32 [[ADD394]], 1
-// CHECK2-NEXT:    [[MUL396:%.*]] = mul i32 1, [[DIV395]]
-// CHECK2-NEXT:    [[CONV397:%.*]] = zext i32 [[MUL396]] to i64
-// CHECK2-NEXT:    [[MUL398:%.*]] = mul nsw i64 [[DIV391]], [[CONV397]]
-// CHECK2-NEXT:    [[SUB399:%.*]] = sub nsw i64 [[SUB301]], [[MUL398]]
-// CHECK2-NEXT:    [[MUL400:%.*]] = mul nsw i64 [[SUB399]], 1
-// CHECK2-NEXT:    [[ADD401:%.*]] = add nsw i64 [[CONV218]], [[MUL400]]
-// CHECK2-NEXT:    [[CONV402:%.*]] = trunc i64 [[ADD401]] to i32
-// CHECK2-NEXT:    store i32 [[CONV402]], ptr [[DOTTILE_0_IV_K52]], align 4
-// CHECK2-NEXT:    [[TMP202:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
-// CHECK2-NEXT:    [[TMP203:%.*]] = load i32, ptr [[DOTTILE_0_IV_K52]], align 4
-// CHECK2-NEXT:    [[TMP204:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
-// CHECK2-NEXT:    [[MUL403:%.*]] = mul i32 [[TMP203]], [[TMP204]]
-// CHECK2-NEXT:    [[ADD404:%.*]] = add i32 [[TMP202]], [[MUL403]]
-// CHECK2-NEXT:    store i32 [[ADD404]], ptr [[K]], align 4
-// CHECK2-NEXT:    [[TMP205:%.*]] = load i32, ptr [[I49]], align 4
-// CHECK2-NEXT:    [[TMP206:%.*]] = load i32, ptr [[J50]], align 4
-// CHECK2-NEXT:    [[TMP207:%.*]] = load i32, ptr [[K]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP205]], i32 noundef [[TMP206]], i32 noundef [[TMP207]])
+// CHECK2-NEXT:    [[DIV264:%.*]] = sdiv i64 [[SUB258]], [[CONV263]]
+// CHECK2-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB265:%.*]] = sub i32 [[TMP140]], -31
+// CHECK2-NEXT:    [[DIV266:%.*]] = udiv i32 [[SUB265]], 32
+// CHECK2-NEXT:    [[MUL267:%.*]] = mul i32 1, [[DIV266]]
+// CHECK2-NEXT:    [[MUL268:%.*]] = mul i32 [[MUL267]], 32
+// CHECK2-NEXT:    [[CONV269:%.*]] = zext i32 [[MUL268]] to i64
+// CHECK2-NEXT:    [[MUL270:%.*]] = mul nsw i64 [[DIV264]], [[CONV269]]
+// CHECK2-NEXT:    [[SUB271:%.*]] = sub nsw i64 [[SUB235]], [[MUL270]]
+// CHECK2-NEXT:    [[DIV272:%.*]] = sdiv i64 [[SUB271]], 32
+// CHECK2-NEXT:    [[MUL273:%.*]] = mul nsw i64 [[DIV272]], 32
+// CHECK2-NEXT:    [[SUB274:%.*]] = sub nsw i64 [[SUB212]], [[MUL273]]
+// CHECK2-NEXT:    [[MUL275:%.*]] = mul nsw i64 [[SUB274]], 1
+// CHECK2-NEXT:    [[ADD276:%.*]] = add nsw i64 0, [[MUL275]]
+// CHECK2-NEXT:    [[CONV277:%.*]] = trunc i64 [[ADD276]] to i32
+// CHECK2-NEXT:    store i32 [[CONV277]], ptr [[DOTTILE_CNT_0_IV_K38]], align 4
+// CHECK2-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
+// CHECK2-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
+// CHECK2-NEXT:    [[ADD278:%.*]] = add i32 [[TMP141]], [[TMP142]]
+// CHECK2-NEXT:    store i32 [[ADD278]], ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK2-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
+// CHECK2-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK2-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
+// CHECK2-NEXT:    [[MUL279:%.*]] = mul i32 [[TMP144]], [[TMP145]]
+// CHECK2-NEXT:    [[ADD280:%.*]] = add i32 [[TMP143]], [[MUL279]]
+// CHECK2-NEXT:    store i32 [[ADD280]], ptr [[K]], align 4
+// CHECK2-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK2-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// CHECK2-NEXT:    [[ADD281:%.*]] = add i32 [[TMP147]], 1
+// CHECK2-NEXT:    [[CMP282:%.*]] = icmp ult i32 [[TMP146]], [[ADD281]]
+// CHECK2-NEXT:    br i1 [[CMP282]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK2:       if.then:
+// CHECK2-NEXT:    [[TMP148:%.*]] = load i32, ptr [[I35]], align 4
+// CHECK2-NEXT:    [[TMP149:%.*]] = load i32, ptr [[J36]], align 4
+// CHECK2-NEXT:    [[TMP150:%.*]] = load i32, ptr [[K]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP148]], i32 noundef [[TMP149]], i32 noundef [[TMP150]])
+// CHECK2-NEXT:    br label [[IF_END]]
+// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
-// CHECK2-NEXT:    [[TMP208:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[ADD405:%.*]] = add nsw i64 [[TMP208]], 1
-// CHECK2-NEXT:    store i64 [[ADD405]], ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP151:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[ADD283:%.*]] = add nsw i64 [[TMP151]], 1
+// CHECK2-NEXT:    store i64 [[ADD283]], ptr [[DOTOMP_IV]], align 8
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -4635,22 +4099,21 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTNEW_STEP10:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_14:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_17:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_22:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_K:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_K:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[I49:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[J50:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV_K51:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_K52:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[I35:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[J36:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV_K37:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_K38:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
@@ -4688,630 +4151,452 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
 // CHECK2-NEXT:    [[ADD15:%.*]] = add i32 [[TMP15]], 1
 // CHECK2-NEXT:    store i32 [[ADD15]], ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[_TMP2]], align 4
-// CHECK2-NEXT:    store i32 [[TMP16]], ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK2-NEXT:    [[ADD18:%.*]] = add i32 [[TMP17]], 1
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[_TMP2]], align 4
-// CHECK2-NEXT:    [[ADD19:%.*]] = add i32 [[TMP18]], 64
-// CHECK2-NEXT:    [[CMP:%.*]] = icmp ult i32 [[ADD18]], [[ADD19]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK2:       cond.true:
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK2-NEXT:    [[ADD20:%.*]] = add i32 [[TMP19]], 1
-// CHECK2-NEXT:    br label [[COND_END:%.*]]
-// CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[_TMP2]], align 4
-// CHECK2-NEXT:    [[ADD21:%.*]] = add i32 [[TMP20]], 64
-// CHECK2-NEXT:    br label [[COND_END]]
-// CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD20]], [[COND_TRUE]] ], [ [[ADD21]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    store i32 [[COND]], ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[SUB23:%.*]] = sub i32 [[TMP21]], [[TMP22]]
-// CHECK2-NEXT:    [[SUB24:%.*]] = sub i32 [[SUB23]], 1
-// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[ADD25:%.*]] = add i32 [[SUB24]], [[TMP23]]
-// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[DIV26:%.*]] = udiv i32 [[ADD25]], [[TMP24]]
-// CHECK2-NEXT:    [[CONV:%.*]] = zext i32 [[DIV26]] to i64
-// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[SUB17:%.*]] = sub i32 [[TMP16]], [[TMP17]]
+// CHECK2-NEXT:    [[SUB18:%.*]] = sub i32 [[SUB17]], 1
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[ADD19:%.*]] = add i32 [[SUB18]], [[TMP18]]
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[DIV20:%.*]] = udiv i32 [[ADD19]], [[TMP19]]
+// CHECK2-NEXT:    [[CONV:%.*]] = zext i32 [[DIV20]] to i64
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB21:%.*]] = sub i32 [[TMP20]], [[TMP21]]
+// CHECK2-NEXT:    [[SUB22:%.*]] = sub i32 [[SUB21]], 1
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD23:%.*]] = add i32 [[SUB22]], [[TMP22]]
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV24:%.*]] = udiv i32 [[ADD23]], [[TMP23]]
+// CHECK2-NEXT:    [[CONV25:%.*]] = zext i32 [[DIV24]] to i64
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV25]]
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB26:%.*]] = sub i32 [[TMP24]], -63
+// CHECK2-NEXT:    [[DIV27:%.*]] = udiv i32 [[SUB26]], 64
+// CHECK2-NEXT:    [[CONV28:%.*]] = zext i32 [[DIV27]] to i64
+// CHECK2-NEXT:    [[MUL29:%.*]] = mul nsw i64 [[MUL]], [[CONV28]]
+// CHECK2-NEXT:    [[MUL30:%.*]] = mul nsw i64 [[MUL29]], 64
+// CHECK2-NEXT:    [[SUB31:%.*]] = sub nsw i64 [[MUL30]], 1
+// CHECK2-NEXT:    store i64 [[SUB31]], ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    store i32 [[TMP25]], ptr [[I]], align 4
 // CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB27:%.*]] = sub i32 [[TMP25]], [[TMP26]]
-// CHECK2-NEXT:    [[SUB28:%.*]] = sub i32 [[SUB27]], 1
-// CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD29:%.*]] = add i32 [[SUB28]], [[TMP27]]
-// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV30:%.*]] = udiv i32 [[ADD29]], [[TMP28]]
-// CHECK2-NEXT:    [[CONV31:%.*]] = zext i32 [[DIV30]] to i64
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV31]]
-// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB32:%.*]] = sub i32 [[TMP29]], -63
-// CHECK2-NEXT:    [[DIV33:%.*]] = udiv i32 [[SUB32]], 64
-// CHECK2-NEXT:    [[CONV34:%.*]] = zext i32 [[DIV33]] to i64
-// CHECK2-NEXT:    [[MUL35:%.*]] = mul nsw i64 [[MUL]], [[CONV34]]
-// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB36:%.*]] = sub i32 [[TMP30]], [[TMP31]]
-// CHECK2-NEXT:    [[SUB37:%.*]] = sub i32 [[SUB36]], 1
-// CHECK2-NEXT:    [[ADD38:%.*]] = add i32 [[SUB37]], 1
-// CHECK2-NEXT:    [[DIV39:%.*]] = udiv i32 [[ADD38]], 1
-// CHECK2-NEXT:    [[CONV40:%.*]] = zext i32 [[DIV39]] to i64
-// CHECK2-NEXT:    [[MUL41:%.*]] = mul nsw i64 [[MUL35]], [[CONV40]]
-// CHECK2-NEXT:    [[SUB42:%.*]] = sub nsw i64 [[MUL41]], 1
-// CHECK2-NEXT:    store i64 [[SUB42]], ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK2-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    store i32 [[TMP32]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    store i32 [[TMP33]], ptr [[J]], align 4
+// CHECK2-NEXT:    store i32 [[TMP26]], ptr [[J]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_K]], align 4
-// CHECK2-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    store i32 [[TMP34]], ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK2-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK2-NEXT:    [[CMP43:%.*]] = icmp slt i32 [[TMP35]], [[TMP36]]
-// CHECK2-NEXT:    br i1 [[CMP43]], label [[LAND_LHS_TRUE:%.*]], label [[OMP_PRECOND_END:%.*]]
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_K]], align 4
+// CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP27]], [[TMP28]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[LAND_LHS_TRUE:%.*]], label [[OMP_PRECOND_END:%.*]]
 // CHECK2:       land.lhs.true:
-// CHECK2-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[CMP44:%.*]] = icmp slt i32 [[TMP37]], [[TMP38]]
-// CHECK2-NEXT:    br i1 [[CMP44]], label [[LAND_LHS_TRUE45:%.*]], label [[OMP_PRECOND_END]]
-// CHECK2:       land.lhs.true45:
-// CHECK2-NEXT:    [[TMP39:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[CMP46:%.*]] = icmp ult i32 0, [[TMP39]]
-// CHECK2-NEXT:    br i1 [[CMP46]], label [[LAND_LHS_TRUE47:%.*]], label [[OMP_PRECOND_END]]
-// CHECK2:       land.lhs.true47:
-// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[CMP48:%.*]] = icmp ult i32 [[TMP40]], [[TMP41]]
-// CHECK2-NEXT:    br i1 [[CMP48]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
+// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[CMP32:%.*]] = icmp slt i32 [[TMP29]], [[TMP30]]
+// CHECK2-NEXT:    br i1 [[CMP32]], label [[LAND_LHS_TRUE33:%.*]], label [[OMP_PRECOND_END]]
+// CHECK2:       land.lhs.true33:
+// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[CMP34:%.*]] = icmp ult i32 0, [[TMP31]]
+// CHECK2-NEXT:    br i1 [[CMP34]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
 // CHECK2:       omp.precond.then:
 // CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
-// CHECK2-NEXT:    [[TMP42:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK2-NEXT:    store i64 [[TMP42]], ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK2-NEXT:    store i64 [[TMP32]], ptr [[DOTOMP_UB]], align 8
 // CHECK2-NEXT:    store i64 1, ptr [[DOTOMP_STRIDE]], align 8
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
 // CHECK2-NEXT:    call void @__kmpc_for_static_init_8(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i64 1, i64 1)
-// CHECK2-NEXT:    [[TMP43:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[TMP44:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK2-NEXT:    [[CMP53:%.*]] = icmp sgt i64 [[TMP43]], [[TMP44]]
-// CHECK2-NEXT:    br i1 [[CMP53]], label [[COND_TRUE54:%.*]], label [[COND_FALSE55:%.*]]
-// CHECK2:       cond.true54:
-// CHECK2-NEXT:    [[TMP45:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_22]], align 8
-// CHECK2-NEXT:    br label [[COND_END56:%.*]]
-// CHECK2:       cond.false55:
-// CHECK2-NEXT:    [[TMP46:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    br label [[COND_END56]]
-// CHECK2:       cond.end56:
-// CHECK2-NEXT:    [[COND57:%.*]] = phi i64 [ [[TMP45]], [[COND_TRUE54]] ], [ [[TMP46]], [[COND_FALSE55]] ]
-// CHECK2-NEXT:    store i64 [[COND57]], ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[TMP47:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
-// CHECK2-NEXT:    store i64 [[TMP47]], ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK2-NEXT:    [[CMP39:%.*]] = icmp sgt i64 [[TMP33]], [[TMP34]]
+// CHECK2-NEXT:    br i1 [[CMP39]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i64 [ [[TMP35]], [[COND_TRUE]] ], [ [[TMP36]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    store i64 [[COND]], ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
+// CHECK2-NEXT:    store i64 [[TMP37]], ptr [[DOTOMP_IV]], align 8
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK2:       omp.inner.for.cond:
-// CHECK2-NEXT:    [[TMP48:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[CMP58:%.*]] = icmp sle i64 [[TMP48]], [[TMP49]]
-// CHECK2-NEXT:    br i1 [[CMP58]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK2-NEXT:    [[TMP38:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP39:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[CMP40:%.*]] = icmp sle i64 [[TMP38]], [[TMP39]]
+// CHECK2-NEXT:    br i1 [[CMP40]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
 // CHECK2:       omp.inner.for.body:
-// CHECK2-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[CONV59:%.*]] = sext i32 [[TMP50]] to i64
-// CHECK2-NEXT:    [[TMP51:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB60:%.*]] = sub i32 [[TMP52]], [[TMP53]]
-// CHECK2-NEXT:    [[SUB61:%.*]] = sub i32 [[SUB60]], 1
+// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[CONV41:%.*]] = sext i32 [[TMP40]] to i64
+// CHECK2-NEXT:    [[TMP41:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB42:%.*]] = sub i32 [[TMP42]], [[TMP43]]
+// CHECK2-NEXT:    [[SUB43:%.*]] = sub i32 [[SUB42]], 1
+// CHECK2-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD44:%.*]] = add i32 [[SUB43]], [[TMP44]]
+// CHECK2-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV45:%.*]] = udiv i32 [[ADD44]], [[TMP45]]
+// CHECK2-NEXT:    [[MUL46:%.*]] = mul i32 1, [[DIV45]]
+// CHECK2-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB47:%.*]] = sub i32 [[TMP46]], -63
+// CHECK2-NEXT:    [[DIV48:%.*]] = udiv i32 [[SUB47]], 64
+// CHECK2-NEXT:    [[MUL49:%.*]] = mul i32 [[MUL46]], [[DIV48]]
+// CHECK2-NEXT:    [[MUL50:%.*]] = mul i32 [[MUL49]], 64
+// CHECK2-NEXT:    [[CONV51:%.*]] = zext i32 [[MUL50]] to i64
+// CHECK2-NEXT:    [[DIV52:%.*]] = sdiv i64 [[TMP41]], [[CONV51]]
+// CHECK2-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[CONV53:%.*]] = sext i32 [[TMP47]] to i64
+// CHECK2-NEXT:    [[MUL54:%.*]] = mul nsw i64 [[DIV52]], [[CONV53]]
+// CHECK2-NEXT:    [[ADD55:%.*]] = add nsw i64 [[CONV41]], [[MUL54]]
+// CHECK2-NEXT:    [[CONV56:%.*]] = trunc i64 [[ADD55]] to i32
+// CHECK2-NEXT:    store i32 [[CONV56]], ptr [[I35]], align 4
+// CHECK2-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP48]] to i64
+// CHECK2-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP50:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB58:%.*]] = sub i32 [[TMP51]], [[TMP52]]
+// CHECK2-NEXT:    [[SUB59:%.*]] = sub i32 [[SUB58]], 1
+// CHECK2-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD60:%.*]] = add i32 [[SUB59]], [[TMP53]]
 // CHECK2-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD62:%.*]] = add i32 [[SUB61]], [[TMP54]]
-// CHECK2-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV63:%.*]] = udiv i32 [[ADD62]], [[TMP55]]
-// CHECK2-NEXT:    [[MUL64:%.*]] = mul i32 1, [[DIV63]]
-// CHECK2-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB65:%.*]] = sub i32 [[TMP56]], -63
-// CHECK2-NEXT:    [[DIV66:%.*]] = udiv i32 [[SUB65]], 64
-// CHECK2-NEXT:    [[MUL67:%.*]] = mul i32 [[MUL64]], [[DIV66]]
-// CHECK2-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB68:%.*]] = sub i32 [[TMP57]], [[TMP58]]
-// CHECK2-NEXT:    [[SUB69:%.*]] = sub i32 [[SUB68]], 1
-// CHECK2-NEXT:    [[ADD70:%.*]] = add i32 [[SUB69]], 1
-// CHECK2-NEXT:    [[DIV71:%.*]] = udiv i32 [[ADD70]], 1
-// CHECK2-NEXT:    [[MUL72:%.*]] = mul i32 [[MUL67]], [[DIV71]]
-// CHECK2-NEXT:    [[CONV73:%.*]] = zext i32 [[MUL72]] to i64
-// CHECK2-NEXT:    [[DIV74:%.*]] = sdiv i64 [[TMP51]], [[CONV73]]
-// CHECK2-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[CONV75:%.*]] = sext i32 [[TMP59]] to i64
-// CHECK2-NEXT:    [[MUL76:%.*]] = mul nsw i64 [[DIV74]], [[CONV75]]
-// CHECK2-NEXT:    [[ADD77:%.*]] = add nsw i64 [[CONV59]], [[MUL76]]
-// CHECK2-NEXT:    [[CONV78:%.*]] = trunc i64 [[ADD77]] to i32
-// CHECK2-NEXT:    store i32 [[CONV78]], ptr [[I49]], align 4
-// CHECK2-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[CONV79:%.*]] = sext i32 [[TMP60]] to i64
-// CHECK2-NEXT:    [[TMP61:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP62:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP63:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP64:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB80:%.*]] = sub i32 [[TMP63]], [[TMP64]]
-// CHECK2-NEXT:    [[SUB81:%.*]] = sub i32 [[SUB80]], 1
-// CHECK2-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD82:%.*]] = add i32 [[SUB81]], [[TMP65]]
-// CHECK2-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV83:%.*]] = udiv i32 [[ADD82]], [[TMP66]]
-// CHECK2-NEXT:    [[MUL84:%.*]] = mul i32 1, [[DIV83]]
-// CHECK2-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB85:%.*]] = sub i32 [[TMP67]], -63
-// CHECK2-NEXT:    [[DIV86:%.*]] = udiv i32 [[SUB85]], 64
-// CHECK2-NEXT:    [[MUL87:%.*]] = mul i32 [[MUL84]], [[DIV86]]
-// CHECK2-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB88:%.*]] = sub i32 [[TMP68]], [[TMP69]]
-// CHECK2-NEXT:    [[SUB89:%.*]] = sub i32 [[SUB88]], 1
-// CHECK2-NEXT:    [[ADD90:%.*]] = add i32 [[SUB89]], 1
-// CHECK2-NEXT:    [[DIV91:%.*]] = udiv i32 [[ADD90]], 1
-// CHECK2-NEXT:    [[MUL92:%.*]] = mul i32 [[MUL87]], [[DIV91]]
-// CHECK2-NEXT:    [[CONV93:%.*]] = zext i32 [[MUL92]] to i64
-// CHECK2-NEXT:    [[DIV94:%.*]] = sdiv i64 [[TMP62]], [[CONV93]]
+// CHECK2-NEXT:    [[DIV61:%.*]] = udiv i32 [[ADD60]], [[TMP54]]
+// CHECK2-NEXT:    [[MUL62:%.*]] = mul i32 1, [[DIV61]]
+// CHECK2-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB63:%.*]] = sub i32 [[TMP55]], -63
+// CHECK2-NEXT:    [[DIV64:%.*]] = udiv i32 [[SUB63]], 64
+// CHECK2-NEXT:    [[MUL65:%.*]] = mul i32 [[MUL62]], [[DIV64]]
+// CHECK2-NEXT:    [[MUL66:%.*]] = mul i32 [[MUL65]], 64
+// CHECK2-NEXT:    [[CONV67:%.*]] = zext i32 [[MUL66]] to i64
+// CHECK2-NEXT:    [[DIV68:%.*]] = sdiv i64 [[TMP50]], [[CONV67]]
+// CHECK2-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB69:%.*]] = sub i32 [[TMP56]], [[TMP57]]
+// CHECK2-NEXT:    [[SUB70:%.*]] = sub i32 [[SUB69]], 1
+// CHECK2-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD71:%.*]] = add i32 [[SUB70]], [[TMP58]]
+// CHECK2-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV72:%.*]] = udiv i32 [[ADD71]], [[TMP59]]
+// CHECK2-NEXT:    [[MUL73:%.*]] = mul i32 1, [[DIV72]]
+// CHECK2-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB74:%.*]] = sub i32 [[TMP60]], -63
+// CHECK2-NEXT:    [[DIV75:%.*]] = udiv i32 [[SUB74]], 64
+// CHECK2-NEXT:    [[MUL76:%.*]] = mul i32 [[MUL73]], [[DIV75]]
+// CHECK2-NEXT:    [[MUL77:%.*]] = mul i32 [[MUL76]], 64
+// CHECK2-NEXT:    [[CONV78:%.*]] = zext i32 [[MUL77]] to i64
+// CHECK2-NEXT:    [[MUL79:%.*]] = mul nsw i64 [[DIV68]], [[CONV78]]
+// CHECK2-NEXT:    [[SUB80:%.*]] = sub nsw i64 [[TMP49]], [[MUL79]]
+// CHECK2-NEXT:    [[TMP61:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB81:%.*]] = sub i32 [[TMP61]], -63
+// CHECK2-NEXT:    [[DIV82:%.*]] = udiv i32 [[SUB81]], 64
+// CHECK2-NEXT:    [[MUL83:%.*]] = mul i32 1, [[DIV82]]
+// CHECK2-NEXT:    [[MUL84:%.*]] = mul i32 [[MUL83]], 64
+// CHECK2-NEXT:    [[CONV85:%.*]] = zext i32 [[MUL84]] to i64
+// CHECK2-NEXT:    [[DIV86:%.*]] = sdiv i64 [[SUB80]], [[CONV85]]
+// CHECK2-NEXT:    [[TMP62:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[CONV87:%.*]] = sext i32 [[TMP62]] to i64
+// CHECK2-NEXT:    [[MUL88:%.*]] = mul nsw i64 [[DIV86]], [[CONV87]]
+// CHECK2-NEXT:    [[ADD89:%.*]] = add nsw i64 [[CONV57]], [[MUL88]]
+// CHECK2-NEXT:    [[CONV90:%.*]] = trunc i64 [[ADD89]] to i32
+// CHECK2-NEXT:    store i32 [[CONV90]], ptr [[J36]], align 4
+// CHECK2-NEXT:    [[TMP63:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP64:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB91:%.*]] = sub i32 [[TMP65]], [[TMP66]]
+// CHECK2-NEXT:    [[SUB92:%.*]] = sub i32 [[SUB91]], 1
+// CHECK2-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD93:%.*]] = add i32 [[SUB92]], [[TMP67]]
+// CHECK2-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV94:%.*]] = udiv i32 [[ADD93]], [[TMP68]]
+// CHECK2-NEXT:    [[MUL95:%.*]] = mul i32 1, [[DIV94]]
+// CHECK2-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB96:%.*]] = sub i32 [[TMP69]], -63
+// CHECK2-NEXT:    [[DIV97:%.*]] = udiv i32 [[SUB96]], 64
+// CHECK2-NEXT:    [[MUL98:%.*]] = mul i32 [[MUL95]], [[DIV97]]
+// CHECK2-NEXT:    [[MUL99:%.*]] = mul i32 [[MUL98]], 64
+// CHECK2-NEXT:    [[CONV100:%.*]] = zext i32 [[MUL99]] to i64
+// CHECK2-NEXT:    [[DIV101:%.*]] = sdiv i64 [[TMP64]], [[CONV100]]
 // CHECK2-NEXT:    [[TMP70:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
 // CHECK2-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB95:%.*]] = sub i32 [[TMP70]], [[TMP71]]
-// CHECK2-NEXT:    [[SUB96:%.*]] = sub i32 [[SUB95]], 1
+// CHECK2-NEXT:    [[SUB102:%.*]] = sub i32 [[TMP70]], [[TMP71]]
+// CHECK2-NEXT:    [[SUB103:%.*]] = sub i32 [[SUB102]], 1
 // CHECK2-NEXT:    [[TMP72:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD97:%.*]] = add i32 [[SUB96]], [[TMP72]]
+// CHECK2-NEXT:    [[ADD104:%.*]] = add i32 [[SUB103]], [[TMP72]]
 // CHECK2-NEXT:    [[TMP73:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV98:%.*]] = udiv i32 [[ADD97]], [[TMP73]]
-// CHECK2-NEXT:    [[MUL99:%.*]] = mul i32 1, [[DIV98]]
+// CHECK2-NEXT:    [[DIV105:%.*]] = udiv i32 [[ADD104]], [[TMP73]]
+// CHECK2-NEXT:    [[MUL106:%.*]] = mul i32 1, [[DIV105]]
 // CHECK2-NEXT:    [[TMP74:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB100:%.*]] = sub i32 [[TMP74]], -63
-// CHECK2-NEXT:    [[DIV101:%.*]] = udiv i32 [[SUB100]], 64
-// CHECK2-NEXT:    [[MUL102:%.*]] = mul i32 [[MUL99]], [[DIV101]]
-// CHECK2-NEXT:    [[TMP75:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP76:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB103:%.*]] = sub i32 [[TMP75]], [[TMP76]]
-// CHECK2-NEXT:    [[SUB104:%.*]] = sub i32 [[SUB103]], 1
-// CHECK2-NEXT:    [[ADD105:%.*]] = add i32 [[SUB104]], 1
-// CHECK2-NEXT:    [[DIV106:%.*]] = udiv i32 [[ADD105]], 1
-// CHECK2-NEXT:    [[MUL107:%.*]] = mul i32 [[MUL102]], [[DIV106]]
-// CHECK2-NEXT:    [[CONV108:%.*]] = zext i32 [[MUL107]] to i64
-// CHECK2-NEXT:    [[MUL109:%.*]] = mul nsw i64 [[DIV94]], [[CONV108]]
-// CHECK2-NEXT:    [[SUB110:%.*]] = sub nsw i64 [[TMP61]], [[MUL109]]
-// CHECK2-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB111:%.*]] = sub i32 [[TMP77]], -63
-// CHECK2-NEXT:    [[DIV112:%.*]] = udiv i32 [[SUB111]], 64
-// CHECK2-NEXT:    [[MUL113:%.*]] = mul i32 1, [[DIV112]]
-// CHECK2-NEXT:    [[TMP78:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP79:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB114:%.*]] = sub i32 [[TMP78]], [[TMP79]]
+// CHECK2-NEXT:    [[SUB107:%.*]] = sub i32 [[TMP74]], -63
+// CHECK2-NEXT:    [[DIV108:%.*]] = udiv i32 [[SUB107]], 64
+// CHECK2-NEXT:    [[MUL109:%.*]] = mul i32 [[MUL106]], [[DIV108]]
+// CHECK2-NEXT:    [[MUL110:%.*]] = mul i32 [[MUL109]], 64
+// CHECK2-NEXT:    [[CONV111:%.*]] = zext i32 [[MUL110]] to i64
+// CHECK2-NEXT:    [[MUL112:%.*]] = mul nsw i64 [[DIV101]], [[CONV111]]
+// CHECK2-NEXT:    [[SUB113:%.*]] = sub nsw i64 [[TMP63]], [[MUL112]]
+// CHECK2-NEXT:    [[TMP75:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP76:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP78:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB114:%.*]] = sub i32 [[TMP77]], [[TMP78]]
 // CHECK2-NEXT:    [[SUB115:%.*]] = sub i32 [[SUB114]], 1
-// CHECK2-NEXT:    [[ADD116:%.*]] = add i32 [[SUB115]], 1
-// CHECK2-NEXT:    [[DIV117:%.*]] = udiv i32 [[ADD116]], 1
-// CHECK2-NEXT:    [[MUL118:%.*]] = mul i32 [[MUL113]], [[DIV117]]
-// CHECK2-NEXT:    [[CONV119:%.*]] = zext i32 [[MUL118]] to i64
-// CHECK2-NEXT:    [[DIV120:%.*]] = sdiv i64 [[SUB110]], [[CONV119]]
+// CHECK2-NEXT:    [[TMP79:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD116:%.*]] = add i32 [[SUB115]], [[TMP79]]
 // CHECK2-NEXT:    [[TMP80:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[CONV121:%.*]] = sext i32 [[TMP80]] to i64
-// CHECK2-NEXT:    [[MUL122:%.*]] = mul nsw i64 [[DIV120]], [[CONV121]]
-// CHECK2-NEXT:    [[ADD123:%.*]] = add nsw i64 [[CONV79]], [[MUL122]]
-// CHECK2-NEXT:    [[CONV124:%.*]] = trunc i64 [[ADD123]] to i32
-// CHECK2-NEXT:    store i32 [[CONV124]], ptr [[J50]], align 4
-// CHECK2-NEXT:    [[TMP81:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP82:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB125:%.*]] = sub i32 [[TMP83]], [[TMP84]]
+// CHECK2-NEXT:    [[DIV117:%.*]] = udiv i32 [[ADD116]], [[TMP80]]
+// CHECK2-NEXT:    [[MUL118:%.*]] = mul i32 1, [[DIV117]]
+// CHECK2-NEXT:    [[TMP81:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB119:%.*]] = sub i32 [[TMP81]], -63
+// CHECK2-NEXT:    [[DIV120:%.*]] = udiv i32 [[SUB119]], 64
+// CHECK2-NEXT:    [[MUL121:%.*]] = mul i32 [[MUL118]], [[DIV120]]
+// CHECK2-NEXT:    [[MUL122:%.*]] = mul i32 [[MUL121]], 64
+// CHECK2-NEXT:    [[CONV123:%.*]] = zext i32 [[MUL122]] to i64
+// CHECK2-NEXT:    [[DIV124:%.*]] = sdiv i64 [[TMP76]], [[CONV123]]
+// CHECK2-NEXT:    [[TMP82:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB125:%.*]] = sub i32 [[TMP82]], [[TMP83]]
 // CHECK2-NEXT:    [[SUB126:%.*]] = sub i32 [[SUB125]], 1
+// CHECK2-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD127:%.*]] = add i32 [[SUB126]], [[TMP84]]
 // CHECK2-NEXT:    [[TMP85:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD127:%.*]] = add i32 [[SUB126]], [[TMP85]]
-// CHECK2-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV128:%.*]] = udiv i32 [[ADD127]], [[TMP86]]
+// CHECK2-NEXT:    [[DIV128:%.*]] = udiv i32 [[ADD127]], [[TMP85]]
 // CHECK2-NEXT:    [[MUL129:%.*]] = mul i32 1, [[DIV128]]
-// CHECK2-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB130:%.*]] = sub i32 [[TMP87]], -63
+// CHECK2-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB130:%.*]] = sub i32 [[TMP86]], -63
 // CHECK2-NEXT:    [[DIV131:%.*]] = udiv i32 [[SUB130]], 64
 // CHECK2-NEXT:    [[MUL132:%.*]] = mul i32 [[MUL129]], [[DIV131]]
-// CHECK2-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP89:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB133:%.*]] = sub i32 [[TMP88]], [[TMP89]]
-// CHECK2-NEXT:    [[SUB134:%.*]] = sub i32 [[SUB133]], 1
-// CHECK2-NEXT:    [[ADD135:%.*]] = add i32 [[SUB134]], 1
-// CHECK2-NEXT:    [[DIV136:%.*]] = udiv i32 [[ADD135]], 1
-// CHECK2-NEXT:    [[MUL137:%.*]] = mul i32 [[MUL132]], [[DIV136]]
-// CHECK2-NEXT:    [[CONV138:%.*]] = zext i32 [[MUL137]] to i64
-// CHECK2-NEXT:    [[DIV139:%.*]] = sdiv i64 [[TMP82]], [[CONV138]]
-// CHECK2-NEXT:    [[TMP90:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB140:%.*]] = sub i32 [[TMP90]], [[TMP91]]
-// CHECK2-NEXT:    [[SUB141:%.*]] = sub i32 [[SUB140]], 1
-// CHECK2-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD142:%.*]] = add i32 [[SUB141]], [[TMP92]]
+// CHECK2-NEXT:    [[MUL133:%.*]] = mul i32 [[MUL132]], 64
+// CHECK2-NEXT:    [[CONV134:%.*]] = zext i32 [[MUL133]] to i64
+// CHECK2-NEXT:    [[MUL135:%.*]] = mul nsw i64 [[DIV124]], [[CONV134]]
+// CHECK2-NEXT:    [[SUB136:%.*]] = sub nsw i64 [[TMP75]], [[MUL135]]
+// CHECK2-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB137:%.*]] = sub i32 [[TMP87]], -63
+// CHECK2-NEXT:    [[DIV138:%.*]] = udiv i32 [[SUB137]], 64
+// CHECK2-NEXT:    [[MUL139:%.*]] = mul i32 1, [[DIV138]]
+// CHECK2-NEXT:    [[MUL140:%.*]] = mul i32 [[MUL139]], 64
+// CHECK2-NEXT:    [[CONV141:%.*]] = zext i32 [[MUL140]] to i64
+// CHECK2-NEXT:    [[DIV142:%.*]] = sdiv i64 [[SUB136]], [[CONV141]]
+// CHECK2-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB143:%.*]] = sub i32 [[TMP88]], -63
+// CHECK2-NEXT:    [[DIV144:%.*]] = udiv i32 [[SUB143]], 64
+// CHECK2-NEXT:    [[MUL145:%.*]] = mul i32 1, [[DIV144]]
+// CHECK2-NEXT:    [[MUL146:%.*]] = mul i32 [[MUL145]], 64
+// CHECK2-NEXT:    [[CONV147:%.*]] = zext i32 [[MUL146]] to i64
+// CHECK2-NEXT:    [[MUL148:%.*]] = mul nsw i64 [[DIV142]], [[CONV147]]
+// CHECK2-NEXT:    [[SUB149:%.*]] = sub nsw i64 [[SUB113]], [[MUL148]]
+// CHECK2-NEXT:    [[DIV150:%.*]] = sdiv i64 [[SUB149]], 64
+// CHECK2-NEXT:    [[MUL151:%.*]] = mul nsw i64 [[DIV150]], 64
+// CHECK2-NEXT:    [[ADD152:%.*]] = add nsw i64 0, [[MUL151]]
+// CHECK2-NEXT:    [[CONV153:%.*]] = trunc i64 [[ADD152]] to i32
+// CHECK2-NEXT:    store i32 [[CONV153]], ptr [[DOTFLOOR_0_IV_K37]], align 4
+// CHECK2-NEXT:    [[TMP89:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP90:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB154:%.*]] = sub i32 [[TMP91]], [[TMP92]]
+// CHECK2-NEXT:    [[SUB155:%.*]] = sub i32 [[SUB154]], 1
 // CHECK2-NEXT:    [[TMP93:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV143:%.*]] = udiv i32 [[ADD142]], [[TMP93]]
-// CHECK2-NEXT:    [[MUL144:%.*]] = mul i32 1, [[DIV143]]
-// CHECK2-NEXT:    [[TMP94:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB145:%.*]] = sub i32 [[TMP94]], -63
-// CHECK2-NEXT:    [[DIV146:%.*]] = udiv i32 [[SUB145]], 64
-// CHECK2-NEXT:    [[MUL147:%.*]] = mul i32 [[MUL144]], [[DIV146]]
-// CHECK2-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB148:%.*]] = sub i32 [[TMP95]], [[TMP96]]
-// CHECK2-NEXT:    [[SUB149:%.*]] = sub i32 [[SUB148]], 1
-// CHECK2-NEXT:    [[ADD150:%.*]] = add i32 [[SUB149]], 1
-// CHECK2-NEXT:    [[DIV151:%.*]] = udiv i32 [[ADD150]], 1
-// CHECK2-NEXT:    [[MUL152:%.*]] = mul i32 [[MUL147]], [[DIV151]]
-// CHECK2-NEXT:    [[CONV153:%.*]] = zext i32 [[MUL152]] to i64
-// CHECK2-NEXT:    [[MUL154:%.*]] = mul nsw i64 [[DIV139]], [[CONV153]]
-// CHECK2-NEXT:    [[SUB155:%.*]] = sub nsw i64 [[TMP81]], [[MUL154]]
-// CHECK2-NEXT:    [[TMP97:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP98:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB156:%.*]] = sub i32 [[TMP99]], [[TMP100]]
-// CHECK2-NEXT:    [[SUB157:%.*]] = sub i32 [[SUB156]], 1
-// CHECK2-NEXT:    [[TMP101:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD158:%.*]] = add i32 [[SUB157]], [[TMP101]]
-// CHECK2-NEXT:    [[TMP102:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV159:%.*]] = udiv i32 [[ADD158]], [[TMP102]]
-// CHECK2-NEXT:    [[MUL160:%.*]] = mul i32 1, [[DIV159]]
-// CHECK2-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB161:%.*]] = sub i32 [[TMP103]], -63
-// CHECK2-NEXT:    [[DIV162:%.*]] = udiv i32 [[SUB161]], 64
-// CHECK2-NEXT:    [[MUL163:%.*]] = mul i32 [[MUL160]], [[DIV162]]
-// CHECK2-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP105:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB164:%.*]] = sub i32 [[TMP104]], [[TMP105]]
-// CHECK2-NEXT:    [[SUB165:%.*]] = sub i32 [[SUB164]], 1
-// CHECK2-NEXT:    [[ADD166:%.*]] = add i32 [[SUB165]], 1
-// CHECK2-NEXT:    [[DIV167:%.*]] = udiv i32 [[ADD166]], 1
-// CHECK2-NEXT:    [[MUL168:%.*]] = mul i32 [[MUL163]], [[DIV167]]
-// CHECK2-NEXT:    [[CONV169:%.*]] = zext i32 [[MUL168]] to i64
-// CHECK2-NEXT:    [[DIV170:%.*]] = sdiv i64 [[TMP98]], [[CONV169]]
-// CHECK2-NEXT:    [[TMP106:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB171:%.*]] = sub i32 [[TMP106]], [[TMP107]]
-// CHECK2-NEXT:    [[SUB172:%.*]] = sub i32 [[SUB171]], 1
-// CHECK2-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD173:%.*]] = add i32 [[SUB172]], [[TMP108]]
-// CHECK2-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV174:%.*]] = udiv i32 [[ADD173]], [[TMP109]]
-// CHECK2-NEXT:    [[MUL175:%.*]] = mul i32 1, [[DIV174]]
-// CHECK2-NEXT:    [[TMP110:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB176:%.*]] = sub i32 [[TMP110]], -63
-// CHECK2-NEXT:    [[DIV177:%.*]] = udiv i32 [[SUB176]], 64
-// CHECK2-NEXT:    [[MUL178:%.*]] = mul i32 [[MUL175]], [[DIV177]]
-// CHECK2-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB179:%.*]] = sub i32 [[TMP111]], [[TMP112]]
-// CHECK2-NEXT:    [[SUB180:%.*]] = sub i32 [[SUB179]], 1
-// CHECK2-NEXT:    [[ADD181:%.*]] = add i32 [[SUB180]], 1
-// CHECK2-NEXT:    [[DIV182:%.*]] = udiv i32 [[ADD181]], 1
-// CHECK2-NEXT:    [[MUL183:%.*]] = mul i32 [[MUL178]], [[DIV182]]
-// CHECK2-NEXT:    [[CONV184:%.*]] = zext i32 [[MUL183]] to i64
-// CHECK2-NEXT:    [[MUL185:%.*]] = mul nsw i64 [[DIV170]], [[CONV184]]
-// CHECK2-NEXT:    [[SUB186:%.*]] = sub nsw i64 [[TMP97]], [[MUL185]]
+// CHECK2-NEXT:    [[ADD156:%.*]] = add i32 [[SUB155]], [[TMP93]]
+// CHECK2-NEXT:    [[TMP94:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV157:%.*]] = udiv i32 [[ADD156]], [[TMP94]]
+// CHECK2-NEXT:    [[MUL158:%.*]] = mul i32 1, [[DIV157]]
+// CHECK2-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB159:%.*]] = sub i32 [[TMP95]], -63
+// CHECK2-NEXT:    [[DIV160:%.*]] = udiv i32 [[SUB159]], 64
+// CHECK2-NEXT:    [[MUL161:%.*]] = mul i32 [[MUL158]], [[DIV160]]
+// CHECK2-NEXT:    [[MUL162:%.*]] = mul i32 [[MUL161]], 64
+// CHECK2-NEXT:    [[CONV163:%.*]] = zext i32 [[MUL162]] to i64
+// CHECK2-NEXT:    [[DIV164:%.*]] = sdiv i64 [[TMP90]], [[CONV163]]
+// CHECK2-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP97:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB165:%.*]] = sub i32 [[TMP96]], [[TMP97]]
+// CHECK2-NEXT:    [[SUB166:%.*]] = sub i32 [[SUB165]], 1
+// CHECK2-NEXT:    [[TMP98:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD167:%.*]] = add i32 [[SUB166]], [[TMP98]]
+// CHECK2-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV168:%.*]] = udiv i32 [[ADD167]], [[TMP99]]
+// CHECK2-NEXT:    [[MUL169:%.*]] = mul i32 1, [[DIV168]]
+// CHECK2-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB170:%.*]] = sub i32 [[TMP100]], -63
+// CHECK2-NEXT:    [[DIV171:%.*]] = udiv i32 [[SUB170]], 64
+// CHECK2-NEXT:    [[MUL172:%.*]] = mul i32 [[MUL169]], [[DIV171]]
+// CHECK2-NEXT:    [[MUL173:%.*]] = mul i32 [[MUL172]], 64
+// CHECK2-NEXT:    [[CONV174:%.*]] = zext i32 [[MUL173]] to i64
+// CHECK2-NEXT:    [[MUL175:%.*]] = mul nsw i64 [[DIV164]], [[CONV174]]
+// CHECK2-NEXT:    [[SUB176:%.*]] = sub nsw i64 [[TMP89]], [[MUL175]]
+// CHECK2-NEXT:    [[TMP101:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP102:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB177:%.*]] = sub i32 [[TMP103]], [[TMP104]]
+// CHECK2-NEXT:    [[SUB178:%.*]] = sub i32 [[SUB177]], 1
+// CHECK2-NEXT:    [[TMP105:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD179:%.*]] = add i32 [[SUB178]], [[TMP105]]
+// CHECK2-NEXT:    [[TMP106:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV180:%.*]] = udiv i32 [[ADD179]], [[TMP106]]
+// CHECK2-NEXT:    [[MUL181:%.*]] = mul i32 1, [[DIV180]]
+// CHECK2-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB182:%.*]] = sub i32 [[TMP107]], -63
+// CHECK2-NEXT:    [[DIV183:%.*]] = udiv i32 [[SUB182]], 64
+// CHECK2-NEXT:    [[MUL184:%.*]] = mul i32 [[MUL181]], [[DIV183]]
+// CHECK2-NEXT:    [[MUL185:%.*]] = mul i32 [[MUL184]], 64
+// CHECK2-NEXT:    [[CONV186:%.*]] = zext i32 [[MUL185]] to i64
+// CHECK2-NEXT:    [[DIV187:%.*]] = sdiv i64 [[TMP102]], [[CONV186]]
+// CHECK2-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB188:%.*]] = sub i32 [[TMP108]], [[TMP109]]
+// CHECK2-NEXT:    [[SUB189:%.*]] = sub i32 [[SUB188]], 1
+// CHECK2-NEXT:    [[TMP110:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD190:%.*]] = add i32 [[SUB189]], [[TMP110]]
+// CHECK2-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV191:%.*]] = udiv i32 [[ADD190]], [[TMP111]]
+// CHECK2-NEXT:    [[MUL192:%.*]] = mul i32 1, [[DIV191]]
+// CHECK2-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB193:%.*]] = sub i32 [[TMP112]], -63
+// CHECK2-NEXT:    [[DIV194:%.*]] = udiv i32 [[SUB193]], 64
+// CHECK2-NEXT:    [[MUL195:%.*]] = mul i32 [[MUL192]], [[DIV194]]
+// CHECK2-NEXT:    [[MUL196:%.*]] = mul i32 [[MUL195]], 64
+// CHECK2-NEXT:    [[CONV197:%.*]] = zext i32 [[MUL196]] to i64
+// CHECK2-NEXT:    [[MUL198:%.*]] = mul nsw i64 [[DIV187]], [[CONV197]]
+// CHECK2-NEXT:    [[SUB199:%.*]] = sub nsw i64 [[TMP101]], [[MUL198]]
 // CHECK2-NEXT:    [[TMP113:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB187:%.*]] = sub i32 [[TMP113]], -63
-// CHECK2-NEXT:    [[DIV188:%.*]] = udiv i32 [[SUB187]], 64
-// CHECK2-NEXT:    [[MUL189:%.*]] = mul i32 1, [[DIV188]]
-// CHECK2-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP115:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB190:%.*]] = sub i32 [[TMP114]], [[TMP115]]
-// CHECK2-NEXT:    [[SUB191:%.*]] = sub i32 [[SUB190]], 1
-// CHECK2-NEXT:    [[ADD192:%.*]] = add i32 [[SUB191]], 1
-// CHECK2-NEXT:    [[DIV193:%.*]] = udiv i32 [[ADD192]], 1
-// CHECK2-NEXT:    [[MUL194:%.*]] = mul i32 [[MUL189]], [[DIV193]]
-// CHECK2-NEXT:    [[CONV195:%.*]] = zext i32 [[MUL194]] to i64
-// CHECK2-NEXT:    [[DIV196:%.*]] = sdiv i64 [[SUB186]], [[CONV195]]
-// CHECK2-NEXT:    [[TMP116:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB197:%.*]] = sub i32 [[TMP116]], -63
-// CHECK2-NEXT:    [[DIV198:%.*]] = udiv i32 [[SUB197]], 64
-// CHECK2-NEXT:    [[MUL199:%.*]] = mul i32 1, [[DIV198]]
-// CHECK2-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB200:%.*]] = sub i32 [[TMP117]], [[TMP118]]
-// CHECK2-NEXT:    [[SUB201:%.*]] = sub i32 [[SUB200]], 1
-// CHECK2-NEXT:    [[ADD202:%.*]] = add i32 [[SUB201]], 1
-// CHECK2-NEXT:    [[DIV203:%.*]] = udiv i32 [[ADD202]], 1
-// CHECK2-NEXT:    [[MUL204:%.*]] = mul i32 [[MUL199]], [[DIV203]]
-// CHECK2-NEXT:    [[CONV205:%.*]] = zext i32 [[MUL204]] to i64
-// CHECK2-NEXT:    [[MUL206:%.*]] = mul nsw i64 [[DIV196]], [[CONV205]]
-// CHECK2-NEXT:    [[SUB207:%.*]] = sub nsw i64 [[SUB155]], [[MUL206]]
-// CHECK2-NEXT:    [[TMP119:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP120:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB208:%.*]] = sub i32 [[TMP119]], [[TMP120]]
-// CHECK2-NEXT:    [[SUB209:%.*]] = sub i32 [[SUB208]], 1
-// CHECK2-NEXT:    [[ADD210:%.*]] = add i32 [[SUB209]], 1
-// CHECK2-NEXT:    [[DIV211:%.*]] = udiv i32 [[ADD210]], 1
-// CHECK2-NEXT:    [[MUL212:%.*]] = mul i32 1, [[DIV211]]
-// CHECK2-NEXT:    [[CONV213:%.*]] = zext i32 [[MUL212]] to i64
-// CHECK2-NEXT:    [[DIV214:%.*]] = sdiv i64 [[SUB207]], [[CONV213]]
-// CHECK2-NEXT:    [[MUL215:%.*]] = mul nsw i64 [[DIV214]], 64
-// CHECK2-NEXT:    [[ADD216:%.*]] = add nsw i64 0, [[MUL215]]
-// CHECK2-NEXT:    [[CONV217:%.*]] = trunc i64 [[ADD216]] to i32
-// CHECK2-NEXT:    store i32 [[CONV217]], ptr [[DOTFLOOR_0_IV_K51]], align 4
-// CHECK2-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[CONV218:%.*]] = zext i32 [[TMP121]] to i64
-// CHECK2-NEXT:    [[TMP122:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP123:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP124:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB219:%.*]] = sub i32 [[TMP124]], [[TMP125]]
-// CHECK2-NEXT:    [[SUB220:%.*]] = sub i32 [[SUB219]], 1
-// CHECK2-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD221:%.*]] = add i32 [[SUB220]], [[TMP126]]
-// CHECK2-NEXT:    [[TMP127:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV222:%.*]] = udiv i32 [[ADD221]], [[TMP127]]
-// CHECK2-NEXT:    [[MUL223:%.*]] = mul i32 1, [[DIV222]]
-// CHECK2-NEXT:    [[TMP128:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB224:%.*]] = sub i32 [[TMP128]], -63
-// CHECK2-NEXT:    [[DIV225:%.*]] = udiv i32 [[SUB224]], 64
-// CHECK2-NEXT:    [[MUL226:%.*]] = mul i32 [[MUL223]], [[DIV225]]
-// CHECK2-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB227:%.*]] = sub i32 [[TMP129]], [[TMP130]]
-// CHECK2-NEXT:    [[SUB228:%.*]] = sub i32 [[SUB227]], 1
-// CHECK2-NEXT:    [[ADD229:%.*]] = add i32 [[SUB228]], 1
-// CHECK2-NEXT:    [[DIV230:%.*]] = udiv i32 [[ADD229]], 1
-// CHECK2-NEXT:    [[MUL231:%.*]] = mul i32 [[MUL226]], [[DIV230]]
-// CHECK2-NEXT:    [[CONV232:%.*]] = zext i32 [[MUL231]] to i64
-// CHECK2-NEXT:    [[DIV233:%.*]] = sdiv i64 [[TMP123]], [[CONV232]]
-// CHECK2-NEXT:    [[TMP131:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP132:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB234:%.*]] = sub i32 [[TMP131]], [[TMP132]]
-// CHECK2-NEXT:    [[SUB235:%.*]] = sub i32 [[SUB234]], 1
-// CHECK2-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD236:%.*]] = add i32 [[SUB235]], [[TMP133]]
-// CHECK2-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV237:%.*]] = udiv i32 [[ADD236]], [[TMP134]]
-// CHECK2-NEXT:    [[MUL238:%.*]] = mul i32 1, [[DIV237]]
-// CHECK2-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB239:%.*]] = sub i32 [[TMP135]], -63
-// CHECK2-NEXT:    [[DIV240:%.*]] = udiv i32 [[SUB239]], 64
-// CHECK2-NEXT:    [[MUL241:%.*]] = mul i32 [[MUL238]], [[DIV240]]
-// CHECK2-NEXT:    [[TMP136:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB242:%.*]] = sub i32 [[TMP136]], [[TMP137]]
-// CHECK2-NEXT:    [[SUB243:%.*]] = sub i32 [[SUB242]], 1
-// CHECK2-NEXT:    [[ADD244:%.*]] = add i32 [[SUB243]], 1
-// CHECK2-NEXT:    [[DIV245:%.*]] = udiv i32 [[ADD244]], 1
-// CHECK2-NEXT:    [[MUL246:%.*]] = mul i32 [[MUL241]], [[DIV245]]
-// CHECK2-NEXT:    [[CONV247:%.*]] = zext i32 [[MUL246]] to i64
-// CHECK2-NEXT:    [[MUL248:%.*]] = mul nsw i64 [[DIV233]], [[CONV247]]
-// CHECK2-NEXT:    [[SUB249:%.*]] = sub nsw i64 [[TMP122]], [[MUL248]]
-// CHECK2-NEXT:    [[TMP138:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP139:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB250:%.*]] = sub i32 [[TMP140]], [[TMP141]]
-// CHECK2-NEXT:    [[SUB251:%.*]] = sub i32 [[SUB250]], 1
-// CHECK2-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD252:%.*]] = add i32 [[SUB251]], [[TMP142]]
-// CHECK2-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV253:%.*]] = udiv i32 [[ADD252]], [[TMP143]]
-// CHECK2-NEXT:    [[MUL254:%.*]] = mul i32 1, [[DIV253]]
-// CHECK2-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB255:%.*]] = sub i32 [[TMP144]], -63
-// CHECK2-NEXT:    [[DIV256:%.*]] = udiv i32 [[SUB255]], 64
-// CHECK2-NEXT:    [[MUL257:%.*]] = mul i32 [[MUL254]], [[DIV256]]
-// CHECK2-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB258:%.*]] = sub i32 [[TMP145]], [[TMP146]]
-// CHECK2-NEXT:    [[SUB259:%.*]] = sub i32 [[SUB258]], 1
-// CHECK2-NEXT:    [[ADD260:%.*]] = add i32 [[SUB259]], 1
-// CHECK2-NEXT:    [[DIV261:%.*]] = udiv i32 [[ADD260]], 1
-// CHECK2-NEXT:    [[MUL262:%.*]] = mul i32 [[MUL257]], [[DIV261]]
+// CHECK2-NEXT:    [[SUB200:%.*]] = sub i32 [[TMP113]], -63
+// CHECK2-NEXT:    [[DIV201:%.*]] = udiv i32 [[SUB200]], 64
+// CHECK2-NEXT:    [[MUL202:%.*]] = mul i32 1, [[DIV201]]
+// CHECK2-NEXT:    [[MUL203:%.*]] = mul i32 [[MUL202]], 64
+// CHECK2-NEXT:    [[CONV204:%.*]] = zext i32 [[MUL203]] to i64
+// CHECK2-NEXT:    [[DIV205:%.*]] = sdiv i64 [[SUB199]], [[CONV204]]
+// CHECK2-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB206:%.*]] = sub i32 [[TMP114]], -63
+// CHECK2-NEXT:    [[DIV207:%.*]] = udiv i32 [[SUB206]], 64
+// CHECK2-NEXT:    [[MUL208:%.*]] = mul i32 1, [[DIV207]]
+// CHECK2-NEXT:    [[MUL209:%.*]] = mul i32 [[MUL208]], 64
+// CHECK2-NEXT:    [[CONV210:%.*]] = zext i32 [[MUL209]] to i64
+// CHECK2-NEXT:    [[MUL211:%.*]] = mul nsw i64 [[DIV205]], [[CONV210]]
+// CHECK2-NEXT:    [[SUB212:%.*]] = sub nsw i64 [[SUB176]], [[MUL211]]
+// CHECK2-NEXT:    [[TMP115:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP116:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB213:%.*]] = sub i32 [[TMP117]], [[TMP118]]
+// CHECK2-NEXT:    [[SUB214:%.*]] = sub i32 [[SUB213]], 1
+// CHECK2-NEXT:    [[TMP119:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD215:%.*]] = add i32 [[SUB214]], [[TMP119]]
+// CHECK2-NEXT:    [[TMP120:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV216:%.*]] = udiv i32 [[ADD215]], [[TMP120]]
+// CHECK2-NEXT:    [[MUL217:%.*]] = mul i32 1, [[DIV216]]
+// CHECK2-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB218:%.*]] = sub i32 [[TMP121]], -63
+// CHECK2-NEXT:    [[DIV219:%.*]] = udiv i32 [[SUB218]], 64
+// CHECK2-NEXT:    [[MUL220:%.*]] = mul i32 [[MUL217]], [[DIV219]]
+// CHECK2-NEXT:    [[MUL221:%.*]] = mul i32 [[MUL220]], 64
+// CHECK2-NEXT:    [[CONV222:%.*]] = zext i32 [[MUL221]] to i64
+// CHECK2-NEXT:    [[DIV223:%.*]] = sdiv i64 [[TMP116]], [[CONV222]]
+// CHECK2-NEXT:    [[TMP122:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP123:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB224:%.*]] = sub i32 [[TMP122]], [[TMP123]]
+// CHECK2-NEXT:    [[SUB225:%.*]] = sub i32 [[SUB224]], 1
+// CHECK2-NEXT:    [[TMP124:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD226:%.*]] = add i32 [[SUB225]], [[TMP124]]
+// CHECK2-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV227:%.*]] = udiv i32 [[ADD226]], [[TMP125]]
+// CHECK2-NEXT:    [[MUL228:%.*]] = mul i32 1, [[DIV227]]
+// CHECK2-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB229:%.*]] = sub i32 [[TMP126]], -63
+// CHECK2-NEXT:    [[DIV230:%.*]] = udiv i32 [[SUB229]], 64
+// CHECK2-NEXT:    [[MUL231:%.*]] = mul i32 [[MUL228]], [[DIV230]]
+// CHECK2-NEXT:    [[MUL232:%.*]] = mul i32 [[MUL231]], 64
+// CHECK2-NEXT:    [[CONV233:%.*]] = zext i32 [[MUL232]] to i64
+// CHECK2-NEXT:    [[MUL234:%.*]] = mul nsw i64 [[DIV223]], [[CONV233]]
+// CHECK2-NEXT:    [[SUB235:%.*]] = sub nsw i64 [[TMP115]], [[MUL234]]
+// CHECK2-NEXT:    [[TMP127:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP128:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB236:%.*]] = sub i32 [[TMP129]], [[TMP130]]
+// CHECK2-NEXT:    [[SUB237:%.*]] = sub i32 [[SUB236]], 1
+// CHECK2-NEXT:    [[TMP131:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD238:%.*]] = add i32 [[SUB237]], [[TMP131]]
+// CHECK2-NEXT:    [[TMP132:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV239:%.*]] = udiv i32 [[ADD238]], [[TMP132]]
+// CHECK2-NEXT:    [[MUL240:%.*]] = mul i32 1, [[DIV239]]
+// CHECK2-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB241:%.*]] = sub i32 [[TMP133]], -63
+// CHECK2-NEXT:    [[DIV242:%.*]] = udiv i32 [[SUB241]], 64
+// CHECK2-NEXT:    [[MUL243:%.*]] = mul i32 [[MUL240]], [[DIV242]]
+// CHECK2-NEXT:    [[MUL244:%.*]] = mul i32 [[MUL243]], 64
+// CHECK2-NEXT:    [[CONV245:%.*]] = zext i32 [[MUL244]] to i64
+// CHECK2-NEXT:    [[DIV246:%.*]] = sdiv i64 [[TMP128]], [[CONV245]]
+// CHECK2-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB247:%.*]] = sub i32 [[TMP134]], [[TMP135]]
+// CHECK2-NEXT:    [[SUB248:%.*]] = sub i32 [[SUB247]], 1
+// CHECK2-NEXT:    [[TMP136:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD249:%.*]] = add i32 [[SUB248]], [[TMP136]]
+// CHECK2-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV250:%.*]] = udiv i32 [[ADD249]], [[TMP137]]
+// CHECK2-NEXT:    [[MUL251:%.*]] = mul i32 1, [[DIV250]]
+// CHECK2-NEXT:    [[TMP138:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB252:%.*]] = sub i32 [[TMP138]], -63
+// CHECK2-NEXT:    [[DIV253:%.*]] = udiv i32 [[SUB252]], 64
+// CHECK2-NEXT:    [[MUL254:%.*]] = mul i32 [[MUL251]], [[DIV253]]
+// CHECK2-NEXT:    [[MUL255:%.*]] = mul i32 [[MUL254]], 64
+// CHECK2-NEXT:    [[CONV256:%.*]] = zext i32 [[MUL255]] to i64
+// CHECK2-NEXT:    [[MUL257:%.*]] = mul nsw i64 [[DIV246]], [[CONV256]]
+// CHECK2-NEXT:    [[SUB258:%.*]] = sub nsw i64 [[TMP127]], [[MUL257]]
+// CHECK2-NEXT:    [[TMP139:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB259:%.*]] = sub i32 [[TMP139]], -63
+// CHECK2-NEXT:    [[DIV260:%.*]] = udiv i32 [[SUB259]], 64
+// CHECK2-NEXT:    [[MUL261:%.*]] = mul i32 1, [[DIV260]]
+// CHECK2-NEXT:    [[MUL262:%.*]] = mul i32 [[MUL261]], 64
 // CHECK2-NEXT:    [[CONV263:%.*]] = zext i32 [[MUL262]] to i64
-// CHECK2-NEXT:    [[DIV264:%.*]] = sdiv i64 [[TMP139]], [[CONV263]]
-// CHECK2-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB265:%.*]] = sub i32 [[TMP147]], [[TMP148]]
-// CHECK2-NEXT:    [[SUB266:%.*]] = sub i32 [[SUB265]], 1
-// CHECK2-NEXT:    [[TMP149:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD267:%.*]] = add i32 [[SUB266]], [[TMP149]]
-// CHECK2-NEXT:    [[TMP150:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV268:%.*]] = udiv i32 [[ADD267]], [[TMP150]]
-// CHECK2-NEXT:    [[MUL269:%.*]] = mul i32 1, [[DIV268]]
-// CHECK2-NEXT:    [[TMP151:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB270:%.*]] = sub i32 [[TMP151]], -63
-// CHECK2-NEXT:    [[DIV271:%.*]] = udiv i32 [[SUB270]], 64
-// CHECK2-NEXT:    [[MUL272:%.*]] = mul i32 [[MUL269]], [[DIV271]]
-// CHECK2-NEXT:    [[TMP152:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP153:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB273:%.*]] = sub i32 [[TMP152]], [[TMP153]]
-// CHECK2-NEXT:    [[SUB274:%.*]] = sub i32 [[SUB273]], 1
-// CHECK2-NEXT:    [[ADD275:%.*]] = add i32 [[SUB274]], 1
-// CHECK2-NEXT:    [[DIV276:%.*]] = udiv i32 [[ADD275]], 1
-// CHECK2-NEXT:    [[MUL277:%.*]] = mul i32 [[MUL272]], [[DIV276]]
-// CHECK2-NEXT:    [[CONV278:%.*]] = zext i32 [[MUL277]] to i64
-// CHECK2-NEXT:    [[MUL279:%.*]] = mul nsw i64 [[DIV264]], [[CONV278]]
-// CHECK2-NEXT:    [[SUB280:%.*]] = sub nsw i64 [[TMP138]], [[MUL279]]
-// CHECK2-NEXT:    [[TMP154:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB281:%.*]] = sub i32 [[TMP154]], -63
-// CHECK2-NEXT:    [[DIV282:%.*]] = udiv i32 [[SUB281]], 64
-// CHECK2-NEXT:    [[MUL283:%.*]] = mul i32 1, [[DIV282]]
-// CHECK2-NEXT:    [[TMP155:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP156:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB284:%.*]] = sub i32 [[TMP155]], [[TMP156]]
-// CHECK2-NEXT:    [[SUB285:%.*]] = sub i32 [[SUB284]], 1
-// CHECK2-NEXT:    [[ADD286:%.*]] = add i32 [[SUB285]], 1
-// CHECK2-NEXT:    [[DIV287:%.*]] = udiv i32 [[ADD286]], 1
-// CHECK2-NEXT:    [[MUL288:%.*]] = mul i32 [[MUL283]], [[DIV287]]
-// CHECK2-NEXT:    [[CONV289:%.*]] = zext i32 [[MUL288]] to i64
-// CHECK2-NEXT:    [[DIV290:%.*]] = sdiv i64 [[SUB280]], [[CONV289]]
-// CHECK2-NEXT:    [[TMP157:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB291:%.*]] = sub i32 [[TMP157]], -63
-// CHECK2-NEXT:    [[DIV292:%.*]] = udiv i32 [[SUB291]], 64
-// CHECK2-NEXT:    [[MUL293:%.*]] = mul i32 1, [[DIV292]]
-// CHECK2-NEXT:    [[TMP158:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP159:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB294:%.*]] = sub i32 [[TMP158]], [[TMP159]]
-// CHECK2-NEXT:    [[SUB295:%.*]] = sub i32 [[SUB294]], 1
-// CHECK2-NEXT:    [[ADD296:%.*]] = add i32 [[SUB295]], 1
-// CHECK2-NEXT:    [[DIV297:%.*]] = udiv i32 [[ADD296]], 1
-// CHECK2-NEXT:    [[MUL298:%.*]] = mul i32 [[MUL293]], [[DIV297]]
-// CHECK2-NEXT:    [[CONV299:%.*]] = zext i32 [[MUL298]] to i64
-// CHECK2-NEXT:    [[MUL300:%.*]] = mul nsw i64 [[DIV290]], [[CONV299]]
-// CHECK2-NEXT:    [[SUB301:%.*]] = sub nsw i64 [[SUB249]], [[MUL300]]
-// CHECK2-NEXT:    [[TMP160:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP161:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP162:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP163:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB302:%.*]] = sub i32 [[TMP162]], [[TMP163]]
-// CHECK2-NEXT:    [[SUB303:%.*]] = sub i32 [[SUB302]], 1
-// CHECK2-NEXT:    [[TMP164:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD304:%.*]] = add i32 [[SUB303]], [[TMP164]]
-// CHECK2-NEXT:    [[TMP165:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV305:%.*]] = udiv i32 [[ADD304]], [[TMP165]]
-// CHECK2-NEXT:    [[MUL306:%.*]] = mul i32 1, [[DIV305]]
-// CHECK2-NEXT:    [[TMP166:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB307:%.*]] = sub i32 [[TMP166]], -63
-// CHECK2-NEXT:    [[DIV308:%.*]] = udiv i32 [[SUB307]], 64
-// CHECK2-NEXT:    [[MUL309:%.*]] = mul i32 [[MUL306]], [[DIV308]]
-// CHECK2-NEXT:    [[TMP167:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP168:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB310:%.*]] = sub i32 [[TMP167]], [[TMP168]]
-// CHECK2-NEXT:    [[SUB311:%.*]] = sub i32 [[SUB310]], 1
-// CHECK2-NEXT:    [[ADD312:%.*]] = add i32 [[SUB311]], 1
-// CHECK2-NEXT:    [[DIV313:%.*]] = udiv i32 [[ADD312]], 1
-// CHECK2-NEXT:    [[MUL314:%.*]] = mul i32 [[MUL309]], [[DIV313]]
-// CHECK2-NEXT:    [[CONV315:%.*]] = zext i32 [[MUL314]] to i64
-// CHECK2-NEXT:    [[DIV316:%.*]] = sdiv i64 [[TMP161]], [[CONV315]]
-// CHECK2-NEXT:    [[TMP169:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP170:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB317:%.*]] = sub i32 [[TMP169]], [[TMP170]]
-// CHECK2-NEXT:    [[SUB318:%.*]] = sub i32 [[SUB317]], 1
-// CHECK2-NEXT:    [[TMP171:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD319:%.*]] = add i32 [[SUB318]], [[TMP171]]
-// CHECK2-NEXT:    [[TMP172:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV320:%.*]] = udiv i32 [[ADD319]], [[TMP172]]
-// CHECK2-NEXT:    [[MUL321:%.*]] = mul i32 1, [[DIV320]]
-// CHECK2-NEXT:    [[TMP173:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB322:%.*]] = sub i32 [[TMP173]], -63
-// CHECK2-NEXT:    [[DIV323:%.*]] = udiv i32 [[SUB322]], 64
-// CHECK2-NEXT:    [[MUL324:%.*]] = mul i32 [[MUL321]], [[DIV323]]
-// CHECK2-NEXT:    [[TMP174:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP175:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB325:%.*]] = sub i32 [[TMP174]], [[TMP175]]
-// CHECK2-NEXT:    [[SUB326:%.*]] = sub i32 [[SUB325]], 1
-// CHECK2-NEXT:    [[ADD327:%.*]] = add i32 [[SUB326]], 1
-// CHECK2-NEXT:    [[DIV328:%.*]] = udiv i32 [[ADD327]], 1
-// CHECK2-NEXT:    [[MUL329:%.*]] = mul i32 [[MUL324]], [[DIV328]]
-// CHECK2-NEXT:    [[CONV330:%.*]] = zext i32 [[MUL329]] to i64
-// CHECK2-NEXT:    [[MUL331:%.*]] = mul nsw i64 [[DIV316]], [[CONV330]]
-// CHECK2-NEXT:    [[SUB332:%.*]] = sub nsw i64 [[TMP160]], [[MUL331]]
-// CHECK2-NEXT:    [[TMP176:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP177:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP178:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP179:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB333:%.*]] = sub i32 [[TMP178]], [[TMP179]]
-// CHECK2-NEXT:    [[SUB334:%.*]] = sub i32 [[SUB333]], 1
-// CHECK2-NEXT:    [[TMP180:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD335:%.*]] = add i32 [[SUB334]], [[TMP180]]
-// CHECK2-NEXT:    [[TMP181:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV336:%.*]] = udiv i32 [[ADD335]], [[TMP181]]
-// CHECK2-NEXT:    [[MUL337:%.*]] = mul i32 1, [[DIV336]]
-// CHECK2-NEXT:    [[TMP182:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB338:%.*]] = sub i32 [[TMP182]], -63
-// CHECK2-NEXT:    [[DIV339:%.*]] = udiv i32 [[SUB338]], 64
-// CHECK2-NEXT:    [[MUL340:%.*]] = mul i32 [[MUL337]], [[DIV339]]
-// CHECK2-NEXT:    [[TMP183:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP184:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB341:%.*]] = sub i32 [[TMP183]], [[TMP184]]
-// CHECK2-NEXT:    [[SUB342:%.*]] = sub i32 [[SUB341]], 1
-// CHECK2-NEXT:    [[ADD343:%.*]] = add i32 [[SUB342]], 1
-// CHECK2-NEXT:    [[DIV344:%.*]] = udiv i32 [[ADD343]], 1
-// CHECK2-NEXT:    [[MUL345:%.*]] = mul i32 [[MUL340]], [[DIV344]]
-// CHECK2-NEXT:    [[CONV346:%.*]] = zext i32 [[MUL345]] to i64
-// CHECK2-NEXT:    [[DIV347:%.*]] = sdiv i64 [[TMP177]], [[CONV346]]
-// CHECK2-NEXT:    [[TMP185:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP186:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB348:%.*]] = sub i32 [[TMP185]], [[TMP186]]
-// CHECK2-NEXT:    [[SUB349:%.*]] = sub i32 [[SUB348]], 1
-// CHECK2-NEXT:    [[TMP187:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD350:%.*]] = add i32 [[SUB349]], [[TMP187]]
-// CHECK2-NEXT:    [[TMP188:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV351:%.*]] = udiv i32 [[ADD350]], [[TMP188]]
-// CHECK2-NEXT:    [[MUL352:%.*]] = mul i32 1, [[DIV351]]
-// CHECK2-NEXT:    [[TMP189:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB353:%.*]] = sub i32 [[TMP189]], -63
-// CHECK2-NEXT:    [[DIV354:%.*]] = udiv i32 [[SUB353]], 64
-// CHECK2-NEXT:    [[MUL355:%.*]] = mul i32 [[MUL352]], [[DIV354]]
-// CHECK2-NEXT:    [[TMP190:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP191:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB356:%.*]] = sub i32 [[TMP190]], [[TMP191]]
-// CHECK2-NEXT:    [[SUB357:%.*]] = sub i32 [[SUB356]], 1
-// CHECK2-NEXT:    [[ADD358:%.*]] = add i32 [[SUB357]], 1
-// CHECK2-NEXT:    [[DIV359:%.*]] = udiv i32 [[ADD358]], 1
-// CHECK2-NEXT:    [[MUL360:%.*]] = mul i32 [[MUL355]], [[DIV359]]
-// CHECK2-NEXT:    [[CONV361:%.*]] = zext i32 [[MUL360]] to i64
-// CHECK2-NEXT:    [[MUL362:%.*]] = mul nsw i64 [[DIV347]], [[CONV361]]
-// CHECK2-NEXT:    [[SUB363:%.*]] = sub nsw i64 [[TMP176]], [[MUL362]]
-// CHECK2-NEXT:    [[TMP192:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB364:%.*]] = sub i32 [[TMP192]], -63
-// CHECK2-NEXT:    [[DIV365:%.*]] = udiv i32 [[SUB364]], 64
-// CHECK2-NEXT:    [[MUL366:%.*]] = mul i32 1, [[DIV365]]
-// CHECK2-NEXT:    [[TMP193:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP194:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB367:%.*]] = sub i32 [[TMP193]], [[TMP194]]
-// CHECK2-NEXT:    [[SUB368:%.*]] = sub i32 [[SUB367]], 1
-// CHECK2-NEXT:    [[ADD369:%.*]] = add i32 [[SUB368]], 1
-// CHECK2-NEXT:    [[DIV370:%.*]] = udiv i32 [[ADD369]], 1
-// CHECK2-NEXT:    [[MUL371:%.*]] = mul i32 [[MUL366]], [[DIV370]]
-// CHECK2-NEXT:    [[CONV372:%.*]] = zext i32 [[MUL371]] to i64
-// CHECK2-NEXT:    [[DIV373:%.*]] = sdiv i64 [[SUB363]], [[CONV372]]
-// CHECK2-NEXT:    [[TMP195:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB374:%.*]] = sub i32 [[TMP195]], -63
-// CHECK2-NEXT:    [[DIV375:%.*]] = udiv i32 [[SUB374]], 64
-// CHECK2-NEXT:    [[MUL376:%.*]] = mul i32 1, [[DIV375]]
-// CHECK2-NEXT:    [[TMP196:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP197:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB377:%.*]] = sub i32 [[TMP196]], [[TMP197]]
-// CHECK2-NEXT:    [[SUB378:%.*]] = sub i32 [[SUB377]], 1
-// CHECK2-NEXT:    [[ADD379:%.*]] = add i32 [[SUB378]], 1
-// CHECK2-NEXT:    [[DIV380:%.*]] = udiv i32 [[ADD379]], 1
-// CHECK2-NEXT:    [[MUL381:%.*]] = mul i32 [[MUL376]], [[DIV380]]
-// CHECK2-NEXT:    [[CONV382:%.*]] = zext i32 [[MUL381]] to i64
-// CHECK2-NEXT:    [[MUL383:%.*]] = mul nsw i64 [[DIV373]], [[CONV382]]
-// CHECK2-NEXT:    [[SUB384:%.*]] = sub nsw i64 [[SUB332]], [[MUL383]]
-// CHECK2-NEXT:    [[TMP198:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP199:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB385:%.*]] = sub i32 [[TMP198]], [[TMP199]]
-// CHECK2-NEXT:    [[SUB386:%.*]] = sub i32 [[SUB385]], 1
-// CHECK2-NEXT:    [[ADD387:%.*]] = add i32 [[SUB386]], 1
-// CHECK2-NEXT:    [[DIV388:%.*]] = udiv i32 [[ADD387]], 1
-// CHECK2-NEXT:    [[MUL389:%.*]] = mul i32 1, [[DIV388]]
-// CHECK2-NEXT:    [[CONV390:%.*]] = zext i32 [[MUL389]] to i64
-// CHECK2-NEXT:    [[DIV391:%.*]] = sdiv i64 [[SUB384]], [[CONV390]]
-// CHECK2-NEXT:    [[TMP200:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_17]], align 4
-// CHECK2-NEXT:    [[TMP201:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_16]], align 4
-// CHECK2-NEXT:    [[SUB392:%.*]] = sub i32 [[TMP200]], [[TMP201]]
-// CHECK2-NEXT:    [[SUB393:%.*]] = sub i32 [[SUB392]], 1
-// CHECK2-NEXT:    [[ADD394:%.*]] = add i32 [[SUB393]], 1
-// CHECK2-NEXT:    [[DIV395:%.*]] = udiv i32 [[ADD394]], 1
-// CHECK2-NEXT:    [[MUL396:%.*]] = mul i32 1, [[DIV395]]
-// CHECK2-NEXT:    [[CONV397:%.*]] = zext i32 [[MUL396]] to i64
-// CHECK2-NEXT:    [[MUL398:%.*]] = mul nsw i64 [[DIV391]], [[CONV397]]
-// CHECK2-NEXT:    [[SUB399:%.*]] = sub nsw i64 [[SUB301]], [[MUL398]]
-// CHECK2-NEXT:    [[MUL400:%.*]] = mul nsw i64 [[SUB399]], 1
-// CHECK2-NEXT:    [[ADD401:%.*]] = add nsw i64 [[CONV218]], [[MUL400]]
-// CHECK2-NEXT:    [[CONV402:%.*]] = trunc i64 [[ADD401]] to i32
-// CHECK2-NEXT:    store i32 [[CONV402]], ptr [[DOTTILE_0_IV_K52]], align 4
-// CHECK2-NEXT:    [[TMP202:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
-// CHECK2-NEXT:    [[TMP203:%.*]] = load i32, ptr [[DOTTILE_0_IV_K52]], align 4
-// CHECK2-NEXT:    [[TMP204:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
-// CHECK2-NEXT:    [[MUL403:%.*]] = mul i32 [[TMP203]], [[TMP204]]
-// CHECK2-NEXT:    [[ADD404:%.*]] = add i32 [[TMP202]], [[MUL403]]
-// CHECK2-NEXT:    store i32 [[ADD404]], ptr [[K]], align 4
-// CHECK2-NEXT:    [[TMP205:%.*]] = load i32, ptr [[I49]], align 4
-// CHECK2-NEXT:    [[TMP206:%.*]] = load i32, ptr [[J50]], align 4
-// CHECK2-NEXT:    [[TMP207:%.*]] = load i32, ptr [[K]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP205]], i32 noundef [[TMP206]], i32 noundef [[TMP207]])
+// CHECK2-NEXT:    [[DIV264:%.*]] = sdiv i64 [[SUB258]], [[CONV263]]
+// CHECK2-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB265:%.*]] = sub i32 [[TMP140]], -63
+// CHECK2-NEXT:    [[DIV266:%.*]] = udiv i32 [[SUB265]], 64
+// CHECK2-NEXT:    [[MUL267:%.*]] = mul i32 1, [[DIV266]]
+// CHECK2-NEXT:    [[MUL268:%.*]] = mul i32 [[MUL267]], 64
+// CHECK2-NEXT:    [[CONV269:%.*]] = zext i32 [[MUL268]] to i64
+// CHECK2-NEXT:    [[MUL270:%.*]] = mul nsw i64 [[DIV264]], [[CONV269]]
+// CHECK2-NEXT:    [[SUB271:%.*]] = sub nsw i64 [[SUB235]], [[MUL270]]
+// CHECK2-NEXT:    [[DIV272:%.*]] = sdiv i64 [[SUB271]], 64
+// CHECK2-NEXT:    [[MUL273:%.*]] = mul nsw i64 [[DIV272]], 64
+// CHECK2-NEXT:    [[SUB274:%.*]] = sub nsw i64 [[SUB212]], [[MUL273]]
+// CHECK2-NEXT:    [[MUL275:%.*]] = mul nsw i64 [[SUB274]], 1
+// CHECK2-NEXT:    [[ADD276:%.*]] = add nsw i64 0, [[MUL275]]
+// CHECK2-NEXT:    [[CONV277:%.*]] = trunc i64 [[ADD276]] to i32
+// CHECK2-NEXT:    store i32 [[CONV277]], ptr [[DOTTILE_CNT_0_IV_K38]], align 4
+// CHECK2-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
+// CHECK2-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
+// CHECK2-NEXT:    [[ADD278:%.*]] = add i32 [[TMP141]], [[TMP142]]
+// CHECK2-NEXT:    store i32 [[ADD278]], ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK2-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
+// CHECK2-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK2-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
+// CHECK2-NEXT:    [[MUL279:%.*]] = mul i32 [[TMP144]], [[TMP145]]
+// CHECK2-NEXT:    [[ADD280:%.*]] = add i32 [[TMP143]], [[MUL279]]
+// CHECK2-NEXT:    store i32 [[ADD280]], ptr [[K]], align 4
+// CHECK2-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK2-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// CHECK2-NEXT:    [[ADD281:%.*]] = add i32 [[TMP147]], 1
+// CHECK2-NEXT:    [[CMP282:%.*]] = icmp ult i32 [[TMP146]], [[ADD281]]
+// CHECK2-NEXT:    br i1 [[CMP282]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK2:       if.then:
+// CHECK2-NEXT:    [[TMP148:%.*]] = load i32, ptr [[I35]], align 4
+// CHECK2-NEXT:    [[TMP149:%.*]] = load i32, ptr [[J36]], align 4
+// CHECK2-NEXT:    [[TMP150:%.*]] = load i32, ptr [[K]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP148]], i32 noundef [[TMP149]], i32 noundef [[TMP150]])
+// CHECK2-NEXT:    br label [[IF_END]]
+// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
-// CHECK2-NEXT:    [[TMP208:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[ADD405:%.*]] = add nsw i64 [[TMP208]], 1
-// CHECK2-NEXT:    store i64 [[ADD405]], ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP151:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[ADD283:%.*]] = add nsw i64 [[TMP151]], 1
+// CHECK2-NEXT:    store i64 [[ADD283]], ptr [[DOTOMP_IV]], align 8
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -5321,4 +4606,4 @@ extern "C" void foo10() {
 // CHECK2:       omp.precond.end:
 // CHECK2-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3]], i32 [[TMP0]])
 // CHECK2-NEXT:    ret void
-
+//
diff --git a/clang/test/OpenMP/irbuilder_unroll_partial_factor_for.c b/clang/test/OpenMP/irbuilder_unroll_partial_factor_for.c
index cf4fa21c9a31c..43711af5f2e33 100644
--- a/clang/test/OpenMP/irbuilder_unroll_partial_factor_for.c
+++ b/clang/test/OpenMP/irbuilder_unroll_partial_factor_for.c
@@ -79,8 +79,6 @@ void unroll_partial_heuristic_for(int n, float *a, float *b, float *c, float *d)
 // CHECK-NEXT:    br i1 [[OMP_FLOOR0_CMP]], label [[OMP_FLOOR0_BODY:%.*]], label [[OMP_FLOOR0_EXIT:%.*]]
 // CHECK:       omp_floor0.body:
 // CHECK-NEXT:    [[TMP13:%.*]] = add i32 [[OMP_FLOOR0_IV]], [[TMP9]]
-// CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP13]], [[TMP4]]
-// CHECK-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i32 [[TMP5]], i32 13
 // CHECK-NEXT:    br label [[OMP_TILE0_PREHEADER:%.*]]
 // CHECK:       omp_tile0.preheader:
 // CHECK-NEXT:    br label [[OMP_TILE0_HEADER:%.*]]
@@ -88,36 +86,39 @@ void unroll_partial_heuristic_for(int n, float *a, float *b, float *c, float *d)
 // CHECK-NEXT:    [[OMP_TILE0_IV:%.*]] = phi i32 [ 0, [[OMP_TILE0_PREHEADER]] ], [ [[OMP_TILE0_NEXT:%.*]], [[OMP_TILE0_INC:%.*]] ]
 // CHECK-NEXT:    br label [[OMP_TILE0_COND:%.*]]
 // CHECK:       omp_tile0.cond:
-// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], [[TMP15]]
+// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], 13
 // CHECK-NEXT:    br i1 [[OMP_TILE0_CMP]], label [[OMP_TILE0_BODY:%.*]], label [[OMP_TILE0_EXIT:%.*]]
 // CHECK:       omp_tile0.body:
-// CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i32 13, [[TMP13]]
-// CHECK-NEXT:    [[TMP17:%.*]] = add nuw i32 [[TMP16]], [[OMP_TILE0_IV]]
-// CHECK-NEXT:    br label [[OMP_LOOP_BODY:%.*]]
+// CHECK-NEXT:    [[TMP14:%.*]] = mul nuw i32 13, [[TMP13]]
+// CHECK-NEXT:    [[TMP15:%.*]] = add nuw i32 [[TMP14]], [[OMP_TILE0_IV]]
+// CHECK-NEXT:    [[OMP_TILE0_INBOUNDS:%.*]] = icmp ult i32 [[TMP15]], [[DOTCOUNT]]
+// CHECK-NEXT:    br i1 [[OMP_TILE0_INBOUNDS]], label [[OMP_LOOP_BODY:%.*]], label [[OMP_TILE_BODY_MERGE:%.*]]
 // CHECK:       omp_loop.body:
-// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP17]], ptr [[AGG_CAPTURED1]])
-// CHECK-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[B_ADDR]], align 8
-// CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP19]] to i64
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[IDXPROM]]
-// CHECK-NEXT:    [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-// CHECK-NEXT:    [[TMP21:%.*]] = load ptr, ptr [[C_ADDR]], align 8
-// CHECK-NEXT:    [[TMP22:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP22]] to i64
-// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP21]], i64 [[IDXPROM2]]
-// CHECK-NEXT:    [[TMP23:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
-// CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP20]], [[TMP23]]
-// CHECK-NEXT:    [[TMP24:%.*]] = load ptr, ptr [[D_ADDR]], align 8
-// CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[TMP25]] to i64
-// CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[TMP24]], i64 [[IDXPROM4]]
-// CHECK-NEXT:    [[TMP26:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
-// CHECK-NEXT:    [[MUL6:%.*]] = fmul float [[MUL]], [[TMP26]]
-// CHECK-NEXT:    [[TMP27:%.*]] = load ptr, ptr [[A_ADDR]], align 8
-// CHECK-NEXT:    [[TMP28:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM7:%.*]] = sext i32 [[TMP28]] to i64
-// CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[TMP27]], i64 [[IDXPROM7]]
+// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP15]], ptr [[AGG_CAPTURED1]])
+// CHECK-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[B_ADDR]], align 8
+// CHECK-NEXT:    [[TMP17:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP17]] to i64
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[IDXPROM]]
+// CHECK-NEXT:    [[TMP18:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+// CHECK-NEXT:    [[TMP19:%.*]] = load ptr, ptr [[C_ADDR]], align 8
+// CHECK-NEXT:    [[TMP20:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP20]] to i64
+// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP19]], i64 [[IDXPROM2]]
+// CHECK-NEXT:    [[TMP21:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
+// CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP18]], [[TMP21]]
+// CHECK-NEXT:    [[TMP22:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK-NEXT:    [[TMP23:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[TMP23]] to i64
+// CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[TMP22]], i64 [[IDXPROM4]]
+// CHECK-NEXT:    [[TMP24:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
+// CHECK-NEXT:    [[MUL6:%.*]] = fmul float [[MUL]], [[TMP24]]
+// CHECK-NEXT:    [[TMP25:%.*]] = load ptr, ptr [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP26:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM7:%.*]] = sext i32 [[TMP26]] to i64
+// CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[TMP25]], i64 [[IDXPROM7]]
 // CHECK-NEXT:    store float [[MUL6]], ptr [[ARRAYIDX8]], align 4
+// CHECK-NEXT:    br label [[OMP_TILE_BODY_MERGE]]
+// CHECK:       omp_tile.body.merge:
 // CHECK-NEXT:    br label [[OMP_TILE0_INC]]
 // CHECK:       omp_tile0.inc:
 // CHECK-NEXT:    [[OMP_TILE0_NEXT]] = add nuw i32 [[OMP_TILE0_IV]], 1
diff --git a/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_constant_for.c b/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_constant_for.c
index acf0e5164411c..b8a993c9cd8a7 100644
--- a/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_constant_for.c
+++ b/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_constant_for.c
@@ -85,8 +85,6 @@ void unroll_partial_heuristic_constant_for(float *a, float *b, float *c, float *
 // CHECK-NEXT:    br i1 [[OMP_FLOOR0_CMP]], label [[OMP_FLOOR0_BODY:%.*]], label [[OMP_FLOOR0_EXIT:%.*]]
 // CHECK:       omp_floor0.body:
 // CHECK-NEXT:    [[TMP12:%.*]] = add i32 [[OMP_FLOOR0_IV]], [[TMP8]]
-// CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP12]], [[TMP3]]
-// CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP4]], i32 4
 // CHECK-NEXT:    br label [[OMP_TILE0_PREHEADER:%.*]]
 // CHECK:       omp_tile0.preheader:
 // CHECK-NEXT:    br label [[OMP_TILE0_HEADER:%.*]]
@@ -94,54 +92,57 @@ void unroll_partial_heuristic_constant_for(float *a, float *b, float *c, float *
 // CHECK-NEXT:    [[OMP_TILE0_IV:%.*]] = phi i32 [ 0, [[OMP_TILE0_PREHEADER]] ], [ [[OMP_TILE0_NEXT:%.*]], [[OMP_TILE0_INC:%.*]] ]
 // CHECK-NEXT:    br label [[OMP_TILE0_COND:%.*]]
 // CHECK:       omp_tile0.cond:
-// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], [[TMP14]]
+// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], 4
 // CHECK-NEXT:    br i1 [[OMP_TILE0_CMP]], label [[OMP_TILE0_BODY:%.*]], label [[OMP_TILE0_EXIT:%.*]]
 // CHECK:       omp_tile0.body:
-// CHECK-NEXT:    [[TMP15:%.*]] = mul nuw i32 4, [[TMP12]]
-// CHECK-NEXT:    [[TMP16:%.*]] = add nuw i32 [[TMP15]], [[OMP_TILE0_IV]]
-// CHECK-NEXT:    br label [[OMP_LOOP_BODY:%.*]]
+// CHECK-NEXT:    [[TMP13:%.*]] = mul nuw i32 4, [[TMP12]]
+// CHECK-NEXT:    [[TMP14:%.*]] = add nuw i32 [[TMP13]], [[OMP_TILE0_IV]]
+// CHECK-NEXT:    [[OMP_TILE0_INBOUNDS:%.*]] = icmp ult i32 [[TMP14]], [[DOTCOUNT]]
+// CHECK-NEXT:    br i1 [[OMP_TILE0_INBOUNDS]], label [[OMP_LOOP_BODY:%.*]], label [[OMP_TILE_BODY_MERGE:%.*]]
 // CHECK:       omp_loop.body:
-// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP16]], ptr [[AGG_CAPTURED1]])
-// CHECK-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[B_ADDR]], align 8
-// CHECK-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP18]] to i64
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[IDXPROM]]
-// CHECK-NEXT:    [[TMP19:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-// CHECK-NEXT:    [[CONV:%.*]] = fpext float [[TMP19]] to double
+// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP14]], ptr [[AGG_CAPTURED1]])
+// CHECK-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[B_ADDR]], align 8
+// CHECK-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP16]] to i64
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP15]], i64 [[IDXPROM]]
+// CHECK-NEXT:    [[TMP17:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+// CHECK-NEXT:    [[CONV:%.*]] = fpext float [[TMP17]] to double
 // CHECK-NEXT:    [[CALL:%.*]] = call double @sind(double noundef [[CONV]])
-// CHECK-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[C_ADDR]], align 8
-// CHECK-NEXT:    [[TMP21:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP21]] to i64
-// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP20]], i64 [[IDXPROM2]]
-// CHECK-NEXT:    [[TMP22:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
-// CHECK-NEXT:    [[CONV4:%.*]] = fpext float [[TMP22]] to double
+// CHECK-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[C_ADDR]], align 8
+// CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP19]] to i64
+// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[IDXPROM2]]
+// CHECK-NEXT:    [[TMP20:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
+// CHECK-NEXT:    [[CONV4:%.*]] = fpext float [[TMP20]] to double
 // CHECK-NEXT:    [[MUL:%.*]] = fmul double [[CALL]], [[CONV4]]
-// CHECK-NEXT:    [[TMP23:%.*]] = load ptr, ptr [[D_ADDR]], align 8
-// CHECK-NEXT:    [[TMP24:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM5:%.*]] = sext i32 [[TMP24]] to i64
-// CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, ptr [[TMP23]], i64 [[IDXPROM5]]
-// CHECK-NEXT:    [[TMP25:%.*]] = load float, ptr [[ARRAYIDX6]], align 4
-// CHECK-NEXT:    [[CONV7:%.*]] = fpext float [[TMP25]] to double
+// CHECK-NEXT:    [[TMP21:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK-NEXT:    [[TMP22:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM5:%.*]] = sext i32 [[TMP22]] to i64
+// CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, ptr [[TMP21]], i64 [[IDXPROM5]]
+// CHECK-NEXT:    [[TMP23:%.*]] = load float, ptr [[ARRAYIDX6]], align 4
+// CHECK-NEXT:    [[CONV7:%.*]] = fpext float [[TMP23]] to double
 // CHECK-NEXT:    [[MUL8:%.*]] = fmul double [[MUL]], [[CONV7]]
-// CHECK-NEXT:    [[TMP26:%.*]] = load ptr, ptr [[E_ADDR]], align 8
-// CHECK-NEXT:    [[TMP27:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM9:%.*]] = sext i32 [[TMP27]] to i64
-// CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[TMP26]], i64 [[IDXPROM9]]
-// CHECK-NEXT:    [[TMP28:%.*]] = load float, ptr [[ARRAYIDX10]], align 4
-// CHECK-NEXT:    [[CONV11:%.*]] = fpext float [[TMP28]] to double
+// CHECK-NEXT:    [[TMP24:%.*]] = load ptr, ptr [[E_ADDR]], align 8
+// CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM9:%.*]] = sext i32 [[TMP25]] to i64
+// CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[TMP24]], i64 [[IDXPROM9]]
+// CHECK-NEXT:    [[TMP26:%.*]] = load float, ptr [[ARRAYIDX10]], align 4
+// CHECK-NEXT:    [[CONV11:%.*]] = fpext float [[TMP26]] to double
 // CHECK-NEXT:    [[MUL12:%.*]] = fmul double [[MUL8]], [[CONV11]]
-// CHECK-NEXT:    [[TMP29:%.*]] = load float, ptr [[OFFSET_ADDR]], align 4
-// CHECK-NEXT:    [[CONV13:%.*]] = fpext float [[TMP29]] to double
+// CHECK-NEXT:    [[TMP27:%.*]] = load float, ptr [[OFFSET_ADDR]], align 4
+// CHECK-NEXT:    [[CONV13:%.*]] = fpext float [[TMP27]] to double
 // CHECK-NEXT:    [[ADD:%.*]] = fadd double [[MUL12]], [[CONV13]]
-// CHECK-NEXT:    [[TMP30:%.*]] = load ptr, ptr [[A_ADDR]], align 8
-// CHECK-NEXT:    [[TMP31:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM14:%.*]] = sext i32 [[TMP31]] to i64
-// CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds float, ptr [[TMP30]], i64 [[IDXPROM14]]
-// CHECK-NEXT:    [[TMP32:%.*]] = load float, ptr [[ARRAYIDX15]], align 4
-// CHECK-NEXT:    [[CONV16:%.*]] = fpext float [[TMP32]] to double
+// CHECK-NEXT:    [[TMP28:%.*]] = load ptr, ptr [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP29:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM14:%.*]] = sext i32 [[TMP29]] to i64
+// CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds float, ptr [[TMP28]], i64 [[IDXPROM14]]
+// CHECK-NEXT:    [[TMP30:%.*]] = load float, ptr [[ARRAYIDX15]], align 4
+// CHECK-NEXT:    [[CONV16:%.*]] = fpext float [[TMP30]] to double
 // CHECK-NEXT:    [[ADD17:%.*]] = fadd double [[CONV16]], [[ADD]]
 // CHECK-NEXT:    [[CONV18:%.*]] = fptrunc double [[ADD17]] to float
 // CHECK-NEXT:    store float [[CONV18]], ptr [[ARRAYIDX15]], align 4
+// CHECK-NEXT:    br label [[OMP_TILE_BODY_MERGE]]
+// CHECK:       omp_tile.body.merge:
 // CHECK-NEXT:    br label [[OMP_TILE0_INC]]
 // CHECK:       omp_tile0.inc:
 // CHECK-NEXT:    [[OMP_TILE0_NEXT]] = add nuw i32 [[OMP_TILE0_IV]], 1
diff --git a/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_runtime_for.c b/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_runtime_for.c
index 8fee5773e53b4..6b48e3ca22888 100644
--- a/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_runtime_for.c
+++ b/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_runtime_for.c
@@ -87,8 +87,6 @@ void unroll_partial_heuristic_runtime_for(int n, float *a, float *b, float *c, f
 // CHECK-NEXT:    br i1 [[OMP_FLOOR0_CMP]], label [[OMP_FLOOR0_BODY:%.*]], label [[OMP_FLOOR0_EXIT:%.*]]
 // CHECK:       omp_floor0.body:
 // CHECK-NEXT:    [[TMP13:%.*]] = add i32 [[OMP_FLOOR0_IV]], [[TMP9]]
-// CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP13]], [[TMP4]]
-// CHECK-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i32 [[TMP5]], i32 4
 // CHECK-NEXT:    br label [[OMP_TILE0_PREHEADER:%.*]]
 // CHECK:       omp_tile0.preheader:
 // CHECK-NEXT:    br label [[OMP_TILE0_HEADER:%.*]]
@@ -96,54 +94,57 @@ void unroll_partial_heuristic_runtime_for(int n, float *a, float *b, float *c, f
 // CHECK-NEXT:    [[OMP_TILE0_IV:%.*]] = phi i32 [ 0, [[OMP_TILE0_PREHEADER]] ], [ [[OMP_TILE0_NEXT:%.*]], [[OMP_TILE0_INC:%.*]] ]
 // CHECK-NEXT:    br label [[OMP_TILE0_COND:%.*]]
 // CHECK:       omp_tile0.cond:
-// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], [[TMP15]]
+// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], 4
 // CHECK-NEXT:    br i1 [[OMP_TILE0_CMP]], label [[OMP_TILE0_BODY:%.*]], label [[OMP_TILE0_EXIT:%.*]]
 // CHECK:       omp_tile0.body:
-// CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i32 4, [[TMP13]]
-// CHECK-NEXT:    [[TMP17:%.*]] = add nuw i32 [[TMP16]], [[OMP_TILE0_IV]]
-// CHECK-NEXT:    br label [[OMP_LOOP_BODY:%.*]]
+// CHECK-NEXT:    [[TMP14:%.*]] = mul nuw i32 4, [[TMP13]]
+// CHECK-NEXT:    [[TMP15:%.*]] = add nuw i32 [[TMP14]], [[OMP_TILE0_IV]]
+// CHECK-NEXT:    [[OMP_TILE0_INBOUNDS:%.*]] = icmp ult i32 [[TMP15]], [[DOTCOUNT]]
+// CHECK-NEXT:    br i1 [[OMP_TILE0_INBOUNDS]], label [[OMP_LOOP_BODY:%.*]], label [[OMP_TILE_BODY_MERGE:%.*]]
 // CHECK:       omp_loop.body:
-// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP17]], ptr [[AGG_CAPTURED1]])
-// CHECK-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[B_ADDR]], align 8
-// CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP19]] to i64
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[IDXPROM]]
-// CHECK-NEXT:    [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-// CHECK-NEXT:    [[CONV:%.*]] = fpext float [[TMP20]] to double
+// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP15]], ptr [[AGG_CAPTURED1]])
+// CHECK-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[B_ADDR]], align 8
+// CHECK-NEXT:    [[TMP17:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP17]] to i64
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[IDXPROM]]
+// CHECK-NEXT:    [[TMP18:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+// CHECK-NEXT:    [[CONV:%.*]] = fpext float [[TMP18]] to double
 // CHECK-NEXT:    [[CALL:%.*]] = call double @sind(double noundef [[CONV]])
-// CHECK-NEXT:    [[TMP21:%.*]] = load ptr, ptr [[C_ADDR]], align 8
-// CHECK-NEXT:    [[TMP22:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP22]] to i64
-// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP21]], i64 [[IDXPROM2]]
-// CHECK-NEXT:    [[TMP23:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
-// CHECK-NEXT:    [[CONV4:%.*]] = fpext float [[TMP23]] to double
+// CHECK-NEXT:    [[TMP19:%.*]] = load ptr, ptr [[C_ADDR]], align 8
+// CHECK-NEXT:    [[TMP20:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP20]] to i64
+// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP19]], i64 [[IDXPROM2]]
+// CHECK-NEXT:    [[TMP21:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
+// CHECK-NEXT:    [[CONV4:%.*]] = fpext float [[TMP21]] to double
 // CHECK-NEXT:    [[MUL:%.*]] = fmul double [[CALL]], [[CONV4]]
-// CHECK-NEXT:    [[TMP24:%.*]] = load ptr, ptr [[D_ADDR]], align 8
-// CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM5:%.*]] = sext i32 [[TMP25]] to i64
-// CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, ptr [[TMP24]], i64 [[IDXPROM5]]
-// CHECK-NEXT:    [[TMP26:%.*]] = load float, ptr [[ARRAYIDX6]], align 4
-// CHECK-NEXT:    [[CONV7:%.*]] = fpext float [[TMP26]] to double
+// CHECK-NEXT:    [[TMP22:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK-NEXT:    [[TMP23:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM5:%.*]] = sext i32 [[TMP23]] to i64
+// CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, ptr [[TMP22]], i64 [[IDXPROM5]]
+// CHECK-NEXT:    [[TMP24:%.*]] = load float, ptr [[ARRAYIDX6]], align 4
+// CHECK-NEXT:    [[CONV7:%.*]] = fpext float [[TMP24]] to double
 // CHECK-NEXT:    [[MUL8:%.*]] = fmul double [[MUL]], [[CONV7]]
-// CHECK-NEXT:    [[TMP27:%.*]] = load ptr, ptr [[E_ADDR]], align 8
-// CHECK-NEXT:    [[TMP28:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM9:%.*]] = sext i32 [[TMP28]] to i64
-// CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[TMP27]], i64 [[IDXPROM9]]
-// CHECK-NEXT:    [[TMP29:%.*]] = load float, ptr [[ARRAYIDX10]], align 4
-// CHECK-NEXT:    [[CONV11:%.*]] = fpext float [[TMP29]] to double
+// CHECK-NEXT:    [[TMP25:%.*]] = load ptr, ptr [[E_ADDR]], align 8
+// CHECK-NEXT:    [[TMP26:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM9:%.*]] = sext i32 [[TMP26]] to i64
+// CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[TMP25]], i64 [[IDXPROM9]]
+// CHECK-NEXT:    [[TMP27:%.*]] = load float, ptr [[ARRAYIDX10]], align 4
+// CHECK-NEXT:    [[CONV11:%.*]] = fpext float [[TMP27]] to double
 // CHECK-NEXT:    [[MUL12:%.*]] = fmul double [[MUL8]], [[CONV11]]
-// CHECK-NEXT:    [[TMP30:%.*]] = load float, ptr [[OFFSET_ADDR]], align 4
-// CHECK-NEXT:    [[CONV13:%.*]] = fpext float [[TMP30]] to double
+// CHECK-NEXT:    [[TMP28:%.*]] = load float, ptr [[OFFSET_ADDR]], align 4
+// CHECK-NEXT:    [[CONV13:%.*]] = fpext float [[TMP28]] to double
 // CHECK-NEXT:    [[ADD:%.*]] = fadd double [[MUL12]], [[CONV13]]
-// CHECK-NEXT:    [[TMP31:%.*]] = load ptr, ptr [[A_ADDR]], align 8
-// CHECK-NEXT:    [[TMP32:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM14:%.*]] = sext i32 [[TMP32]] to i64
-// CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds float, ptr [[TMP31]], i64 [[IDXPROM14]]
-// CHECK-NEXT:    [[TMP33:%.*]] = load float, ptr [[ARRAYIDX15]], align 4
-// CHECK-NEXT:    [[CONV16:%.*]] = fpext float [[TMP33]] to double
+// CHECK-NEXT:    [[TMP29:%.*]] = load ptr, ptr [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP30:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM14:%.*]] = sext i32 [[TMP30]] to i64
+// CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds float, ptr [[TMP29]], i64 [[IDXPROM14]]
+// CHECK-NEXT:    [[TMP31:%.*]] = load float, ptr [[ARRAYIDX15]], align 4
+// CHECK-NEXT:    [[CONV16:%.*]] = fpext float [[TMP31]] to double
 // CHECK-NEXT:    [[ADD17:%.*]] = fadd double [[CONV16]], [[ADD]]
 // CHECK-NEXT:    [[CONV18:%.*]] = fptrunc double [[ADD17]] to float
 // CHECK-NEXT:    store float [[CONV18]], ptr [[ARRAYIDX15]], align 4
+// CHECK-NEXT:    br label [[OMP_TILE_BODY_MERGE]]
+// CHECK:       omp_tile.body.merge:
 // CHECK-NEXT:    br label [[OMP_TILE0_INC]]
 // CHECK:       omp_tile0.inc:
 // CHECK-NEXT:    [[OMP_TILE0_NEXT]] = add nuw i32 [[OMP_TILE0_IV]], 1
diff --git a/clang/test/OpenMP/irbuilder_unroll_unroll_partial_factor.c b/clang/test/OpenMP/irbuilder_unroll_unroll_partial_factor.c
index c4ebcddb097bc..65bd495ca0426 100644
--- a/clang/test/OpenMP/irbuilder_unroll_unroll_partial_factor.c
+++ b/clang/test/OpenMP/irbuilder_unroll_unroll_partial_factor.c
@@ -75,8 +75,6 @@ void unroll_partial_factor_for(float *a, float *b, float *c, float *d) {
 // CHECK-NEXT:    br i1 [[OMP_FLOOR0_CMP]], label [[OMP_FLOOR0_BODY:%.*]], label [[OMP_FLOOR0_EXIT:%.*]]
 // CHECK:       omp_floor0.body:
 // CHECK-NEXT:    [[TMP12:%.*]] = add i32 [[OMP_FLOOR0_IV]], [[TMP8]]
-// CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP12]], [[TMP3]]
-// CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP4]], i32 2
 // CHECK-NEXT:    br label [[OMP_TILE0_PREHEADER:%.*]]
 // CHECK:       omp_tile0.preheader:
 // CHECK-NEXT:    br label [[OMP_TILE0_HEADER:%.*]]
@@ -84,36 +82,39 @@ void unroll_partial_factor_for(float *a, float *b, float *c, float *d) {
 // CHECK-NEXT:    [[OMP_TILE0_IV:%.*]] = phi i32 [ 0, [[OMP_TILE0_PREHEADER]] ], [ [[OMP_TILE0_NEXT:%.*]], [[OMP_TILE0_INC:%.*]] ]
 // CHECK-NEXT:    br label [[OMP_TILE0_COND:%.*]]
 // CHECK:       omp_tile0.cond:
-// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], [[TMP14]]
+// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], 2
 // CHECK-NEXT:    br i1 [[OMP_TILE0_CMP]], label [[OMP_TILE0_BODY:%.*]], label [[OMP_TILE0_EXIT:%.*]]
 // CHECK:       omp_tile0.body:
-// CHECK-NEXT:    [[TMP15:%.*]] = mul nuw i32 2, [[TMP12]]
-// CHECK-NEXT:    [[TMP16:%.*]] = add nuw i32 [[TMP15]], [[OMP_TILE0_IV]]
-// CHECK-NEXT:    br label [[OMP_LOOP_BODY:%.*]]
+// CHECK-NEXT:    [[TMP13:%.*]] = mul nuw i32 2, [[TMP12]]
+// CHECK-NEXT:    [[TMP14:%.*]] = add nuw i32 [[TMP13]], [[OMP_TILE0_IV]]
+// CHECK-NEXT:    [[OMP_TILE0_INBOUNDS:%.*]] = icmp ult i32 [[TMP14]], [[DOTCOUNT]]
+// CHECK-NEXT:    br i1 [[OMP_TILE0_INBOUNDS]], label [[OMP_LOOP_BODY:%.*]], label [[OMP_TILE_BODY_MERGE:%.*]]
 // CHECK:       omp_loop.body:
-// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP16]], ptr [[AGG_CAPTURED1]])
-// CHECK-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[B_ADDR]], align 8
-// CHECK-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP18]] to i64
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[IDXPROM]]
-// CHECK-NEXT:    [[TMP19:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-// CHECK-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[C_ADDR]], align 8
-// CHECK-NEXT:    [[TMP21:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP21]] to i64
-// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP20]], i64 [[IDXPROM2]]
-// CHECK-NEXT:    [[TMP22:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
-// CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP19]], [[TMP22]]
-// CHECK-NEXT:    [[TMP23:%.*]] = load ptr, ptr [[D_ADDR]], align 8
-// CHECK-NEXT:    [[TMP24:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[TMP24]] to i64
-// CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[TMP23]], i64 [[IDXPROM4]]
-// CHECK-NEXT:    [[TMP25:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
-// CHECK-NEXT:    [[MUL6:%.*]] = fmul float [[MUL]], [[TMP25]]
-// CHECK-NEXT:    [[TMP26:%.*]] = load ptr, ptr [[A_ADDR]], align 8
-// CHECK-NEXT:    [[TMP27:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM7:%.*]] = sext i32 [[TMP27]] to i64
-// CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[TMP26]], i64 [[IDXPROM7]]
+// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP14]], ptr [[AGG_CAPTURED1]])
+// CHECK-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[B_ADDR]], align 8
+// CHECK-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP16]] to i64
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP15]], i64 [[IDXPROM]]
+// CHECK-NEXT:    [[TMP17:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+// CHECK-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[C_ADDR]], align 8
+// CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP19]] to i64
+// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[IDXPROM2]]
+// CHECK-NEXT:    [[TMP20:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
+// CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP17]], [[TMP20]]
+// CHECK-NEXT:    [[TMP21:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK-NEXT:    [[TMP22:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[TMP22]] to i64
+// CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[TMP21]], i64 [[IDXPROM4]]
+// CHECK-NEXT:    [[TMP23:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
+// CHECK-NEXT:    [[MUL6:%.*]] = fmul float [[MUL]], [[TMP23]]
+// CHECK-NEXT:    [[TMP24:%.*]] = load ptr, ptr [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM7:%.*]] = sext i32 [[TMP25]] to i64
+// CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[TMP24]], i64 [[IDXPROM7]]
 // CHECK-NEXT:    store float [[MUL6]], ptr [[ARRAYIDX8]], align 4
+// CHECK-NEXT:    br label [[OMP_TILE_BODY_MERGE]]
+// CHECK:       omp_tile.body.merge:
 // CHECK-NEXT:    br label [[OMP_TILE0_INC]]
 // CHECK:       omp_tile0.inc:
 // CHECK-NEXT:    [[OMP_TILE0_NEXT]] = add nuw i32 [[OMP_TILE0_IV]], 1
diff --git a/clang/test/OpenMP/irbuilder_unroll_unroll_partial_heuristic.c b/clang/test/OpenMP/irbuilder_unroll_unroll_partial_heuristic.c
index 24d42d265d6a6..c2b7d46588f66 100644
--- a/clang/test/OpenMP/irbuilder_unroll_unroll_partial_heuristic.c
+++ b/clang/test/OpenMP/irbuilder_unroll_unroll_partial_heuristic.c
@@ -59,8 +59,6 @@ void unroll_unroll_partial_heuristic(float *a, float *b, float *c, float *d) {
 // CHECK-NEXT:    [[OMP_FLOOR0_CMP:%.*]] = icmp ult i32 [[OMP_FLOOR0_IV]], [[OMP_FLOOR0_TRIPCOUNT]]
 // CHECK-NEXT:    br i1 [[OMP_FLOOR0_CMP]], label [[OMP_FLOOR0_BODY:%.*]], label [[OMP_FLOOR0_EXIT:%.*]]
 // CHECK:       omp_floor0.body:
-// CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[OMP_FLOOR0_IV]], [[TMP3]]
-// CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP4]], i32 8
 // CHECK-NEXT:    br label [[OMP_TILE0_PREHEADER:%.*]]
 // CHECK:       omp_tile0.preheader:
 // CHECK-NEXT:    br label [[OMP_TILE0_HEADER:%.*]]
@@ -68,47 +66,50 @@ void unroll_unroll_partial_heuristic(float *a, float *b, float *c, float *d) {
 // CHECK-NEXT:    [[OMP_TILE0_IV:%.*]] = phi i32 [ 0, [[OMP_TILE0_PREHEADER]] ], [ [[OMP_TILE0_NEXT:%.*]], [[OMP_TILE0_INC:%.*]] ]
 // CHECK-NEXT:    br label [[OMP_TILE0_COND:%.*]]
 // CHECK:       omp_tile0.cond:
-// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], [[TMP8]]
+// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], 8
 // CHECK-NEXT:    br i1 [[OMP_TILE0_CMP]], label [[OMP_TILE0_BODY:%.*]], label [[OMP_TILE0_EXIT:%.*]]
 // CHECK:       omp_tile0.body:
-// CHECK-NEXT:    [[TMP9:%.*]] = mul nuw i32 8, [[OMP_FLOOR0_IV]]
-// CHECK-NEXT:    [[TMP10:%.*]] = add nuw i32 [[TMP9]], [[OMP_TILE0_IV]]
-// CHECK-NEXT:    br label [[OMP_LOOP_BODY:%.*]]
+// CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i32 8, [[OMP_FLOOR0_IV]]
+// CHECK-NEXT:    [[TMP8:%.*]] = add nuw i32 [[TMP7]], [[OMP_TILE0_IV]]
+// CHECK-NEXT:    [[OMP_TILE0_INBOUNDS:%.*]] = icmp ult i32 [[TMP8]], [[DOTCOUNT]]
+// CHECK-NEXT:    br i1 [[OMP_TILE0_INBOUNDS]], label [[OMP_LOOP_BODY:%.*]], label [[OMP_TILE_BODY_MERGE:%.*]]
 // CHECK:       omp_loop.body:
-// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP10]], ptr [[AGG_CAPTURED1]])
-// CHECK-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[B_ADDR]], align 8
-// CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP12]] to i64
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP11]], i64 [[IDXPROM]]
-// CHECK-NEXT:    [[TMP13:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-// CHECK-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[C_ADDR]], align 8
-// CHECK-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP15]] to i64
-// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP14]], i64 [[IDXPROM2]]
-// CHECK-NEXT:    [[TMP16:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
-// CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP13]], [[TMP16]]
-// CHECK-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[D_ADDR]], align 8
-// CHECK-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[TMP18]] to i64
-// CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[IDXPROM4]]
-// CHECK-NEXT:    [[TMP19:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
-// CHECK-NEXT:    [[MUL6:%.*]] = fmul float [[MUL]], [[TMP19]]
-// CHECK-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[A_ADDR]], align 8
-// CHECK-NEXT:    [[TMP21:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM7:%.*]] = sext i32 [[TMP21]] to i64
-// CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[TMP20]], i64 [[IDXPROM7]]
+// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP8]], ptr [[AGG_CAPTURED1]])
+// CHECK-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[B_ADDR]], align 8
+// CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP10]] to i64
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP9]], i64 [[IDXPROM]]
+// CHECK-NEXT:    [[TMP11:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+// CHECK-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[C_ADDR]], align 8
+// CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP13]] to i64
+// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP12]], i64 [[IDXPROM2]]
+// CHECK-NEXT:    [[TMP14:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
+// CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP11]], [[TMP14]]
+// CHECK-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[TMP16]] to i64
+// CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[TMP15]], i64 [[IDXPROM4]]
+// CHECK-NEXT:    [[TMP17:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
+// CHECK-NEXT:    [[MUL6:%.*]] = fmul float [[MUL]], [[TMP17]]
+// CHECK-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM7:%.*]] = sext i32 [[TMP19]] to i64
+// CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[IDXPROM7]]
 // CHECK-NEXT:    store float [[MUL6]], ptr [[ARRAYIDX8]], align 4
+// CHECK-NEXT:    br label [[OMP_TILE_BODY_MERGE]]
+// CHECK:       omp_tile.body.merge:
 // CHECK-NEXT:    br label [[OMP_TILE0_INC]]
 // CHECK:       omp_tile0.inc:
 // CHECK-NEXT:    [[OMP_TILE0_NEXT]] = add nuw i32 [[OMP_TILE0_IV]], 1
-// CHECK-NEXT:    br label [[OMP_TILE0_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
+// CHECK-NEXT:    br label [[OMP_TILE0_HEADER]], !llvm.loop [[LOOP2:![0-9]+]]
 // CHECK:       omp_tile0.exit:
 // CHECK-NEXT:    br label [[OMP_TILE0_AFTER:%.*]]
 // CHECK:       omp_tile0.after:
 // CHECK-NEXT:    br label [[OMP_FLOOR0_INC]]
 // CHECK:       omp_floor0.inc:
 // CHECK-NEXT:    [[OMP_FLOOR0_NEXT]] = add nuw i32 [[OMP_FLOOR0_IV]], 1
-// CHECK-NEXT:    br label [[OMP_FLOOR0_HEADER]], !llvm.loop [[LOOP6:![0-9]+]]
+// CHECK-NEXT:    br label [[OMP_FLOOR0_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
 // CHECK:       omp_floor0.exit:
 // CHECK-NEXT:    br label [[OMP_FLOOR0_AFTER:%.*]]
 // CHECK:       omp_floor0.after:
@@ -129,7 +130,7 @@ void unroll_unroll_partial_heuristic(float *a, float *b, float *c, float *d) {
 // CHECK-NEXT:    store ptr [[__CONTEXT]], ptr [[__CONTEXT_ADDR]], align 8
 // CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__CONTEXT_ADDR]], align 8
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw [[STRUCT_ANON:%.*]], ptr [[TMP0]], i32 0, i32 0
-// CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP1]], align 8
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP1]], align 8, !nonnull [[META6:![0-9]+]], !align [[META7:![0-9]+]]
 // CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
 // CHECK-NEXT:    store i32 [[TMP3]], ptr [[DOTSTART]], align 4
 // CHECK-NEXT:    store i32 2, ptr [[DOTSTOP]], align 4
@@ -152,7 +153,7 @@ void unroll_unroll_partial_heuristic(float *a, float *b, float *c, float *d) {
 // CHECK-NEXT:    br label [[COND_END]]
 // CHECK:       cond.end:
 // CHECK-NEXT:    [[COND:%.*]] = phi i32 [ [[DIV]], [[COND_TRUE]] ], [ 0, [[COND_FALSE]] ]
-// CHECK-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[DISTANCE_ADDR]], align 8
+// CHECK-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[DISTANCE_ADDR]], align 8, !nonnull [[META6]], !align [[META7]]
 // CHECK-NEXT:    store i32 [[COND]], ptr [[TMP10]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -172,7 +173,7 @@ void unroll_unroll_partial_heuristic(float *a, float *b, float *c, float *d) {
 // CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[LOGICAL_ADDR]], align 4
 // CHECK-NEXT:    [[MUL:%.*]] = mul i32 1, [[TMP3]]
 // CHECK-NEXT:    [[ADD:%.*]] = add i32 [[TMP2]], [[MUL]]
-// CHECK-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[LOOPVAR_ADDR]], align 8
+// CHECK-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[LOOPVAR_ADDR]], align 8, !nonnull [[META6]], !align [[META7]]
 // CHECK-NEXT:    store i32 [[ADD]], ptr [[TMP4]], align 4
 // CHECK-NEXT:    ret void
 //
diff --git a/clang/test/OpenMP/unroll_codegen_tile_for.cpp b/clang/test/OpenMP/unroll_codegen_tile_for.cpp
index ccfc9f38f5a24..6f13f6b67c8cd 100644
--- a/clang/test/OpenMP/unroll_codegen_tile_for.cpp
+++ b/clang/test/OpenMP/unroll_codegen_tile_for.cpp
@@ -1,3 +1,4 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
 // Check code generation
 // RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fopenmp -emit-llvm %s -o - | FileCheck %s --check-prefix=IR
 
@@ -13,221 +14,194 @@
 extern "C" void body(...) {}
 
 
-// IR-LABEL: @func(
-// IR-NEXT:  [[ENTRY:.*]]:
-// IR-NEXT:    %[[START_ADDR:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[END_ADDR:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[STEP_ADDR:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_IV:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[TMP:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[I:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_1:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_2:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_3:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTUNROLLED_IV_I:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_6:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_8:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_12:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTCAPTURE_EXPR_14:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTFLOOR_0_IV__UNROLLED_IV_I:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_LB:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_UB:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_STRIDE:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTOMP_IS_LAST:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTFLOOR_0_IV__UNROLLED_IV_I18:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTTILE_0_IV__UNROLLED_IV_I:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[DOTUNROLL_INNER_IV_I:.+]] = alloca i32, align 4
-// IR-NEXT:    %[[TMP0:.+]] = call i32 @__kmpc_global_thread_num(ptr @2)
-// IR-NEXT:    store i32 %[[START:.+]], ptr %[[START_ADDR]], align 4
-// IR-NEXT:    store i32 %[[END:.+]], ptr %[[END_ADDR]], align 4
-// IR-NEXT:    store i32 %[[STEP:.+]], ptr %[[STEP_ADDR]], align 4
-// IR-NEXT:    %[[TMP1:.+]] = load i32, ptr %[[START_ADDR]], align 4
-// IR-NEXT:    store i32 %[[TMP1]], ptr %[[I]], align 4
-// IR-NEXT:    %[[TMP2:.+]] = load i32, ptr %[[START_ADDR]], align 4
-// IR-NEXT:    store i32 %[[TMP2]], ptr %[[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    %[[TMP3:.+]] = load i32, ptr %[[END_ADDR]], align 4
-// IR-NEXT:    store i32 %[[TMP3]], ptr %[[DOTCAPTURE_EXPR_1]], align 4
-// IR-NEXT:    %[[TMP4:.+]] = load i32, ptr %[[STEP_ADDR]], align 4
-// IR-NEXT:    store i32 %[[TMP4]], ptr %[[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    %[[TMP5:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_1]], align 4
-// IR-NEXT:    %[[TMP6:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    %[[SUB:.+]] = sub i32 %[[TMP5]], %[[TMP6]]
-// IR-NEXT:    %[[SUB4:.+]] = sub i32 %[[SUB]], 1
-// IR-NEXT:    %[[TMP7:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    %[[ADD:.+]] = add i32 %[[SUB4]], %[[TMP7]]
-// IR-NEXT:    %[[TMP8:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    %[[DIV:.+]] = udiv i32 %[[ADD]], %[[TMP8]]
-// IR-NEXT:    %[[SUB5:.+]] = sub i32 %[[DIV]], 1
-// IR-NEXT:    store i32 %[[SUB5]], ptr %[[DOTCAPTURE_EXPR_3]], align 4
-// IR-NEXT:    store i32 0, ptr %[[DOTUNROLLED_IV_I]], align 4
-// IR-NEXT:    %[[TMP9:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_3]], align 4
-// IR-NEXT:    %[[ADD7:.+]] = add i32 %[[TMP9]], 1
-// IR-NEXT:    store i32 %[[ADD7]], ptr %[[DOTCAPTURE_EXPR_6]], align 4
-// IR-NEXT:    %[[TMP10:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_6]], align 4
-// IR-NEXT:    %[[SUB9:.+]] = sub i32 %[[TMP10]], -1
-// IR-NEXT:    %[[DIV10:.+]] = udiv i32 %[[SUB9]], 2
-// IR-NEXT:    %[[SUB11:.+]] = sub i32 %[[DIV10]], 1
-// IR-NEXT:    store i32 %[[SUB11]], ptr %[[DOTCAPTURE_EXPR_8]], align 4
-// IR-NEXT:    %[[TMP11:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_8]], align 4
-// IR-NEXT:    %[[ADD13:.+]] = add i32 %[[TMP11]], 1
-// IR-NEXT:    store i32 %[[ADD13]], ptr %[[DOTCAPTURE_EXPR_12]], align 4
-// IR-NEXT:    %[[TMP12:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_12]], align 4
-// IR-NEXT:    %[[SUB15:.+]] = sub i32 %[[TMP12]], -3
-// IR-NEXT:    %[[DIV16:.+]] = udiv i32 %[[SUB15]], 4
-// IR-NEXT:    %[[SUB17:.+]] = sub i32 %[[DIV16]], 1
-// IR-NEXT:    store i32 %[[SUB17]], ptr %[[DOTCAPTURE_EXPR_14]], align 4
-// IR-NEXT:    store i32 0, ptr %[[DOTFLOOR_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    %[[TMP13:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_12]], align 4
-// IR-NEXT:    %[[CMP:.+]] = icmp ult i32 0, %[[TMP13]]
-// IR-NEXT:    br i1 %[[CMP]], label %[[OMP_PRECOND_THEN:.+]], label %[[OMP_PRECOND_END:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_PRECOND_THEN]]:
-// IR-NEXT:    store i32 0, ptr %[[DOTOMP_LB]], align 4
-// IR-NEXT:    %[[TMP14:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_14]], align 4
-// IR-NEXT:    store i32 %[[TMP14]], ptr %[[DOTOMP_UB]], align 4
-// IR-NEXT:    store i32 1, ptr %[[DOTOMP_STRIDE]], align 4
-// IR-NEXT:    store i32 0, ptr %[[DOTOMP_IS_LAST]], align 4
-// IR-NEXT:    call void @__kmpc_for_static_init_4u(ptr @1, i32 %[[TMP0]], i32 34, ptr %[[DOTOMP_IS_LAST]], ptr %[[DOTOMP_LB]], ptr %[[DOTOMP_UB]], ptr %[[DOTOMP_STRIDE]], i32 1, i32 1)
-// IR-NEXT:    %[[TMP15:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
-// IR-NEXT:    %[[TMP16:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_14]], align 4
-// IR-NEXT:    %[[CMP19:.+]] = icmp ugt i32 %[[TMP15]], %[[TMP16]]
-// IR-NEXT:    br i1 %[[CMP19]], label %[[COND_TRUE:.+]], label %[[COND_FALSE:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_TRUE]]:
-// IR-NEXT:    %[[TMP17:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_14]], align 4
-// IR-NEXT:    br label %[[COND_END:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_FALSE]]:
-// IR-NEXT:    %[[TMP18:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
+// IR-LABEL: define dso_local void @func(
+// IR-SAME: i32 noundef [[START:%.*]], i32 noundef [[END:%.*]], i32 noundef [[STEP:%.*]]) #[[ATTR0:[0-9]+]] {
+// IR-NEXT:  [[ENTRY:.*:]]
+// IR-NEXT:    [[START_ADDR:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[TMP:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTUNROLLED_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_5:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV__UNROLLED_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTCAPTURE_EXPR_13:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV__UNROLLED_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV__UNROLLED_IV_I17:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_CNT_0_IV__UNROLLED_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTUNROLL_INNER_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
+// IR-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
+// IR-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
+// IR-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// IR-NEXT:    store i32 [[TMP1]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// IR-NEXT:    store i32 [[TMP2]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// IR-NEXT:    store i32 [[TMP3]], ptr [[DOTCAPTURE_EXPR_1]], align 4
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// IR-NEXT:    store i32 [[TMP4]], ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    [[SUB:%.*]] = sub i32 [[TMP5]], [[TMP6]]
+// IR-NEXT:    [[SUB3:%.*]] = sub i32 [[SUB]], 1
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add i32 [[SUB3]], [[TMP7]]
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP8]]
+// IR-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
+// IR-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTUNROLLED_IV_I]], align 4
+// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    [[ADD6:%.*]] = add i32 [[TMP9]], 1
+// IR-NEXT:    store i32 [[ADD6]], ptr [[DOTCAPTURE_EXPR_5]], align 4
+// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// IR-NEXT:    [[SUB8:%.*]] = sub i32 [[TMP10]], -1
+// IR-NEXT:    [[DIV9:%.*]] = udiv i32 [[SUB8]], 2
+// IR-NEXT:    [[SUB10:%.*]] = sub i32 [[DIV9]], 1
+// IR-NEXT:    store i32 [[SUB10]], ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    [[ADD12:%.*]] = add i32 [[TMP11]], 1
+// IR-NEXT:    store i32 [[ADD12]], ptr [[DOTCAPTURE_EXPR_11]], align 4
+// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// IR-NEXT:    [[SUB14:%.*]] = sub i32 [[TMP12]], -3
+// IR-NEXT:    [[DIV15:%.*]] = udiv i32 [[SUB14]], 4
+// IR-NEXT:    [[SUB16:%.*]] = sub i32 [[DIV15]], 1
+// IR-NEXT:    store i32 [[SUB16]], ptr [[DOTCAPTURE_EXPR_13]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// IR-NEXT:    [[CMP:%.*]] = icmp ult i32 0, [[TMP13]]
+// IR-NEXT:    br i1 [[CMP]], label %[[OMP_PRECOND_THEN:.*]], label %[[OMP_PRECOND_END:.*]]
+// IR:       [[OMP_PRECOND_THEN]]:
+// IR-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
+// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_13]], align 4
+// IR-NEXT:    store i32 [[TMP14]], ptr [[DOTOMP_UB]], align 4
+// IR-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
+// IR-NEXT:    call void @__kmpc_for_static_init_4u(ptr @[[GLOB1:[0-9]+]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
+// IR-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// IR-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_13]], align 4
+// IR-NEXT:    [[CMP18:%.*]] = icmp ugt i32 [[TMP15]], [[TMP16]]
+// IR-NEXT:    br i1 [[CMP18]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_13]], align 4
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
 // IR-NEXT:    br label %[[COND_END]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_END]]:
-// IR-NEXT:    %[[COND:.+]] = phi i32 [ %[[TMP17]], %[[COND_TRUE]] ], [ %[[TMP18]], %[[COND_FALSE]] ]
-// IR-NEXT:    store i32 %[[COND]], ptr %[[DOTOMP_UB]], align 4
-// IR-NEXT:    %[[TMP19:.+]] = load i32, ptr %[[DOTOMP_LB]], align 4
-// IR-NEXT:    store i32 %[[TMP19]], ptr %[[DOTOMP_IV]], align 4
-// IR-NEXT:    br label %[[OMP_INNER_FOR_COND:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_INNER_FOR_COND]]:
-// IR-NEXT:    %[[TMP20:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
-// IR-NEXT:    %[[TMP21:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
-// IR-NEXT:    %[[ADD20:.+]] = add i32 %[[TMP21]], 1
-// IR-NEXT:    %[[CMP21:.+]] = icmp ult i32 %[[TMP20]], %[[ADD20]]
-// IR-NEXT:    br i1 %[[CMP21]], label %[[OMP_INNER_FOR_BODY:.+]], label %[[OMP_INNER_FOR_END:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_INNER_FOR_BODY]]:
-// IR-NEXT:    %[[TMP22:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
-// IR-NEXT:    %[[MUL:.+]] = mul i32 %[[TMP22]], 4
-// IR-NEXT:    %[[ADD22:.+]] = add i32 0, %[[MUL]]
-// IR-NEXT:    store i32 %[[ADD22]], ptr %[[DOTFLOOR_0_IV__UNROLLED_IV_I18]], align 4
-// IR-NEXT:    %[[TMP23:.+]] = load i32, ptr %[[DOTFLOOR_0_IV__UNROLLED_IV_I18]], align 4
-// IR-NEXT:    store i32 %[[TMP23]], ptr %[[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_COND]]:
-// IR-NEXT:    %[[TMP24:.+]] = load i32, ptr %[[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    %[[TMP25:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_8]], align 4
-// IR-NEXT:    %[[ADD23:.+]] = add i32 %[[TMP25]], 1
-// IR-NEXT:    %[[TMP26:.+]] = load i32, ptr %[[DOTFLOOR_0_IV__UNROLLED_IV_I18]], align 4
-// IR-NEXT:    %[[ADD24:.+]] = add i32 %[[TMP26]], 4
-// IR-NEXT:    %[[CMP25:.+]] = icmp ult i32 %[[ADD23]], %[[ADD24]]
-// IR-NEXT:    br i1 %[[CMP25]], label %[[COND_TRUE26:.+]], label %[[COND_FALSE28:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_TRUE26]]:
-// IR-NEXT:    %[[TMP27:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_8]], align 4
-// IR-NEXT:    %[[ADD27:.+]] = add i32 %[[TMP27]], 1
-// IR-NEXT:    br label %[[COND_END30:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_FALSE28]]:
-// IR-NEXT:    %[[TMP28:.+]] = load i32, ptr %[[DOTFLOOR_0_IV__UNROLLED_IV_I18]], align 4
-// IR-NEXT:    %[[ADD29:.+]] = add i32 %[[TMP28]], 4
-// IR-NEXT:    br label %[[COND_END30]]
-// IR-EMPTY:
-// IR-NEXT:  [[COND_END30]]:
-// IR-NEXT:    %[[COND31:.+]] = phi i32 [ %[[ADD27]], %[[COND_TRUE26]] ], [ %[[ADD29]], %[[COND_FALSE28]] ]
-// IR-NEXT:    %[[CMP32:.+]] = icmp ult i32 %[[TMP24]], %[[COND31]]
-// IR-NEXT:    br i1 %[[CMP32]], label %[[FOR_BODY:.+]], label %[[FOR_END45:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_BODY]]:
-// IR-NEXT:    %[[TMP29:.+]] = load i32, ptr %[[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    %[[MUL33:.+]] = mul i32 %[[TMP29]], 2
-// IR-NEXT:    %[[ADD34:.+]] = add i32 0, %[[MUL33]]
-// IR-NEXT:    store i32 %[[ADD34]], ptr %[[DOTUNROLLED_IV_I]], align 4
-// IR-NEXT:    %[[TMP30:.+]] = load i32, ptr %[[DOTUNROLLED_IV_I]], align 4
-// IR-NEXT:    store i32 %[[TMP30]], ptr %[[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND35:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_COND35]]:
-// IR-NEXT:    %[[TMP31:.+]] = load i32, ptr %[[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    %[[TMP32:.+]] = load i32, ptr %[[DOTUNROLLED_IV_I]], align 4
-// IR-NEXT:    %[[ADD36:.+]] = add i32 %[[TMP32]], 2
-// IR-NEXT:    %[[CMP37:.+]] = icmp ult i32 %[[TMP31]], %[[ADD36]]
-// IR-NEXT:    br i1 %[[CMP37]], label %[[LAND_RHS:.+]], label %[[LAND_END:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[LAND_RHS]]:
-// IR-NEXT:    %[[TMP33:.+]] = load i32, ptr %[[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    %[[TMP34:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_3]], align 4
-// IR-NEXT:    %[[ADD38:.+]] = add i32 %[[TMP34]], 1
-// IR-NEXT:    %[[CMP39:.+]] = icmp ult i32 %[[TMP33]], %[[ADD38]]
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP17]], %[[COND_TRUE]] ], [ [[TMP18]], %[[COND_FALSE]] ]
+// IR-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
+// IR-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
+// IR-NEXT:    store i32 [[TMP19]], ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    br label %[[OMP_INNER_FOR_COND:.*]]
+// IR:       [[OMP_INNER_FOR_COND]]:
+// IR-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// IR-NEXT:    [[ADD19:%.*]] = add i32 [[TMP21]], 1
+// IR-NEXT:    [[CMP20:%.*]] = icmp ult i32 [[TMP20]], [[ADD19]]
+// IR-NEXT:    br i1 [[CMP20]], label %[[OMP_INNER_FOR_BODY:.*]], label %[[OMP_INNER_FOR_END:.*]]
+// IR:       [[OMP_INNER_FOR_BODY]]:
+// IR-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul i32 [[TMP22]], 4
+// IR-NEXT:    [[ADD21:%.*]] = add i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD21]], ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I17]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND:.*]]
+// IR:       [[FOR_COND]]:
+// IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    [[CMP22:%.*]] = icmp ult i32 [[TMP23]], 4
+// IR-NEXT:    br i1 [[CMP22]], label %[[FOR_BODY:.*]], label %[[FOR_END38:.*]]
+// IR:       [[FOR_BODY]]:
+// IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I17]], align 4
+// IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    [[ADD23:%.*]] = add i32 [[TMP24]], [[TMP25]]
+// IR-NEXT:    store i32 [[ADD23]], ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    [[MUL24:%.*]] = mul i32 [[TMP26]], 2
+// IR-NEXT:    [[ADD25:%.*]] = add i32 0, [[MUL24]]
+// IR-NEXT:    store i32 [[ADD25]], ptr [[DOTUNROLLED_IV_I]], align 4
+// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    [[ADD26:%.*]] = add i32 [[TMP28]], 1
+// IR-NEXT:    [[CMP27:%.*]] = icmp ult i32 [[TMP27]], [[ADD26]]
+// IR-NEXT:    br i1 [[CMP27]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
+// IR:       [[IF_THEN]]:
+// IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTUNROLLED_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP29]], ptr [[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND28:.*]]
+// IR:       [[FOR_COND28]]:
+// IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTUNROLLED_IV_I]], align 4
+// IR-NEXT:    [[ADD29:%.*]] = add i32 [[TMP31]], 2
+// IR-NEXT:    [[CMP30:%.*]] = icmp ult i32 [[TMP30]], [[ADD29]]
+// IR-NEXT:    br i1 [[CMP30]], label %[[LAND_RHS:.*]], label %[[LAND_END:.*]]
+// IR:       [[LAND_RHS]]:
+// IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    [[ADD31:%.*]] = add i32 [[TMP33]], 1
+// IR-NEXT:    [[CMP32:%.*]] = icmp ult i32 [[TMP32]], [[ADD31]]
 // IR-NEXT:    br label %[[LAND_END]]
-// IR-EMPTY:
-// IR-NEXT:  [[LAND_END]]:
-// IR-NEXT:    %[[TMP35:.+]] = phi i1 [ false, %[[FOR_COND35]] ], [ %[[CMP39]], %[[LAND_RHS]] ]
-// IR-NEXT:    br i1 %[[TMP35]], label %[[FOR_BODY40:.+]], label %[[FOR_END:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_BODY40]]:
-// IR-NEXT:    %[[TMP36:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    %[[TMP37:.+]] = load i32, ptr %[[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    %[[TMP38:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    %[[MUL41:.+]] = mul i32 %[[TMP37]], %[[TMP38]]
-// IR-NEXT:    %[[ADD42:.+]] = add i32 %[[TMP36]], %[[MUL41]]
-// IR-NEXT:    store i32 %[[ADD42]], ptr %[[I]], align 4
-// IR-NEXT:    %[[TMP39:.+]] = load i32, ptr %[[START_ADDR]], align 4
-// IR-NEXT:    %[[TMP40:.+]] = load i32, ptr %[[END_ADDR]], align 4
-// IR-NEXT:    %[[TMP41:.+]] = load i32, ptr %[[STEP_ADDR]], align 4
-// IR-NEXT:    %[[TMP42:.+]] = load i32, ptr %[[I]], align 4
-// IR-NEXT:    call void (...) @body(i32 noundef %[[TMP39]], i32 noundef %[[TMP40]], i32 noundef %[[TMP41]], i32 noundef %[[TMP42]])
-// IR-NEXT:    br label %[[FOR_INC:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_INC]]:
-// IR-NEXT:    %[[TMP43:.+]] = load i32, ptr %[[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    %[[INC:.+]] = add i32 %[[TMP43]], 1
-// IR-NEXT:    store i32 %[[INC]], ptr %[[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND35]], !llvm.loop ![[LOOP2:[0-9]+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC43:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_INC43]]:
-// IR-NEXT:    %[[TMP44:.+]] = load i32, ptr %[[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    %[[INC44:.+]] = add i32 %[[TMP44]], 1
-// IR-NEXT:    store i32 %[[INC44]], ptr %[[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop ![[LOOP5:[0-9]+]]
-// IR-EMPTY:
-// IR-NEXT:  [[FOR_END45]]:
-// IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_BODY_CONTINUE]]:
-// IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_INNER_FOR_INC]]:
-// IR-NEXT:    %[[TMP45:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
-// IR-NEXT:    %[[ADD46:.+]] = add i32 %[[TMP45]], 1
-// IR-NEXT:    store i32 %[[ADD46]], ptr %[[DOTOMP_IV]], align 4
+// IR:       [[LAND_END]]:
+// IR-NEXT:    [[TMP34:%.*]] = phi i1 [ false, %[[FOR_COND28]] ], [ [[CMP32]], %[[LAND_RHS]] ]
+// IR-NEXT:    br i1 [[TMP34]], label %[[FOR_BODY33:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY33]]:
+// IR-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[MUL34:%.*]] = mul i32 [[TMP36]], [[TMP37]]
+// IR-NEXT:    [[ADD35:%.*]] = add i32 [[TMP35]], [[MUL34]]
+// IR-NEXT:    store i32 [[ADD35]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP38:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// IR-NEXT:    [[TMP39:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// IR-NEXT:    [[TMP40:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// IR-NEXT:    [[TMP41:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void (...) @body(i32 noundef [[TMP38]], i32 noundef [[TMP39]], i32 noundef [[TMP40]], i32 noundef [[TMP41]])
+// IR-NEXT:    br label %[[FOR_INC:.*]]
+// IR:       [[FOR_INC]]:
+// IR-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add i32 [[TMP42]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND28]], !llvm.loop [[LOOP2:![0-9]+]]
+// IR:       [[FOR_END]]:
+// IR-NEXT:    br label %[[IF_END]]
+// IR:       [[IF_END]]:
+// IR-NEXT:    br label %[[FOR_INC36:.*]]
+// IR:       [[FOR_INC36]]:
+// IR-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    [[INC37:%.*]] = add i32 [[TMP43]], 1
+// IR-NEXT:    store i32 [[INC37]], ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP5:![0-9]+]]
+// IR:       [[FOR_END38]]:
+// IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.*]]
+// IR:       [[OMP_BODY_CONTINUE]]:
+// IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.*]]
+// IR:       [[OMP_INNER_FOR_INC]]:
+// IR-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[ADD39:%.*]] = add i32 [[TMP44]], 1
+// IR-NEXT:    store i32 [[ADD39]], ptr [[DOTOMP_IV]], align 4
 // IR-NEXT:    br label %[[OMP_INNER_FOR_COND]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_INNER_FOR_END]]:
-// IR-NEXT:    br label %[[OMP_LOOP_EXIT:.+]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_LOOP_EXIT]]:
-// IR-NEXT:    call void @__kmpc_for_static_fini(ptr @1, i32 %[[TMP0]])
+// IR:       [[OMP_INNER_FOR_END]]:
+// IR-NEXT:    br label %[[OMP_LOOP_EXIT:.*]]
+// IR:       [[OMP_LOOP_EXIT]]:
+// IR-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP0]])
 // IR-NEXT:    br label %[[OMP_PRECOND_END]]
-// IR-EMPTY:
-// IR-NEXT:  [[OMP_PRECOND_END]]:
-// IR-NEXT:    call void @__kmpc_barrier(ptr @3, i32 %[[TMP0]])
+// IR:       [[OMP_PRECOND_END]]:
+// IR-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3:[0-9]+]], i32 [[TMP0]])
 // IR-NEXT:    ret void
-// IR-NEXT:  }
+//
 extern "C" void func(int start, int end, int step) {
   #pragma omp for
   #pragma omp tile sizes(4)
@@ -239,7 +213,9 @@ extern "C" void func(int start, int end, int step) {
 #endif /* HEADER */
 
 
-// IR: ![[LOOP2]] = distinct !{![[LOOP2]], ![[LOOPPROP3:[0-9]+]], ![[LOOPPROP4:[0-9]+]]}
-// IR: ![[LOOPPROP3]] = !{!"llvm.loop.mustprogress"}
-// IR: ![[LOOPPROP4]] = !{!"llvm.loop.unroll.count", i32 2}
-// IR: ![[LOOP5]] = distinct !{![[LOOP5]], ![[LOOPPROP3]]}
+//.
+// IR: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]], [[META4:![0-9]+]]}
+// IR: [[META3]] = !{!"llvm.loop.mustprogress"}
+// IR: [[META4]] = !{!"llvm.loop.unroll.count", i32 2}
+// IR: [[LOOP5]] = distinct !{[[LOOP5]], [[META3]]}
+//.

>From 9548b4a2aaebf93be69f0d6a342539acaca84f66 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 15 Apr 2026 07:10:22 -0400
Subject: [PATCH 07/19] rect-codegen

---
 clang/test/OpenMP/tile_rect_codegen.cpp    | 50 +++++++++++++
 clang/test/OpenMP/tile_rect_codegen_ir.cpp | 84 ++++++++++++++++++++++
 2 files changed, 134 insertions(+)
 create mode 100644 clang/test/OpenMP/tile_rect_codegen.cpp
 create mode 100644 clang/test/OpenMP/tile_rect_codegen_ir.cpp

diff --git a/clang/test/OpenMP/tile_rect_codegen.cpp b/clang/test/OpenMP/tile_rect_codegen.cpp
new file mode 100644
index 0000000000000..589914eb9d63e
--- /dev/null
+++ b/clang/test/OpenMP/tile_rect_codegen.cpp
@@ -0,0 +1,50 @@
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fclang-abi-compat=latest -std=c++20 -fopenmp -ast-print %s | FileCheck %s
+// expected-no-diagnostics
+
+extern "C" void body(...) {}
+
+// CHECK-LABEL: void rect_tile_1d(
+void rect_tile_1d() {
+  // Tile size 5, trip count 4: 4 % 5 != 0, so predicate is needed.
+  // The tile loop should have rectangular bound (the tile size) and
+  // the body should be guarded by a validity predicate.
+  //
+  // CHECK: #pragma omp tile sizes(5)
+  // CHECK-NEXT: for (int i = 7; i < 17; i += 3)
+  #pragma omp tile sizes(5)
+  for (int i = 7; i < 17; i += 3)
+    body(i);
+}
+
+// CHECK-LABEL: void rect_tile_2d(
+void rect_tile_2d() {
+  // CHECK: #pragma omp tile sizes(5, 5)
+  // CHECK-NEXT: for (int i = 7; i < 17; i += 3)
+  // CHECK-NEXT:   for (int j = 7; j < 17; j += 3)
+  #pragma omp tile sizes(5, 5)
+  for (int i = 7; i < 17; i += 3)
+    for (int j = 7; j < 17; j += 3)
+      body(i, j);
+}
+
+// CHECK-LABEL: void rect_tile_exact_div(
+void rect_tile_exact_div() {
+  // Tile size 5, trip count 10: 10 % 5 == 0, so predicate is NOT needed.
+  // CHECK: #pragma omp tile sizes(5)
+  // CHECK-NEXT: for (int i = 0; i < 10; i += 1)
+  #pragma omp tile sizes(5)
+  for (int i = 0; i < 10; i += 1)
+    body(i);
+}
+
+// CHECK-LABEL: void rect_tile_nested_body_loop(
+void rect_tile_nested_body_loop(int n) {
+  // After tiling i, the j loop (from the associated body) should be part
+  // of the same canonical nest as the floor and tile loops.
+  // CHECK: #pragma omp tile sizes(4)
+  // CHECK-NEXT: for (int i = 0; i < 6; i += 1)
+  #pragma omp tile sizes(4)
+  for (int i = 0; i < 6; i += 1)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+}
diff --git a/clang/test/OpenMP/tile_rect_codegen_ir.cpp b/clang/test/OpenMP/tile_rect_codegen_ir.cpp
new file mode 100644
index 0000000000000..f35c581c8b8ed
--- /dev/null
+++ b/clang/test/OpenMP/tile_rect_codegen_ir.cpp
@@ -0,0 +1,84 @@
+// Regression tests for rectangular #pragma omp tile lowering:
+// - Inner loop counter .tile.cnt.* with bound == tile size (constant).
+// - Validity guard (icmp + branch) when remainder iterations exist or trip count
+//   is not proven divisible at compile time.
+//
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fopenmp -emit-llvm -o - %s | FileCheck %s --check-prefix=IR
+// expected-no-diagnostics
+
+extern "C" void body(int);
+
+// Trip count 6, tile 4 → remainder; body must be guarded.
+// IR-LABEL: remainder_6_tile_4v(
+void remainder_6_tile_4(void) {
+  // Rectangular tile counter and constant tile-size bound.
+  // IR-DAG: %.tile.cnt{{.*}} = alloca i32
+  // IR: icmp {{.*}} i32 {{.*}}, 4
+  // IR: icmp {{.*}} i32 {{.*}}, 6
+  // IR: br i1 {{.*}}, label %if.then{{.*}}, label %if.end{{.*}}
+  // IR: call {{.*}} @body(
+#pragma omp tile sizes(4)
+  for (int i = 0; i < 6; ++i)
+    body(i);
+}
+
+// Simple stride-1 loop; tile size matches full tiles only (10, tile 5).
+// IR-LABEL: full_tiles_10_5v(
+void full_tiles_10_5(void) {
+  // IR-DAG: %.tile.cnt{{.*}} = alloca i32
+  // IR: icmp {{.*}} i32 {{.*}}, 5
+  // IR: call {{.*}} @body(
+#pragma omp tile sizes(5)
+  for (int i = 0; i < 10; ++i)
+    body(i);
+}
+
+// Variable tile size: clamp (TS<=0 ? 1 : TS) is inlined; bound compares to %cond.
+// IR-LABEL: var_tilei(
+void var_tile(int ts) {
+  // IR-DAG: %.tile.cnt{{.*}} = alloca i32
+  // IR: %cond = phi i32
+  // IR: icmp {{.*}} i32 {{.*}}, %cond
+  // IR: call {{.*}} @body(
+#pragma omp tile sizes(ts)
+  for (int i = 0; i < 12; ++i)
+    body(i);
+}
+
+// Two tiled dimensions → two tile counters.
+// IR-LABEL: two_dimv(
+void two_dim(void) {
+  // IR-DAG: %.tile.cnt.0.iv.i{{.*}} = alloca i32
+  // IR-DAG: %.tile.cnt.1.iv.j{{.*}} = alloca i32
+  // IR: icmp {{.*}} i32 {{.*}}, 2
+  // IR: icmp {{.*}} i32 {{.*}}, 2
+  // IR: call {{.*}} @body(
+#pragma omp tile sizes(2, 2)
+  for (int i = 0; i < 3; ++i)
+    for (int j = 0; j < 3; ++j)
+      body(i + j);
+}
+
+// Nested loop in tile body: outer nest still uses .tile.cnt for tiled dim.
+// IR-LABEL: nested_inner_ji(
+void nested_inner_j(int n) {
+  // IR-DAG: %.tile.cnt{{.*}} = alloca i32
+  // IR: icmp {{.*}} i32 {{.*}}, 4
+#pragma omp tile sizes(4)
+  for (int i = 0; i < 6; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i + j);
+}
+
+// Predicate elision: when trip count is evenly divisible by constant tile size,
+// no if-guard is needed (no icmp against trip count in tile body).
+// IR-LABEL: elided_predicatev(
+void elided_predicate(void) {
+  // Trip count 12, tile size 4 → 12 % 4 == 0, no predicate.
+  // IR-NOT: if.then
+  // IR-NOT: if.end
+  // IR: call {{.*}} @body(
+#pragma omp tile sizes(4)
+  for (int i = 0; i < 12; ++i)
+    body(i);
+}

>From 8c03f84c5cd9553258779c31e31e85de85b4094b Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 15 Apr 2026 07:29:31 -0400
Subject: [PATCH 08/19] openmp-ir

---
 .../Frontend/OpenMPIRBuilderTest.cpp          | 35 ++++++++++++++++---
 1 file changed, 30 insertions(+), 5 deletions(-)

diff --git a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
index b96103c82c185..2502c37ba28b3 100644
--- a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
@@ -1643,12 +1643,27 @@ TEST_F(OpenMPIRBuilderTest, TileSingleLoop) {
   CanonicalLoopInfo *Floor = GenLoops[0];
   CanonicalLoopInfo *Tile = GenLoops[1];
 
+  // The merge block sits between the body code and the tile latch.
+  BasicBlock *MergeBB = Tile->getLatch()->getSinglePredecessor();
+  ASSERT_NE(MergeBB, nullptr);
+
   BasicBlock *RefOrder[] = {
-      Floor->getPreheader(), Floor->getHeader(),   Floor->getCond(),
-      Floor->getBody(),      Tile->getPreheader(), Tile->getHeader(),
-      Tile->getCond(),       Tile->getBody(),      BodyCode,
-      Tile->getLatch(),      Tile->getExit(),      Tile->getAfter(),
-      Floor->getLatch(),     Floor->getExit(),     Floor->getAfter(),
+      Floor->getPreheader(),
+      Floor->getHeader(),
+      Floor->getCond(),
+      Floor->getBody(),
+      Tile->getPreheader(),
+      Tile->getHeader(),
+      Tile->getCond(),
+      Tile->getBody(),
+      BodyCode,
+      MergeBB,
+      Tile->getLatch(),
+      Tile->getExit(),
+      Tile->getAfter(),
+      Floor->getLatch(),
+      Floor->getExit(),
+      Floor->getAfter(),
   };
   EXPECT_TRUE(verifyDFSOrder(F, RefOrder));
   EXPECT_TRUE(verifyListOrder(F, RefOrder));
@@ -1718,6 +1733,10 @@ TEST_F(OpenMPIRBuilderTest, TileNestedLoops) {
   CanonicalLoopInfo *Tile1 = GenLoops[2];
   CanonicalLoopInfo *Tile2 = GenLoops[3];
 
+  // The merge block sits between the body code and the innermost tile latch.
+  BasicBlock *MergeBB = Tile2->getLatch()->getSinglePredecessor();
+  ASSERT_NE(MergeBB, nullptr);
+
   BasicBlock *RefOrder[] = {
       Floor1->getPreheader(),
       Floor1->getHeader(),
@@ -1736,6 +1755,7 @@ TEST_F(OpenMPIRBuilderTest, TileNestedLoops) {
       Tile2->getCond(),
       Tile2->getBody(),
       BodyCode,
+      MergeBB,
       Tile2->getLatch(),
       Tile2->getExit(),
       Tile2->getAfter(),
@@ -1826,6 +1846,10 @@ TEST_F(OpenMPIRBuilderTest, TileNestedLoopsWithBounds) {
   CanonicalLoopInfo *Tile0 = GenLoops[2];
   CanonicalLoopInfo *Tile1 = GenLoops[3];
 
+  // The merge block sits between the body code and the innermost tile latch.
+  BasicBlock *MergeBB = Tile1->getLatch()->getSinglePredecessor();
+  ASSERT_NE(MergeBB, nullptr);
+
   BasicBlock *RefOrder[] = {
       Floor0->getPreheader(),
       Floor0->getHeader(),
@@ -1844,6 +1868,7 @@ TEST_F(OpenMPIRBuilderTest, TileNestedLoopsWithBounds) {
       Tile1->getCond(),
       Tile1->getBody(),
       BodyCode,
+      MergeBB,
       Tile1->getLatch(),
       Tile1->getExit(),
       Tile1->getAfter(),

>From f7b15a56ed47ef6e68bedd74ccc3cb4a3c927464 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 15 Apr 2026 07:30:29 -0400
Subject: [PATCH 09/19] runtime-tests

---
 .../runtime/test/transform/tile/foreach.cpp   |  36 ++++++
 openmp/runtime/test/transform/tile/intfor.c   |  78 ++++++-------
 .../runtime/test/transform/tile/iterfor.cpp   |  27 +++++
 .../tile/parallel-wsloop-collapse-foreach.cpp | 108 ++++++++++++++++++
 4 files changed, 210 insertions(+), 39 deletions(-)

diff --git a/openmp/runtime/test/transform/tile/foreach.cpp b/openmp/runtime/test/transform/tile/foreach.cpp
index 4fb3595760974..17031a4acf187 100644
--- a/openmp/runtime/test/transform/tile/foreach.cpp
+++ b/openmp/runtime/test/transform/tile/foreach.cpp
@@ -182,6 +182,10 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: v=0 w=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 1
 // CHECK-NEXT: [A] iterator move assign
@@ -191,6 +195,10 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: v=1 w=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -206,6 +214,18 @@ int main() {
 // CHECK-NEXT: v=2 w=1
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 3
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 3
+// CHECK-NEXT: [B] iterator advance: 0 += 0
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 0
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 1
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 1
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 2
@@ -214,6 +234,22 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: v=2 w=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 3
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 3
+// CHECK-NEXT: [B] iterator advance: 0 += 2
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 2
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
diff --git a/openmp/runtime/test/transform/tile/intfor.c b/openmp/runtime/test/transform/tile/intfor.c
index 4a930eab6730a..e980165277b79 100644
--- a/openmp/runtime/test/transform/tile/intfor.c
+++ b/openmp/runtime/test/transform/tile/intfor.c
@@ -30,77 +30,85 @@ int main() {
 // CHECK:      do
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=7 j=7
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=7 j=7
+// CHECK-NEXT: i=7 j=10
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=7 j=13
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=7 j=10
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=10 j=7
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=7 j=13
+// CHECK-NEXT: i=10 j=10
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=10 j=13
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=13 j=7
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=10 j=7
+// CHECK-NEXT: i=13 j=10
+// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=13 j=13
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=10 j=10
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=7 j=16
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=10 j=13
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=7 j=19
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=10 j=16
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=13 j=7
+// CHECK-NEXT: i=10 j=19
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=13 j=10
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=13 j=16
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=13 j=13
+// CHECK-NEXT: i=13 j=19
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
@@ -109,61 +117,57 @@ int main() {
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=7 j=16
+// CHECK-NEXT: i=16 j=7
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=7 j=19
+// CHECK-NEXT: i=16 j=10
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=16 j=13
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=10 j=16
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=10 j=19
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=13 j=16
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=13 j=19
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: i=16 j=16
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=16 j=7
+// CHECK-NEXT: i=16 j=19
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=16 j=10
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=16 j=13
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
@@ -172,14 +176,10 @@ int main() {
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=16 j=16
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=16 j=19
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
@@ -188,4 +188,4 @@ int main() {
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: done
\ No newline at end of file
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/tile/iterfor.cpp b/openmp/runtime/test/transform/tile/iterfor.cpp
index 05b5677bf9c4f..d07b10b3e841a 100644
--- a/openmp/runtime/test/transform/tile/iterfor.cpp
+++ b/openmp/runtime/test/transform/tile/iterfor.cpp
@@ -190,6 +190,9 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=0 j=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 1
 // CHECK-NEXT: [A] iterator move assign
@@ -199,6 +202,9 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=1 j=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -215,6 +221,15 @@ int main() {
 // CHECK-NEXT: i=2 j=1
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 3
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [B] iterator advance: 0 += 0
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 1
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [B] iterator advance: 0 += 2
@@ -223,6 +238,18 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=2 j=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 3
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [B] iterator advance: 0 += 2
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
diff --git a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-foreach.cpp b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-foreach.cpp
index b1f4d98a52ddc..966ff7f84f548 100644
--- a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-foreach.cpp
+++ b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-foreach.cpp
@@ -182,6 +182,10 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=0 v=0 w=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 1
 // CHECK-NEXT: [A] iterator move assign
@@ -191,6 +195,10 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=0 v=1 w=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -206,6 +214,18 @@ int main() {
 // CHECK-NEXT: i=0 v=2 w=1
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 3
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 3
+// CHECK-NEXT: [B] iterator advance: 0 += 0
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 0
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 1
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 1
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 2
@@ -214,6 +234,22 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=0 v=2 w=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 3
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 3
+// CHECK-NEXT: [B] iterator advance: 0 += 2
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 2
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 0
 // CHECK-NEXT: [A] iterator move assign
@@ -251,6 +287,10 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=1 v=0 w=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 1
 // CHECK-NEXT: [A] iterator move assign
@@ -260,6 +300,10 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=1 v=1 w=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -275,6 +319,18 @@ int main() {
 // CHECK-NEXT: i=1 v=2 w=1
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 3
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 3
+// CHECK-NEXT: [B] iterator advance: 0 += 0
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 0
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 1
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 1
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 2
@@ -283,6 +339,22 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=1 v=2 w=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 3
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 3
+// CHECK-NEXT: [B] iterator advance: 0 += 2
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 2
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 0
 // CHECK-NEXT: [A] iterator move assign
@@ -320,6 +392,10 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=2 v=0 w=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 1
 // CHECK-NEXT: [A] iterator move assign
@@ -329,6 +405,10 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=2 v=1 w=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -344,6 +424,18 @@ int main() {
 // CHECK-NEXT: i=2 v=2 w=1
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 3
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 3
+// CHECK-NEXT: [B] iterator advance: 0 += 0
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 0
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 1
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 1
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 2
@@ -352,6 +444,22 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=2 v=2 w=2
 // CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [A] iterator dtor
+// CHECK-NEXT: [A] iterator advance: 0 += 3
+// CHECK-NEXT: [A] iterator move assign
+// CHECK-NEXT: [A] iterator deref: 3
+// CHECK-NEXT: [B] iterator advance: 0 += 2
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 2
+// CHECK-NEXT: [B] iterator dtor
+// CHECK-NEXT: [B] iterator advance: 0 += 3
+// CHECK-NEXT: [B] iterator move assign
+// CHECK-NEXT: [B] iterator deref: 3
+// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [B] iterator dtor

>From 52b17c217c114fc92e0793ebd0da638010993c60 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Fri, 24 Apr 2026 08:11:24 -0400
Subject: [PATCH 10/19] optimize-guard

---
 clang/lib/Sema/SemaOpenMP.cpp | 150 +++++++++++++++++++++++-----------
 1 file changed, 104 insertions(+), 46 deletions(-)

diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index c8d75a0b333b4..df602caafe5cd 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -15078,13 +15078,22 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
   SmallVector<Stmt *, 4> PreInits;
   CaptureVars CopyTransformer(SemaRef);
 
+  // Emit the intra-tile loop in canonical (rectangular + predicate) form only
+  // when a consuming OpenMP construct requires canonical input; otherwise keep
+  // the original min(.floor.iv + T, N) bound.
+  OpenMPDirectiveKind ParentDirective = DSAStack->getParentDirective();
+  const bool IntraTileMustBeCanonical =
+      isOpenMPLoopTransformationDirective(ParentDirective) ||
+      isOpenMPLoopDirective(ParentDirective);
+
   // Create iteration variables for the generated loops.
   SmallVector<VarDecl *, 4> FloorIndVars;
   SmallVector<VarDecl *, 4> TileIndVars;
   SmallVector<VarDecl *, 4> TileCntVars;
   FloorIndVars.resize(NumLoops);
   TileIndVars.resize(NumLoops);
-  TileCntVars.resize(NumLoops);
+  if (IntraTileMustBeCanonical)
+    TileCntVars.resize(NumLoops);
   for (unsigned I = 0; I < NumLoops; ++I) {
     OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
 
@@ -15117,7 +15126,7 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     }
 
     // Loop counter for the rectangular tile loop [0, TileSize).
-    {
+    if (IntraTileMustBeCanonical) {
       std::string TileCntName =
           (Twine(".tile.cnt.") + llvm::utostr(I) + ".iv." + OrigVarName).str();
       VarDecl *TileCntDecl =
@@ -15130,9 +15139,11 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
 
     // Declare the logical tile IV in PreInits so it is in scope for the
     // entire loop nest (it will be assigned in each tile loop body).
-    Decl *TileIVDeclPtr = TileIndVars[I];
-    PreInits.push_back(new (Context) DeclStmt(
-        DeclGroupRef::Create(Context, &TileIVDeclPtr, 1), {}, {}));
+    if (IntraTileMustBeCanonical) {
+      Decl *TileIVDeclPtr = TileIndVars[I];
+      PreInits.push_back(new (Context) DeclStmt(
+          DeclGroupRef::Create(Context, &TileIVDeclPtr, 1), {}, {}));
+    }
   }
 
   // Once the original iteration values are set, append the innermost body.
@@ -15147,23 +15158,25 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
   // Optimization: if every dimension's trip count is a compile-time constant
   // that is evenly divisible by the corresponding tile size (also a constant),
   // then the remainder tile is empty and the predicate is trivially true.
-  {
+  //
+  // The min-bounded path clamps the tile loop's upper bound directly so no
+  // predicate is needed.
+  if (IntraTileMustBeCanonical) {
     bool PredicateNeeded = false;
     for (unsigned I = 0; I < NumLoops; ++I) {
       Expr *TSExpr = SizesClause->getSizesRefs()[I];
       Expr *NExpr = LoopHelpers[I].NumIterations;
-      llvm::APSInt TileVal, TripVal;
       bool TSConst =
           !TSExpr->containsErrors() && TSExpr->isIntegerConstantExpr(Context);
       bool NConst = NExpr->isIntegerConstantExpr(Context);
       if (TSConst && NConst) {
         Expr::EvalResult TSResult;
         TSExpr->EvaluateAsInt(TSResult, Context);
-        TileVal = TSResult.Val.getInt();
+        llvm::APSInt TileVal = TSResult.Val.getInt();
         Expr::EvalResult NResult;
         NExpr->EvaluateAsInt(NResult, Context);
-        TripVal = NResult.Val.getInt();
-        if (TileVal.isStrictlyPositive() && (TripVal.srem(TileVal)).isZero())
+        llvm::APSInt TripVal = NResult.Val.getInt();
+        if (TileVal.isStrictlyPositive() && TripVal.srem(TileVal).isZero())
           continue;
       }
       PredicateNeeded = true;
@@ -15245,6 +15258,8 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
   // Each tile loop uses .tile.cnt as its counter with rectangular bounds
   // [0, TileSize), and computes .tile.iv = .floor.iv + .tile.cnt to set
   // the logical iteration number for LoopHelper.Updates.
+  // The min-bounded path instead uses .tile.iv directly, initialized to
+  // .floor.iv and bounded by min(.floor.iv + DimTileSize, NumIterations).
   for (int I = NumLoops - 1; I >= 0; --I) {
     OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
     Expr *NumIterations = LoopHelper.NumIterations;
@@ -15266,46 +15281,88 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
                               OrigCntVar->getExprLoc());
     };
 
-    // For init-statement: auto .tile.cnt = 0
-    SemaRef.AddInitializerToDecl(
-        TileCntVars[I],
-        SemaRef.ActOnIntegerConstant(LoopHelper.Init->getExprLoc(), 0).get(),
-        /*DirectInit=*/false);
-    Decl *CounterDecl = TileCntVars[I];
-    StmtResult InitStmt = new (Context)
-        DeclStmt(DeclGroupRef::Create(Context, &CounterDecl, 1),
-                 OrigCntVar->getBeginLoc(), OrigCntVar->getEndLoc());
-    if (!InitStmt.isUsable())
-      return StmtError();
-
-    // For cond-expression: .tile.cnt < DimTileSize  (rectangular bound)
     Expr *DimTileSize = MakeDimTileSize(I);
     if (!DimTileSize)
       return StmtError();
-    ExprResult CondExpr =
-        SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
-                           MakeTileCntRef(), DimTileSize);
-    if (!CondExpr.isUsable())
-      return StmtError();
 
-    // For incr-statement: ++.tile.cnt
-    ExprResult IncrStmt = SemaRef.BuildUnaryOp(
-        CurScope, LoopHelper.Inc->getExprLoc(), UO_PreInc, MakeTileCntRef());
-    if (!IncrStmt.isUsable())
-      return StmtError();
+    StmtResult InitStmt;
+    ExprResult CondExpr;
+    ExprResult IncrStmt;
+    Stmt *TileIVAssignStmt = nullptr;
 
-    // Compute the logical iteration number:
-    //   .tile.iv = .floor.iv + .tile.cnt
-    ExprResult FloorPlusCnt = SemaRef.BuildBinOp(
-        CurScope, OrigCntVar->getExprLoc(), BO_Add,
-        makeFloorIVRef(SemaRef, FloorIndVars, I, IVTy, OrigCntVar),
-        MakeTileCntRef());
-    if (!FloorPlusCnt.isUsable())
-      return StmtError();
-    ExprResult TileIVAssign =
-        SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(), BO_Assign,
-                           MakeTileIVRef(), FloorPlusCnt.get());
-    if (!TileIVAssign.isUsable())
+    if (IntraTileMustBeCanonical) {
+      // For init-statement: auto .tile.cnt = 0
+      SemaRef.AddInitializerToDecl(
+          TileCntVars[I],
+          SemaRef.ActOnIntegerConstant(LoopHelper.Init->getExprLoc(), 0).get(),
+          /*DirectInit=*/false);
+      Decl *CounterDecl = TileCntVars[I];
+      InitStmt = new (Context)
+          DeclStmt(DeclGroupRef::Create(Context, &CounterDecl, 1),
+                   OrigCntVar->getBeginLoc(), OrigCntVar->getEndLoc());
+
+      // For cond-expression: .tile.cnt < DimTileSize  (rectangular bound)
+      CondExpr = SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(),
+                                    BO_LT, MakeTileCntRef(), DimTileSize);
+
+      // For incr-statement: ++.tile.cnt
+      IncrStmt = SemaRef.BuildUnaryOp(CurScope, LoopHelper.Inc->getExprLoc(),
+                                      UO_PreInc, MakeTileCntRef());
+
+      // Compute the logical iteration number:
+      //   .tile.iv = .floor.iv + .tile.cnt
+      ExprResult FloorPlusCnt = SemaRef.BuildBinOp(
+          CurScope, OrigCntVar->getExprLoc(), BO_Add,
+          makeFloorIVRef(SemaRef, FloorIndVars, I, IVTy, OrigCntVar),
+          MakeTileCntRef());
+      if (!FloorPlusCnt.isUsable())
+        return StmtError();
+      ExprResult TileIVAssign =
+          SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(), BO_Assign,
+                             MakeTileIVRef(), FloorPlusCnt.get());
+      if (!TileIVAssign.isUsable())
+        return StmtError();
+      TileIVAssignStmt = TileIVAssign.get();
+    } else {
+      // Min-bounded path: .tile.iv is itself the loop counter, initialized to
+      // .floor.iv and bounded by min(.floor.iv + DimTileSize, NumIterations).
+      SemaRef.AddInitializerToDecl(
+          TileIndVars[I],
+          SemaRef
+              .DefaultLvalueConversion(
+                  makeFloorIVRef(SemaRef, FloorIndVars, I, IVTy, OrigCntVar))
+              .get(),
+          /*DirectInit=*/false);
+      Decl *CounterDecl = TileIndVars[I];
+      InitStmt = new (Context)
+          DeclStmt(DeclGroupRef::Create(Context, &CounterDecl, 1),
+                   OrigCntVar->getBeginLoc(), OrigCntVar->getEndLoc());
+
+      ExprResult EndOfTile = SemaRef.BuildBinOp(
+          CurScope, LoopHelper.Cond->getExprLoc(), BO_Add,
+          makeFloorIVRef(SemaRef, FloorIndVars, I, IVTy, OrigCntVar),
+          DimTileSize);
+      if (!EndOfTile.isUsable())
+        return StmtError();
+      ExprResult IsPartialTile =
+          SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
+                             NumIterations, EndOfTile.get());
+      if (!IsPartialTile.isUsable())
+        return StmtError();
+      ExprResult MinTileAndIterSpace = SemaRef.ActOnConditionalOp(
+          LoopHelper.Cond->getBeginLoc(), LoopHelper.Cond->getEndLoc(),
+          IsPartialTile.get(), NumIterations, EndOfTile.get());
+      if (!MinTileAndIterSpace.isUsable())
+        return StmtError();
+      CondExpr =
+          SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
+                             MakeTileIVRef(), MinTileAndIterSpace.get());
+
+      IncrStmt = SemaRef.BuildUnaryOp(CurScope, LoopHelper.Inc->getExprLoc(),
+                                      UO_PreInc, MakeTileIVRef());
+    }
+
+    if (!InitStmt.isUsable() || !CondExpr.isUsable() || !IncrStmt.isUsable())
       return StmtError();
 
     // Generated for loop is:
@@ -15321,7 +15378,8 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     // for applying tiling again). If this is the case, sink the expressions
     // further into the inner loop.
     SmallVector<Stmt *, 4> BodyParts;
-    BodyParts.push_back(TileIVAssign.get());
+    if (TileIVAssignStmt)
+      BodyParts.push_back(TileIVAssignStmt);
     BodyParts.append(LoopHelper.Updates.begin(), LoopHelper.Updates.end());
     if (auto *SourceCXXFor = dyn_cast<CXXForRangeStmt>(LoopStmt))
       BodyParts.push_back(SourceCXXFor->getLoopVarStmt());

>From 5057f6b2f80bffbc9ff5f072fa496512893945f6 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Fri, 24 Apr 2026 10:08:20 -0400
Subject: [PATCH 11/19] test_updated

---
 clang/test/OpenMP/tile_codegen.cpp         | 1224 +++++++++++---------
 clang/test/OpenMP/tile_rect_codegen_ir.cpp |  485 +++++++-
 2 files changed, 1105 insertions(+), 604 deletions(-)

diff --git a/clang/test/OpenMP/tile_codegen.cpp b/clang/test/OpenMP/tile_codegen.cpp
index 1fd02d74a1209..65fd3a327ad64 100644
--- a/clang/test/OpenMP/tile_codegen.cpp
+++ b/clang/test/OpenMP/tile_codegen.cpp
@@ -136,9 +136,8 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[I2:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[I:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
@@ -150,47 +149,50 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END11:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END12:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK1:       for.cond4:
-// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP1]], 5
-// CHECK1-NEXT:    br i1 [[CMP5]], label [[FOR_BODY6:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body6:
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
-// CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP4]], 3
-// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK1-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
-// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[TMP5]], align 4
-// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP8:%.*]] = icmp slt i32 [[TMP6]], 4
-// CHECK1-NEXT:    br i1 [[CMP8]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK1:       if.then:
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 5
+// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK1-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP4]], 5
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
+// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body8:
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
+// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
+// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[TMP6]], align 4
 // CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2]], !align [[META3]]
 // CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
 // CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP8]])
-// CHECK1-NEXT:    br label [[IF_END]]
-// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP9]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP4:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC9:%.*]]
-// CHECK1:       for.inc9:
+// CHECK1-NEXT:    br label [[FOR_INC10:%.*]]
+// CHECK1:       for.inc10:
 // CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP10]], 5
-// CHECK1-NEXT:    store i32 [[ADD10]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP10]], 5
+// CHECK1-NEXT:    store i32 [[ADD11]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
-// CHECK1:       for.end11:
+// CHECK1:       for.end12:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -205,9 +207,8 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -236,49 +237,54 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK1-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP8]], [[ADD5]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END15:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK1:       for.cond6:
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP10]], 5
-// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body8:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD9:%.*]] = add i32 [[TMP11]], [[TMP12]]
-// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP14]], [[TMP15]]
-// CHECK1-NEXT:    [[ADD10:%.*]] = add i32 [[TMP13]], [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD10]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[ADD11:%.*]] = add i32 [[TMP17]], 1
-// CHECK1-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP16]], [[ADD11]]
-// CHECK1-NEXT:    br i1 [[CMP12]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK1:       if.then:
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP18]])
-// CHECK1-NEXT:    br label [[IF_END]]
-// CHECK1:       if.end:
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD7:%.*]] = add i32 [[TMP12]], 1
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD8:%.*]] = add i32 [[TMP13]], 5
+// CHECK1-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
+// CHECK1-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD10:%.*]] = add i32 [[TMP14]], 1
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD11:%.*]] = add i32 [[TMP15]], 5
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP11]], [[COND]]
+// CHECK1-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body13:
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP17]], [[TMP18]]
+// CHECK1-NEXT:    [[ADD14:%.*]] = add i32 [[TMP16]], [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP19]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP19]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP20]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP7:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC13:%.*]]
-// CHECK1:       for.inc13:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD14:%.*]] = add i32 [[TMP20]], 5
-// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_INC15:%.*]]
+// CHECK1:       for.inc15:
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD16:%.*]] = add i32 [[TMP21]], 5
+// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
-// CHECK1:       for.end15:
+// CHECK1:       for.end17:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -289,13 +295,11 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -306,86 +310,94 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END30:%.*]]
 // CHECK1:       for.body:
 // CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK1:       for.cond1:
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 4
-// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END21:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END27:%.*]]
 // CHECK1:       for.body3:
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[TMP2]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK1:       for.cond4:
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP2]], 5
-// CHECK1-NEXT:    br i1 [[CMP5]], label [[FOR_BODY6:%.*]], label [[FOR_END18:%.*]]
-// CHECK1:       for.body6:
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[TMP4]]
-// CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
-// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND8:%.*]]
-// CHECK1:       for.cond8:
-// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP6]], 5
-// CHECK1-NEXT:    br i1 [[CMP9]], label [[FOR_BODY10:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body10:
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], 5
+// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK1-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP5]], 5
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP3]], [[COND]]
+// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END24:%.*]]
+// CHECK1:       for.body8:
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
+// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP7]], [[TMP8]]
-// CHECK1-NEXT:    store i32 [[ADD11]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[MUL12:%.*]] = mul nsw i32 [[TMP9]], 3
-// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i32 7, [[MUL12]]
-// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[J]], align 4
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP10]], 4
-// CHECK1-NEXT:    br i1 [[CMP14]], label [[LAND_LHS_TRUE:%.*]], label [[IF_END:%.*]]
-// CHECK1:       land.lhs.true:
+// CHECK1-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND10:%.*]]
+// CHECK1:       for.cond10:
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP9]], 5
+// CHECK1-NEXT:    [[CMP12:%.*]] = icmp slt i32 4, [[ADD11]]
+// CHECK1-NEXT:    br i1 [[CMP12]], label [[COND_TRUE13:%.*]], label [[COND_FALSE14:%.*]]
+// CHECK1:       cond.true13:
+// CHECK1-NEXT:    br label [[COND_END16:%.*]]
+// CHECK1:       cond.false14:
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP10]], 5
+// CHECK1-NEXT:    br label [[COND_END16]]
+// CHECK1:       cond.end16:
+// CHECK1-NEXT:    [[COND17:%.*]] = phi i32 [ 4, [[COND_TRUE13]] ], [ [[ADD15]], [[COND_FALSE14]] ]
+// CHECK1-NEXT:    [[CMP18:%.*]] = icmp slt i32 [[TMP8]], [[COND17]]
+// CHECK1-NEXT:    br i1 [[CMP18]], label [[FOR_BODY19:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body19:
 // CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[CMP15:%.*]] = icmp slt i32 [[TMP11]], 4
-// CHECK1-NEXT:    br i1 [[CMP15]], label [[IF_THEN:%.*]], label [[IF_END]]
-// CHECK1:       if.then:
+// CHECK1-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[TMP11]], 3
+// CHECK1-NEXT:    [[ADD21:%.*]] = add nsw i32 7, [[MUL20]]
+// CHECK1-NEXT:    store i32 [[ADD21]], ptr [[J]], align 4
 // CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
 // CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP12]], i32 noundef [[TMP13]])
-// CHECK1-NEXT:    br label [[IF_END]]
-// CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP9:![0-9]+]]
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND10]], !llvm.loop [[LOOP9:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC16:%.*]]
-// CHECK1:       for.inc16:
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC17:%.*]] = add nsw i32 [[TMP15]], 1
-// CHECK1-NEXT:    store i32 [[INC17]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_INC22:%.*]]
+// CHECK1:       for.inc22:
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC23:%.*]] = add nsw i32 [[TMP15]], 1
+// CHECK1-NEXT:    store i32 [[INC23]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP10:![0-9]+]]
-// CHECK1:       for.end18:
-// CHECK1-NEXT:    br label [[FOR_INC19:%.*]]
-// CHECK1:       for.inc19:
+// CHECK1:       for.end24:
+// CHECK1-NEXT:    br label [[FOR_INC25:%.*]]
+// CHECK1:       for.inc25:
 // CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP16]], 5
-// CHECK1-NEXT:    store i32 [[ADD20]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD26:%.*]] = add nsw i32 [[TMP16]], 5
+// CHECK1-NEXT:    store i32 [[ADD26]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP11:![0-9]+]]
-// CHECK1:       for.end21:
-// CHECK1-NEXT:    br label [[FOR_INC22:%.*]]
-// CHECK1:       for.inc22:
+// CHECK1:       for.end27:
+// CHECK1-NEXT:    br label [[FOR_INC28:%.*]]
+// CHECK1:       for.inc28:
 // CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP17]], 5
-// CHECK1-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD29:%.*]] = add nsw i32 [[TMP17]], 5
+// CHECK1-NEXT:    store i32 [[ADD29]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP12:![0-9]+]]
-// CHECK1:       for.end24:
+// CHECK1:       for.end30:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -891,9 +903,8 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[START_ADDR]], align 4
@@ -917,48 +928,53 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK1-NEXT:    [[ADD5:%.*]] = add i32 [[TMP6]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP5]], [[ADD5]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END15:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK1:       for.cond6:
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP7]], 5
-// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body8:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD9:%.*]] = add i32 [[TMP8]], [[TMP9]]
-// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP11]], 3
-// CHECK1-NEXT:    [[ADD10:%.*]] = add i32 [[TMP10]], [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD10]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[ADD11:%.*]] = add i32 [[TMP13]], 1
-// CHECK1-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP12]], [[ADD11]]
-// CHECK1-NEXT:    br i1 [[CMP12]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK1:       if.then:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP14]])
-// CHECK1-NEXT:    br label [[IF_END]]
-// CHECK1:       if.end:
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD7:%.*]] = add i32 [[TMP9]], 1
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD8:%.*]] = add i32 [[TMP10]], 5
+// CHECK1-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
+// CHECK1-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD10:%.*]] = add i32 [[TMP11]], 1
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD11:%.*]] = add i32 [[TMP12]], 5
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP8]], [[COND]]
+// CHECK1-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body13:
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP14]], 3
+// CHECK1-NEXT:    [[ADD14:%.*]] = add i32 [[TMP13]], [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP15]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP15]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP16]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP23:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC13:%.*]]
-// CHECK1:       for.inc13:
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD14:%.*]] = add i32 [[TMP16]], 5
-// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_INC15:%.*]]
+// CHECK1:       for.inc15:
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD16:%.*]] = add i32 [[TMP17]], 5
+// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP24:![0-9]+]]
-// CHECK1:       for.end15:
+// CHECK1:       for.end17:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -967,9 +983,8 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
 // CHECK1-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
@@ -977,65 +992,78 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END14:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK1:       for.cond1:
-// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP2]], 0
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], 0
 // CHECK1-NEXT:    br i1 [[CMP2]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK1:       cond.true:
 // CHECK1-NEXT:    br label [[COND_END:%.*]]
 // CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[A_ADDR]], align 4
 // CHECK1-NEXT:    br label [[COND_END]]
 // CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP3]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[TMP1]], [[COND]]
-// CHECK1-NEXT:    br i1 [[CMP3]], label [[FOR_BODY4:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body4:
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
-// CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
-// CHECK1-NEXT:    [[ADD5:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD5]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP6:%.*]] = icmp slt i32 [[TMP7]], 4
-// CHECK1-NEXT:    br i1 [[CMP6]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK1:       if.then:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP8]])
-// CHECK1-NEXT:    br label [[IF_END]]
-// CHECK1:       if.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP5]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[COND]]
+// CHECK1-NEXT:    [[CMP3:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK1-NEXT:    br i1 [[CMP3]], label [[COND_TRUE4:%.*]], label [[COND_FALSE5:%.*]]
+// CHECK1:       cond.true4:
+// CHECK1-NEXT:    br label [[COND_END12:%.*]]
+// CHECK1:       cond.false5:
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    [[CMP6:%.*]] = icmp sle i32 [[TMP7]], 0
+// CHECK1-NEXT:    br i1 [[CMP6]], label [[COND_TRUE7:%.*]], label [[COND_FALSE8:%.*]]
+// CHECK1:       cond.true7:
+// CHECK1-NEXT:    br label [[COND_END9:%.*]]
+// CHECK1:       cond.false8:
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    br label [[COND_END9]]
+// CHECK1:       cond.end9:
+// CHECK1-NEXT:    [[COND10:%.*]] = phi i32 [ 1, [[COND_TRUE7]] ], [ [[TMP8]], [[COND_FALSE8]] ]
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP6]], [[COND10]]
+// CHECK1-NEXT:    br label [[COND_END12]]
+// CHECK1:       cond.end12:
+// CHECK1-NEXT:    [[COND13:%.*]] = phi i32 [ 4, [[COND_TRUE4]] ], [ [[ADD11]], [[COND_END9]] ]
+// CHECK1-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP2]], [[COND13]]
+// CHECK1-NEXT:    br i1 [[CMP14]], label [[FOR_BODY15:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body15:
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP9]], 3
+// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP10]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP9]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP25:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC7:%.*]]
-// CHECK1:       for.inc7:
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    [[CMP8:%.*]] = icmp sle i32 [[TMP10]], 0
-// CHECK1-NEXT:    br i1 [[CMP8]], label [[COND_TRUE9:%.*]], label [[COND_FALSE10:%.*]]
-// CHECK1:       cond.true9:
-// CHECK1-NEXT:    br label [[COND_END11:%.*]]
-// CHECK1:       cond.false10:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    br label [[COND_END11]]
-// CHECK1:       cond.end11:
-// CHECK1-NEXT:    [[COND12:%.*]] = phi i32 [ 1, [[COND_TRUE9]] ], [ [[TMP11]], [[COND_FALSE10]] ]
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP12]], [[COND12]]
-// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_INC17:%.*]]
+// CHECK1:       for.inc17:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    [[CMP18:%.*]] = icmp sle i32 [[TMP12]], 0
+// CHECK1-NEXT:    br i1 [[CMP18]], label [[COND_TRUE19:%.*]], label [[COND_FALSE20:%.*]]
+// CHECK1:       cond.true19:
+// CHECK1-NEXT:    br label [[COND_END21:%.*]]
+// CHECK1:       cond.false20:
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    br label [[COND_END21]]
+// CHECK1:       cond.end21:
+// CHECK1-NEXT:    [[COND22:%.*]] = phi i32 [ 1, [[COND_TRUE19]] ], [ [[TMP13]], [[COND_FALSE20]] ]
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP14]], [[COND22]]
+// CHECK1-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP26:![0-9]+]]
-// CHECK1:       for.end14:
+// CHECK1:       for.end24:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -1048,9 +1076,8 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK1-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
 // CHECK1-NEXT:    store ptr [[ARRAY]], ptr [[__RANGE2]], align 8
@@ -1084,51 +1111,56 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK1:       for.cond7:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
-// CHECK1-NEXT:    br i1 [[CMP8]], label [[FOR_BODY9:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body9:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
-// CHECK1-NEXT:    store i64 [[ADD10]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP12]], 1
-// CHECK1-NEXT:    [[ADD_PTR11:%.*]] = getelementptr inbounds double, ptr [[TMP11]], i64 [[MUL]]
-// CHECK1-NEXT:    store ptr [[ADD_PTR11]], ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP14:%.*]] = load double, ptr [[TMP13]], align 8
-// CHECK1-NEXT:    store double [[TMP14]], ptr [[V]], align 8
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
+// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
+// CHECK1-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
+// CHECK1-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body14:
+// CHECK1-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP16]], 1
-// CHECK1-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP15]], [[ADD12]]
-// CHECK1-NEXT:    br i1 [[CMP13]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK1:       if.then:
-// CHECK1-NEXT:    [[TMP17:%.*]] = load double, ptr [[V]], align 8
-// CHECK1-NEXT:    call void (...) @body(double noundef [[TMP17]])
-// CHECK1-NEXT:    br label [[IF_END]]
-// CHECK1:       if.end:
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
+// CHECK1-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
+// CHECK1-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
+// CHECK1-NEXT:    store double [[TMP17]], ptr [[V]], align 8
+// CHECK1-NEXT:    [[TMP18:%.*]] = load double, ptr [[V]], align 8
+// CHECK1-NEXT:    call void (...) @body(double noundef [[TMP18]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP18]], 1
-// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP19]], 1
+// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP28:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC14:%.*]]
-// CHECK1:       for.inc14:
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD15:%.*]] = add nsw i64 [[TMP19]], 5
-// CHECK1-NEXT:    store i64 [[ADD15]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    br label [[FOR_INC16:%.*]]
+// CHECK1:       for.inc16:
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP20]], 5
+// CHECK1-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP29:![0-9]+]]
-// CHECK1:       for.end16:
+// CHECK1:       for.end18:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -1142,9 +1174,8 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK1-NEXT:    store double 4.200000e+01, ptr [[C]], align 8
 // CHECK1-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
@@ -1179,52 +1210,57 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK1:       for.cond7:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
-// CHECK1-NEXT:    br i1 [[CMP8]], label [[FOR_BODY9:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body9:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
-// CHECK1-NEXT:    store i64 [[ADD10]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP12]], 1
-// CHECK1-NEXT:    [[ADD_PTR11:%.*]] = getelementptr inbounds double, ptr [[TMP11]], i64 [[MUL]]
-// CHECK1-NEXT:    store ptr [[ADD_PTR11]], ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP14:%.*]] = load double, ptr [[TMP13]], align 8
-// CHECK1-NEXT:    store double [[TMP14]], ptr [[V]], align 8
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
+// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
+// CHECK1-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
+// CHECK1-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body14:
+// CHECK1-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP16]], 1
-// CHECK1-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP15]], [[ADD12]]
-// CHECK1-NEXT:    br i1 [[CMP13]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK1:       if.then:
-// CHECK1-NEXT:    [[TMP17:%.*]] = load double, ptr [[C]], align 8
-// CHECK1-NEXT:    [[TMP18:%.*]] = load double, ptr [[V]], align 8
-// CHECK1-NEXT:    call void (...) @body(double noundef [[TMP17]], double noundef [[TMP18]])
-// CHECK1-NEXT:    br label [[IF_END]]
-// CHECK1:       if.end:
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
+// CHECK1-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
+// CHECK1-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
+// CHECK1-NEXT:    store double [[TMP17]], ptr [[V]], align 8
+// CHECK1-NEXT:    [[TMP18:%.*]] = load double, ptr [[C]], align 8
+// CHECK1-NEXT:    [[TMP19:%.*]] = load double, ptr [[V]], align 8
+// CHECK1-NEXT:    call void (...) @body(double noundef [[TMP18]], double noundef [[TMP19]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP19]], 1
-// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP20]], 1
+// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP30:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC14:%.*]]
-// CHECK1:       for.inc14:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD15:%.*]] = add nsw i64 [[TMP20]], 5
-// CHECK1-NEXT:    store i64 [[ADD15]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    br label [[FOR_INC16:%.*]]
+// CHECK1:       for.inc16:
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP21]], 5
+// CHECK1-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP31:![0-9]+]]
-// CHECK1:       for.end16:
+// CHECK1:       for.end18:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -1257,9 +1293,8 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:  entry:
 // CHECK2-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[I2:%.*]] = alloca ptr, align 8
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK2-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK2-NEXT:    [[I:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
@@ -1271,47 +1306,50 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END11:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END12:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK2:       for.cond4:
-// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP1]], 5
-// CHECK2-NEXT:    br i1 [[CMP5]], label [[FOR_BODY6:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body6:
-// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
-// CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP4]], 3
-// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK2-NEXT:    [[TMP5:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
-// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[TMP5]], align 4
-// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP8:%.*]] = icmp slt i32 [[TMP6]], 4
-// CHECK2-NEXT:    br i1 [[CMP8]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK2:       if.then:
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 5
+// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK2-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP4]], 5
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
+// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body8:
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
+// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK2-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
+// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[TMP6]], align 4
 // CHECK2-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2]], !align [[META3]]
 // CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
 // CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP8]])
-// CHECK2-NEXT:    br label [[IF_END]]
-// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP9]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP4:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC9:%.*]]
-// CHECK2:       for.inc9:
+// CHECK2-NEXT:    br label [[FOR_INC10:%.*]]
+// CHECK2:       for.inc10:
 // CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP10]], 5
-// CHECK2-NEXT:    store i32 [[ADD10]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP10]], 5
+// CHECK2-NEXT:    store i32 [[ADD11]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
-// CHECK2:       for.end11:
+// CHECK2:       for.end12:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -1332,9 +1370,8 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -1363,49 +1400,54 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK2-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP8]], [[ADD5]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END15:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[TMP10]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK2:       for.cond6:
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP10]], 5
-// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body8:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD9:%.*]] = add i32 [[TMP11]], [[TMP12]]
-// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP14]], [[TMP15]]
-// CHECK2-NEXT:    [[ADD10:%.*]] = add i32 [[TMP13]], [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD10]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[ADD11:%.*]] = add i32 [[TMP17]], 1
-// CHECK2-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP16]], [[ADD11]]
-// CHECK2-NEXT:    br i1 [[CMP12]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK2:       if.then:
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP18]])
-// CHECK2-NEXT:    br label [[IF_END]]
-// CHECK2:       if.end:
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD7:%.*]] = add i32 [[TMP12]], 1
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD8:%.*]] = add i32 [[TMP13]], 5
+// CHECK2-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
+// CHECK2-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD10:%.*]] = add i32 [[TMP14]], 1
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD11:%.*]] = add i32 [[TMP15]], 5
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP11]], [[COND]]
+// CHECK2-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body13:
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP17]], [[TMP18]]
+// CHECK2-NEXT:    [[ADD14:%.*]] = add i32 [[TMP16]], [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP19]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP19]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP20]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP7:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC13:%.*]]
-// CHECK2:       for.inc13:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD14:%.*]] = add i32 [[TMP20]], 5
-// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_INC15:%.*]]
+// CHECK2:       for.inc15:
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD16:%.*]] = add i32 [[TMP21]], 5
+// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
-// CHECK2:       for.end15:
+// CHECK2:       for.end17:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -1419,9 +1461,8 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK2-NEXT:    store double 4.200000e+01, ptr [[C]], align 8
 // CHECK2-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
@@ -1456,52 +1497,57 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK2:       for.cond7:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
-// CHECK2-NEXT:    br i1 [[CMP8]], label [[FOR_BODY9:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body9:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
-// CHECK2-NEXT:    store i64 [[ADD10]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP12]], 1
-// CHECK2-NEXT:    [[ADD_PTR11:%.*]] = getelementptr inbounds double, ptr [[TMP11]], i64 [[MUL]]
-// CHECK2-NEXT:    store ptr [[ADD_PTR11]], ptr [[__BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP14:%.*]] = load double, ptr [[TMP13]], align 8
-// CHECK2-NEXT:    store double [[TMP14]], ptr [[V]], align 8
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
+// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
+// CHECK2-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
+// CHECK2-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body14:
+// CHECK2-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP16]], 1
-// CHECK2-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP15]], [[ADD12]]
-// CHECK2-NEXT:    br i1 [[CMP13]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK2:       if.then:
-// CHECK2-NEXT:    [[TMP17:%.*]] = load double, ptr [[C]], align 8
-// CHECK2-NEXT:    [[TMP18:%.*]] = load double, ptr [[V]], align 8
-// CHECK2-NEXT:    call void (...) @body(double noundef [[TMP17]], double noundef [[TMP18]])
-// CHECK2-NEXT:    br label [[IF_END]]
-// CHECK2:       if.end:
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
+// CHECK2-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
+// CHECK2-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
+// CHECK2-NEXT:    store double [[TMP17]], ptr [[V]], align 8
+// CHECK2-NEXT:    [[TMP18:%.*]] = load double, ptr [[C]], align 8
+// CHECK2-NEXT:    [[TMP19:%.*]] = load double, ptr [[V]], align 8
+// CHECK2-NEXT:    call void (...) @body(double noundef [[TMP18]], double noundef [[TMP19]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP19]], 1
-// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP20]], 1
+// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP10:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC14:%.*]]
-// CHECK2:       for.inc14:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD15:%.*]] = add nsw i64 [[TMP20]], 5
-// CHECK2-NEXT:    store i64 [[ADD15]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    br label [[FOR_INC16:%.*]]
+// CHECK2:       for.inc16:
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP21]], 5
+// CHECK2-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP11:![0-9]+]]
-// CHECK2:       for.end16:
+// CHECK2:       for.end18:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -1512,13 +1558,11 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -1529,86 +1573,94 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END30:%.*]]
 // CHECK2:       for.body:
 // CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK2:       for.cond1:
 // CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 4
-// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END21:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END27:%.*]]
 // CHECK2:       for.body3:
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[TMP2]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK2:       for.cond4:
-// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP2]], 5
-// CHECK2-NEXT:    br i1 [[CMP5]], label [[FOR_BODY6:%.*]], label [[FOR_END18:%.*]]
-// CHECK2:       for.body6:
-// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[TMP4]]
-// CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
-// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND8:%.*]]
-// CHECK2:       for.cond8:
-// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP6]], 5
-// CHECK2-NEXT:    br i1 [[CMP9]], label [[FOR_BODY10:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body10:
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], 5
+// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK2-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP5]], 5
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP3]], [[COND]]
+// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END24:%.*]]
+// CHECK2:       for.body8:
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
+// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP7]], [[TMP8]]
-// CHECK2-NEXT:    store i32 [[ADD11]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[MUL12:%.*]] = mul nsw i32 [[TMP9]], 3
-// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i32 7, [[MUL12]]
-// CHECK2-NEXT:    store i32 [[ADD13]], ptr [[J]], align 4
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP10]], 4
-// CHECK2-NEXT:    br i1 [[CMP14]], label [[LAND_LHS_TRUE:%.*]], label [[IF_END:%.*]]
-// CHECK2:       land.lhs.true:
+// CHECK2-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND10:%.*]]
+// CHECK2:       for.cond10:
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP9]], 5
+// CHECK2-NEXT:    [[CMP12:%.*]] = icmp slt i32 4, [[ADD11]]
+// CHECK2-NEXT:    br i1 [[CMP12]], label [[COND_TRUE13:%.*]], label [[COND_FALSE14:%.*]]
+// CHECK2:       cond.true13:
+// CHECK2-NEXT:    br label [[COND_END16:%.*]]
+// CHECK2:       cond.false14:
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP10]], 5
+// CHECK2-NEXT:    br label [[COND_END16]]
+// CHECK2:       cond.end16:
+// CHECK2-NEXT:    [[COND17:%.*]] = phi i32 [ 4, [[COND_TRUE13]] ], [ [[ADD15]], [[COND_FALSE14]] ]
+// CHECK2-NEXT:    [[CMP18:%.*]] = icmp slt i32 [[TMP8]], [[COND17]]
+// CHECK2-NEXT:    br i1 [[CMP18]], label [[FOR_BODY19:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body19:
 // CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[CMP15:%.*]] = icmp slt i32 [[TMP11]], 4
-// CHECK2-NEXT:    br i1 [[CMP15]], label [[IF_THEN:%.*]], label [[IF_END]]
-// CHECK2:       if.then:
+// CHECK2-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[TMP11]], 3
+// CHECK2-NEXT:    [[ADD21:%.*]] = add nsw i32 7, [[MUL20]]
+// CHECK2-NEXT:    store i32 [[ADD21]], ptr [[J]], align 4
 // CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
 // CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
 // CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP12]], i32 noundef [[TMP13]])
-// CHECK2-NEXT:    br label [[IF_END]]
-// CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP12:![0-9]+]]
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND10]], !llvm.loop [[LOOP12:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC16:%.*]]
-// CHECK2:       for.inc16:
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC17:%.*]] = add nsw i32 [[TMP15]], 1
-// CHECK2-NEXT:    store i32 [[INC17]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_INC22:%.*]]
+// CHECK2:       for.inc22:
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC23:%.*]] = add nsw i32 [[TMP15]], 1
+// CHECK2-NEXT:    store i32 [[INC23]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP13:![0-9]+]]
-// CHECK2:       for.end18:
-// CHECK2-NEXT:    br label [[FOR_INC19:%.*]]
-// CHECK2:       for.inc19:
+// CHECK2:       for.end24:
+// CHECK2-NEXT:    br label [[FOR_INC25:%.*]]
+// CHECK2:       for.inc25:
 // CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP16]], 5
-// CHECK2-NEXT:    store i32 [[ADD20]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD26:%.*]] = add nsw i32 [[TMP16]], 5
+// CHECK2-NEXT:    store i32 [[ADD26]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP14:![0-9]+]]
-// CHECK2:       for.end21:
-// CHECK2-NEXT:    br label [[FOR_INC22:%.*]]
-// CHECK2:       for.inc22:
+// CHECK2:       for.end27:
+// CHECK2-NEXT:    br label [[FOR_INC28:%.*]]
+// CHECK2:       for.inc28:
 // CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP17]], 5
-// CHECK2-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD29:%.*]] = add nsw i32 [[TMP17]], 5
+// CHECK2-NEXT:    store i32 [[ADD29]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP15:![0-9]+]]
-// CHECK2:       for.end24:
+// CHECK2:       for.end30:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -2103,9 +2155,8 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:  entry:
 // CHECK2-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
 // CHECK2-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
@@ -2113,65 +2164,78 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END14:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK2:       for.cond1:
-// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP2]], 0
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], 0
 // CHECK2-NEXT:    br i1 [[CMP2]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK2:       cond.true:
 // CHECK2-NEXT:    br label [[COND_END:%.*]]
 // CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[A_ADDR]], align 4
 // CHECK2-NEXT:    br label [[COND_END]]
 // CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP3]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[TMP1]], [[COND]]
-// CHECK2-NEXT:    br i1 [[CMP3]], label [[FOR_BODY4:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body4:
-// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
-// CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
-// CHECK2-NEXT:    [[ADD5:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD5]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP6:%.*]] = icmp slt i32 [[TMP7]], 4
-// CHECK2-NEXT:    br i1 [[CMP6]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK2:       if.then:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP8]])
-// CHECK2-NEXT:    br label [[IF_END]]
-// CHECK2:       if.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP5]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[COND]]
+// CHECK2-NEXT:    [[CMP3:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK2-NEXT:    br i1 [[CMP3]], label [[COND_TRUE4:%.*]], label [[COND_FALSE5:%.*]]
+// CHECK2:       cond.true4:
+// CHECK2-NEXT:    br label [[COND_END12:%.*]]
+// CHECK2:       cond.false5:
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    [[CMP6:%.*]] = icmp sle i32 [[TMP7]], 0
+// CHECK2-NEXT:    br i1 [[CMP6]], label [[COND_TRUE7:%.*]], label [[COND_FALSE8:%.*]]
+// CHECK2:       cond.true7:
+// CHECK2-NEXT:    br label [[COND_END9:%.*]]
+// CHECK2:       cond.false8:
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    br label [[COND_END9]]
+// CHECK2:       cond.end9:
+// CHECK2-NEXT:    [[COND10:%.*]] = phi i32 [ 1, [[COND_TRUE7]] ], [ [[TMP8]], [[COND_FALSE8]] ]
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP6]], [[COND10]]
+// CHECK2-NEXT:    br label [[COND_END12]]
+// CHECK2:       cond.end12:
+// CHECK2-NEXT:    [[COND13:%.*]] = phi i32 [ 4, [[COND_TRUE4]] ], [ [[ADD11]], [[COND_END9]] ]
+// CHECK2-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP2]], [[COND13]]
+// CHECK2-NEXT:    br i1 [[CMP14]], label [[FOR_BODY15:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body15:
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP9]], 3
+// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP10]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP9]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP26:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC7:%.*]]
-// CHECK2:       for.inc7:
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    [[CMP8:%.*]] = icmp sle i32 [[TMP10]], 0
-// CHECK2-NEXT:    br i1 [[CMP8]], label [[COND_TRUE9:%.*]], label [[COND_FALSE10:%.*]]
-// CHECK2:       cond.true9:
-// CHECK2-NEXT:    br label [[COND_END11:%.*]]
-// CHECK2:       cond.false10:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    br label [[COND_END11]]
-// CHECK2:       cond.end11:
-// CHECK2-NEXT:    [[COND12:%.*]] = phi i32 [ 1, [[COND_TRUE9]] ], [ [[TMP11]], [[COND_FALSE10]] ]
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP12]], [[COND12]]
-// CHECK2-NEXT:    store i32 [[ADD13]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_INC17:%.*]]
+// CHECK2:       for.inc17:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    [[CMP18:%.*]] = icmp sle i32 [[TMP12]], 0
+// CHECK2-NEXT:    br i1 [[CMP18]], label [[COND_TRUE19:%.*]], label [[COND_FALSE20:%.*]]
+// CHECK2:       cond.true19:
+// CHECK2-NEXT:    br label [[COND_END21:%.*]]
+// CHECK2:       cond.false20:
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    br label [[COND_END21]]
+// CHECK2:       cond.end21:
+// CHECK2-NEXT:    [[COND22:%.*]] = phi i32 [ 1, [[COND_TRUE19]] ], [ [[TMP13]], [[COND_FALSE20]] ]
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP14]], [[COND22]]
+// CHECK2-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP27:![0-9]+]]
-// CHECK2:       for.end14:
+// CHECK2:       for.end24:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -2184,9 +2248,8 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK2-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
 // CHECK2-NEXT:    store ptr [[ARRAY]], ptr [[__RANGE2]], align 8
@@ -2220,51 +2283,56 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK2:       for.cond7:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
-// CHECK2-NEXT:    br i1 [[CMP8]], label [[FOR_BODY9:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body9:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
-// CHECK2-NEXT:    store i64 [[ADD10]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP12]], 1
-// CHECK2-NEXT:    [[ADD_PTR11:%.*]] = getelementptr inbounds double, ptr [[TMP11]], i64 [[MUL]]
-// CHECK2-NEXT:    store ptr [[ADD_PTR11]], ptr [[__BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP13:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP14:%.*]] = load double, ptr [[TMP13]], align 8
-// CHECK2-NEXT:    store double [[TMP14]], ptr [[V]], align 8
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
+// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
+// CHECK2-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
+// CHECK2-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body14:
+// CHECK2-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP16]], 1
-// CHECK2-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP15]], [[ADD12]]
-// CHECK2-NEXT:    br i1 [[CMP13]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK2:       if.then:
-// CHECK2-NEXT:    [[TMP17:%.*]] = load double, ptr [[V]], align 8
-// CHECK2-NEXT:    call void (...) @body(double noundef [[TMP17]])
-// CHECK2-NEXT:    br label [[IF_END]]
-// CHECK2:       if.end:
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
+// CHECK2-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
+// CHECK2-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
+// CHECK2-NEXT:    store double [[TMP17]], ptr [[V]], align 8
+// CHECK2-NEXT:    [[TMP18:%.*]] = load double, ptr [[V]], align 8
+// CHECK2-NEXT:    call void (...) @body(double noundef [[TMP18]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP18]], 1
-// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP19]], 1
+// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP28:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC14:%.*]]
-// CHECK2:       for.inc14:
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD15:%.*]] = add nsw i64 [[TMP19]], 5
-// CHECK2-NEXT:    store i64 [[ADD15]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    br label [[FOR_INC16:%.*]]
+// CHECK2:       for.inc16:
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP20]], 5
+// CHECK2-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP29:![0-9]+]]
-// CHECK2:       for.end16:
+// CHECK2:       for.end18:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -2284,9 +2352,8 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[START_ADDR]], align 4
@@ -2310,48 +2377,53 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK2-NEXT:    [[ADD5:%.*]] = add i32 [[TMP6]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP5]], [[ADD5]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END15:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK2:       for.cond6:
-// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP7]], 5
-// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body8:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD9:%.*]] = add i32 [[TMP8]], [[TMP9]]
-// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP11]], 3
-// CHECK2-NEXT:    [[ADD10:%.*]] = add i32 [[TMP10]], [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD10]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[ADD11:%.*]] = add i32 [[TMP13]], 1
-// CHECK2-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP12]], [[ADD11]]
-// CHECK2-NEXT:    br i1 [[CMP12]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK2:       if.then:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP14]])
-// CHECK2-NEXT:    br label [[IF_END]]
-// CHECK2:       if.end:
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD7:%.*]] = add i32 [[TMP9]], 1
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD8:%.*]] = add i32 [[TMP10]], 5
+// CHECK2-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
+// CHECK2-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD10:%.*]] = add i32 [[TMP11]], 1
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD11:%.*]] = add i32 [[TMP12]], 5
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP8]], [[COND]]
+// CHECK2-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body13:
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP14]], 3
+// CHECK2-NEXT:    [[ADD14:%.*]] = add i32 [[TMP13]], [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP15]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP15]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP16]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP30:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC13:%.*]]
-// CHECK2:       for.inc13:
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD14:%.*]] = add i32 [[TMP16]], 5
-// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_INC15:%.*]]
+// CHECK2:       for.inc15:
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD16:%.*]] = add i32 [[TMP17]], 5
+// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP31:![0-9]+]]
-// CHECK2:       for.end15:
+// CHECK2:       for.end17:
 // CHECK2-NEXT:    ret void
 //
 //
diff --git a/clang/test/OpenMP/tile_rect_codegen_ir.cpp b/clang/test/OpenMP/tile_rect_codegen_ir.cpp
index f35c581c8b8ed..808f1cf0a3d0c 100644
--- a/clang/test/OpenMP/tile_rect_codegen_ir.cpp
+++ b/clang/test/OpenMP/tile_rect_codegen_ir.cpp
@@ -9,50 +9,336 @@
 extern "C" void body(int);
 
 // Trip count 6, tile 4 → remainder; body must be guarded.
-// IR-LABEL: remainder_6_tile_4v(
+// IR-LABEL: define dso_local void @_Z18remainder_6_tile_4v(
+// IR-SAME: ) #[[ATTR0:[0-9]+]] {
+// IR-NEXT:  [[ENTRY:.*:]]
+// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    store i32 0, ptr [[I]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND:.*]]
+// IR:       [[FOR_COND]]:
+// IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 6
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END9:.*]]
+// IR:       [[FOR_BODY]]:
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND1:.*]]
+// IR:       [[FOR_COND1]]:
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 6, [[ADD]]
+// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 4
+// IR-NEXT:    br label %[[COND_END]]
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ 6, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
+// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
+// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY5]]:
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
+// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void @body(i32 noundef [[TMP6]])
+// IR-NEXT:    br label %[[FOR_INC:.*]]
+// IR:       [[FOR_INC]]:
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP7]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP2:![0-9]+]]
+// IR:       [[FOR_END]]:
+// IR-NEXT:    br label %[[FOR_INC7:.*]]
+// IR:       [[FOR_INC7]]:
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP8]], 4
+// IR-NEXT:    store i32 [[ADD8]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP4:![0-9]+]]
+// IR:       [[FOR_END9]]:
+// IR-NEXT:    ret void
+//
 void remainder_6_tile_4(void) {
   // Rectangular tile counter and constant tile-size bound.
-  // IR-DAG: %.tile.cnt{{.*}} = alloca i32
-  // IR: icmp {{.*}} i32 {{.*}}, 4
-  // IR: icmp {{.*}} i32 {{.*}}, 6
-  // IR: br i1 {{.*}}, label %if.then{{.*}}, label %if.end{{.*}}
-  // IR: call {{.*}} @body(
 #pragma omp tile sizes(4)
   for (int i = 0; i < 6; ++i)
     body(i);
 }
 
 // Simple stride-1 loop; tile size matches full tiles only (10, tile 5).
-// IR-LABEL: full_tiles_10_5v(
+// IR-LABEL: define dso_local void @_Z15full_tiles_10_5v(
+// IR-SAME: ) #[[ATTR0]] {
+// IR-NEXT:  [[ENTRY:.*:]]
+// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    store i32 0, ptr [[I]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND:.*]]
+// IR:       [[FOR_COND]]:
+// IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 10
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END9:.*]]
+// IR:       [[FOR_BODY]]:
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND1:.*]]
+// IR:       [[FOR_COND1]]:
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 5
+// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 10, [[ADD]]
+// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 5
+// IR-NEXT:    br label %[[COND_END]]
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ 10, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
+// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
+// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY5]]:
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
+// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void @body(i32 noundef [[TMP6]])
+// IR-NEXT:    br label %[[FOR_INC:.*]]
+// IR:       [[FOR_INC]]:
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP7]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP5:![0-9]+]]
+// IR:       [[FOR_END]]:
+// IR-NEXT:    br label %[[FOR_INC7:.*]]
+// IR:       [[FOR_INC7]]:
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP8]], 5
+// IR-NEXT:    store i32 [[ADD8]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
+// IR:       [[FOR_END9]]:
+// IR-NEXT:    ret void
+//
 void full_tiles_10_5(void) {
-  // IR-DAG: %.tile.cnt{{.*}} = alloca i32
-  // IR: icmp {{.*}} i32 {{.*}}, 5
-  // IR: call {{.*}} @body(
 #pragma omp tile sizes(5)
   for (int i = 0; i < 10; ++i)
     body(i);
 }
 
 // Variable tile size: clamp (TS<=0 ? 1 : TS) is inlined; bound compares to %cond.
-// IR-LABEL: var_tilei(
+// IR-LABEL: define dso_local void @_Z8var_tilei(
+// IR-SAME: i32 noundef [[TS:%.*]]) #[[ATTR0]] {
+// IR-NEXT:  [[ENTRY:.*:]]
+// IR-NEXT:    [[TS_ADDR:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    store i32 [[TS]], ptr [[TS_ADDR]], align 4
+// IR-NEXT:    store i32 0, ptr [[I]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND:.*]]
+// IR:       [[FOR_COND]]:
+// IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 12
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END24:.*]]
+// IR:       [[FOR_BODY]]:
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND1:.*]]
+// IR:       [[FOR_COND1]]:
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TS_ADDR]], align 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], 0
+// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TS_ADDR]], align 4
+// IR-NEXT:    br label %[[COND_END]]
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ 1, %[[COND_TRUE]] ], [ [[TMP5]], %[[COND_FALSE]] ]
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[COND]]
+// IR-NEXT:    [[CMP3:%.*]] = icmp slt i32 12, [[ADD]]
+// IR-NEXT:    br i1 [[CMP3]], label %[[COND_TRUE4:.*]], label %[[COND_FALSE5:.*]]
+// IR:       [[COND_TRUE4]]:
+// IR-NEXT:    br label %[[COND_END12:.*]]
+// IR:       [[COND_FALSE5]]:
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TS_ADDR]], align 4
+// IR-NEXT:    [[CMP6:%.*]] = icmp sle i32 [[TMP7]], 0
+// IR-NEXT:    br i1 [[CMP6]], label %[[COND_TRUE7:.*]], label %[[COND_FALSE8:.*]]
+// IR:       [[COND_TRUE7]]:
+// IR-NEXT:    br label %[[COND_END9:.*]]
+// IR:       [[COND_FALSE8]]:
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TS_ADDR]], align 4
+// IR-NEXT:    br label %[[COND_END9]]
+// IR:       [[COND_END9]]:
+// IR-NEXT:    [[COND10:%.*]] = phi i32 [ 1, %[[COND_TRUE7]] ], [ [[TMP8]], %[[COND_FALSE8]] ]
+// IR-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP6]], [[COND10]]
+// IR-NEXT:    br label %[[COND_END12]]
+// IR:       [[COND_END12]]:
+// IR-NEXT:    [[COND13:%.*]] = phi i32 [ 12, %[[COND_TRUE4]] ], [ [[ADD11]], %[[COND_END9]] ]
+// IR-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP2]], [[COND13]]
+// IR-NEXT:    br i1 [[CMP14]], label %[[FOR_BODY15:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY15]]:
+// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP9]], 1
+// IR-NEXT:    [[ADD16:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD16]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void @body(i32 noundef [[TMP10]])
+// IR-NEXT:    br label %[[FOR_INC:.*]]
+// IR:       [[FOR_INC]]:
+// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP7:![0-9]+]]
+// IR:       [[FOR_END]]:
+// IR-NEXT:    br label %[[FOR_INC17:.*]]
+// IR:       [[FOR_INC17]]:
+// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TS_ADDR]], align 4
+// IR-NEXT:    [[CMP18:%.*]] = icmp sle i32 [[TMP12]], 0
+// IR-NEXT:    br i1 [[CMP18]], label %[[COND_TRUE19:.*]], label %[[COND_FALSE20:.*]]
+// IR:       [[COND_TRUE19]]:
+// IR-NEXT:    br label %[[COND_END21:.*]]
+// IR:       [[COND_FALSE20]]:
+// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TS_ADDR]], align 4
+// IR-NEXT:    br label %[[COND_END21]]
+// IR:       [[COND_END21]]:
+// IR-NEXT:    [[COND22:%.*]] = phi i32 [ 1, %[[COND_TRUE19]] ], [ [[TMP13]], %[[COND_FALSE20]] ]
+// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP14]], [[COND22]]
+// IR-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
+// IR:       [[FOR_END24]]:
+// IR-NEXT:    ret void
+//
 void var_tile(int ts) {
-  // IR-DAG: %.tile.cnt{{.*}} = alloca i32
-  // IR: %cond = phi i32
-  // IR: icmp {{.*}} i32 {{.*}}, %cond
-  // IR: call {{.*}} @body(
 #pragma omp tile sizes(ts)
   for (int i = 0; i < 12; ++i)
     body(i);
 }
 
 // Two tiled dimensions → two tile counters.
-// IR-LABEL: two_dimv(
+// IR-LABEL: define dso_local void @_Z7two_dimv(
+// IR-SAME: ) #[[ATTR0]] {
+// IR-NEXT:  [[ENTRY:.*:]]
+// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[J:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
+// IR-NEXT:    store i32 0, ptr [[I]], align 4
+// IR-NEXT:    store i32 0, ptr [[J]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND:.*]]
+// IR:       [[FOR_COND]]:
+// IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 3
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END31:.*]]
+// IR:       [[FOR_BODY]]:
+// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_1_IV_J]], align 4
+// IR-NEXT:    br label %[[FOR_COND1:.*]]
+// IR:       [[FOR_COND1]]:
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 3
+// IR-NEXT:    br i1 [[CMP2]], label %[[FOR_BODY3:.*]], label %[[FOR_END28:.*]]
+// IR:       [[FOR_BODY3]]:
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP2]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND4:.*]]
+// IR:       [[FOR_COND4]]:
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], 2
+// IR-NEXT:    [[CMP5:%.*]] = icmp slt i32 3, [[ADD]]
+// IR-NEXT:    br i1 [[CMP5]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP5]], 2
+// IR-NEXT:    br label %[[COND_END]]
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ 3, %[[COND_TRUE]] ], [ [[ADD6]], %[[COND_FALSE]] ]
+// IR-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP3]], [[COND]]
+// IR-NEXT:    br i1 [[CMP7]], label %[[FOR_BODY8:.*]], label %[[FOR_END25:.*]]
+// IR:       [[FOR_BODY8]]:
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 1
+// IR-NEXT:    [[ADD9:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// IR-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_1_IV_J]], align 4
+// IR-NEXT:    br label %[[FOR_COND10:.*]]
+// IR:       [[FOR_COND10]]:
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// IR-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP9]], 2
+// IR-NEXT:    [[CMP12:%.*]] = icmp slt i32 3, [[ADD11]]
+// IR-NEXT:    br i1 [[CMP12]], label %[[COND_TRUE13:.*]], label %[[COND_FALSE14:.*]]
+// IR:       [[COND_TRUE13]]:
+// IR-NEXT:    br label %[[COND_END16:.*]]
+// IR:       [[COND_FALSE14]]:
+// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// IR-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP10]], 2
+// IR-NEXT:    br label %[[COND_END16]]
+// IR:       [[COND_END16]]:
+// IR-NEXT:    [[COND17:%.*]] = phi i32 [ 3, %[[COND_TRUE13]] ], [ [[ADD15]], %[[COND_FALSE14]] ]
+// IR-NEXT:    [[CMP18:%.*]] = icmp slt i32 [[TMP8]], [[COND17]]
+// IR-NEXT:    br i1 [[CMP18]], label %[[FOR_BODY19:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY19]]:
+// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// IR-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[TMP11]], 1
+// IR-NEXT:    [[ADD21:%.*]] = add nsw i32 0, [[MUL20]]
+// IR-NEXT:    store i32 [[ADD21]], ptr [[J]], align 4
+// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
+// IR-NEXT:    [[ADD22:%.*]] = add nsw i32 [[TMP12]], [[TMP13]]
+// IR-NEXT:    call void @body(i32 noundef [[ADD22]])
+// IR-NEXT:    br label %[[FOR_INC:.*]]
+// IR:       [[FOR_INC]]:
+// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
+// IR-NEXT:    br label %[[FOR_COND10]], !llvm.loop [[LOOP9:![0-9]+]]
+// IR:       [[FOR_END]]:
+// IR-NEXT:    br label %[[FOR_INC23:.*]]
+// IR:       [[FOR_INC23]]:
+// IR-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[INC24:%.*]] = add nsw i32 [[TMP15]], 1
+// IR-NEXT:    store i32 [[INC24]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND4]], !llvm.loop [[LOOP10:![0-9]+]]
+// IR:       [[FOR_END25]]:
+// IR-NEXT:    br label %[[FOR_INC26:.*]]
+// IR:       [[FOR_INC26]]:
+// IR-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// IR-NEXT:    [[ADD27:%.*]] = add nsw i32 [[TMP16]], 2
+// IR-NEXT:    store i32 [[ADD27]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP11:![0-9]+]]
+// IR:       [[FOR_END28]]:
+// IR-NEXT:    br label %[[FOR_INC29:.*]]
+// IR:       [[FOR_INC29]]:
+// IR-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD30:%.*]] = add nsw i32 [[TMP17]], 2
+// IR-NEXT:    store i32 [[ADD30]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP12:![0-9]+]]
+// IR:       [[FOR_END31]]:
+// IR-NEXT:    ret void
+//
 void two_dim(void) {
-  // IR-DAG: %.tile.cnt.0.iv.i{{.*}} = alloca i32
-  // IR-DAG: %.tile.cnt.1.iv.j{{.*}} = alloca i32
-  // IR: icmp {{.*}} i32 {{.*}}, 2
-  // IR: icmp {{.*}} i32 {{.*}}, 2
-  // IR: call {{.*}} @body(
 #pragma omp tile sizes(2, 2)
   for (int i = 0; i < 3; ++i)
     for (int j = 0; j < 3; ++j)
@@ -60,10 +346,83 @@ void two_dim(void) {
 }
 
 // Nested loop in tile body: outer nest still uses .tile.cnt for tiled dim.
-// IR-LABEL: nested_inner_ji(
+// IR-LABEL: define dso_local void @_Z14nested_inner_ji(
+// IR-SAME: i32 noundef [[N:%.*]]) #[[ATTR0]] {
+// IR-NEXT:  [[ENTRY:.*:]]
+// IR-NEXT:    [[N_ADDR:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[J:%.*]] = alloca i32, align 4
+// IR-NEXT:    store i32 [[N]], ptr [[N_ADDR]], align 4
+// IR-NEXT:    store i32 0, ptr [[I]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND:.*]]
+// IR:       [[FOR_COND]]:
+// IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 6
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END16:.*]]
+// IR:       [[FOR_BODY]]:
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND1:.*]]
+// IR:       [[FOR_COND1]]:
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 6, [[ADD]]
+// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 4
+// IR-NEXT:    br label %[[COND_END]]
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ 6, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
+// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
+// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END13:.*]]
+// IR:       [[FOR_BODY5]]:
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
+// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
+// IR-NEXT:    store i32 0, ptr [[J]], align 4
+// IR-NEXT:    br label %[[FOR_COND7:.*]]
+// IR:       [[FOR_COND7]]:
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[J]], align 4
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[N_ADDR]], align 4
+// IR-NEXT:    [[CMP8:%.*]] = icmp slt i32 [[TMP6]], [[TMP7]]
+// IR-NEXT:    br i1 [[CMP8]], label %[[FOR_BODY9:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY9]]:
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[J]], align 4
+// IR-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP8]], [[TMP9]]
+// IR-NEXT:    call void @body(i32 noundef [[ADD10]])
+// IR-NEXT:    br label %[[FOR_INC:.*]]
+// IR:       [[FOR_INC]]:
+// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[J]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[J]], align 4
+// IR-NEXT:    br label %[[FOR_COND7]], !llvm.loop [[LOOP13:![0-9]+]]
+// IR:       [[FOR_END]]:
+// IR-NEXT:    br label %[[FOR_INC11:.*]]
+// IR:       [[FOR_INC11]]:
+// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[INC12:%.*]] = add nsw i32 [[TMP11]], 1
+// IR-NEXT:    store i32 [[INC12]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP14:![0-9]+]]
+// IR:       [[FOR_END13]]:
+// IR-NEXT:    br label %[[FOR_INC14:.*]]
+// IR:       [[FOR_INC14]]:
+// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP12]], 4
+// IR-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP15:![0-9]+]]
+// IR:       [[FOR_END16]]:
+// IR-NEXT:    ret void
+//
 void nested_inner_j(int n) {
-  // IR-DAG: %.tile.cnt{{.*}} = alloca i32
-  // IR: icmp {{.*}} i32 {{.*}}, 4
 #pragma omp tile sizes(4)
   for (int i = 0; i < 6; ++i)
     for (int j = 0; j < n; ++j)
@@ -72,13 +431,83 @@ void nested_inner_j(int n) {
 
 // Predicate elision: when trip count is evenly divisible by constant tile size,
 // no if-guard is needed (no icmp against trip count in tile body).
-// IR-LABEL: elided_predicatev(
+// IR-LABEL: define dso_local void @_Z16elided_predicatev(
+// IR-SAME: ) #[[ATTR0]] {
+// IR-NEXT:  [[ENTRY:.*:]]
+// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    store i32 0, ptr [[I]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND:.*]]
+// IR:       [[FOR_COND]]:
+// IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 12
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END9:.*]]
+// IR:       [[FOR_BODY]]:
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND1:.*]]
+// IR:       [[FOR_COND1]]:
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 12, [[ADD]]
+// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 4
+// IR-NEXT:    br label %[[COND_END]]
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ 12, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
+// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
+// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY5]]:
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
+// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void @body(i32 noundef [[TMP6]])
+// IR-NEXT:    br label %[[FOR_INC:.*]]
+// IR:       [[FOR_INC]]:
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP7]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP16:![0-9]+]]
+// IR:       [[FOR_END]]:
+// IR-NEXT:    br label %[[FOR_INC7:.*]]
+// IR:       [[FOR_INC7]]:
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP8]], 4
+// IR-NEXT:    store i32 [[ADD8]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP17:![0-9]+]]
+// IR:       [[FOR_END9]]:
+// IR-NEXT:    ret void
+//
 void elided_predicate(void) {
   // Trip count 12, tile size 4 → 12 % 4 == 0, no predicate.
-  // IR-NOT: if.then
-  // IR-NOT: if.end
-  // IR: call {{.*}} @body(
 #pragma omp tile sizes(4)
   for (int i = 0; i < 12; ++i)
     body(i);
 }
+//.
+// IR: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]]}
+// IR: [[META3]] = !{!"llvm.loop.mustprogress"}
+// IR: [[LOOP4]] = distinct !{[[LOOP4]], [[META3]]}
+// IR: [[LOOP5]] = distinct !{[[LOOP5]], [[META3]]}
+// IR: [[LOOP6]] = distinct !{[[LOOP6]], [[META3]]}
+// IR: [[LOOP7]] = distinct !{[[LOOP7]], [[META3]]}
+// IR: [[LOOP8]] = distinct !{[[LOOP8]], [[META3]]}
+// IR: [[LOOP9]] = distinct !{[[LOOP9]], [[META3]]}
+// IR: [[LOOP10]] = distinct !{[[LOOP10]], [[META3]]}
+// IR: [[LOOP11]] = distinct !{[[LOOP11]], [[META3]]}
+// IR: [[LOOP12]] = distinct !{[[LOOP12]], [[META3]]}
+// IR: [[LOOP13]] = distinct !{[[LOOP13]], [[META3]]}
+// IR: [[LOOP14]] = distinct !{[[LOOP14]], [[META3]]}
+// IR: [[LOOP15]] = distinct !{[[LOOP15]], [[META3]]}
+// IR: [[LOOP16]] = distinct !{[[LOOP16]], [[META3]]}
+// IR: [[LOOP17]] = distinct !{[[LOOP17]], [[META3]]}
+//.

>From 78bed3e78c5f69cc3e4373366971b4ff2cebfde0 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Tue, 2 Jun 2026 16:18:57 -0400
Subject: [PATCH 12/19] reverted-mlir

---
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     |  64 ++++------
 .../Frontend/OpenMPIRBuilderTest.cpp          |  35 +----
 .../test/Target/LLVMIR/openmp-cli-tile01.mlir |  18 ++-
 .../test/Target/LLVMIR/openmp-cli-tile02.mlir |  30 ++---
 .../test/Target/LLVMIR/openmp-cli-tile03.mlir | 120 +++++++++---------
 5 files changed, 109 insertions(+), 158 deletions(-)

diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index a38ecede63376..fc3812502fb2e 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -6984,7 +6984,7 @@ OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
   // Compute the trip counts of the floor loops.
   Builder.SetCurrentDebugLocation(DL);
   Builder.restoreIP(OutermostLoop->getPreheaderIP());
-  SmallVector<Value *, 4> FloorCount;
+  SmallVector<Value *, 4> FloorCompleteCount, FloorCount, FloorRems;
   for (int i = 0; i < NumLoops; ++i) {
     Value *TileSize = TileSizes[i];
     Value *OrigTripCount = OrigTripCounts[i];
@@ -7008,7 +7008,10 @@ OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
         Builder.CreateAdd(FloorCompleteTripCount, FloorTripOverflow,
                           "omp_floor" + Twine(i) + ".tripcount", true);
 
+    // Remember some values for later use.
+    FloorCompleteCount.push_back(FloorCompleteTripCount);
     FloorCount.push_back(FloorTripCount);
+    FloorRems.push_back(FloorTripRem);
   }
 
   // Generate the new loop nest, from the outermost to the innermost.
@@ -7052,10 +7055,23 @@ OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
 
   EmbeddNewLoops(FloorCount, "floor");
 
-  // Create the tile loops with rectangular (constant) trip counts equal to
-  // the tile sizes. A validity predicate in the body guards against
-  // out-of-bounds iterations in the remainder tile.
-  SmallVector<Value *, 4> TileCounts(TileSizes.begin(), TileSizes.end());
+  // Within the innermost floor loop, emit the code that computes the tile
+  // sizes.
+  Builder.SetInsertPoint(Enter->getTerminator());
+  SmallVector<Value *, 4> TileCounts;
+  for (int i = 0; i < NumLoops; ++i) {
+    CanonicalLoopInfo *FloorLoop = Result[i];
+    Value *TileSize = TileSizes[i];
+
+    Value *FloorIsEpilogue =
+        Builder.CreateICmpEQ(FloorLoop->getIndVar(), FloorCompleteCount[i]);
+    Value *TileTripCount =
+        Builder.CreateSelect(FloorIsEpilogue, FloorRems[i], TileSize);
+
+    TileCounts.push_back(TileTripCount);
+  }
+
+  // Create the tile loops.
   EmbeddNewLoops(TileCounts, "tile");
 
   // Insert the inbetween code into the body.
@@ -7084,7 +7100,6 @@ OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
   // Replace the original induction variable with an induction variable computed
   // from the tile and floor induction variables.
   Builder.restoreIP(Result.back()->getBodyIP());
-  SmallVector<Value *, 4> ShiftValues;
   for (int i = 0; i < NumLoops; ++i) {
     CanonicalLoopInfo *FloorLoop = Result[i];
     CanonicalLoopInfo *TileLoop = Result[NumLoops + i];
@@ -7096,47 +7111,18 @@ OpenMPIRBuilder::tileLoops(DebugLoc DL, ArrayRef<CanonicalLoopInfo *> Loops,
     Value *Shift =
         Builder.CreateAdd(Scale, TileLoop->getIndVar(), {}, /*HasNUW=*/true);
     OrigIndVar->replaceAllUsesWith(Shift);
-    ShiftValues.push_back(Shift);
-  }
-
-  // Build a validity predicate: for each tiled dimension, check that the
-  // computed original index is within the original trip count. This guards
-  // against executing out-of-bounds iterations in the remainder tile.
-  Value *Pred = nullptr;
-  for (int i = 0; i < NumLoops; ++i) {
-    Value *DimPred = Builder.CreateICmpULT(ShiftValues[i], OrigTripCounts[i],
-                                           "omp_tile" + Twine(i) + ".inbounds");
-    Pred = Pred ? Builder.CreateAnd(Pred, DimPred) : DimPred;
   }
 
-  // Insert the conditional guard: split the body flow so that out-of-bounds
-  // iterations skip directly to a merge block before the tile latch.
-  BasicBlock *TileBodyBB = Builder.GetInsertBlock();
-  Instruction *BodyTerm = TileBodyBB->getTerminator();
-  BasicBlock *BodySucc = cast<UncondBrInst>(BodyTerm)->getSuccessor(0);
-  BasicBlock *TileLatch = Result.back()->getLatch();
-
-  BasicBlock *MergeBB =
-      BasicBlock::Create(F->getContext(), "omp_tile.body.merge", F, TileLatch);
-  Builder.SetInsertPoint(MergeBB);
-  Builder.CreateBr(TileLatch);
-
-  // Redirect the body chain's exit from the tile latch to the merge block.
-  for (BasicBlock *PredBB : llvm::make_early_inc_range(predecessors(TileLatch)))
-    if (PredBB != MergeBB)
-      PredBB->getTerminator()->replaceUsesOfWith(TileLatch, MergeBB);
-
-  // Replace the tile body's unconditional branch with a conditional one.
-  BodyTerm->eraseFromParent();
-  Builder.SetInsertPoint(TileBodyBB);
-  Builder.CreateCondBr(Pred, BodySucc, MergeBB);
-
   // Remove unused parts of the original loops.
   removeUnusedBlocksFromParent(OldControlBBs);
 
   for (CanonicalLoopInfo *L : Loops)
     L->invalidate();
 
+#ifndef NDEBUG
+  for (CanonicalLoopInfo *GenL : Result)
+    GenL->assertOK();
+#endif
   return Result;
 }
 
diff --git a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
index 2502c37ba28b3..b96103c82c185 100644
--- a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
@@ -1643,27 +1643,12 @@ TEST_F(OpenMPIRBuilderTest, TileSingleLoop) {
   CanonicalLoopInfo *Floor = GenLoops[0];
   CanonicalLoopInfo *Tile = GenLoops[1];
 
-  // The merge block sits between the body code and the tile latch.
-  BasicBlock *MergeBB = Tile->getLatch()->getSinglePredecessor();
-  ASSERT_NE(MergeBB, nullptr);
-
   BasicBlock *RefOrder[] = {
-      Floor->getPreheader(),
-      Floor->getHeader(),
-      Floor->getCond(),
-      Floor->getBody(),
-      Tile->getPreheader(),
-      Tile->getHeader(),
-      Tile->getCond(),
-      Tile->getBody(),
-      BodyCode,
-      MergeBB,
-      Tile->getLatch(),
-      Tile->getExit(),
-      Tile->getAfter(),
-      Floor->getLatch(),
-      Floor->getExit(),
-      Floor->getAfter(),
+      Floor->getPreheader(), Floor->getHeader(),   Floor->getCond(),
+      Floor->getBody(),      Tile->getPreheader(), Tile->getHeader(),
+      Tile->getCond(),       Tile->getBody(),      BodyCode,
+      Tile->getLatch(),      Tile->getExit(),      Tile->getAfter(),
+      Floor->getLatch(),     Floor->getExit(),     Floor->getAfter(),
   };
   EXPECT_TRUE(verifyDFSOrder(F, RefOrder));
   EXPECT_TRUE(verifyListOrder(F, RefOrder));
@@ -1733,10 +1718,6 @@ TEST_F(OpenMPIRBuilderTest, TileNestedLoops) {
   CanonicalLoopInfo *Tile1 = GenLoops[2];
   CanonicalLoopInfo *Tile2 = GenLoops[3];
 
-  // The merge block sits between the body code and the innermost tile latch.
-  BasicBlock *MergeBB = Tile2->getLatch()->getSinglePredecessor();
-  ASSERT_NE(MergeBB, nullptr);
-
   BasicBlock *RefOrder[] = {
       Floor1->getPreheader(),
       Floor1->getHeader(),
@@ -1755,7 +1736,6 @@ TEST_F(OpenMPIRBuilderTest, TileNestedLoops) {
       Tile2->getCond(),
       Tile2->getBody(),
       BodyCode,
-      MergeBB,
       Tile2->getLatch(),
       Tile2->getExit(),
       Tile2->getAfter(),
@@ -1846,10 +1826,6 @@ TEST_F(OpenMPIRBuilderTest, TileNestedLoopsWithBounds) {
   CanonicalLoopInfo *Tile0 = GenLoops[2];
   CanonicalLoopInfo *Tile1 = GenLoops[3];
 
-  // The merge block sits between the body code and the innermost tile latch.
-  BasicBlock *MergeBB = Tile1->getLatch()->getSinglePredecessor();
-  ASSERT_NE(MergeBB, nullptr);
-
   BasicBlock *RefOrder[] = {
       Floor0->getPreheader(),
       Floor0->getHeader(),
@@ -1868,7 +1844,6 @@ TEST_F(OpenMPIRBuilderTest, TileNestedLoopsWithBounds) {
       Tile1->getCond(),
       Tile1->getBody(),
       BodyCode,
-      MergeBB,
       Tile1->getLatch(),
       Tile1->getExit(),
       Tile1->getAfter(),
diff --git a/mlir/test/Target/LLVMIR/openmp-cli-tile01.mlir b/mlir/test/Target/LLVMIR/openmp-cli-tile01.mlir
index 948a9815851a5..0d559b69a3ad1 100644
--- a/mlir/test/Target/LLVMIR/openmp-cli-tile01.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-cli-tile01.mlir
@@ -38,6 +38,8 @@ llvm.func @tile_trivial_loop(%baseptr: !llvm.ptr, %tc: i32, %ts: i32) -> () {
 // CHECK-NEXT:    br i1 %[[OMP_FLOOR0_CMP:.+]], label %[[OMP_FLOOR0_BODY:.+]], label %[[OMP_FLOOR0_EXIT:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_FLOOR0_BODY]]:
+// CHECK-NEXT:    %[[TMP8:.+]] = icmp eq i32 %[[OMP_FLOOR0_IV:.+]], %[[TMP4:.+]]
+// CHECK-NEXT:    %[[TMP9:.+]] = select i1 %[[TMP8:.+]], i32 %[[TMP5:.+]], i32 %[[TMP2:.+]]
 // CHECK-NEXT:    br label %[[OMP_TILE0_PREHEADER:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_PREHEADER]]:
@@ -48,27 +50,23 @@ llvm.func @tile_trivial_loop(%baseptr: !llvm.ptr, %tc: i32, %ts: i32) -> () {
 // CHECK-NEXT:    br label %[[OMP_TILE0_COND:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_COND]]:
-// CHECK-NEXT:    %[[OMP_TILE0_CMP:.+]] = icmp ult i32 %[[OMP_TILE0_IV:.+]], %[[TMP2:.+]]
+// CHECK-NEXT:    %[[OMP_TILE0_CMP:.+]] = icmp ult i32 %[[OMP_TILE0_IV:.+]], %[[TMP9:.+]]
 // CHECK-NEXT:    br i1 %[[OMP_TILE0_CMP:.+]], label %[[OMP_TILE0_BODY:.+]], label %[[OMP_TILE0_EXIT:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_BODY]]:
-// CHECK-NEXT:    %[[TMP8:.+]] = mul nuw i32 %[[TMP2:.+]], %[[OMP_FLOOR0_IV:.+]]
-// CHECK-NEXT:    %[[TMP9:.+]] = add nuw i32 %[[TMP8:.+]], %[[OMP_TILE0_IV:.+]]
-// CHECK-NEXT:    %[[OMP_TILE0_INBOUNDS:.+]] = icmp ult i32 %[[TMP9:.+]], %[[TMP1:.+]]
-// CHECK-NEXT:    br i1 %[[OMP_TILE0_INBOUNDS:.+]], label %[[OMP_OMP_LOOP_BODY:.+]], label %[[OMP_TILE_BODY_MERGE:.+]]
+// CHECK-NEXT:    %[[TMP10:.+]] = mul nuw i32 %[[TMP2:.+]], %[[OMP_FLOOR0_IV:.+]]
+// CHECK-NEXT:    %[[TMP11:.+]] = add nuw i32 %[[TMP10:.+]], %[[OMP_TILE0_IV:.+]]
+// CHECK-NEXT:    br label %[[OMP_OMP_LOOP_BODY:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_OMP_LOOP_BODY]]:
 // CHECK-NEXT:    br label %[[OMP_LOOP_REGION:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_LOOP_REGION]]:
-// CHECK-NEXT:    %[[TMP10:.+]] = getelementptr inbounds float, ptr %[[TMP0:.+]], i32 %[[TMP9:.+]]
-// CHECK-NEXT:    store float 4.200000e+01, ptr %[[TMP10:.+]], align 4
+// CHECK-NEXT:    %[[TMP12:.+]] = getelementptr inbounds float, ptr %[[TMP0:.+]], i32 %[[TMP11:.+]]
+// CHECK-NEXT:    store float 4.200000e+01, ptr %[[TMP12:.+]], align 4
 // CHECK-NEXT:    br label %[[OMP_REGION_CONT:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_REGION_CONT]]:
-// CHECK-NEXT:    br label %[[OMP_TILE_BODY_MERGE:.+]]
-// CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE_BODY_MERGE]]:
 // CHECK-NEXT:    br label %[[OMP_TILE0_INC:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_INC]]:
diff --git a/mlir/test/Target/LLVMIR/openmp-cli-tile02.mlir b/mlir/test/Target/LLVMIR/openmp-cli-tile02.mlir
index 59ffa50037f4b..22c2973164159 100644
--- a/mlir/test/Target/LLVMIR/openmp-cli-tile02.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-cli-tile02.mlir
@@ -79,6 +79,10 @@ llvm.func @tile_2d_loop(%baseptr: !llvm.ptr, %tc1: i32, %tc2: i32, %ts1: i32, %t
 // CHECK-NEXT:    br i1 %[[OMP_FLOOR1_CMP:.+]], label %[[OMP_FLOOR1_BODY:.+]], label %[[OMP_FLOOR1_EXIT:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_FLOOR1_BODY]]:
+// CHECK-NEXT:    %[[TMP14:.+]] = icmp eq i32 %[[OMP_FLOOR0_IV:.+]], %[[TMP6:.+]]
+// CHECK-NEXT:    %[[TMP15:.+]] = select i1 %[[TMP14:.+]], i32 %[[TMP7:.+]], i32 %[[TMP3:.+]]
+// CHECK-NEXT:    %[[TMP16:.+]] = icmp eq i32 %[[OMP_FLOOR1_IV:.+]], %[[TMP10:.+]]
+// CHECK-NEXT:    %[[TMP17:.+]] = select i1 %[[TMP16:.+]], i32 %[[TMP11:.+]], i32 %[[TMP4:.+]]
 // CHECK-NEXT:    br label %[[OMP_TILE0_PREHEADER:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_PREHEADER]]:
@@ -89,7 +93,7 @@ llvm.func @tile_2d_loop(%baseptr: !llvm.ptr, %tc1: i32, %tc2: i32, %ts1: i32, %t
 // CHECK-NEXT:    br label %[[OMP_TILE0_COND:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_COND]]:
-// CHECK-NEXT:    %[[OMP_TILE0_CMP:.+]] = icmp ult i32 %[[OMP_TILE0_IV:.+]], %[[TMP3:.+]]
+// CHECK-NEXT:    %[[OMP_TILE0_CMP:.+]] = icmp ult i32 %[[OMP_TILE0_IV:.+]], %[[TMP15:.+]]
 // CHECK-NEXT:    br i1 %[[OMP_TILE0_CMP:.+]], label %[[OMP_TILE0_BODY:.+]], label %[[OMP_TILE0_EXIT:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_BODY]]:
@@ -103,32 +107,26 @@ llvm.func @tile_2d_loop(%baseptr: !llvm.ptr, %tc1: i32, %tc2: i32, %ts1: i32, %t
 // CHECK-NEXT:    br label %[[OMP_TILE1_COND:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE1_COND]]:
-// CHECK-NEXT:    %[[OMP_TILE1_CMP:.+]] = icmp ult i32 %[[OMP_TILE1_IV:.+]], %[[TMP4:.+]]
+// CHECK-NEXT:    %[[OMP_TILE1_CMP:.+]] = icmp ult i32 %[[OMP_TILE1_IV:.+]], %[[TMP17:.+]]
 // CHECK-NEXT:    br i1 %[[OMP_TILE1_CMP:.+]], label %[[OMP_TILE1_BODY:.+]], label %[[OMP_TILE1_EXIT:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE1_BODY]]:
-// CHECK-NEXT:    %[[TMP14:.+]] = mul nuw i32 %[[TMP3:.+]], %[[OMP_FLOOR0_IV:.+]]
-// CHECK-NEXT:    %[[TMP15:.+]] = add nuw i32 %[[TMP14:.+]], %[[OMP_TILE0_IV:.+]]
-// CHECK-NEXT:    %[[TMP16:.+]] = mul nuw i32 %[[TMP4:.+]], %[[OMP_FLOOR1_IV:.+]]
-// CHECK-NEXT:    %[[TMP17:.+]] = add nuw i32 %[[TMP16:.+]], %[[OMP_TILE1_IV:.+]]
-// CHECK-NEXT:    %[[OMP_TILE0_INBOUNDS:.+]] = icmp ult i32 %[[TMP15:.+]], %[[TMP1:.+]]
-// CHECK-NEXT:    %[[OMP_TILE1_INBOUNDS:.+]] = icmp ult i32 %[[TMP17:.+]], %[[TMP2:.+]]
-// CHECK-NEXT:    %[[OMP_TILE_PRED:.+]] = and i1 %[[OMP_TILE0_INBOUNDS:.+]], %[[OMP_TILE1_INBOUNDS:.+]]
-// CHECK-NEXT:    br i1 %[[OMP_TILE_PRED:.+]], label %[[OMP_OMP_LOOP_BODY:.+]], label %[[OMP_TILE_BODY_MERGE:.+]]
+// CHECK-NEXT:    %[[TMP18:.+]] = mul nuw i32 %[[TMP3:.+]], %[[OMP_FLOOR0_IV:.+]]
+// CHECK-NEXT:    %[[TMP19:.+]] = add nuw i32 %[[TMP18:.+]], %[[OMP_TILE0_IV:.+]]
+// CHECK-NEXT:    %[[TMP20:.+]] = mul nuw i32 %[[TMP4:.+]], %[[OMP_FLOOR1_IV:.+]]
+// CHECK-NEXT:    %[[TMP21:.+]] = add nuw i32 %[[TMP20:.+]], %[[OMP_TILE1_IV:.+]]
+// CHECK-NEXT:    br label %[[OMP_OMP_LOOP_BODY:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_OMP_LOOP_BODY4]]:
 // CHECK-NEXT:    br label %[[OMP_LOOP_REGION12:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_LOOP_REGION12]]:
-// CHECK-NEXT:    %[[TMP18:.+]] = add i32 %[[TMP15:.+]], %[[TMP17:.+]]
-// CHECK-NEXT:    %[[TMP19:.+]] = getelementptr inbounds float, ptr %[[TMP0:.+]], i32 %[[TMP18:.+]]
-// CHECK-NEXT:    store float 4.200000e+01, ptr %[[TMP19:.+]], align 4
+// CHECK-NEXT:    %[[TMP22:.+]] = add i32 %[[TMP19:.+]], %[[TMP21:.+]]
+// CHECK-NEXT:    %[[TMP23:.+]] = getelementptr inbounds float, ptr %[[TMP0:.+]], i32 %[[TMP22:.+]]
+// CHECK-NEXT:    store float 4.200000e+01, ptr %[[TMP23:.+]], align 4
 // CHECK-NEXT:    br label %[[OMP_REGION_CONT11:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_REGION_CONT11]]:
-// CHECK-NEXT:    br label %[[OMP_TILE_BODY_MERGE:.+]]
-// CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE_BODY_MERGE]]:
 // CHECK-NEXT:    br label %[[OMP_TILE1_INC:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE1_INC]]:
diff --git a/mlir/test/Target/LLVMIR/openmp-cli-tile03.mlir b/mlir/test/Target/LLVMIR/openmp-cli-tile03.mlir
index fcc37af2ca8f3..0747ff8909362 100644
--- a/mlir/test/Target/LLVMIR/openmp-cli-tile03.mlir
+++ b/mlir/test/Target/LLVMIR/openmp-cli-tile03.mlir
@@ -56,6 +56,8 @@ llvm.func @tile_composition(%baseptr: !llvm.ptr, %tc: i32, %ts: i32, %grid_ts: i
 // CHECK-NEXT:    br i1 %[[OMP_FLOOR0_CMP10:.+]], label %[[OMP_FLOOR0_BODY5:.+]], label %[[OMP_FLOOR0_EXIT7:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_FLOOR0_BODY5]]:
+// CHECK-NEXT:    %[[TMP14:.+]] = icmp eq i32 %[[OMP_FLOOR0_IV9:.+]], %[[TMP10:.+]]
+// CHECK-NEXT:    %[[TMP15:.+]] = select i1 %[[TMP14:.+]], i32 %[[TMP11:.+]], i32 %[[TMP3:.+]]
 // CHECK-NEXT:    br label %[[OMP_TILE0_PREHEADER12:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_PREHEADER12]]:
@@ -66,88 +68,83 @@ llvm.func @tile_composition(%baseptr: !llvm.ptr, %tc: i32, %ts: i32, %grid_ts: i
 // CHECK-NEXT:    br label %[[OMP_TILE0_COND14:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_COND14]]:
-// CHECK-NEXT:    %[[OMP_TILE0_CMP20:.+]] = icmp ult i32 %[[OMP_TILE0_IV19:.+]], %[[TMP3:.+]]
+// CHECK-NEXT:    %[[OMP_TILE0_CMP20:.+]] = icmp ult i32 %[[OMP_TILE0_IV19:.+]], %[[TMP15:.+]]
 // CHECK-NEXT:    br i1 %[[OMP_TILE0_CMP20:.+]], label %[[OMP_TILE0_BODY15:.+]], label %[[OMP_TILE0_EXIT17:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_BODY15]]:
-// CHECK-NEXT:    %[[TMP14:.+]] = mul nuw i32 %[[TMP3:.+]], %[[OMP_FLOOR0_IV9:.+]]
-// CHECK-NEXT:    %[[TMP15:.+]] = add nuw i32 %[[TMP14:.+]], %[[OMP_TILE0_IV19:.+]]
-// CHECK-NEXT:    %[[OMP_TILE0_INBOUNDS22:.+]] = icmp ult i32 %[[TMP15:.+]], %[[OMP_FLOOR0_TRIPCOUNT:.+]]
-// CHECK-NEXT:    br i1 %[[OMP_TILE0_INBOUNDS22:.+]], label %[[OMP_FLOOR_INNER_ENTRY:.+]], label %[[OMP_TILE_BODY_MERGE23:.+]]
-// CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_FLOOR_INNER_ENTRY]]:
-// CHECK-NEXT:    br label %[[OMP_TILE0_PREHEADER_INNER:.+]]
-// CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_PREHEADER_INNER]]:
-// CHECK-NEXT:    %[[TMP16:.+]] = udiv i32 %[[TMP2:.+]], %[[TMP4:.+]]
-// CHECK-NEXT:    %[[TMP17:.+]] = urem i32 %[[TMP2:.+]], %[[TMP4:.+]]
-// CHECK-NEXT:    %[[TMP18:.+]] = icmp ne i32 %[[TMP17:.+]], 0
-// CHECK-NEXT:    %[[TMP19:.+]] = zext i1 %[[TMP18:.+]] to i32
-// CHECK-NEXT:    %[[OMP_FLOOR0_TRIPCOUNT24:.+]] = add nuw i32 %[[TMP16:.+]], %[[TMP19:.+]]
-// CHECK-NEXT:    br label %[[OMP_FLOOR0_PREHEADER25:.+]]
-// CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_FLOOR0_PREHEADER25]]:
+// CHECK-NEXT:    %[[TMP16:.+]] = mul nuw i32 %[[TMP3:.+]], %[[OMP_FLOOR0_IV9:.+]]
+// CHECK-NEXT:    %[[TMP17:.+]] = add nuw i32 %[[TMP16:.+]], %[[OMP_TILE0_IV19:.+]]
+// CHECK-NEXT:    br label %[[OMP_FLOOR0_BODY:.+]]
+// CHECK-EMPTY:
+// CHECK-NEXT:  [[OMP_FLOOR0_BODY]]:
+// CHECK-NEXT:    %[[TMP18:.+]] = icmp eq i32 %[[TMP17:.+]], %[[TMP6:.+]]
+// CHECK-NEXT:    %[[TMP19:.+]] = select i1 %[[TMP18:.+]], i32 %[[TMP7:.+]], i32 %[[TMP2:.+]]
+// CHECK-NEXT:    br label %[[OMP_TILE0_PREHEADER:.+]]
+// CHECK-EMPTY:
+// CHECK-NEXT:  [[OMP_TILE0_PREHEADER]]:
+// CHECK-NEXT:    %[[TMP20:.+]] = udiv i32 %[[TMP19:.+]], %[[TMP4:.+]]
+// CHECK-NEXT:    %[[TMP21:.+]] = urem i32 %[[TMP19:.+]], %[[TMP4:.+]]
+// CHECK-NEXT:    %[[TMP22:.+]] = icmp ne i32 %[[TMP21:.+]], 0
+// CHECK-NEXT:    %[[TMP23:.+]] = zext i1 %[[TMP22:.+]] to i32
+// CHECK-NEXT:    %[[OMP_FLOOR0_TRIPCOUNT22:.+]] = add nuw i32 %[[TMP20:.+]], %[[TMP23:.+]]
+// CHECK-NEXT:    br label %[[OMP_FLOOR0_PREHEADER23:.+]]
+// CHECK-EMPTY:
+// CHECK-NEXT:  [[OMP_FLOOR0_PREHEADER23]]:
 // CHECK-NEXT:    br label %[[OMP_FLOOR0_HEADER:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_FLOOR0_HEADER]]:
-// CHECK-NEXT:    %[[OMP_FLOOR0_IV:.+]] = phi i32 [ 0, %[[OMP_FLOOR0_PREHEADER25:.+]] ], [ %[[OMP_FLOOR0_NEXT:.+]], %[[OMP_FLOOR0_INC:.+]] ]
+// CHECK-NEXT:    %[[OMP_FLOOR0_IV:.+]] = phi i32 [ 0, %[[OMP_FLOOR0_PREHEADER23:.+]] ], [ %[[OMP_FLOOR0_NEXT:.+]], %[[OMP_FLOOR0_INC:.+]] ]
 // CHECK-NEXT:    br label %[[OMP_FLOOR0_COND:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_FLOOR0_COND]]:
-// CHECK-NEXT:    %[[OMP_FLOOR0_CMP:.+]] = icmp ult i32 %[[OMP_FLOOR0_IV:.+]], %[[OMP_FLOOR0_TRIPCOUNT24:.+]]
-// CHECK-NEXT:    br i1 %[[OMP_FLOOR0_CMP:.+]], label %[[OMP_FLOOR0_BODY26:.+]], label %[[OMP_FLOOR0_EXIT:.+]]
+// CHECK-NEXT:    %[[OMP_FLOOR0_CMP:.+]] = icmp ult i32 %[[OMP_FLOOR0_IV:.+]], %[[OMP_FLOOR0_TRIPCOUNT22:.+]]
+// CHECK-NEXT:    br i1 %[[OMP_FLOOR0_CMP:.+]], label %[[OMP_FLOOR0_BODY24:.+]], label %[[OMP_FLOOR0_EXIT:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_FLOOR0_BODY26]]:
-// CHECK-NEXT:    br label %[[OMP_TILE0_PREHEADER28:.+]]
+// CHECK-NEXT:  [[OMP_FLOOR0_BODY24]]:
+// CHECK-NEXT:    %[[TMP24:.+]] = icmp eq i32 %[[OMP_FLOOR0_IV:.+]], %[[TMP20:.+]]
+// CHECK-NEXT:    %[[TMP25:.+]] = select i1 %[[TMP24:.+]], i32 %[[TMP21:.+]], i32 %[[TMP4:.+]]
+// CHECK-NEXT:    br label %[[OMP_TILE0_PREHEADER26:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_PREHEADER28]]:
-// CHECK-NEXT:    br label %[[OMP_TILE0_HEADER29:.+]]
+// CHECK-NEXT:  [[OMP_TILE0_PREHEADER26]]:
+// CHECK-NEXT:    br label %[[OMP_TILE0_HEADER27:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_HEADER29]]:
-// CHECK-NEXT:    %[[OMP_TILE0_IV35:.+]] = phi i32 [ 0, %[[OMP_TILE0_PREHEADER28:.+]] ], [ %[[OMP_TILE0_NEXT37:.+]], %[[OMP_TILE0_INC32:.+]] ]
-// CHECK-NEXT:    br label %[[OMP_TILE0_COND30:.+]]
+// CHECK-NEXT:  [[OMP_TILE0_HEADER27]]:
+// CHECK-NEXT:    %[[OMP_TILE0_IV33:.+]] = phi i32 [ 0, %[[OMP_TILE0_PREHEADER26:.+]] ], [ %[[OMP_TILE0_NEXT35:.+]], %[[OMP_TILE0_INC30:.+]] ]
+// CHECK-NEXT:    br label %[[OMP_TILE0_COND28:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_COND30]]:
-// CHECK-NEXT:    %[[OMP_TILE0_CMP36:.+]] = icmp ult i32 %[[OMP_TILE0_IV35:.+]], %[[TMP4:.+]]
-// CHECK-NEXT:    br i1 %[[OMP_TILE0_CMP36:.+]], label %[[OMP_TILE0_BODY31:.+]], label %[[OMP_TILE0_EXIT33:.+]]
+// CHECK-NEXT:  [[OMP_TILE0_COND28]]:
+// CHECK-NEXT:    %[[OMP_TILE0_CMP34:.+]] = icmp ult i32 %[[OMP_TILE0_IV33:.+]], %[[TMP25:.+]]
+// CHECK-NEXT:    br i1 %[[OMP_TILE0_CMP34:.+]], label %[[OMP_TILE0_BODY29:.+]], label %[[OMP_TILE0_EXIT31:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_BODY31]]:
-// CHECK-NEXT:    %[[TMP20:.+]] = mul nuw i32 %[[TMP4:.+]], %[[OMP_FLOOR0_IV:.+]]
-// CHECK-NEXT:    %[[TMP21:.+]] = add nuw i32 %[[TMP20:.+]], %[[OMP_TILE0_IV35:.+]]
-// CHECK-NEXT:    %[[OMP_TILE0_INBOUNDS38:.+]] = icmp ult i32 %[[TMP21:.+]], %[[TMP2:.+]]
-// CHECK-NEXT:    br i1 %[[OMP_TILE0_INBOUNDS38:.+]], label %[[OMP_TILE0_BODY_STORE:.+]], label %[[OMP_TILE_BODY_MERGE39:.+]]
+// CHECK-NEXT:  [[OMP_TILE0_BODY29]]:
+// CHECK-NEXT:    %[[TMP26:.+]] = mul nuw i32 %[[TMP4:.+]], %[[OMP_FLOOR0_IV:.+]]
+// CHECK-NEXT:    %[[TMP27:.+]] = add nuw i32 %[[TMP26:.+]], %[[OMP_TILE0_IV33:.+]]
+// CHECK-NEXT:    br label %[[OMP_TILE0_BODY:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_BODY_STORE]]:
-// CHECK-NEXT:    %[[TMP22:.+]] = mul nuw i32 %[[TMP2:.+]], %[[TMP15:.+]]
-// CHECK-NEXT:    %[[TMP23:.+]] = add nuw i32 %[[TMP22:.+]], %[[TMP21:.+]]
-// CHECK-NEXT:    %[[OMP_TILE0_INBOUNDS:.+]] = icmp ult i32 %[[TMP23:.+]], %[[TMP1:.+]]
-// CHECK-NEXT:    br i1 %[[OMP_TILE0_INBOUNDS:.+]], label %[[OMP_OMP_LOOP_BODY:.+]], label %[[OMP_TILE_BODY_MERGE:.+]]
+// CHECK-NEXT:  [[OMP_TILE0_BODY]]:
+// CHECK-NEXT:    %[[TMP28:.+]] = mul nuw i32 %[[TMP2:.+]], %[[TMP17:.+]]
+// CHECK-NEXT:    %[[TMP29:.+]] = add nuw i32 %[[TMP28:.+]], %[[TMP27:.+]]
+// CHECK-NEXT:    br label %[[OMP_OMP_LOOP_BODY:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_OMP_LOOP_BODY]]:
 // CHECK-NEXT:    br label %[[OMP_LOOP_REGION:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_LOOP_REGION]]:
-// CHECK-NEXT:    %[[TMP24:.+]] = getelementptr inbounds float, ptr %[[TMP0:.+]], i32 %[[TMP23:.+]]
-// CHECK-NEXT:    store float 4.200000e+01, ptr %[[TMP24:.+]], align 4
+// CHECK-NEXT:    %[[TMP30:.+]] = getelementptr inbounds float, ptr %[[TMP0:.+]], i32 %[[TMP29:.+]]
+// CHECK-NEXT:    store float 4.200000e+01, ptr %[[TMP30:.+]], align 4
 // CHECK-NEXT:    br label %[[OMP_REGION_CONT:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_REGION_CONT]]:
-// CHECK-NEXT:    br label %[[OMP_TILE_BODY_MERGE:.+]]
+// CHECK-NEXT:    br label %[[OMP_TILE0_INC30:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE_BODY_MERGE]]:
-// CHECK-NEXT:    br label %[[OMP_TILE_BODY_MERGE39:.+]]
+// CHECK-NEXT:  [[OMP_TILE0_INC30]]:
+// CHECK-NEXT:    %[[OMP_TILE0_NEXT35:.+]] = add nuw i32 %[[OMP_TILE0_IV33:.+]], 1
+// CHECK-NEXT:    br label %[[OMP_TILE0_HEADER27:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE_BODY_MERGE39]]:
-// CHECK-NEXT:    br label %[[OMP_TILE0_INC32:.+]]
+// CHECK-NEXT:  [[OMP_TILE0_EXIT31]]:
+// CHECK-NEXT:    br label %[[OMP_TILE0_AFTER32:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_INC32]]:
-// CHECK-NEXT:    %[[OMP_TILE0_NEXT37:.+]] = add nuw i32 %[[OMP_TILE0_IV35:.+]], 1
-// CHECK-NEXT:    br label %[[OMP_TILE0_HEADER29:.+]]
-// CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_EXIT33]]:
-// CHECK-NEXT:    br label %[[OMP_TILE0_AFTER34:.+]]
-// CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_AFTER34]]:
+// CHECK-NEXT:  [[OMP_TILE0_AFTER32]]:
 // CHECK-NEXT:    br label %[[OMP_FLOOR0_INC:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_FLOOR0_INC]]:
@@ -155,15 +152,12 @@ llvm.func @tile_composition(%baseptr: !llvm.ptr, %tc: i32, %ts: i32, %grid_ts: i
 // CHECK-NEXT:    br label %[[OMP_FLOOR0_HEADER:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_FLOOR0_EXIT]]:
-// CHECK-NEXT:    br label %[[OMP_FLOOR0_AFTER27:.+]]
-// CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_FLOOR0_AFTER27]]:
-// CHECK-NEXT:    br label %[[OMP_TILE0_AFTER_INNER:.+]]
+// CHECK-NEXT:    br label %[[OMP_FLOOR0_AFTER25:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE0_AFTER_INNER]]:
-// CHECK-NEXT:    br label %[[OMP_TILE_BODY_MERGE23:.+]]
+// CHECK-NEXT:  [[OMP_FLOOR0_AFTER25]]:
+// CHECK-NEXT:    br label %[[OMP_TILE0_AFTER:.+]]
 // CHECK-EMPTY:
-// CHECK-NEXT:  [[OMP_TILE_BODY_MERGE23]]:
+// CHECK-NEXT:  [[OMP_TILE0_AFTER]]:
 // CHECK-NEXT:    br label %[[OMP_TILE0_INC16:.+]]
 // CHECK-EMPTY:
 // CHECK-NEXT:  [[OMP_TILE0_INC16]]:

>From d48aac101e6486a85f494db321da504b8a0d063c Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 3 Jun 2026 05:40:05 -0400
Subject: [PATCH 13/19] guard-attribute-marker

---
 clang/include/clang/Basic/Attr.td  |   7 +
 clang/lib/AST/StmtOpenMP.cpp       |  46 ++++++-
 clang/lib/CodeGen/CGStmt.cpp       |  48 ++++++-
 clang/lib/CodeGen/CGStmtOpenMP.cpp |  45 ++++++
 clang/lib/Sema/SemaOpenMP.cpp      | 213 ++++++++++++-----------------
 5 files changed, 227 insertions(+), 132 deletions(-)

diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 39c672322d515..22f5e0fd33433 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -4877,6 +4877,13 @@ def OMPAssume : InheritableAttr {
   let Args = [StringArgument<"Assumption">];
 }
 
+def OMPInvariantPredicateBound : StmtAttr {
+  let Spellings = [];
+  let SemaHandler = 0;
+  let Subjects = SubjectList<[Stmt], ErrorDiag>;
+  let Documentation = [InternalOnly];
+}
+
 def InternalLinkage : InheritableAttr {
   let Spellings = [Clang<"internal_linkage">];
   let Subjects = SubjectList<[Var, Function, CXXRecord]>;
diff --git a/clang/lib/AST/StmtOpenMP.cpp b/clang/lib/AST/StmtOpenMP.cpp
index 9d6b315effb41..94eeddc01d1a3 100644
--- a/clang/lib/AST/StmtOpenMP.cpp
+++ b/clang/lib/AST/StmtOpenMP.cpp
@@ -10,12 +10,48 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "clang/AST/ASTContext.h"
 #include "clang/AST/StmtOpenMP.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Attr.h"
+#include "clang/AST/Stmt.h"
 
 using namespace clang;
 using namespace llvm::omp;
 
+// True if the attributed statement carries the internal `omp tile` intra-tile
+// body-guard marker.
+static bool hasOMPInvariantPredicateBound(const AttributedStmt *AS) {
+  for (const Attr *A : AS->getAttrs())
+    if (isa<OMPInvariantPredicateBoundAttr>(A))
+      return true;
+  return false;
+}
+
+// Like `Stmt::IgnoreContainers`, but also treats the `omp tile` body-guard
+static Stmt *IgnoreContainersAndOMPTileBodyGuard(Stmt *S) {
+  while (true) {
+    if (auto *AS = dyn_cast_or_null<AttributedStmt>(S)) {
+      if (hasOMPInvariantPredicateBound(AS)) {
+        if (auto *If = dyn_cast<IfStmt>(AS->getSubStmt());
+            If && !If->getElse()) {
+          S = If->getThen();
+          continue;
+        }
+      }
+      S = AS->getSubStmt();
+      continue;
+    }
+    if (auto *CS = dyn_cast_or_null<CompoundStmt>(S)) {
+      if (CS->size() != 1)
+        break;
+      S = CS->body_back();
+      continue;
+    }
+    break;
+  }
+  return S;
+}
+
 size_t OMPChildren::size(unsigned NumClauses, bool HasAssociatedStmt,
                          unsigned NumChildren) {
   return llvm::alignTo(
@@ -78,7 +114,7 @@ Stmt *
 OMPLoopBasedDirective::tryToFindNextInnerLoop(Stmt *CurStmt,
                                               bool TryImperfectlyNestedLoops) {
   Stmt *OrigStmt = CurStmt;
-  CurStmt = CurStmt->IgnoreContainers();
+  CurStmt = IgnoreContainersAndOMPTileBodyGuard(CurStmt);
   // Additional work for imperfectly nested loops, introduced in OpenMP 5.0.
   if (TryImperfectlyNestedLoops) {
     if (auto *CS = dyn_cast<CompoundStmt>(CurStmt)) {
@@ -94,6 +130,10 @@ OMPLoopBasedDirective::tryToFindNextInnerLoop(Stmt *CurStmt,
             continue;
           if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(S))
             S = CanonLoop->getLoopStmt();
+          // Check for the `omp tile` body-guard marker.
+          if (auto *AS = dyn_cast<AttributedStmt>(S);
+              AS && hasOMPInvariantPredicateBound(AS))
+            S = IgnoreContainersAndOMPTileBodyGuard(S);
           if (isa<ForStmt>(S) || isa<CXXForRangeStmt>(S) ||
               (isa<OMPLoopBasedDirective>(S) && !isa<OMPLoopDirective>(S))) {
             // Only single loop construct is allowed.
@@ -127,7 +167,7 @@ bool OMPLoopBasedDirective::doForAllLoops(
     llvm::function_ref<bool(unsigned, Stmt *)> Callback,
     llvm::function_ref<void(OMPLoopTransformationDirective *)>
         OnTransformationCallback) {
-  CurStmt = CurStmt->IgnoreContainers();
+  CurStmt = IgnoreContainersAndOMPTileBodyGuard(CurStmt);
   for (unsigned Cnt = 0; Cnt < NumLoops; ++Cnt) {
     while (true) {
       auto *Dir = dyn_cast<OMPLoopTransformationDirective>(CurStmt);
diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index 27e74d966eca1..da7a1b561b207 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -1271,9 +1271,35 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S,
     ConvergenceTokenStack.pop_back();
 }
 
+// Helper function to find and return the omp-tile body-guard marker on the
+// for-body's last item.
+static std::pair<const AttributedStmt *, const IfStmt *>
+findOMPTileBodyGuard(const Stmt *Body) {
+  const auto *CS = dyn_cast_or_null<CompoundStmt>(Body);
+  if (!CS || CS->body().empty())
+    return {nullptr, nullptr};
+  const auto *AS = dyn_cast<AttributedStmt>(CS->body_back());
+  if (!AS)
+    return {nullptr, nullptr};
+  bool IsMarker = false;
+  for (const Attr *A : AS->getAttrs()) {
+    if (!isa<OMPInvariantPredicateBoundAttr>(A))
+      continue;
+    IsMarker = true;
+    break;
+  }
+  if (!IsMarker)
+    return {nullptr, nullptr};
+  const auto *If = dyn_cast<IfStmt>(AS->getSubStmt());
+  if (!If || If->getElse())
+    return {nullptr, nullptr};
+  return {AS, If};
+}
+
 void CodeGenFunction::EmitForStmt(const ForStmt &S,
                                   ArrayRef<const Attr *> ForAttrs) {
   JumpDest LoopExit = getJumpDestInCurrentScope("for.end");
+  const auto [MarkerAS, MarkerIf] = findOMPTileBodyGuard(S.getBody());
 
   std::optional<LexicalScope> ForScope;
   if (getLangOpts().C99 || getLangOpts().CPlusPlus)
@@ -1341,6 +1367,14 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S,
     // compares unequal to 0.  The condition must be a scalar type.
     llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
 
+    // If the for-body has a marker, evaluate the predicate and AND it into the
+    // loop condition.
+    if (MarkerIf) {
+      llvm::Value *PredVal = EvaluateExprAsBool(MarkerIf->getCond());
+      BoolCondVal = Builder.CreateAnd(BoolCondVal, PredVal,
+                                      "omp_tile.invariant_pred_bound");
+    }
+
     MaybeEmitDeferredVarDeclInit(S.getConditionVariable());
 
     llvm::MDNode *Weights =
@@ -1377,7 +1411,19 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S,
     // Create a separate cleanup scope for the body, in case it is not
     // a compound statement.
     RunCleanupsScope BodyScope(*this);
-    EmitStmt(S.getBody());
+    if (MarkerAS) {
+      // If the for-body has a marker, walk the body and emit the then-block of
+      // the marker's predicate.
+      const auto *CS = cast<CompoundStmt>(S.getBody());
+      for (const Stmt *Sub : CS->body()) {
+        if (Sub == MarkerAS)
+          EmitStmt(MarkerIf->getThen());
+        else
+          EmitStmt(Sub);
+      }
+    } else {
+      EmitStmt(S.getBody());
+    }
   }
 
   // The last block in the loop's body (which unconditionally branches to the
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 97c43b4c0b384..93805f7b54432 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -2231,6 +2231,34 @@ class OMPTransformDirectiveScopeRAII {
 };
 } // namespace
 
+// Helper function to look through the statement and return the special marked
+// `omp tile` guard `if`, or nullptr if this is not that shape.
+static const IfStmt *getOMPTileBodyGuardIf(const Stmt *S) {
+  while (S) {
+    if (const auto *AS = dyn_cast<AttributedStmt>(S)) {
+      bool IsMarker = llvm::any_of(AS->getAttrs(), [](const Attr *A) {
+        return isa<OMPInvariantPredicateBoundAttr>(A);
+      });
+      if (!IsMarker) {
+        S = AS->getSubStmt();
+        continue;
+      }
+      const auto *If = dyn_cast<IfStmt>(AS->getSubStmt());
+      if (!If || If->getElse())
+        return nullptr;
+      return If;
+    }
+    if (const auto *CS = dyn_cast<CompoundStmt>(S)) {
+      if (CS->size() != 1)
+        return nullptr;
+      S = CS->body_back();
+      continue;
+    }
+    return nullptr;
+  }
+  return nullptr;
+}
+
 static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop,
                      int MaxLevel, int Level = 0) {
   assert(Level < MaxLevel && "Too deep lookup during loop body codegen.");
@@ -2246,6 +2274,23 @@ static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop,
       emitBody(CGF, CurStmt, NextLoop, MaxLevel, Level);
     return;
   }
+
+  // If this is the marked `omp tile` guard, emit its predicate as a runtime
+  // body guard and recurse into `Then` so consumed loops are still walked.
+  if (const IfStmt *Guard = getOMPTileBodyGuardIf(S)) {
+    assert(!Guard->getInit() && !Guard->getConditionVariable() &&
+           "omp tile body-guard must be a plain `if (P) Then`");
+    llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_tile.pred.then");
+    llvm::BasicBlock *EndBlock = CGF.createBasicBlock("omp_tile.pred.end");
+    CGF.EmitBranchOnBoolExpr(
+        Guard->getCond(), ThenBlock, EndBlock, /*TrueCount=*/0,
+        Stmt::getLikelihood(Guard->getThen(), Guard->getElse()));
+    CGF.EmitBlock(ThenBlock);
+    emitBody(CGF, Guard->getThen(), NextLoop, MaxLevel, Level);
+    CGF.EmitBranch(EndBlock);
+    CGF.EmitBlock(EndBlock);
+    return;
+  }
   if (SimplifiedS == NextLoop) {
     if (auto *Dir = dyn_cast<OMPLoopTransformationDirective>(SimplifiedS))
       SimplifiedS = Dir->getTransformedStmt();
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index df602caafe5cd..20b8c99a1def2 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -15078,22 +15078,13 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
   SmallVector<Stmt *, 4> PreInits;
   CaptureVars CopyTransformer(SemaRef);
 
-  // Emit the intra-tile loop in canonical (rectangular + predicate) form only
-  // when a consuming OpenMP construct requires canonical input; otherwise keep
-  // the original min(.floor.iv + T, N) bound.
-  OpenMPDirectiveKind ParentDirective = DSAStack->getParentDirective();
-  const bool IntraTileMustBeCanonical =
-      isOpenMPLoopTransformationDirective(ParentDirective) ||
-      isOpenMPLoopDirective(ParentDirective);
-
   // Create iteration variables for the generated loops.
   SmallVector<VarDecl *, 4> FloorIndVars;
   SmallVector<VarDecl *, 4> TileIndVars;
   SmallVector<VarDecl *, 4> TileCntVars;
   FloorIndVars.resize(NumLoops);
   TileIndVars.resize(NumLoops);
-  if (IntraTileMustBeCanonical)
-    TileCntVars.resize(NumLoops);
+  TileCntVars.resize(NumLoops);
   for (unsigned I = 0; I < NumLoops; ++I) {
     OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
 
@@ -15126,7 +15117,7 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     }
 
     // Loop counter for the rectangular tile loop [0, TileSize).
-    if (IntraTileMustBeCanonical) {
+    {
       std::string TileCntName =
           (Twine(".tile.cnt.") + llvm::utostr(I) + ".iv." + OrigVarName).str();
       VarDecl *TileCntDecl =
@@ -15139,7 +15130,7 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
 
     // Declare the logical tile IV in PreInits so it is in scope for the
     // entire loop nest (it will be assigned in each tile loop body).
-    if (IntraTileMustBeCanonical) {
+    {
       Decl *TileIVDeclPtr = TileIndVars[I];
       PreInits.push_back(new (Context) DeclStmt(
           DeclGroupRef::Create(Context, &TileIVDeclPtr, 1), {}, {}));
@@ -15159,57 +15150,62 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
   // that is evenly divisible by the corresponding tile size (also a constant),
   // then the remainder tile is empty and the predicate is trivially true.
   //
-  // The min-bounded path clamps the tile loop's upper bound directly so no
-  // predicate is needed.
-  if (IntraTileMustBeCanonical) {
-    bool PredicateNeeded = false;
-    for (unsigned I = 0; I < NumLoops; ++I) {
-      Expr *TSExpr = SizesClause->getSizesRefs()[I];
-      Expr *NExpr = LoopHelpers[I].NumIterations;
-      bool TSConst =
-          !TSExpr->containsErrors() && TSExpr->isIntegerConstantExpr(Context);
-      bool NConst = NExpr->isIntegerConstantExpr(Context);
-      if (TSConst && NConst) {
-        Expr::EvalResult TSResult;
-        TSExpr->EvaluateAsInt(TSResult, Context);
-        llvm::APSInt TileVal = TSResult.Val.getInt();
-        Expr::EvalResult NResult;
-        NExpr->EvaluateAsInt(NResult, Context);
-        llvm::APSInt TripVal = NResult.Val.getInt();
-        if (TileVal.isStrictlyPositive() && TripVal.srem(TileVal).isZero())
-          continue;
-      }
-      PredicateNeeded = true;
-      break;
+  bool PredicateNeeded = false;
+  for (unsigned I = 0; I < NumLoops; ++I) {
+    Expr *TSExpr = SizesClause->getSizesRefs()[I];
+    Expr *NExpr = LoopHelpers[I].NumIterations;
+    Expr::EvalResult TSResult;
+    Expr::EvalResult NResult;
+    if (TSExpr->EvaluateAsInt(TSResult, Context) &&
+        NExpr->EvaluateAsInt(NResult, Context)) {
+      llvm::APSInt TileVal = TSResult.Val.getInt();
+      llvm::APSInt TripVal = NResult.Val.getInt();
+      if (TileVal.isStrictlyPositive() && TripVal.srem(TileVal).isZero())
+        continue;
     }
+    PredicateNeeded = true;
+    break;
+  }
 
-    if (PredicateNeeded) {
-      Expr *CombinedPred = nullptr;
-      for (unsigned I = 0; I < NumLoops; ++I) {
-        auto *OrigCntVar = cast<DeclRefExpr>(LoopHelpers[I].Counters[0]);
-        QualType IVTy = LoopHelpers[I].NumIterations->getType();
-        Expr *TileIVRef = buildDeclRefExpr(SemaRef, TileIndVars[I], IVTy,
-                                           OrigCntVar->getExprLoc());
-        ExprResult DimPred =
-            SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(), BO_LT,
-                               TileIVRef, LoopHelpers[I].NumIterations);
-        if (!DimPred.isUsable())
+  if (PredicateNeeded) {
+    Expr *CombinedPred = nullptr;
+    for (unsigned I = 0; I < NumLoops; ++I) {
+      auto *OrigCntVar = cast<DeclRefExpr>(LoopHelpers[I].Counters[0]);
+      QualType IVTy = LoopHelpers[I].NumIterations->getType();
+      Expr *FloorRef = buildDeclRefExpr(SemaRef, FloorIndVars[I], IVTy,
+                                        OrigCntVar->getExprLoc());
+      Expr *TileCntRef = buildDeclRefExpr(SemaRef, TileCntVars[I], IVTy,
+                                          OrigCntVar->getExprLoc());
+      ExprResult Sum = SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(),
+                                          BO_Add, FloorRef, TileCntRef);
+      if (!Sum.isUsable())
+        return StmtError();
+      ExprResult DimPred =
+          SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(), BO_LT,
+                             Sum.get(), LoopHelpers[I].NumIterations);
+      if (!DimPred.isUsable())
+        return StmtError();
+      if (CombinedPred) {
+        ExprResult Combined =
+            SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(), BO_LAnd,
+                               CombinedPred, DimPred.get());
+        if (!Combined.isUsable())
           return StmtError();
-        if (CombinedPred) {
-          ExprResult Combined =
-              SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(), BO_LAnd,
-                                 CombinedPred, DimPred.get());
-          if (!Combined.isUsable())
-            return StmtError();
-          CombinedPred = Combined.get();
-        } else {
-          CombinedPred = DimPred.get();
-        }
+        CombinedPred = Combined.get();
+      } else {
+        CombinedPred = DimPred.get();
       }
-      Inner = IfStmt::Create(
-          Context, SourceLocation(), IfStatementKind::Ordinary, nullptr,
-          nullptr, CombinedPred, SourceLocation(), SourceLocation(), Inner);
     }
+    // Use the source body's location for the synthetic if-statement and
+    // AttributedStmt so any later diagnostic points at the original body
+    // location.
+    SourceLocation SyntheticLoc =
+        Body ? Body->getBeginLoc() : Inner->getBeginLoc();
+    Stmt *PredIf = IfStmt::Create(
+        Context, SyntheticLoc, IfStatementKind::Ordinary, nullptr, nullptr,
+        CombinedPred, SyntheticLoc, SyntheticLoc, Inner);
+    auto *Marker = OMPInvariantPredicateBoundAttr::CreateImplicit(Context);
+    Inner = AttributedStmt::Create(Context, SyntheticLoc, {Marker}, PredIf);
   }
 
   auto MakeDimTileSize = [&SemaRef = this->SemaRef, &CopyTransformer, &Context,
@@ -15290,77 +15286,38 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     ExprResult IncrStmt;
     Stmt *TileIVAssignStmt = nullptr;
 
-    if (IntraTileMustBeCanonical) {
-      // For init-statement: auto .tile.cnt = 0
-      SemaRef.AddInitializerToDecl(
-          TileCntVars[I],
-          SemaRef.ActOnIntegerConstant(LoopHelper.Init->getExprLoc(), 0).get(),
-          /*DirectInit=*/false);
-      Decl *CounterDecl = TileCntVars[I];
-      InitStmt = new (Context)
-          DeclStmt(DeclGroupRef::Create(Context, &CounterDecl, 1),
-                   OrigCntVar->getBeginLoc(), OrigCntVar->getEndLoc());
-
-      // For cond-expression: .tile.cnt < DimTileSize  (rectangular bound)
-      CondExpr = SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(),
-                                    BO_LT, MakeTileCntRef(), DimTileSize);
-
-      // For incr-statement: ++.tile.cnt
-      IncrStmt = SemaRef.BuildUnaryOp(CurScope, LoopHelper.Inc->getExprLoc(),
-                                      UO_PreInc, MakeTileCntRef());
-
-      // Compute the logical iteration number:
-      //   .tile.iv = .floor.iv + .tile.cnt
-      ExprResult FloorPlusCnt = SemaRef.BuildBinOp(
-          CurScope, OrigCntVar->getExprLoc(), BO_Add,
-          makeFloorIVRef(SemaRef, FloorIndVars, I, IVTy, OrigCntVar),
-          MakeTileCntRef());
-      if (!FloorPlusCnt.isUsable())
-        return StmtError();
-      ExprResult TileIVAssign =
-          SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(), BO_Assign,
-                             MakeTileIVRef(), FloorPlusCnt.get());
-      if (!TileIVAssign.isUsable())
-        return StmtError();
-      TileIVAssignStmt = TileIVAssign.get();
-    } else {
-      // Min-bounded path: .tile.iv is itself the loop counter, initialized to
-      // .floor.iv and bounded by min(.floor.iv + DimTileSize, NumIterations).
-      SemaRef.AddInitializerToDecl(
-          TileIndVars[I],
-          SemaRef
-              .DefaultLvalueConversion(
-                  makeFloorIVRef(SemaRef, FloorIndVars, I, IVTy, OrigCntVar))
-              .get(),
-          /*DirectInit=*/false);
-      Decl *CounterDecl = TileIndVars[I];
-      InitStmt = new (Context)
-          DeclStmt(DeclGroupRef::Create(Context, &CounterDecl, 1),
-                   OrigCntVar->getBeginLoc(), OrigCntVar->getEndLoc());
-
-      ExprResult EndOfTile = SemaRef.BuildBinOp(
-          CurScope, LoopHelper.Cond->getExprLoc(), BO_Add,
-          makeFloorIVRef(SemaRef, FloorIndVars, I, IVTy, OrigCntVar),
-          DimTileSize);
-      if (!EndOfTile.isUsable())
-        return StmtError();
-      ExprResult IsPartialTile =
-          SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
-                             NumIterations, EndOfTile.get());
-      if (!IsPartialTile.isUsable())
-        return StmtError();
-      ExprResult MinTileAndIterSpace = SemaRef.ActOnConditionalOp(
-          LoopHelper.Cond->getBeginLoc(), LoopHelper.Cond->getEndLoc(),
-          IsPartialTile.get(), NumIterations, EndOfTile.get());
-      if (!MinTileAndIterSpace.isUsable())
-        return StmtError();
-      CondExpr =
-          SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
-                             MakeTileIVRef(), MinTileAndIterSpace.get());
+    // For init-statement: auto .tile.cnt = 0
+    SemaRef.AddInitializerToDecl(
+        TileCntVars[I],
+        SemaRef.ActOnIntegerConstant(LoopHelper.Init->getExprLoc(), 0).get(),
+        /*DirectInit=*/false);
+    Decl *CounterDecl = TileCntVars[I];
+    InitStmt = new (Context)
+        DeclStmt(DeclGroupRef::Create(Context, &CounterDecl, 1),
+                 OrigCntVar->getBeginLoc(), OrigCntVar->getEndLoc());
 
-      IncrStmt = SemaRef.BuildUnaryOp(CurScope, LoopHelper.Inc->getExprLoc(),
-                                      UO_PreInc, MakeTileIVRef());
-    }
+    // For cond-expression: .tile.cnt < DimTileSize  (rectangular bound)
+    CondExpr = SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(),
+                                  BO_LT, MakeTileCntRef(), DimTileSize);
+
+    // For incr-statement: ++.tile.cnt
+    IncrStmt = SemaRef.BuildUnaryOp(CurScope, LoopHelper.Inc->getExprLoc(),
+                                    UO_PreInc, MakeTileCntRef());
+
+    // Compute the logical iteration number:
+    //   .tile.iv = .floor.iv + .tile.cnt
+    ExprResult FloorPlusCnt = SemaRef.BuildBinOp(
+        CurScope, OrigCntVar->getExprLoc(), BO_Add,
+        makeFloorIVRef(SemaRef, FloorIndVars, I, IVTy, OrigCntVar),
+        MakeTileCntRef());
+    if (!FloorPlusCnt.isUsable())
+      return StmtError();
+    ExprResult TileIVAssign =
+        SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(), BO_Assign,
+                           MakeTileIVRef(), FloorPlusCnt.get());
+    if (!TileIVAssign.isUsable())
+      return StmtError();
+    TileIVAssignStmt = TileIVAssign.get();
 
     if (!InitStmt.isUsable() || !CondExpr.isUsable() || !IncrStmt.isUsable())
       return StmtError();

>From 972170e353ed3a4e41c18b3c4b3e7836cbe0316b Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 3 Jun 2026 05:42:14 -0400
Subject: [PATCH 14/19] tests-update

---
 clang/test/OpenMP/interchange_codegen.cpp     |  104 +-
 .../irbuilder_unroll_partial_factor_for.c     |   57 +-
 ...er_unroll_partial_heuristic_constant_for.c |   79 +-
 ...der_unroll_partial_heuristic_runtime_for.c |   79 +-
 .../irbuilder_unroll_unroll_partial_factor.c  |   57 +-
 ...rbuilder_unroll_unroll_partial_heuristic.c |   67 +-
 .../tile_canonical_required_codegen.cpp       |   86 +
 clang/test/OpenMP/tile_codegen.cpp            | 1778 ++++++++---------
 .../OpenMP/tile_codegen_for_dependent.cpp     |   53 +-
 clang/test/OpenMP/tile_codegen_tile_for.cpp   |  110 +-
 clang/test/OpenMP/tile_messages.cpp           |    2 +-
 clang/test/OpenMP/tile_rect_codegen_ir.cpp    |  488 +++--
 clang/test/OpenMP/unroll_codegen_tile_for.cpp |  107 +-
 .../runtime/test/transform/tile/foreach.cpp   |   28 -
 openmp/runtime/test/transform/tile/intfor.c   |   32 -
 .../runtime/test/transform/tile/iterfor.cpp   |   21 -
 .../tile/parallel-wsloop-collapse-foreach.cpp |   84 -
 .../parallel-wsloop-collapse-through-tile.cpp |   44 +
 18 files changed, 1576 insertions(+), 1700 deletions(-)
 create mode 100644 clang/test/OpenMP/tile_canonical_required_codegen.cpp
 create mode 100644 openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile.cpp

diff --git a/clang/test/OpenMP/interchange_codegen.cpp b/clang/test/OpenMP/interchange_codegen.cpp
index b062d42c9f162..67ec7e34c5dfd 100644
--- a/clang/test/OpenMP/interchange_codegen.cpp
+++ b/clang/test/OpenMP/interchange_codegen.cpp
@@ -1290,25 +1290,27 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[MUL279:%.*]] = mul i32 [[TMP144]], [[TMP145]]
 // CHECK1-NEXT:    [[ADD280:%.*]] = add i32 [[TMP143]], [[MUL279]]
 // CHECK1-NEXT:    store i32 [[ADD280]], ptr [[K]], align 4
-// CHECK1-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK1-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK1-NEXT:    [[ADD281:%.*]] = add i32 [[TMP147]], 1
-// CHECK1-NEXT:    [[CMP282:%.*]] = icmp ult i32 [[TMP146]], [[ADD281]]
-// CHECK1-NEXT:    br i1 [[CMP282]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK1-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
+// CHECK1-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
+// CHECK1-NEXT:    [[ADD281:%.*]] = add i32 [[TMP146]], [[TMP147]]
+// CHECK1-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// CHECK1-NEXT:    [[ADD282:%.*]] = add i32 [[TMP148]], 1
+// CHECK1-NEXT:    [[CMP283:%.*]] = icmp ult i32 [[ADD281]], [[ADD282]]
+// CHECK1-NEXT:    br i1 [[CMP283]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
 // CHECK1:       if.then:
-// CHECK1-NEXT:    [[TMP148:%.*]] = load i32, ptr [[I35]], align 4
-// CHECK1-NEXT:    [[TMP149:%.*]] = load i32, ptr [[J36]], align 4
-// CHECK1-NEXT:    [[TMP150:%.*]] = load i32, ptr [[K]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP148]], i32 noundef [[TMP149]], i32 noundef [[TMP150]])
+// CHECK1-NEXT:    [[TMP149:%.*]] = load i32, ptr [[I35]], align 4
+// CHECK1-NEXT:    [[TMP150:%.*]] = load i32, ptr [[J36]], align 4
+// CHECK1-NEXT:    [[TMP151:%.*]] = load i32, ptr [[K]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP149]], i32 noundef [[TMP150]], i32 noundef [[TMP151]])
 // CHECK1-NEXT:    br label [[IF_END]]
 // CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP151:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[ADD283:%.*]] = add nsw i64 [[TMP151]], 1
-// CHECK1-NEXT:    store i64 [[ADD283]], ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP152:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[ADD284:%.*]] = add nsw i64 [[TMP152]], 1
+// CHECK1-NEXT:    store i64 [[ADD284]], ptr [[DOTOMP_IV]], align 8
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -1821,25 +1823,27 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[MUL279:%.*]] = mul i32 [[TMP144]], [[TMP145]]
 // CHECK1-NEXT:    [[ADD280:%.*]] = add i32 [[TMP143]], [[MUL279]]
 // CHECK1-NEXT:    store i32 [[ADD280]], ptr [[K]], align 4
-// CHECK1-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK1-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK1-NEXT:    [[ADD281:%.*]] = add i32 [[TMP147]], 1
-// CHECK1-NEXT:    [[CMP282:%.*]] = icmp ult i32 [[TMP146]], [[ADD281]]
-// CHECK1-NEXT:    br i1 [[CMP282]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK1-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
+// CHECK1-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
+// CHECK1-NEXT:    [[ADD281:%.*]] = add i32 [[TMP146]], [[TMP147]]
+// CHECK1-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// CHECK1-NEXT:    [[ADD282:%.*]] = add i32 [[TMP148]], 1
+// CHECK1-NEXT:    [[CMP283:%.*]] = icmp ult i32 [[ADD281]], [[ADD282]]
+// CHECK1-NEXT:    br i1 [[CMP283]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
 // CHECK1:       if.then:
-// CHECK1-NEXT:    [[TMP148:%.*]] = load i32, ptr [[I35]], align 4
-// CHECK1-NEXT:    [[TMP149:%.*]] = load i32, ptr [[J36]], align 4
-// CHECK1-NEXT:    [[TMP150:%.*]] = load i32, ptr [[K]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP148]], i32 noundef [[TMP149]], i32 noundef [[TMP150]])
+// CHECK1-NEXT:    [[TMP149:%.*]] = load i32, ptr [[I35]], align 4
+// CHECK1-NEXT:    [[TMP150:%.*]] = load i32, ptr [[J36]], align 4
+// CHECK1-NEXT:    [[TMP151:%.*]] = load i32, ptr [[K]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP149]], i32 noundef [[TMP150]], i32 noundef [[TMP151]])
 // CHECK1-NEXT:    br label [[IF_END]]
 // CHECK1:       if.end:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP151:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[ADD283:%.*]] = add nsw i64 [[TMP151]], 1
-// CHECK1-NEXT:    store i64 [[ADD283]], ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP152:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[ADD284:%.*]] = add nsw i64 [[TMP152]], 1
+// CHECK1-NEXT:    store i64 [[ADD284]], ptr [[DOTOMP_IV]], align 8
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -4047,25 +4051,27 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[MUL279:%.*]] = mul i32 [[TMP144]], [[TMP145]]
 // CHECK2-NEXT:    [[ADD280:%.*]] = add i32 [[TMP143]], [[MUL279]]
 // CHECK2-NEXT:    store i32 [[ADD280]], ptr [[K]], align 4
-// CHECK2-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK2-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK2-NEXT:    [[ADD281:%.*]] = add i32 [[TMP147]], 1
-// CHECK2-NEXT:    [[CMP282:%.*]] = icmp ult i32 [[TMP146]], [[ADD281]]
-// CHECK2-NEXT:    br i1 [[CMP282]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK2-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
+// CHECK2-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
+// CHECK2-NEXT:    [[ADD281:%.*]] = add i32 [[TMP146]], [[TMP147]]
+// CHECK2-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// CHECK2-NEXT:    [[ADD282:%.*]] = add i32 [[TMP148]], 1
+// CHECK2-NEXT:    [[CMP283:%.*]] = icmp ult i32 [[ADD281]], [[ADD282]]
+// CHECK2-NEXT:    br i1 [[CMP283]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
 // CHECK2:       if.then:
-// CHECK2-NEXT:    [[TMP148:%.*]] = load i32, ptr [[I35]], align 4
-// CHECK2-NEXT:    [[TMP149:%.*]] = load i32, ptr [[J36]], align 4
-// CHECK2-NEXT:    [[TMP150:%.*]] = load i32, ptr [[K]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP148]], i32 noundef [[TMP149]], i32 noundef [[TMP150]])
+// CHECK2-NEXT:    [[TMP149:%.*]] = load i32, ptr [[I35]], align 4
+// CHECK2-NEXT:    [[TMP150:%.*]] = load i32, ptr [[J36]], align 4
+// CHECK2-NEXT:    [[TMP151:%.*]] = load i32, ptr [[K]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP149]], i32 noundef [[TMP150]], i32 noundef [[TMP151]])
 // CHECK2-NEXT:    br label [[IF_END]]
 // CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
-// CHECK2-NEXT:    [[TMP151:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[ADD283:%.*]] = add nsw i64 [[TMP151]], 1
-// CHECK2-NEXT:    store i64 [[ADD283]], ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP152:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[ADD284:%.*]] = add nsw i64 [[TMP152]], 1
+// CHECK2-NEXT:    store i64 [[ADD284]], ptr [[DOTOMP_IV]], align 8
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -4578,25 +4584,27 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[MUL279:%.*]] = mul i32 [[TMP144]], [[TMP145]]
 // CHECK2-NEXT:    [[ADD280:%.*]] = add i32 [[TMP143]], [[MUL279]]
 // CHECK2-NEXT:    store i32 [[ADD280]], ptr [[K]], align 4
-// CHECK2-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK2-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK2-NEXT:    [[ADD281:%.*]] = add i32 [[TMP147]], 1
-// CHECK2-NEXT:    [[CMP282:%.*]] = icmp ult i32 [[TMP146]], [[ADD281]]
-// CHECK2-NEXT:    br i1 [[CMP282]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
+// CHECK2-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
+// CHECK2-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
+// CHECK2-NEXT:    [[ADD281:%.*]] = add i32 [[TMP146]], [[TMP147]]
+// CHECK2-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// CHECK2-NEXT:    [[ADD282:%.*]] = add i32 [[TMP148]], 1
+// CHECK2-NEXT:    [[CMP283:%.*]] = icmp ult i32 [[ADD281]], [[ADD282]]
+// CHECK2-NEXT:    br i1 [[CMP283]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
 // CHECK2:       if.then:
-// CHECK2-NEXT:    [[TMP148:%.*]] = load i32, ptr [[I35]], align 4
-// CHECK2-NEXT:    [[TMP149:%.*]] = load i32, ptr [[J36]], align 4
-// CHECK2-NEXT:    [[TMP150:%.*]] = load i32, ptr [[K]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP148]], i32 noundef [[TMP149]], i32 noundef [[TMP150]])
+// CHECK2-NEXT:    [[TMP149:%.*]] = load i32, ptr [[I35]], align 4
+// CHECK2-NEXT:    [[TMP150:%.*]] = load i32, ptr [[J36]], align 4
+// CHECK2-NEXT:    [[TMP151:%.*]] = load i32, ptr [[K]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP149]], i32 noundef [[TMP150]], i32 noundef [[TMP151]])
 // CHECK2-NEXT:    br label [[IF_END]]
 // CHECK2:       if.end:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
-// CHECK2-NEXT:    [[TMP151:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[ADD283:%.*]] = add nsw i64 [[TMP151]], 1
-// CHECK2-NEXT:    store i64 [[ADD283]], ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP152:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[ADD284:%.*]] = add nsw i64 [[TMP152]], 1
+// CHECK2-NEXT:    store i64 [[ADD284]], ptr [[DOTOMP_IV]], align 8
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
diff --git a/clang/test/OpenMP/irbuilder_unroll_partial_factor_for.c b/clang/test/OpenMP/irbuilder_unroll_partial_factor_for.c
index 43711af5f2e33..cf4fa21c9a31c 100644
--- a/clang/test/OpenMP/irbuilder_unroll_partial_factor_for.c
+++ b/clang/test/OpenMP/irbuilder_unroll_partial_factor_for.c
@@ -79,6 +79,8 @@ void unroll_partial_heuristic_for(int n, float *a, float *b, float *c, float *d)
 // CHECK-NEXT:    br i1 [[OMP_FLOOR0_CMP]], label [[OMP_FLOOR0_BODY:%.*]], label [[OMP_FLOOR0_EXIT:%.*]]
 // CHECK:       omp_floor0.body:
 // CHECK-NEXT:    [[TMP13:%.*]] = add i32 [[OMP_FLOOR0_IV]], [[TMP9]]
+// CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP13]], [[TMP4]]
+// CHECK-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i32 [[TMP5]], i32 13
 // CHECK-NEXT:    br label [[OMP_TILE0_PREHEADER:%.*]]
 // CHECK:       omp_tile0.preheader:
 // CHECK-NEXT:    br label [[OMP_TILE0_HEADER:%.*]]
@@ -86,39 +88,36 @@ void unroll_partial_heuristic_for(int n, float *a, float *b, float *c, float *d)
 // CHECK-NEXT:    [[OMP_TILE0_IV:%.*]] = phi i32 [ 0, [[OMP_TILE0_PREHEADER]] ], [ [[OMP_TILE0_NEXT:%.*]], [[OMP_TILE0_INC:%.*]] ]
 // CHECK-NEXT:    br label [[OMP_TILE0_COND:%.*]]
 // CHECK:       omp_tile0.cond:
-// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], 13
+// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], [[TMP15]]
 // CHECK-NEXT:    br i1 [[OMP_TILE0_CMP]], label [[OMP_TILE0_BODY:%.*]], label [[OMP_TILE0_EXIT:%.*]]
 // CHECK:       omp_tile0.body:
-// CHECK-NEXT:    [[TMP14:%.*]] = mul nuw i32 13, [[TMP13]]
-// CHECK-NEXT:    [[TMP15:%.*]] = add nuw i32 [[TMP14]], [[OMP_TILE0_IV]]
-// CHECK-NEXT:    [[OMP_TILE0_INBOUNDS:%.*]] = icmp ult i32 [[TMP15]], [[DOTCOUNT]]
-// CHECK-NEXT:    br i1 [[OMP_TILE0_INBOUNDS]], label [[OMP_LOOP_BODY:%.*]], label [[OMP_TILE_BODY_MERGE:%.*]]
+// CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i32 13, [[TMP13]]
+// CHECK-NEXT:    [[TMP17:%.*]] = add nuw i32 [[TMP16]], [[OMP_TILE0_IV]]
+// CHECK-NEXT:    br label [[OMP_LOOP_BODY:%.*]]
 // CHECK:       omp_loop.body:
-// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP15]], ptr [[AGG_CAPTURED1]])
-// CHECK-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[B_ADDR]], align 8
-// CHECK-NEXT:    [[TMP17:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP17]] to i64
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[IDXPROM]]
-// CHECK-NEXT:    [[TMP18:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-// CHECK-NEXT:    [[TMP19:%.*]] = load ptr, ptr [[C_ADDR]], align 8
-// CHECK-NEXT:    [[TMP20:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP20]] to i64
-// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP19]], i64 [[IDXPROM2]]
-// CHECK-NEXT:    [[TMP21:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
-// CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP18]], [[TMP21]]
-// CHECK-NEXT:    [[TMP22:%.*]] = load ptr, ptr [[D_ADDR]], align 8
-// CHECK-NEXT:    [[TMP23:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[TMP23]] to i64
-// CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[TMP22]], i64 [[IDXPROM4]]
-// CHECK-NEXT:    [[TMP24:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
-// CHECK-NEXT:    [[MUL6:%.*]] = fmul float [[MUL]], [[TMP24]]
-// CHECK-NEXT:    [[TMP25:%.*]] = load ptr, ptr [[A_ADDR]], align 8
-// CHECK-NEXT:    [[TMP26:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM7:%.*]] = sext i32 [[TMP26]] to i64
-// CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[TMP25]], i64 [[IDXPROM7]]
+// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP17]], ptr [[AGG_CAPTURED1]])
+// CHECK-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[B_ADDR]], align 8
+// CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP19]] to i64
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[IDXPROM]]
+// CHECK-NEXT:    [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+// CHECK-NEXT:    [[TMP21:%.*]] = load ptr, ptr [[C_ADDR]], align 8
+// CHECK-NEXT:    [[TMP22:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP22]] to i64
+// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP21]], i64 [[IDXPROM2]]
+// CHECK-NEXT:    [[TMP23:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
+// CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP20]], [[TMP23]]
+// CHECK-NEXT:    [[TMP24:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[TMP25]] to i64
+// CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[TMP24]], i64 [[IDXPROM4]]
+// CHECK-NEXT:    [[TMP26:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
+// CHECK-NEXT:    [[MUL6:%.*]] = fmul float [[MUL]], [[TMP26]]
+// CHECK-NEXT:    [[TMP27:%.*]] = load ptr, ptr [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP28:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM7:%.*]] = sext i32 [[TMP28]] to i64
+// CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[TMP27]], i64 [[IDXPROM7]]
 // CHECK-NEXT:    store float [[MUL6]], ptr [[ARRAYIDX8]], align 4
-// CHECK-NEXT:    br label [[OMP_TILE_BODY_MERGE]]
-// CHECK:       omp_tile.body.merge:
 // CHECK-NEXT:    br label [[OMP_TILE0_INC]]
 // CHECK:       omp_tile0.inc:
 // CHECK-NEXT:    [[OMP_TILE0_NEXT]] = add nuw i32 [[OMP_TILE0_IV]], 1
diff --git a/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_constant_for.c b/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_constant_for.c
index b8a993c9cd8a7..acf0e5164411c 100644
--- a/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_constant_for.c
+++ b/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_constant_for.c
@@ -85,6 +85,8 @@ void unroll_partial_heuristic_constant_for(float *a, float *b, float *c, float *
 // CHECK-NEXT:    br i1 [[OMP_FLOOR0_CMP]], label [[OMP_FLOOR0_BODY:%.*]], label [[OMP_FLOOR0_EXIT:%.*]]
 // CHECK:       omp_floor0.body:
 // CHECK-NEXT:    [[TMP12:%.*]] = add i32 [[OMP_FLOOR0_IV]], [[TMP8]]
+// CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP12]], [[TMP3]]
+// CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP4]], i32 4
 // CHECK-NEXT:    br label [[OMP_TILE0_PREHEADER:%.*]]
 // CHECK:       omp_tile0.preheader:
 // CHECK-NEXT:    br label [[OMP_TILE0_HEADER:%.*]]
@@ -92,57 +94,54 @@ void unroll_partial_heuristic_constant_for(float *a, float *b, float *c, float *
 // CHECK-NEXT:    [[OMP_TILE0_IV:%.*]] = phi i32 [ 0, [[OMP_TILE0_PREHEADER]] ], [ [[OMP_TILE0_NEXT:%.*]], [[OMP_TILE0_INC:%.*]] ]
 // CHECK-NEXT:    br label [[OMP_TILE0_COND:%.*]]
 // CHECK:       omp_tile0.cond:
-// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], 4
+// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], [[TMP14]]
 // CHECK-NEXT:    br i1 [[OMP_TILE0_CMP]], label [[OMP_TILE0_BODY:%.*]], label [[OMP_TILE0_EXIT:%.*]]
 // CHECK:       omp_tile0.body:
-// CHECK-NEXT:    [[TMP13:%.*]] = mul nuw i32 4, [[TMP12]]
-// CHECK-NEXT:    [[TMP14:%.*]] = add nuw i32 [[TMP13]], [[OMP_TILE0_IV]]
-// CHECK-NEXT:    [[OMP_TILE0_INBOUNDS:%.*]] = icmp ult i32 [[TMP14]], [[DOTCOUNT]]
-// CHECK-NEXT:    br i1 [[OMP_TILE0_INBOUNDS]], label [[OMP_LOOP_BODY:%.*]], label [[OMP_TILE_BODY_MERGE:%.*]]
+// CHECK-NEXT:    [[TMP15:%.*]] = mul nuw i32 4, [[TMP12]]
+// CHECK-NEXT:    [[TMP16:%.*]] = add nuw i32 [[TMP15]], [[OMP_TILE0_IV]]
+// CHECK-NEXT:    br label [[OMP_LOOP_BODY:%.*]]
 // CHECK:       omp_loop.body:
-// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP14]], ptr [[AGG_CAPTURED1]])
-// CHECK-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[B_ADDR]], align 8
-// CHECK-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP16]] to i64
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP15]], i64 [[IDXPROM]]
-// CHECK-NEXT:    [[TMP17:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-// CHECK-NEXT:    [[CONV:%.*]] = fpext float [[TMP17]] to double
+// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP16]], ptr [[AGG_CAPTURED1]])
+// CHECK-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[B_ADDR]], align 8
+// CHECK-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP18]] to i64
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[IDXPROM]]
+// CHECK-NEXT:    [[TMP19:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+// CHECK-NEXT:    [[CONV:%.*]] = fpext float [[TMP19]] to double
 // CHECK-NEXT:    [[CALL:%.*]] = call double @sind(double noundef [[CONV]])
-// CHECK-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[C_ADDR]], align 8
-// CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP19]] to i64
-// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[IDXPROM2]]
-// CHECK-NEXT:    [[TMP20:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
-// CHECK-NEXT:    [[CONV4:%.*]] = fpext float [[TMP20]] to double
+// CHECK-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[C_ADDR]], align 8
+// CHECK-NEXT:    [[TMP21:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP21]] to i64
+// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP20]], i64 [[IDXPROM2]]
+// CHECK-NEXT:    [[TMP22:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
+// CHECK-NEXT:    [[CONV4:%.*]] = fpext float [[TMP22]] to double
 // CHECK-NEXT:    [[MUL:%.*]] = fmul double [[CALL]], [[CONV4]]
-// CHECK-NEXT:    [[TMP21:%.*]] = load ptr, ptr [[D_ADDR]], align 8
-// CHECK-NEXT:    [[TMP22:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM5:%.*]] = sext i32 [[TMP22]] to i64
-// CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, ptr [[TMP21]], i64 [[IDXPROM5]]
-// CHECK-NEXT:    [[TMP23:%.*]] = load float, ptr [[ARRAYIDX6]], align 4
-// CHECK-NEXT:    [[CONV7:%.*]] = fpext float [[TMP23]] to double
+// CHECK-NEXT:    [[TMP23:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK-NEXT:    [[TMP24:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM5:%.*]] = sext i32 [[TMP24]] to i64
+// CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, ptr [[TMP23]], i64 [[IDXPROM5]]
+// CHECK-NEXT:    [[TMP25:%.*]] = load float, ptr [[ARRAYIDX6]], align 4
+// CHECK-NEXT:    [[CONV7:%.*]] = fpext float [[TMP25]] to double
 // CHECK-NEXT:    [[MUL8:%.*]] = fmul double [[MUL]], [[CONV7]]
-// CHECK-NEXT:    [[TMP24:%.*]] = load ptr, ptr [[E_ADDR]], align 8
-// CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM9:%.*]] = sext i32 [[TMP25]] to i64
-// CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[TMP24]], i64 [[IDXPROM9]]
-// CHECK-NEXT:    [[TMP26:%.*]] = load float, ptr [[ARRAYIDX10]], align 4
-// CHECK-NEXT:    [[CONV11:%.*]] = fpext float [[TMP26]] to double
+// CHECK-NEXT:    [[TMP26:%.*]] = load ptr, ptr [[E_ADDR]], align 8
+// CHECK-NEXT:    [[TMP27:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM9:%.*]] = sext i32 [[TMP27]] to i64
+// CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[TMP26]], i64 [[IDXPROM9]]
+// CHECK-NEXT:    [[TMP28:%.*]] = load float, ptr [[ARRAYIDX10]], align 4
+// CHECK-NEXT:    [[CONV11:%.*]] = fpext float [[TMP28]] to double
 // CHECK-NEXT:    [[MUL12:%.*]] = fmul double [[MUL8]], [[CONV11]]
-// CHECK-NEXT:    [[TMP27:%.*]] = load float, ptr [[OFFSET_ADDR]], align 4
-// CHECK-NEXT:    [[CONV13:%.*]] = fpext float [[TMP27]] to double
+// CHECK-NEXT:    [[TMP29:%.*]] = load float, ptr [[OFFSET_ADDR]], align 4
+// CHECK-NEXT:    [[CONV13:%.*]] = fpext float [[TMP29]] to double
 // CHECK-NEXT:    [[ADD:%.*]] = fadd double [[MUL12]], [[CONV13]]
-// CHECK-NEXT:    [[TMP28:%.*]] = load ptr, ptr [[A_ADDR]], align 8
-// CHECK-NEXT:    [[TMP29:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM14:%.*]] = sext i32 [[TMP29]] to i64
-// CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds float, ptr [[TMP28]], i64 [[IDXPROM14]]
-// CHECK-NEXT:    [[TMP30:%.*]] = load float, ptr [[ARRAYIDX15]], align 4
-// CHECK-NEXT:    [[CONV16:%.*]] = fpext float [[TMP30]] to double
+// CHECK-NEXT:    [[TMP30:%.*]] = load ptr, ptr [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP31:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM14:%.*]] = sext i32 [[TMP31]] to i64
+// CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds float, ptr [[TMP30]], i64 [[IDXPROM14]]
+// CHECK-NEXT:    [[TMP32:%.*]] = load float, ptr [[ARRAYIDX15]], align 4
+// CHECK-NEXT:    [[CONV16:%.*]] = fpext float [[TMP32]] to double
 // CHECK-NEXT:    [[ADD17:%.*]] = fadd double [[CONV16]], [[ADD]]
 // CHECK-NEXT:    [[CONV18:%.*]] = fptrunc double [[ADD17]] to float
 // CHECK-NEXT:    store float [[CONV18]], ptr [[ARRAYIDX15]], align 4
-// CHECK-NEXT:    br label [[OMP_TILE_BODY_MERGE]]
-// CHECK:       omp_tile.body.merge:
 // CHECK-NEXT:    br label [[OMP_TILE0_INC]]
 // CHECK:       omp_tile0.inc:
 // CHECK-NEXT:    [[OMP_TILE0_NEXT]] = add nuw i32 [[OMP_TILE0_IV]], 1
diff --git a/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_runtime_for.c b/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_runtime_for.c
index 6b48e3ca22888..8fee5773e53b4 100644
--- a/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_runtime_for.c
+++ b/clang/test/OpenMP/irbuilder_unroll_partial_heuristic_runtime_for.c
@@ -87,6 +87,8 @@ void unroll_partial_heuristic_runtime_for(int n, float *a, float *b, float *c, f
 // CHECK-NEXT:    br i1 [[OMP_FLOOR0_CMP]], label [[OMP_FLOOR0_BODY:%.*]], label [[OMP_FLOOR0_EXIT:%.*]]
 // CHECK:       omp_floor0.body:
 // CHECK-NEXT:    [[TMP13:%.*]] = add i32 [[OMP_FLOOR0_IV]], [[TMP9]]
+// CHECK-NEXT:    [[TMP14:%.*]] = icmp eq i32 [[TMP13]], [[TMP4]]
+// CHECK-NEXT:    [[TMP15:%.*]] = select i1 [[TMP14]], i32 [[TMP5]], i32 4
 // CHECK-NEXT:    br label [[OMP_TILE0_PREHEADER:%.*]]
 // CHECK:       omp_tile0.preheader:
 // CHECK-NEXT:    br label [[OMP_TILE0_HEADER:%.*]]
@@ -94,57 +96,54 @@ void unroll_partial_heuristic_runtime_for(int n, float *a, float *b, float *c, f
 // CHECK-NEXT:    [[OMP_TILE0_IV:%.*]] = phi i32 [ 0, [[OMP_TILE0_PREHEADER]] ], [ [[OMP_TILE0_NEXT:%.*]], [[OMP_TILE0_INC:%.*]] ]
 // CHECK-NEXT:    br label [[OMP_TILE0_COND:%.*]]
 // CHECK:       omp_tile0.cond:
-// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], 4
+// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], [[TMP15]]
 // CHECK-NEXT:    br i1 [[OMP_TILE0_CMP]], label [[OMP_TILE0_BODY:%.*]], label [[OMP_TILE0_EXIT:%.*]]
 // CHECK:       omp_tile0.body:
-// CHECK-NEXT:    [[TMP14:%.*]] = mul nuw i32 4, [[TMP13]]
-// CHECK-NEXT:    [[TMP15:%.*]] = add nuw i32 [[TMP14]], [[OMP_TILE0_IV]]
-// CHECK-NEXT:    [[OMP_TILE0_INBOUNDS:%.*]] = icmp ult i32 [[TMP15]], [[DOTCOUNT]]
-// CHECK-NEXT:    br i1 [[OMP_TILE0_INBOUNDS]], label [[OMP_LOOP_BODY:%.*]], label [[OMP_TILE_BODY_MERGE:%.*]]
+// CHECK-NEXT:    [[TMP16:%.*]] = mul nuw i32 4, [[TMP13]]
+// CHECK-NEXT:    [[TMP17:%.*]] = add nuw i32 [[TMP16]], [[OMP_TILE0_IV]]
+// CHECK-NEXT:    br label [[OMP_LOOP_BODY:%.*]]
 // CHECK:       omp_loop.body:
-// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP15]], ptr [[AGG_CAPTURED1]])
-// CHECK-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[B_ADDR]], align 8
-// CHECK-NEXT:    [[TMP17:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP17]] to i64
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP16]], i64 [[IDXPROM]]
-// CHECK-NEXT:    [[TMP18:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-// CHECK-NEXT:    [[CONV:%.*]] = fpext float [[TMP18]] to double
+// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP17]], ptr [[AGG_CAPTURED1]])
+// CHECK-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[B_ADDR]], align 8
+// CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP19]] to i64
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[IDXPROM]]
+// CHECK-NEXT:    [[TMP20:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+// CHECK-NEXT:    [[CONV:%.*]] = fpext float [[TMP20]] to double
 // CHECK-NEXT:    [[CALL:%.*]] = call double @sind(double noundef [[CONV]])
-// CHECK-NEXT:    [[TMP19:%.*]] = load ptr, ptr [[C_ADDR]], align 8
-// CHECK-NEXT:    [[TMP20:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP20]] to i64
-// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP19]], i64 [[IDXPROM2]]
-// CHECK-NEXT:    [[TMP21:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
-// CHECK-NEXT:    [[CONV4:%.*]] = fpext float [[TMP21]] to double
+// CHECK-NEXT:    [[TMP21:%.*]] = load ptr, ptr [[C_ADDR]], align 8
+// CHECK-NEXT:    [[TMP22:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP22]] to i64
+// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP21]], i64 [[IDXPROM2]]
+// CHECK-NEXT:    [[TMP23:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
+// CHECK-NEXT:    [[CONV4:%.*]] = fpext float [[TMP23]] to double
 // CHECK-NEXT:    [[MUL:%.*]] = fmul double [[CALL]], [[CONV4]]
-// CHECK-NEXT:    [[TMP22:%.*]] = load ptr, ptr [[D_ADDR]], align 8
-// CHECK-NEXT:    [[TMP23:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM5:%.*]] = sext i32 [[TMP23]] to i64
-// CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, ptr [[TMP22]], i64 [[IDXPROM5]]
-// CHECK-NEXT:    [[TMP24:%.*]] = load float, ptr [[ARRAYIDX6]], align 4
-// CHECK-NEXT:    [[CONV7:%.*]] = fpext float [[TMP24]] to double
+// CHECK-NEXT:    [[TMP24:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM5:%.*]] = sext i32 [[TMP25]] to i64
+// CHECK-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds float, ptr [[TMP24]], i64 [[IDXPROM5]]
+// CHECK-NEXT:    [[TMP26:%.*]] = load float, ptr [[ARRAYIDX6]], align 4
+// CHECK-NEXT:    [[CONV7:%.*]] = fpext float [[TMP26]] to double
 // CHECK-NEXT:    [[MUL8:%.*]] = fmul double [[MUL]], [[CONV7]]
-// CHECK-NEXT:    [[TMP25:%.*]] = load ptr, ptr [[E_ADDR]], align 8
-// CHECK-NEXT:    [[TMP26:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM9:%.*]] = sext i32 [[TMP26]] to i64
-// CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[TMP25]], i64 [[IDXPROM9]]
-// CHECK-NEXT:    [[TMP27:%.*]] = load float, ptr [[ARRAYIDX10]], align 4
-// CHECK-NEXT:    [[CONV11:%.*]] = fpext float [[TMP27]] to double
+// CHECK-NEXT:    [[TMP27:%.*]] = load ptr, ptr [[E_ADDR]], align 8
+// CHECK-NEXT:    [[TMP28:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM9:%.*]] = sext i32 [[TMP28]] to i64
+// CHECK-NEXT:    [[ARRAYIDX10:%.*]] = getelementptr inbounds float, ptr [[TMP27]], i64 [[IDXPROM9]]
+// CHECK-NEXT:    [[TMP29:%.*]] = load float, ptr [[ARRAYIDX10]], align 4
+// CHECK-NEXT:    [[CONV11:%.*]] = fpext float [[TMP29]] to double
 // CHECK-NEXT:    [[MUL12:%.*]] = fmul double [[MUL8]], [[CONV11]]
-// CHECK-NEXT:    [[TMP28:%.*]] = load float, ptr [[OFFSET_ADDR]], align 4
-// CHECK-NEXT:    [[CONV13:%.*]] = fpext float [[TMP28]] to double
+// CHECK-NEXT:    [[TMP30:%.*]] = load float, ptr [[OFFSET_ADDR]], align 4
+// CHECK-NEXT:    [[CONV13:%.*]] = fpext float [[TMP30]] to double
 // CHECK-NEXT:    [[ADD:%.*]] = fadd double [[MUL12]], [[CONV13]]
-// CHECK-NEXT:    [[TMP29:%.*]] = load ptr, ptr [[A_ADDR]], align 8
-// CHECK-NEXT:    [[TMP30:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM14:%.*]] = sext i32 [[TMP30]] to i64
-// CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds float, ptr [[TMP29]], i64 [[IDXPROM14]]
-// CHECK-NEXT:    [[TMP31:%.*]] = load float, ptr [[ARRAYIDX15]], align 4
-// CHECK-NEXT:    [[CONV16:%.*]] = fpext float [[TMP31]] to double
+// CHECK-NEXT:    [[TMP31:%.*]] = load ptr, ptr [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP32:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM14:%.*]] = sext i32 [[TMP32]] to i64
+// CHECK-NEXT:    [[ARRAYIDX15:%.*]] = getelementptr inbounds float, ptr [[TMP31]], i64 [[IDXPROM14]]
+// CHECK-NEXT:    [[TMP33:%.*]] = load float, ptr [[ARRAYIDX15]], align 4
+// CHECK-NEXT:    [[CONV16:%.*]] = fpext float [[TMP33]] to double
 // CHECK-NEXT:    [[ADD17:%.*]] = fadd double [[CONV16]], [[ADD]]
 // CHECK-NEXT:    [[CONV18:%.*]] = fptrunc double [[ADD17]] to float
 // CHECK-NEXT:    store float [[CONV18]], ptr [[ARRAYIDX15]], align 4
-// CHECK-NEXT:    br label [[OMP_TILE_BODY_MERGE]]
-// CHECK:       omp_tile.body.merge:
 // CHECK-NEXT:    br label [[OMP_TILE0_INC]]
 // CHECK:       omp_tile0.inc:
 // CHECK-NEXT:    [[OMP_TILE0_NEXT]] = add nuw i32 [[OMP_TILE0_IV]], 1
diff --git a/clang/test/OpenMP/irbuilder_unroll_unroll_partial_factor.c b/clang/test/OpenMP/irbuilder_unroll_unroll_partial_factor.c
index 65bd495ca0426..c4ebcddb097bc 100644
--- a/clang/test/OpenMP/irbuilder_unroll_unroll_partial_factor.c
+++ b/clang/test/OpenMP/irbuilder_unroll_unroll_partial_factor.c
@@ -75,6 +75,8 @@ void unroll_partial_factor_for(float *a, float *b, float *c, float *d) {
 // CHECK-NEXT:    br i1 [[OMP_FLOOR0_CMP]], label [[OMP_FLOOR0_BODY:%.*]], label [[OMP_FLOOR0_EXIT:%.*]]
 // CHECK:       omp_floor0.body:
 // CHECK-NEXT:    [[TMP12:%.*]] = add i32 [[OMP_FLOOR0_IV]], [[TMP8]]
+// CHECK-NEXT:    [[TMP13:%.*]] = icmp eq i32 [[TMP12]], [[TMP3]]
+// CHECK-NEXT:    [[TMP14:%.*]] = select i1 [[TMP13]], i32 [[TMP4]], i32 2
 // CHECK-NEXT:    br label [[OMP_TILE0_PREHEADER:%.*]]
 // CHECK:       omp_tile0.preheader:
 // CHECK-NEXT:    br label [[OMP_TILE0_HEADER:%.*]]
@@ -82,39 +84,36 @@ void unroll_partial_factor_for(float *a, float *b, float *c, float *d) {
 // CHECK-NEXT:    [[OMP_TILE0_IV:%.*]] = phi i32 [ 0, [[OMP_TILE0_PREHEADER]] ], [ [[OMP_TILE0_NEXT:%.*]], [[OMP_TILE0_INC:%.*]] ]
 // CHECK-NEXT:    br label [[OMP_TILE0_COND:%.*]]
 // CHECK:       omp_tile0.cond:
-// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], 2
+// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], [[TMP14]]
 // CHECK-NEXT:    br i1 [[OMP_TILE0_CMP]], label [[OMP_TILE0_BODY:%.*]], label [[OMP_TILE0_EXIT:%.*]]
 // CHECK:       omp_tile0.body:
-// CHECK-NEXT:    [[TMP13:%.*]] = mul nuw i32 2, [[TMP12]]
-// CHECK-NEXT:    [[TMP14:%.*]] = add nuw i32 [[TMP13]], [[OMP_TILE0_IV]]
-// CHECK-NEXT:    [[OMP_TILE0_INBOUNDS:%.*]] = icmp ult i32 [[TMP14]], [[DOTCOUNT]]
-// CHECK-NEXT:    br i1 [[OMP_TILE0_INBOUNDS]], label [[OMP_LOOP_BODY:%.*]], label [[OMP_TILE_BODY_MERGE:%.*]]
+// CHECK-NEXT:    [[TMP15:%.*]] = mul nuw i32 2, [[TMP12]]
+// CHECK-NEXT:    [[TMP16:%.*]] = add nuw i32 [[TMP15]], [[OMP_TILE0_IV]]
+// CHECK-NEXT:    br label [[OMP_LOOP_BODY:%.*]]
 // CHECK:       omp_loop.body:
-// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP14]], ptr [[AGG_CAPTURED1]])
-// CHECK-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[B_ADDR]], align 8
-// CHECK-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP16]] to i64
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP15]], i64 [[IDXPROM]]
-// CHECK-NEXT:    [[TMP17:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-// CHECK-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[C_ADDR]], align 8
-// CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP19]] to i64
-// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[IDXPROM2]]
-// CHECK-NEXT:    [[TMP20:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
-// CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP17]], [[TMP20]]
-// CHECK-NEXT:    [[TMP21:%.*]] = load ptr, ptr [[D_ADDR]], align 8
-// CHECK-NEXT:    [[TMP22:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[TMP22]] to i64
-// CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[TMP21]], i64 [[IDXPROM4]]
-// CHECK-NEXT:    [[TMP23:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
-// CHECK-NEXT:    [[MUL6:%.*]] = fmul float [[MUL]], [[TMP23]]
-// CHECK-NEXT:    [[TMP24:%.*]] = load ptr, ptr [[A_ADDR]], align 8
-// CHECK-NEXT:    [[TMP25:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM7:%.*]] = sext i32 [[TMP25]] to i64
-// CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[TMP24]], i64 [[IDXPROM7]]
+// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP16]], ptr [[AGG_CAPTURED1]])
+// CHECK-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[B_ADDR]], align 8
+// CHECK-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP18]] to i64
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[IDXPROM]]
+// CHECK-NEXT:    [[TMP19:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+// CHECK-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[C_ADDR]], align 8
+// CHECK-NEXT:    [[TMP21:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP21]] to i64
+// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP20]], i64 [[IDXPROM2]]
+// CHECK-NEXT:    [[TMP22:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
+// CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP19]], [[TMP22]]
+// CHECK-NEXT:    [[TMP23:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK-NEXT:    [[TMP24:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[TMP24]] to i64
+// CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[TMP23]], i64 [[IDXPROM4]]
+// CHECK-NEXT:    [[TMP25:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
+// CHECK-NEXT:    [[MUL6:%.*]] = fmul float [[MUL]], [[TMP25]]
+// CHECK-NEXT:    [[TMP26:%.*]] = load ptr, ptr [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP27:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM7:%.*]] = sext i32 [[TMP27]] to i64
+// CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[TMP26]], i64 [[IDXPROM7]]
 // CHECK-NEXT:    store float [[MUL6]], ptr [[ARRAYIDX8]], align 4
-// CHECK-NEXT:    br label [[OMP_TILE_BODY_MERGE]]
-// CHECK:       omp_tile.body.merge:
 // CHECK-NEXT:    br label [[OMP_TILE0_INC]]
 // CHECK:       omp_tile0.inc:
 // CHECK-NEXT:    [[OMP_TILE0_NEXT]] = add nuw i32 [[OMP_TILE0_IV]], 1
diff --git a/clang/test/OpenMP/irbuilder_unroll_unroll_partial_heuristic.c b/clang/test/OpenMP/irbuilder_unroll_unroll_partial_heuristic.c
index c2b7d46588f66..24d42d265d6a6 100644
--- a/clang/test/OpenMP/irbuilder_unroll_unroll_partial_heuristic.c
+++ b/clang/test/OpenMP/irbuilder_unroll_unroll_partial_heuristic.c
@@ -59,6 +59,8 @@ void unroll_unroll_partial_heuristic(float *a, float *b, float *c, float *d) {
 // CHECK-NEXT:    [[OMP_FLOOR0_CMP:%.*]] = icmp ult i32 [[OMP_FLOOR0_IV]], [[OMP_FLOOR0_TRIPCOUNT]]
 // CHECK-NEXT:    br i1 [[OMP_FLOOR0_CMP]], label [[OMP_FLOOR0_BODY:%.*]], label [[OMP_FLOOR0_EXIT:%.*]]
 // CHECK:       omp_floor0.body:
+// CHECK-NEXT:    [[TMP7:%.*]] = icmp eq i32 [[OMP_FLOOR0_IV]], [[TMP3]]
+// CHECK-NEXT:    [[TMP8:%.*]] = select i1 [[TMP7]], i32 [[TMP4]], i32 8
 // CHECK-NEXT:    br label [[OMP_TILE0_PREHEADER:%.*]]
 // CHECK:       omp_tile0.preheader:
 // CHECK-NEXT:    br label [[OMP_TILE0_HEADER:%.*]]
@@ -66,50 +68,47 @@ void unroll_unroll_partial_heuristic(float *a, float *b, float *c, float *d) {
 // CHECK-NEXT:    [[OMP_TILE0_IV:%.*]] = phi i32 [ 0, [[OMP_TILE0_PREHEADER]] ], [ [[OMP_TILE0_NEXT:%.*]], [[OMP_TILE0_INC:%.*]] ]
 // CHECK-NEXT:    br label [[OMP_TILE0_COND:%.*]]
 // CHECK:       omp_tile0.cond:
-// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], 8
+// CHECK-NEXT:    [[OMP_TILE0_CMP:%.*]] = icmp ult i32 [[OMP_TILE0_IV]], [[TMP8]]
 // CHECK-NEXT:    br i1 [[OMP_TILE0_CMP]], label [[OMP_TILE0_BODY:%.*]], label [[OMP_TILE0_EXIT:%.*]]
 // CHECK:       omp_tile0.body:
-// CHECK-NEXT:    [[TMP7:%.*]] = mul nuw i32 8, [[OMP_FLOOR0_IV]]
-// CHECK-NEXT:    [[TMP8:%.*]] = add nuw i32 [[TMP7]], [[OMP_TILE0_IV]]
-// CHECK-NEXT:    [[OMP_TILE0_INBOUNDS:%.*]] = icmp ult i32 [[TMP8]], [[DOTCOUNT]]
-// CHECK-NEXT:    br i1 [[OMP_TILE0_INBOUNDS]], label [[OMP_LOOP_BODY:%.*]], label [[OMP_TILE_BODY_MERGE:%.*]]
+// CHECK-NEXT:    [[TMP9:%.*]] = mul nuw i32 8, [[OMP_FLOOR0_IV]]
+// CHECK-NEXT:    [[TMP10:%.*]] = add nuw i32 [[TMP9]], [[OMP_TILE0_IV]]
+// CHECK-NEXT:    br label [[OMP_LOOP_BODY:%.*]]
 // CHECK:       omp_loop.body:
-// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP8]], ptr [[AGG_CAPTURED1]])
-// CHECK-NEXT:    [[TMP9:%.*]] = load ptr, ptr [[B_ADDR]], align 8
-// CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP10]] to i64
-// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP9]], i64 [[IDXPROM]]
-// CHECK-NEXT:    [[TMP11:%.*]] = load float, ptr [[ARRAYIDX]], align 4
-// CHECK-NEXT:    [[TMP12:%.*]] = load ptr, ptr [[C_ADDR]], align 8
-// CHECK-NEXT:    [[TMP13:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP13]] to i64
-// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP12]], i64 [[IDXPROM2]]
-// CHECK-NEXT:    [[TMP14:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
-// CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP11]], [[TMP14]]
-// CHECK-NEXT:    [[TMP15:%.*]] = load ptr, ptr [[D_ADDR]], align 8
-// CHECK-NEXT:    [[TMP16:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[TMP16]] to i64
-// CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[TMP15]], i64 [[IDXPROM4]]
-// CHECK-NEXT:    [[TMP17:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
-// CHECK-NEXT:    [[MUL6:%.*]] = fmul float [[MUL]], [[TMP17]]
-// CHECK-NEXT:    [[TMP18:%.*]] = load ptr, ptr [[A_ADDR]], align 8
-// CHECK-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
-// CHECK-NEXT:    [[IDXPROM7:%.*]] = sext i32 [[TMP19]] to i64
-// CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[TMP18]], i64 [[IDXPROM7]]
+// CHECK-NEXT:    call void @__captured_stmt.1(ptr [[I]], i32 [[TMP10]], ptr [[AGG_CAPTURED1]])
+// CHECK-NEXT:    [[TMP11:%.*]] = load ptr, ptr [[B_ADDR]], align 8
+// CHECK-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM:%.*]] = sext i32 [[TMP12]] to i64
+// CHECK-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[TMP11]], i64 [[IDXPROM]]
+// CHECK-NEXT:    [[TMP13:%.*]] = load float, ptr [[ARRAYIDX]], align 4
+// CHECK-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[C_ADDR]], align 8
+// CHECK-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM2:%.*]] = sext i32 [[TMP15]] to i64
+// CHECK-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds float, ptr [[TMP14]], i64 [[IDXPROM2]]
+// CHECK-NEXT:    [[TMP16:%.*]] = load float, ptr [[ARRAYIDX3]], align 4
+// CHECK-NEXT:    [[MUL:%.*]] = fmul float [[TMP13]], [[TMP16]]
+// CHECK-NEXT:    [[TMP17:%.*]] = load ptr, ptr [[D_ADDR]], align 8
+// CHECK-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM4:%.*]] = sext i32 [[TMP18]] to i64
+// CHECK-NEXT:    [[ARRAYIDX5:%.*]] = getelementptr inbounds float, ptr [[TMP17]], i64 [[IDXPROM4]]
+// CHECK-NEXT:    [[TMP19:%.*]] = load float, ptr [[ARRAYIDX5]], align 4
+// CHECK-NEXT:    [[MUL6:%.*]] = fmul float [[MUL]], [[TMP19]]
+// CHECK-NEXT:    [[TMP20:%.*]] = load ptr, ptr [[A_ADDR]], align 8
+// CHECK-NEXT:    [[TMP21:%.*]] = load i32, ptr [[I]], align 4
+// CHECK-NEXT:    [[IDXPROM7:%.*]] = sext i32 [[TMP21]] to i64
+// CHECK-NEXT:    [[ARRAYIDX8:%.*]] = getelementptr inbounds float, ptr [[TMP20]], i64 [[IDXPROM7]]
 // CHECK-NEXT:    store float [[MUL6]], ptr [[ARRAYIDX8]], align 4
-// CHECK-NEXT:    br label [[OMP_TILE_BODY_MERGE]]
-// CHECK:       omp_tile.body.merge:
 // CHECK-NEXT:    br label [[OMP_TILE0_INC]]
 // CHECK:       omp_tile0.inc:
 // CHECK-NEXT:    [[OMP_TILE0_NEXT]] = add nuw i32 [[OMP_TILE0_IV]], 1
-// CHECK-NEXT:    br label [[OMP_TILE0_HEADER]], !llvm.loop [[LOOP2:![0-9]+]]
+// CHECK-NEXT:    br label [[OMP_TILE0_HEADER]], !llvm.loop [[LOOP3:![0-9]+]]
 // CHECK:       omp_tile0.exit:
 // CHECK-NEXT:    br label [[OMP_TILE0_AFTER:%.*]]
 // CHECK:       omp_tile0.after:
 // CHECK-NEXT:    br label [[OMP_FLOOR0_INC]]
 // CHECK:       omp_floor0.inc:
 // CHECK-NEXT:    [[OMP_FLOOR0_NEXT]] = add nuw i32 [[OMP_FLOOR0_IV]], 1
-// CHECK-NEXT:    br label [[OMP_FLOOR0_HEADER]], !llvm.loop [[LOOP5:![0-9]+]]
+// CHECK-NEXT:    br label [[OMP_FLOOR0_HEADER]], !llvm.loop [[LOOP6:![0-9]+]]
 // CHECK:       omp_floor0.exit:
 // CHECK-NEXT:    br label [[OMP_FLOOR0_AFTER:%.*]]
 // CHECK:       omp_floor0.after:
@@ -130,7 +129,7 @@ void unroll_unroll_partial_heuristic(float *a, float *b, float *c, float *d) {
 // CHECK-NEXT:    store ptr [[__CONTEXT]], ptr [[__CONTEXT_ADDR]], align 8
 // CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__CONTEXT_ADDR]], align 8
 // CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds nuw [[STRUCT_ANON:%.*]], ptr [[TMP0]], i32 0, i32 0
-// CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP1]], align 8, !nonnull [[META6:![0-9]+]], !align [[META7:![0-9]+]]
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[TMP1]], align 8
 // CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TMP2]], align 4
 // CHECK-NEXT:    store i32 [[TMP3]], ptr [[DOTSTART]], align 4
 // CHECK-NEXT:    store i32 2, ptr [[DOTSTOP]], align 4
@@ -153,7 +152,7 @@ void unroll_unroll_partial_heuristic(float *a, float *b, float *c, float *d) {
 // CHECK-NEXT:    br label [[COND_END]]
 // CHECK:       cond.end:
 // CHECK-NEXT:    [[COND:%.*]] = phi i32 [ [[DIV]], [[COND_TRUE]] ], [ 0, [[COND_FALSE]] ]
-// CHECK-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[DISTANCE_ADDR]], align 8, !nonnull [[META6]], !align [[META7]]
+// CHECK-NEXT:    [[TMP10:%.*]] = load ptr, ptr [[DISTANCE_ADDR]], align 8
 // CHECK-NEXT:    store i32 [[COND]], ptr [[TMP10]], align 4
 // CHECK-NEXT:    ret void
 //
@@ -173,7 +172,7 @@ void unroll_unroll_partial_heuristic(float *a, float *b, float *c, float *d) {
 // CHECK-NEXT:    [[TMP3:%.*]] = load i32, ptr [[LOGICAL_ADDR]], align 4
 // CHECK-NEXT:    [[MUL:%.*]] = mul i32 1, [[TMP3]]
 // CHECK-NEXT:    [[ADD:%.*]] = add i32 [[TMP2]], [[MUL]]
-// CHECK-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[LOOPVAR_ADDR]], align 8, !nonnull [[META6]], !align [[META7]]
+// CHECK-NEXT:    [[TMP4:%.*]] = load ptr, ptr [[LOOPVAR_ADDR]], align 8
 // CHECK-NEXT:    store i32 [[ADD]], ptr [[TMP4]], align 4
 // CHECK-NEXT:    ret void
 //
diff --git a/clang/test/OpenMP/tile_canonical_required_codegen.cpp b/clang/test/OpenMP/tile_canonical_required_codegen.cpp
new file mode 100644
index 0000000000000..ceeaed4cb666a
--- /dev/null
+++ b/clang/test/OpenMP/tile_canonical_required_codegen.cpp
@@ -0,0 +1,86 @@
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fclang-abi-compat=latest -std=c++20 -fopenmp -ast-print %s | FileCheck %s --check-prefix=PRINT
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fclang-abi-compat=latest -std=c++20 -fopenmp -emit-llvm %s -o - | FileCheck %s --check-prefix=IR
+// expected-no-diagnostics
+
+// Positive tests for canonical intra-tile loop shape.
+
+extern "C" void body(...) {}
+
+// PRINT-LABEL: void collapse2_tile1d(
+// IR-LABEL: define {{.*}} @collapse2_tile1d(
+// collapse(2) consumes floor+tile; predicate stays as a body guard.
+// IR should have one flattened loop body and no nested floor/tile loop.
+// IR: %.tile.cnt.0.iv.i
+// IR: omp.inner.for.body:
+// IR: br i1 %{{.*}}, label %if.then, label %if.end
+// IR: if.then:
+// IR: call void (...) @body
+// IR-NEXT: br label %if.end
+// IR: if.end:
+extern "C" void collapse2_tile1d(int n) {
+#pragma omp parallel for collapse(2)
+#pragma omp tile sizes(2)
+  for (int i = 0; i < n; ++i)
+    body(i);
+}
+
+// PRINT-LABEL: void collapse3_tile1d_nested(
+// IR-LABEL: define {{.*}} @collapse3_tile1d_nested(
+// collapse(3) reaches through the tile guard to include `j`.
+// IR: br i1 %{{.*}}, label %omp_tile.pred.then, label %omp_tile.pred.end
+// IR: omp_tile.pred.then:
+// IR: call void (...) @body
+// IR-NEXT: br label %omp_tile.pred.end
+// IR: omp_tile.pred.end:
+extern "C" void collapse3_tile1d_nested(int n) {
+#pragma omp parallel for collapse(3)
+#pragma omp tile sizes(5)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+}
+
+// PRINT-LABEL: void tile2d_inside_tile1d(
+// IR-LABEL: define {{.*}} @tile2d_inside_tile1d(
+// Nested tiling: both intra-tile loops stay canonical and hoist their predicates.
+// IR: %.tile.cnt.0.iv..floor_0.iv.i = alloca i32
+// IR: %.tile.cnt.0.iv.i = alloca i32
+// IR: %omp_tile.invariant_pred_bound = and i1
+// IR: %omp_tile.invariant_pred_bound{{[0-9]+}} = and i1
+extern "C" void tile2d_inside_tile1d(int n) {
+#pragma omp tile sizes(3)
+#pragma omp tile sizes(2)
+  for (int i = 0; i < n; ++i)
+    body(i);
+}
+
+// PRINT-LABEL: void collapse3_tile2d(
+// IR-LABEL: define {{.*}} @collapse3_tile2d(
+// 2-D tile + collapse(3): dim checks combine and are hoisted into the surviving loop.
+// IR: %.tile.cnt.0.iv.i
+// IR: %.tile.cnt.1.iv.j
+// IR: land.rhs:
+// IR: land.end:
+// IR: %omp_tile.invariant_pred_bound = and i1
+extern "C" void collapse3_tile2d(int n) {
+#pragma omp parallel for collapse(3)
+#pragma omp tile sizes(2, 3)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+}
+
+// PRINT-LABEL: void tile1d_constant_div_no_predicate(
+// IR-LABEL: define {{.*}} @tile1d_constant_div_no_predicate(
+// Exact-divisible case: canonical tile-count loop, no predicate.
+// IR: %.tile.cnt.0.iv.i
+// IR: for.cond1:
+// IR: %[[TC:.*]] = icmp slt i32 %{{.*}}, 5
+// IR-NEXT: br i1 %[[TC]], label %for.body3, label %for.end
+// IR-NOT: omp_tile.invariant_pred_bound
+// IR-NOT: omp_tile.pred
+extern "C" void tile1d_constant_div_no_predicate() {
+#pragma omp tile sizes(5)
+  for (int i = 0; i < 10; ++i)
+    body(i);
+}
diff --git a/clang/test/OpenMP/tile_codegen.cpp b/clang/test/OpenMP/tile_codegen.cpp
index 65fd3a327ad64..a76e46bddfb8a 100644
--- a/clang/test/OpenMP/tile_codegen.cpp
+++ b/clang/test/OpenMP/tile_codegen.cpp
@@ -56,7 +56,7 @@ extern "C" void foo4() {
 
 
 extern "C" void foo5() {
-#pragma omp for collapse(2)
+#pragma omp for collapse(3)
 #pragma omp tile sizes(5)
   for (int i = 7; i < 17; i += 3)
     for (int j = 7; j < 17; j += 3)
@@ -136,8 +136,9 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[I2:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[I:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
@@ -151,45 +152,41 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
 // CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END12:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK1:       for.cond4:
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 5
-// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
-// CHECK1-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK1:       cond.true:
-// CHECK1-NEXT:    br label [[COND_END:%.*]]
-// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP1]], 5
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
+// CHECK1-NEXT:    [[CMP6:%.*]] = icmp slt i32 [[ADD]], 4
+// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP5]], [[CMP6]]
+// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY7:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body7:
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP4]], 5
-// CHECK1-NEXT:    br label [[COND_END]]
-// CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
-// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body8:
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
+// CHECK1-NEXT:    store i32 [[ADD8]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
 // CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
-// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[TMP6]], align 4
-// CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2]], !align [[META3]]
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP8]])
+// CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
+// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[TMP7]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2]], !align [[META3]]
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP9]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP9]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP4:![0-9]+]]
 // CHECK1:       for.end:
 // CHECK1-NEXT:    br label [[FOR_INC10:%.*]]
 // CHECK1:       for.inc10:
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP10]], 5
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP11]], 5
 // CHECK1-NEXT:    store i32 [[ADD11]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
 // CHECK1:       for.end12:
@@ -207,8 +204,9 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -237,54 +235,48 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK1-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP8]], [[ADD5]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK1:       for.cond6:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[ADD7:%.*]] = add i32 [[TMP12]], 1
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD8:%.*]] = add i32 [[TMP13]], 5
-// CHECK1-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
-// CHECK1-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK1:       cond.true:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[ADD10:%.*]] = add i32 [[TMP14]], 1
-// CHECK1-NEXT:    br label [[COND_END:%.*]]
-// CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD11:%.*]] = add i32 [[TMP15]], 5
-// CHECK1-NEXT:    br label [[COND_END]]
-// CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP11]], [[COND]]
-// CHECK1-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body13:
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP10]], 5
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD8:%.*]] = add i32 [[TMP11]], [[TMP12]]
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD9:%.*]] = add i32 [[TMP13]], 1
+// CHECK1-NEXT:    [[CMP10:%.*]] = icmp ult i32 [[ADD8]], [[ADD9]]
+// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP7]], [[CMP10]]
+// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY11:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body11:
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD12:%.*]] = add i32 [[TMP14]], [[TMP15]]
+// CHECK1-NEXT:    store i32 [[ADD12]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
 // CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
 // CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP17]], [[TMP18]]
-// CHECK1-NEXT:    [[ADD14:%.*]] = add i32 [[TMP16]], [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[ADD13:%.*]] = add i32 [[TMP16]], [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[I]], align 4
 // CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP19]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP20]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP7:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC15:%.*]]
-// CHECK1:       for.inc15:
+// CHECK1-NEXT:    br label [[FOR_INC14:%.*]]
+// CHECK1:       for.inc14:
 // CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD16:%.*]] = add i32 [[TMP21]], 5
-// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD15:%.*]] = add i32 [[TMP21]], 5
+// CHECK1-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
-// CHECK1:       for.end17:
+// CHECK1:       for.end16:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -295,11 +287,13 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -310,94 +304,90 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END30:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END26:%.*]]
 // CHECK1:       for.body:
 // CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK1:       for.cond1:
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 4
-// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END27:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END23:%.*]]
 // CHECK1:       for.body3:
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 [[TMP2]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK1:       for.cond4:
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], 5
-// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
-// CHECK1-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK1:       cond.true:
-// CHECK1-NEXT:    br label [[COND_END:%.*]]
-// CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP5]], 5
-// CHECK1-NEXT:    br label [[COND_END]]
-// CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP3]], [[COND]]
-// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END24:%.*]]
-// CHECK1:       for.body8:
-// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
-// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND10:%.*]]
-// CHECK1:       for.cond10:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP2]], 5
+// CHECK1-NEXT:    br i1 [[CMP5]], label [[FOR_BODY6:%.*]], label [[FOR_END20:%.*]]
+// CHECK1:       for.body6:
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[TMP4]]
+// CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
+// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND8:%.*]]
+// CHECK1:       for.cond8:
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP6]], 5
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP7]], [[TMP8]]
+// CHECK1-NEXT:    [[CMP11:%.*]] = icmp slt i32 [[ADD10]], 4
+// CHECK1-NEXT:    br i1 [[CMP11]], label [[LAND_RHS:%.*]], label [[LAND_END:%.*]]
+// CHECK1:       land.rhs:
 // CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP9]], 5
-// CHECK1-NEXT:    [[CMP12:%.*]] = icmp slt i32 4, [[ADD11]]
-// CHECK1-NEXT:    br i1 [[CMP12]], label [[COND_TRUE13:%.*]], label [[COND_FALSE14:%.*]]
-// CHECK1:       cond.true13:
-// CHECK1-NEXT:    br label [[COND_END16:%.*]]
-// CHECK1:       cond.false14:
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP10]], 5
-// CHECK1-NEXT:    br label [[COND_END16]]
-// CHECK1:       cond.end16:
-// CHECK1-NEXT:    [[COND17:%.*]] = phi i32 [ 4, [[COND_TRUE13]] ], [ [[ADD15]], [[COND_FALSE14]] ]
-// CHECK1-NEXT:    [[CMP18:%.*]] = icmp slt i32 [[TMP8]], [[COND17]]
-// CHECK1-NEXT:    br i1 [[CMP18]], label [[FOR_BODY19:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body19:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[TMP11]], 3
-// CHECK1-NEXT:    [[ADD21:%.*]] = add nsw i32 7, [[MUL20]]
-// CHECK1-NEXT:    store i32 [[ADD21]], ptr [[J]], align 4
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP12]], i32 noundef [[TMP13]])
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
+// CHECK1-NEXT:    [[CMP13:%.*]] = icmp slt i32 [[ADD12]], 4
+// CHECK1-NEXT:    br label [[LAND_END]]
+// CHECK1:       land.end:
+// CHECK1-NEXT:    [[TMP11:%.*]] = phi i1 [ false, [[FOR_COND8]] ], [ [[CMP13]], [[LAND_RHS]] ]
+// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP9]], [[TMP11]]
+// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body14:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP12]], [[TMP13]]
+// CHECK1-NEXT:    store i32 [[ADD15]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[MUL16:%.*]] = mul nsw i32 [[TMP14]], 3
+// CHECK1-NEXT:    [[ADD17:%.*]] = add nsw i32 7, [[MUL16]]
+// CHECK1-NEXT:    store i32 [[ADD17]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP15]], i32 noundef [[TMP16]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND10]], !llvm.loop [[LOOP9:![0-9]+]]
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP17]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP9:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC22:%.*]]
-// CHECK1:       for.inc22:
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC23:%.*]] = add nsw i32 [[TMP15]], 1
-// CHECK1-NEXT:    store i32 [[INC23]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_INC18:%.*]]
+// CHECK1:       for.inc18:
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC19:%.*]] = add nsw i32 [[TMP18]], 1
+// CHECK1-NEXT:    store i32 [[INC19]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP10:![0-9]+]]
-// CHECK1:       for.end24:
-// CHECK1-NEXT:    br label [[FOR_INC25:%.*]]
-// CHECK1:       for.inc25:
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD26:%.*]] = add nsw i32 [[TMP16]], 5
-// CHECK1-NEXT:    store i32 [[ADD26]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1:       for.end20:
+// CHECK1-NEXT:    br label [[FOR_INC21:%.*]]
+// CHECK1:       for.inc21:
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD22:%.*]] = add nsw i32 [[TMP19]], 5
+// CHECK1-NEXT:    store i32 [[ADD22]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP11:![0-9]+]]
-// CHECK1:       for.end27:
-// CHECK1-NEXT:    br label [[FOR_INC28:%.*]]
-// CHECK1:       for.inc28:
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD29:%.*]] = add nsw i32 [[TMP17]], 5
-// CHECK1-NEXT:    store i32 [[ADD29]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1:       for.end23:
+// CHECK1-NEXT:    br label [[FOR_INC24:%.*]]
+// CHECK1:       for.inc24:
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD25:%.*]] = add nsw i32 [[TMP20]], 5
+// CHECK1-NEXT:    store i32 [[ADD25]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP12:![0-9]+]]
-// CHECK1:       for.end30:
+// CHECK1:       for.end26:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -455,14 +445,14 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP7]], 4
-// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END22:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
 // CHECK1:       for.body:
 // CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND3:%.*]]
 // CHECK1:       for.cond3:
 // CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP8]], 5
-// CHECK1-NEXT:    br i1 [[CMP4]], label [[FOR_BODY5:%.*]], label [[FOR_END19:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP4]], label [[FOR_BODY5:%.*]], label [[FOR_END21:%.*]]
 // CHECK1:       for.body5:
 // CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
@@ -477,57 +467,61 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond9:
 // CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
 // CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i32 [[TMP12]], 5
-// CHECK1-NEXT:    br i1 [[CMP10]], label [[FOR_BODY11:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body11:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP13]], [[TMP14]]
-// CHECK1-NEXT:    store i32 [[ADD12]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[MUL13:%.*]] = mul nsw i32 [[TMP15]], 3
-// CHECK1-NEXT:    [[ADD14:%.*]] = add nsw i32 7, [[MUL13]]
-// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[J]], align 4
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP15:%.*]] = icmp slt i32 [[TMP16]], 4
-// CHECK1-NEXT:    br i1 [[CMP15]], label [[LAND_LHS_TRUE:%.*]], label [[IF_END:%.*]]
-// CHECK1:       land.lhs.true:
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[CMP16:%.*]] = icmp slt i32 [[TMP17]], 4
-// CHECK1-NEXT:    br i1 [[CMP16]], label [[IF_THEN:%.*]], label [[IF_END]]
-// CHECK1:       if.then:
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[J]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP18]], i32 noundef [[TMP19]])
-// CHECK1-NEXT:    br label [[IF_END]]
-// CHECK1:       if.end:
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP13]], [[TMP14]]
+// CHECK1-NEXT:    [[CMP12:%.*]] = icmp slt i32 [[ADD11]], 4
+// CHECK1-NEXT:    br i1 [[CMP12]], label [[LAND_RHS:%.*]], label [[LAND_END:%.*]]
+// CHECK1:       land.rhs:
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP15]], [[TMP16]]
+// CHECK1-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[ADD13]], 4
+// CHECK1-NEXT:    br label [[LAND_END]]
+// CHECK1:       land.end:
+// CHECK1-NEXT:    [[TMP17:%.*]] = phi i1 [ false, [[FOR_COND9]] ], [ [[CMP14]], [[LAND_RHS]] ]
+// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP10]], [[TMP17]]
+// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY15:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body15:
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP18]], [[TMP19]]
+// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[MUL17:%.*]] = mul nsw i32 [[TMP20]], 3
+// CHECK1-NEXT:    [[ADD18:%.*]] = add nsw i32 7, [[MUL17]]
+// CHECK1-NEXT:    store i32 [[ADD18]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP21]], i32 noundef [[TMP22]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP20]], 1
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP23]], 1
 // CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND9]], !llvm.loop [[LOOP13:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC17:%.*]]
-// CHECK1:       for.inc17:
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC18:%.*]] = add nsw i32 [[TMP21]], 1
-// CHECK1-NEXT:    store i32 [[INC18]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_INC19:%.*]]
+// CHECK1:       for.inc19:
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC20:%.*]] = add nsw i32 [[TMP24]], 1
+// CHECK1-NEXT:    store i32 [[INC20]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND3]], !llvm.loop [[LOOP14:![0-9]+]]
-// CHECK1:       for.end19:
-// CHECK1-NEXT:    br label [[FOR_INC20:%.*]]
-// CHECK1:       for.inc20:
-// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD21:%.*]] = add nsw i32 [[TMP22]], 5
-// CHECK1-NEXT:    store i32 [[ADD21]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1:       for.end21:
+// CHECK1-NEXT:    br label [[FOR_INC22:%.*]]
+// CHECK1:       for.inc22:
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP25]], 5
+// CHECK1-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP15:![0-9]+]]
-// CHECK1:       for.end22:
+// CHECK1:       for.end24:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP23]], 1
-// CHECK1-NEXT:    store i32 [[ADD23]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[ADD25:%.*]] = add nsw i32 [[TMP26]], 1
+// CHECK1-NEXT:    store i32 [[ADD25]], ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -602,14 +596,14 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP9]], 4
-// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY:%.*]], label [[FOR_END27:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY:%.*]], label [[FOR_END29:%.*]]
 // CHECK1:       for.body:
 // CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND8:%.*]]
 // CHECK1:       for.cond8:
 // CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP10]], 5
-// CHECK1-NEXT:    br i1 [[CMP9]], label [[FOR_BODY10:%.*]], label [[FOR_END24:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP9]], label [[FOR_BODY10:%.*]], label [[FOR_END26:%.*]]
 // CHECK1:       for.body10:
 // CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
@@ -624,57 +618,61 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond14:
 // CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
 // CHECK1-NEXT:    [[CMP15:%.*]] = icmp slt i32 [[TMP14]], 5
-// CHECK1-NEXT:    br i1 [[CMP15]], label [[FOR_BODY16:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body16:
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD17:%.*]] = add nsw i32 [[TMP15]], [[TMP16]]
-// CHECK1-NEXT:    store i32 [[ADD17]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[MUL18:%.*]] = mul nsw i32 [[TMP17]], 3
-// CHECK1-NEXT:    [[ADD19:%.*]] = add nsw i32 7, [[MUL18]]
-// CHECK1-NEXT:    store i32 [[ADD19]], ptr [[J]], align 4
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP20:%.*]] = icmp slt i32 [[TMP18]], 4
-// CHECK1-NEXT:    br i1 [[CMP20]], label [[LAND_LHS_TRUE:%.*]], label [[IF_END:%.*]]
-// CHECK1:       land.lhs.true:
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[CMP21:%.*]] = icmp slt i32 [[TMP19]], 4
-// CHECK1-NEXT:    br i1 [[CMP21]], label [[IF_THEN:%.*]], label [[IF_END]]
-// CHECK1:       if.then:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[J]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP20]], i32 noundef [[TMP21]])
-// CHECK1-NEXT:    br label [[IF_END]]
-// CHECK1:       if.end:
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP15]], [[TMP16]]
+// CHECK1-NEXT:    [[CMP17:%.*]] = icmp slt i32 [[ADD16]], 4
+// CHECK1-NEXT:    br i1 [[CMP17]], label [[LAND_RHS:%.*]], label [[LAND_END:%.*]]
+// CHECK1:       land.rhs:
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD18:%.*]] = add nsw i32 [[TMP17]], [[TMP18]]
+// CHECK1-NEXT:    [[CMP19:%.*]] = icmp slt i32 [[ADD18]], 4
+// CHECK1-NEXT:    br label [[LAND_END]]
+// CHECK1:       land.end:
+// CHECK1-NEXT:    [[TMP19:%.*]] = phi i1 [ false, [[FOR_COND14]] ], [ [[CMP19]], [[LAND_RHS]] ]
+// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP15]], [[TMP19]]
+// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY20:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body20:
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD21:%.*]] = add nsw i32 [[TMP20]], [[TMP21]]
+// CHECK1-NEXT:    store i32 [[ADD21]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[MUL22:%.*]] = mul nsw i32 [[TMP22]], 3
+// CHECK1-NEXT:    [[ADD23:%.*]] = add nsw i32 7, [[MUL22]]
+// CHECK1-NEXT:    store i32 [[ADD23]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP23]], i32 noundef [[TMP24]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP22]], 1
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP25]], 1
 // CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND14]], !llvm.loop [[LOOP16:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC22:%.*]]
-// CHECK1:       for.inc22:
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC23:%.*]] = add nsw i32 [[TMP23]], 1
-// CHECK1-NEXT:    store i32 [[INC23]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_INC24:%.*]]
+// CHECK1:       for.inc24:
+// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC25:%.*]] = add nsw i32 [[TMP26]], 1
+// CHECK1-NEXT:    store i32 [[INC25]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP17:![0-9]+]]
-// CHECK1:       for.end24:
-// CHECK1-NEXT:    br label [[FOR_INC25:%.*]]
-// CHECK1:       for.inc25:
-// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD26:%.*]] = add nsw i32 [[TMP24]], 5
-// CHECK1-NEXT:    store i32 [[ADD26]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1:       for.end26:
+// CHECK1-NEXT:    br label [[FOR_INC27:%.*]]
+// CHECK1:       for.inc27:
+// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD28:%.*]] = add nsw i32 [[TMP27]], 5
+// CHECK1-NEXT:    store i32 [[ADD28]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP18:![0-9]+]]
-// CHECK1:       for.end27:
+// CHECK1:       for.end29:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[ADD28:%.*]] = add nsw i32 [[TMP25]], 1
-// CHECK1-NEXT:    store i32 [[ADD28]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[ADD30:%.*]] = add nsw i32 [[TMP28]], 1
+// CHECK1-NEXT:    store i32 [[ADD30]], ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -690,6 +688,7 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[_TMP1:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[_TMP2:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
@@ -702,12 +701,12 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK1-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
-// CHECK1-NEXT:    store i32 4, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    store i32 19, ptr [[DOTOMP_UB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
 // CHECK1-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// CHECK1-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], 19
 // CHECK1-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK1:       cond.true:
 // CHECK1-NEXT:    br label [[COND_END:%.*]]
@@ -715,7 +714,7 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
 // CHECK1-NEXT:    br label [[COND_END]]
 // CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[TMP2]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 19, [[COND_TRUE]] ], [ [[TMP2]], [[COND_FALSE]] ]
 // CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
 // CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 [[TMP3]], ptr [[DOTOMP_IV]], align 4
@@ -723,60 +722,65 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       omp.inner.for.cond:
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// CHECK1-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], [[TMP5]]
-// CHECK1-NEXT:    br i1 [[CMP2]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK1-NEXT:    [[CMP3:%.*]] = icmp sle i32 [[TMP4]], [[TMP5]]
+// CHECK1-NEXT:    br i1 [[CMP3]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
 // CHECK1:       omp.inner.for.body:
 // CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP6]], 5
+// CHECK1-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP6]], 20
 // CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[DIV]], 5
 // CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[DIV3:%.*]] = sdiv i32 [[TMP8]], 5
-// CHECK1-NEXT:    [[MUL4:%.*]] = mul nsw i32 [[DIV3]], 5
-// CHECK1-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP7]], [[MUL4]]
-// CHECK1-NEXT:    [[MUL5:%.*]] = mul nsw i32 [[SUB]], 1
-// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL5]]
-// CHECK1-NEXT:    store i32 [[ADD6]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
-// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL8:%.*]] = mul nsw i32 [[TMP11]], 3
-// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL8]]
-// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i32 [[TMP12]], 4
-// CHECK1-NEXT:    br i1 [[CMP10]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK1:       if.then:
-// CHECK1-NEXT:    store i32 7, ptr [[J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND:%.*]]
-// CHECK1:       for.cond:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
-// CHECK1-NEXT:    [[CMP11:%.*]] = icmp slt i32 [[TMP13]], 17
-// CHECK1-NEXT:    br i1 [[CMP11]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[J]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP14]], i32 noundef [[TMP15]])
-// CHECK1-NEXT:    br label [[FOR_INC:%.*]]
-// CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[J]], align 4
-// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP16]], 3
-// CHECK1-NEXT:    store i32 [[ADD12]], ptr [[J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP19:![0-9]+]]
-// CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[IF_END]]
-// CHECK1:       if.end:
+// CHECK1-NEXT:    [[DIV4:%.*]] = sdiv i32 [[TMP8]], 20
+// CHECK1-NEXT:    [[MUL5:%.*]] = mul nsw i32 [[DIV4]], 20
+// CHECK1-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP7]], [[MUL5]]
+// CHECK1-NEXT:    [[DIV6:%.*]] = sdiv i32 [[SUB]], 4
+// CHECK1-NEXT:    [[MUL7:%.*]] = mul nsw i32 [[DIV6]], 1
+// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i32 0, [[MUL7]]
+// CHECK1-NEXT:    store i32 [[ADD8]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[DIV9:%.*]] = sdiv i32 [[TMP10]], 20
+// CHECK1-NEXT:    [[MUL10:%.*]] = mul nsw i32 [[DIV9]], 20
+// CHECK1-NEXT:    [[SUB11:%.*]] = sub nsw i32 [[TMP9]], [[MUL10]]
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[DIV12:%.*]] = sdiv i32 [[TMP12]], 20
+// CHECK1-NEXT:    [[MUL13:%.*]] = mul nsw i32 [[DIV12]], 20
+// CHECK1-NEXT:    [[SUB14:%.*]] = sub nsw i32 [[TMP11]], [[MUL13]]
+// CHECK1-NEXT:    [[DIV15:%.*]] = sdiv i32 [[SUB14]], 4
+// CHECK1-NEXT:    [[MUL16:%.*]] = mul nsw i32 [[DIV15]], 4
+// CHECK1-NEXT:    [[SUB17:%.*]] = sub nsw i32 [[SUB11]], [[MUL16]]
+// CHECK1-NEXT:    [[MUL18:%.*]] = mul nsw i32 [[SUB17]], 3
+// CHECK1-NEXT:    [[ADD19:%.*]] = add nsw i32 7, [[MUL18]]
+// CHECK1-NEXT:    store i32 [[ADD19]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP13]], [[TMP14]]
+// CHECK1-NEXT:    store i32 [[ADD20]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL21:%.*]] = mul nsw i32 [[TMP15]], 3
+// CHECK1-NEXT:    [[ADD22:%.*]] = add nsw i32 7, [[MUL21]]
+// CHECK1-NEXT:    store i32 [[ADD22]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP16]], [[TMP17]]
+// CHECK1-NEXT:    [[CMP24:%.*]] = icmp slt i32 [[ADD23]], 4
+// CHECK1-NEXT:    br i1 [[CMP24]], label [[OMP_TILE_PRED_THEN:%.*]], label [[OMP_TILE_PRED_END:%.*]]
+// CHECK1:       omp_tile.pred.then:
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP18]], i32 noundef [[TMP19]])
+// CHECK1-NEXT:    br label [[OMP_TILE_PRED_END]]
+// CHECK1:       omp_tile.pred.end:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP17]], 1
-// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[ADD25:%.*]] = add nsw i32 [[TMP20]], 1
+// CHECK1-NEXT:    store i32 [[ADD25]], ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -847,38 +851,37 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP8]], 5
-// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body:
 // CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
-// CHECK1-NEXT:    store i32 [[ADD3]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL4:%.*]] = mul nsw i32 [[TMP11]], 3
-// CHECK1-NEXT:    [[ADD5:%.*]] = add nsw i32 7, [[MUL4]]
-// CHECK1-NEXT:    store i32 [[ADD5]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP6:%.*]] = icmp slt i32 [[TMP12]], 4
-// CHECK1-NEXT:    br i1 [[CMP6]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK1:       if.then:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP13]])
-// CHECK1-NEXT:    br label [[IF_END]]
-// CHECK1:       if.end:
+// CHECK1-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[ADD3]], 4
+// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP2]], [[CMP4]]
+// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body:
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD5:%.*]] = add nsw i32 [[TMP11]], [[TMP12]]
+// CHECK1-NEXT:    store i32 [[ADD5]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL6:%.*]] = mul nsw i32 [[TMP13]], 3
+// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL6]]
+// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP14]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP15]], 1
 // CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP20:![0-9]+]]
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP19:![0-9]+]]
 // CHECK1:       for.end:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 [[TMP15]], 1
-// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP16]], 1
+// CHECK1-NEXT:    store i32 [[ADD8]], ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -903,8 +906,9 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[START_ADDR]], align 4
@@ -928,53 +932,47 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK1-NEXT:    [[ADD5:%.*]] = add i32 [[TMP6]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP5]], [[ADD5]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK1:       for.cond6:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[ADD7:%.*]] = add i32 [[TMP9]], 1
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD8:%.*]] = add i32 [[TMP10]], 5
-// CHECK1-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
-// CHECK1-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK1:       cond.true:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[ADD10:%.*]] = add i32 [[TMP11]], 1
-// CHECK1-NEXT:    br label [[COND_END:%.*]]
-// CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD11:%.*]] = add i32 [[TMP12]], 5
-// CHECK1-NEXT:    br label [[COND_END]]
-// CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP8]], [[COND]]
-// CHECK1-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body13:
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP7]], 5
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD8:%.*]] = add i32 [[TMP8]], [[TMP9]]
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD9:%.*]] = add i32 [[TMP10]], 1
+// CHECK1-NEXT:    [[CMP10:%.*]] = icmp ult i32 [[ADD8]], [[ADD9]]
+// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP7]], [[CMP10]]
+// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY11:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body11:
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD12:%.*]] = add i32 [[TMP11]], [[TMP12]]
+// CHECK1-NEXT:    store i32 [[ADD12]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
 // CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP14]], 3
-// CHECK1-NEXT:    [[ADD14:%.*]] = add i32 [[TMP13]], [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[ADD13:%.*]] = add i32 [[TMP13]], [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[I]], align 4
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP15]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP16]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP23:![0-9]+]]
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP22:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC15:%.*]]
-// CHECK1:       for.inc15:
+// CHECK1-NEXT:    br label [[FOR_INC14:%.*]]
+// CHECK1:       for.inc14:
 // CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD16:%.*]] = add i32 [[TMP17]], 5
-// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP24:![0-9]+]]
-// CHECK1:       for.end17:
+// CHECK1-NEXT:    [[ADD15:%.*]] = add i32 [[TMP17]], 5
+// CHECK1-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP23:![0-9]+]]
+// CHECK1:       for.end16:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -983,8 +981,9 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
 // CHECK1-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
@@ -992,78 +991,64 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END15:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK1:       for.cond1:
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], 0
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP2]], 0
 // CHECK1-NEXT:    br i1 [[CMP2]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK1:       cond.true:
 // CHECK1-NEXT:    br label [[COND_END:%.*]]
 // CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[A_ADDR]], align 4
 // CHECK1-NEXT:    br label [[COND_END]]
 // CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP5]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[COND]]
-// CHECK1-NEXT:    [[CMP3:%.*]] = icmp slt i32 4, [[ADD]]
-// CHECK1-NEXT:    br i1 [[CMP3]], label [[COND_TRUE4:%.*]], label [[COND_FALSE5:%.*]]
-// CHECK1:       cond.true4:
-// CHECK1-NEXT:    br label [[COND_END12:%.*]]
-// CHECK1:       cond.false5:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP3]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[TMP1]], [[COND]]
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
+// CHECK1-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[ADD]], 4
+// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP3]], [[CMP4]]
+// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY5:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body5:
 // CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    [[CMP6:%.*]] = icmp sle i32 [[TMP7]], 0
-// CHECK1-NEXT:    br i1 [[CMP6]], label [[COND_TRUE7:%.*]], label [[COND_FALSE8:%.*]]
-// CHECK1:       cond.true7:
-// CHECK1-NEXT:    br label [[COND_END9:%.*]]
-// CHECK1:       cond.false8:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    br label [[COND_END9]]
-// CHECK1:       cond.end9:
-// CHECK1-NEXT:    [[COND10:%.*]] = phi i32 [ 1, [[COND_TRUE7]] ], [ [[TMP8]], [[COND_FALSE8]] ]
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP6]], [[COND10]]
-// CHECK1-NEXT:    br label [[COND_END12]]
-// CHECK1:       cond.end12:
-// CHECK1-NEXT:    [[COND13:%.*]] = phi i32 [ 4, [[COND_TRUE4]] ], [ [[ADD11]], [[COND_END9]] ]
-// CHECK1-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP2]], [[COND13]]
-// CHECK1-NEXT:    br i1 [[CMP14]], label [[FOR_BODY15:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body15:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP9]], 3
-// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP10]])
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP6]], [[TMP7]]
+// CHECK1-NEXT:    store i32 [[ADD6]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP8]], 3
+// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP9]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP25:![0-9]+]]
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP24:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC17:%.*]]
-// CHECK1:       for.inc17:
+// CHECK1-NEXT:    br label [[FOR_INC8:%.*]]
+// CHECK1:       for.inc8:
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    [[CMP9:%.*]] = icmp sle i32 [[TMP11]], 0
+// CHECK1-NEXT:    br i1 [[CMP9]], label [[COND_TRUE10:%.*]], label [[COND_FALSE11:%.*]]
+// CHECK1:       cond.true10:
+// CHECK1-NEXT:    br label [[COND_END12:%.*]]
+// CHECK1:       cond.false11:
 // CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    [[CMP18:%.*]] = icmp sle i32 [[TMP12]], 0
-// CHECK1-NEXT:    br i1 [[CMP18]], label [[COND_TRUE19:%.*]], label [[COND_FALSE20:%.*]]
-// CHECK1:       cond.true19:
-// CHECK1-NEXT:    br label [[COND_END21:%.*]]
-// CHECK1:       cond.false20:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    br label [[COND_END21]]
-// CHECK1:       cond.end21:
-// CHECK1-NEXT:    [[COND22:%.*]] = phi i32 [ 1, [[COND_TRUE19]] ], [ [[TMP13]], [[COND_FALSE20]] ]
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP14]], [[COND22]]
-// CHECK1-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP26:![0-9]+]]
-// CHECK1:       for.end24:
+// CHECK1-NEXT:    br label [[COND_END12]]
+// CHECK1:       cond.end12:
+// CHECK1-NEXT:    [[COND13:%.*]] = phi i32 [ 1, [[COND_TRUE10]] ], [ [[TMP12]], [[COND_FALSE11]] ]
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD14:%.*]] = add nsw i32 [[TMP13]], [[COND13]]
+// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP25:![0-9]+]]
+// CHECK1:       for.end15:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -1076,19 +1061,20 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK1-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
 // CHECK1-NEXT:    store ptr [[ARRAY]], ptr [[__RANGE2]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META27:![0-9]+]]
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META26:![0-9]+]]
 // CHECK1-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP0]], i64 0, i64 0
 // CHECK1-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 12
 // CHECK1-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META27]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META26]]
 // CHECK1-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP1]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[ARRAYDECAY1]], ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META27]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META26]]
 // CHECK1-NEXT:    [[ARRAYDECAY2:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP2]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[ARRAYDECAY2]], ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__END2]], align 8
@@ -1111,37 +1097,31 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK1:       for.cond7:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
-// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
-// CHECK1-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK1:       cond.true:
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
-// CHECK1-NEXT:    br label [[COND_END:%.*]]
-// CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
-// CHECK1-NEXT:    br label [[COND_END]]
-// CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
-// CHECK1-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body14:
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK1-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP11]], 1
+// CHECK1-NEXT:    [[CMP11:%.*]] = icmp slt i64 [[ADD9]], [[ADD10]]
+// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP8]], [[CMP11]]
+// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY12:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body12:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i64 [[TMP12]], [[TMP13]]
+// CHECK1-NEXT:    store i64 [[ADD13]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
-// CHECK1-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
-// CHECK1-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD_PTR14:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
+// CHECK1-NEXT:    store ptr [[ADD_PTR14]], ptr [[__BEGIN2]], align 8
 // CHECK1-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
 // CHECK1-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
 // CHECK1-NEXT:    store double [[TMP17]], ptr [[V]], align 8
@@ -1149,18 +1129,18 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    call void (...) @body(double noundef [[TMP18]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP19]], 1
-// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP28:![0-9]+]]
+// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP27:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC16:%.*]]
-// CHECK1:       for.inc16:
+// CHECK1-NEXT:    br label [[FOR_INC15:%.*]]
+// CHECK1:       for.inc15:
 // CHECK1-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP20]], 5
-// CHECK1-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP29:![0-9]+]]
-// CHECK1:       for.end18:
+// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i64 [[TMP20]], 5
+// CHECK1-NEXT:    store i64 [[ADD16]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP28:![0-9]+]]
+// CHECK1:       for.end17:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -1174,20 +1154,21 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK1-NEXT:    store double 4.200000e+01, ptr [[C]], align 8
 // CHECK1-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
 // CHECK1-NEXT:    store ptr [[ARRAY]], ptr [[__RANGE2]], align 8
-// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META27]]
+// CHECK1-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META26]]
 // CHECK1-NEXT:    [[ARRAYDECAY:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP0]], i64 0, i64 0
 // CHECK1-NEXT:    [[ADD_PTR:%.*]] = getelementptr inbounds double, ptr [[ARRAYDECAY]], i64 12
 // CHECK1-NEXT:    store ptr [[ADD_PTR]], ptr [[__END2]], align 8
-// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META27]]
+// CHECK1-NEXT:    [[TMP1:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META26]]
 // CHECK1-NEXT:    [[ARRAYDECAY1:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP1]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[ARRAYDECAY1]], ptr [[__BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META27]]
+// CHECK1-NEXT:    [[TMP2:%.*]] = load ptr, ptr [[__RANGE2]], align 8, !nonnull [[META2]], !align [[META26]]
 // CHECK1-NEXT:    [[ARRAYDECAY2:%.*]] = getelementptr inbounds [12 x double], ptr [[TMP2]], i64 0, i64 0
 // CHECK1-NEXT:    store ptr [[ARRAYDECAY2]], ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK1-NEXT:    [[TMP3:%.*]] = load ptr, ptr [[__END2]], align 8
@@ -1210,37 +1191,31 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK1:       for.cond7:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
-// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
-// CHECK1-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK1:       cond.true:
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
-// CHECK1-NEXT:    br label [[COND_END:%.*]]
-// CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
-// CHECK1-NEXT:    br label [[COND_END]]
-// CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
-// CHECK1-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body14:
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK1-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP11]], 1
+// CHECK1-NEXT:    [[CMP11:%.*]] = icmp slt i64 [[ADD9]], [[ADD10]]
+// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP8]], [[CMP11]]
+// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY12:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body12:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i64 [[TMP12]], [[TMP13]]
+// CHECK1-NEXT:    store i64 [[ADD13]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
-// CHECK1-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
-// CHECK1-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD_PTR14:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
+// CHECK1-NEXT:    store ptr [[ADD_PTR14]], ptr [[__BEGIN2]], align 8
 // CHECK1-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
 // CHECK1-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
 // CHECK1-NEXT:    store double [[TMP17]], ptr [[V]], align 8
@@ -1249,18 +1224,18 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    call void (...) @body(double noundef [[TMP18]], double noundef [[TMP19]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP20]], 1
-// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP30:![0-9]+]]
+// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP29:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC16:%.*]]
-// CHECK1:       for.inc16:
+// CHECK1-NEXT:    br label [[FOR_INC15:%.*]]
+// CHECK1:       for.inc15:
 // CHECK1-NEXT:    [[TMP21:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP21]], 5
-// CHECK1-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP31:![0-9]+]]
-// CHECK1:       for.end18:
+// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i64 [[TMP21]], 5
+// CHECK1-NEXT:    store i64 [[ADD16]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP30:![0-9]+]]
+// CHECK1:       for.end17:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -1293,8 +1268,9 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:  entry:
 // CHECK2-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[I2:%.*]] = alloca ptr, align 8
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK2-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK2-NEXT:    [[I:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
@@ -1308,45 +1284,41 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
 // CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END12:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK2:       for.cond4:
-// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 5
-// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
-// CHECK2-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK2:       cond.true:
-// CHECK2-NEXT:    br label [[COND_END:%.*]]
-// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP1]], 5
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
+// CHECK2-NEXT:    [[CMP6:%.*]] = icmp slt i32 [[ADD]], 4
+// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP5]], [[CMP6]]
+// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY7:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body7:
 // CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP4]], 5
-// CHECK2-NEXT:    br label [[COND_END]]
-// CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
-// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body8:
-// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
+// CHECK2-NEXT:    store i32 [[ADD8]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
 // CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK2-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
-// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[TMP6]], align 4
-// CHECK2-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2]], !align [[META3]]
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP8]])
+// CHECK2-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
+// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[TMP7]], align 4
+// CHECK2-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2]], !align [[META3]]
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP9]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP9]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP4:![0-9]+]]
 // CHECK2:       for.end:
 // CHECK2-NEXT:    br label [[FOR_INC10:%.*]]
 // CHECK2:       for.inc10:
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP10]], 5
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP11]], 5
 // CHECK2-NEXT:    store i32 [[ADD11]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
 // CHECK2:       for.end12:
@@ -1370,8 +1342,9 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -1400,54 +1373,48 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK2-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP8]], [[ADD5]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 [[TMP10]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK2:       for.cond6:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[ADD7:%.*]] = add i32 [[TMP12]], 1
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD8:%.*]] = add i32 [[TMP13]], 5
-// CHECK2-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
-// CHECK2-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK2:       cond.true:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[ADD10:%.*]] = add i32 [[TMP14]], 1
-// CHECK2-NEXT:    br label [[COND_END:%.*]]
-// CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD11:%.*]] = add i32 [[TMP15]], 5
-// CHECK2-NEXT:    br label [[COND_END]]
-// CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP11]], [[COND]]
-// CHECK2-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body13:
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP10]], 5
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD8:%.*]] = add i32 [[TMP11]], [[TMP12]]
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD9:%.*]] = add i32 [[TMP13]], 1
+// CHECK2-NEXT:    [[CMP10:%.*]] = icmp ult i32 [[ADD8]], [[ADD9]]
+// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP7]], [[CMP10]]
+// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY11:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body11:
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD12:%.*]] = add i32 [[TMP14]], [[TMP15]]
+// CHECK2-NEXT:    store i32 [[ADD12]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
 // CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
 // CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP17]], [[TMP18]]
-// CHECK2-NEXT:    [[ADD14:%.*]] = add i32 [[TMP16]], [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[ADD13:%.*]] = add i32 [[TMP16]], [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD13]], ptr [[I]], align 4
 // CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
 // CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP19]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP20]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP7:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC15:%.*]]
-// CHECK2:       for.inc15:
+// CHECK2-NEXT:    br label [[FOR_INC14:%.*]]
+// CHECK2:       for.inc14:
 // CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD16:%.*]] = add i32 [[TMP21]], 5
-// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD15:%.*]] = add i32 [[TMP21]], 5
+// CHECK2-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
-// CHECK2:       for.end17:
+// CHECK2:       for.end16:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -1461,8 +1428,9 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK2-NEXT:    store double 4.200000e+01, ptr [[C]], align 8
 // CHECK2-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
@@ -1497,37 +1465,31 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK2:       for.cond7:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
-// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
-// CHECK2-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK2:       cond.true:
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
-// CHECK2-NEXT:    br label [[COND_END:%.*]]
-// CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
-// CHECK2-NEXT:    br label [[COND_END]]
-// CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
-// CHECK2-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body14:
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK2-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP11]], 1
+// CHECK2-NEXT:    [[CMP11:%.*]] = icmp slt i64 [[ADD9]], [[ADD10]]
+// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP8]], [[CMP11]]
+// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY12:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body12:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i64 [[TMP12]], [[TMP13]]
+// CHECK2-NEXT:    store i64 [[ADD13]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
-// CHECK2-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
-// CHECK2-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD_PTR14:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
+// CHECK2-NEXT:    store ptr [[ADD_PTR14]], ptr [[__BEGIN2]], align 8
 // CHECK2-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
 // CHECK2-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
 // CHECK2-NEXT:    store double [[TMP17]], ptr [[V]], align 8
@@ -1536,18 +1498,18 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    call void (...) @body(double noundef [[TMP18]], double noundef [[TMP19]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP20]], 1
-// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP10:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC16:%.*]]
-// CHECK2:       for.inc16:
+// CHECK2-NEXT:    br label [[FOR_INC15:%.*]]
+// CHECK2:       for.inc15:
 // CHECK2-NEXT:    [[TMP21:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP21]], 5
-// CHECK2-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i64 [[TMP21]], 5
+// CHECK2-NEXT:    store i64 [[ADD16]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP11:![0-9]+]]
-// CHECK2:       for.end18:
+// CHECK2:       for.end17:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -1558,11 +1520,13 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -1573,94 +1537,90 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END30:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END26:%.*]]
 // CHECK2:       for.body:
 // CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK2:       for.cond1:
 // CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 4
-// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END27:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END23:%.*]]
 // CHECK2:       for.body3:
-// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 [[TMP2]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK2:       for.cond4:
-// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], 5
-// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
-// CHECK2-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK2:       cond.true:
-// CHECK2-NEXT:    br label [[COND_END:%.*]]
-// CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP5]], 5
-// CHECK2-NEXT:    br label [[COND_END]]
-// CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP3]], [[COND]]
-// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END24:%.*]]
-// CHECK2:       for.body8:
-// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
-// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND10:%.*]]
-// CHECK2:       for.cond10:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP2]], 5
+// CHECK2-NEXT:    br i1 [[CMP5]], label [[FOR_BODY6:%.*]], label [[FOR_END20:%.*]]
+// CHECK2:       for.body6:
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[TMP4]]
+// CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
+// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND8:%.*]]
+// CHECK2:       for.cond8:
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP6]], 5
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP7]], [[TMP8]]
+// CHECK2-NEXT:    [[CMP11:%.*]] = icmp slt i32 [[ADD10]], 4
+// CHECK2-NEXT:    br i1 [[CMP11]], label [[LAND_RHS:%.*]], label [[LAND_END:%.*]]
+// CHECK2:       land.rhs:
 // CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP9]], 5
-// CHECK2-NEXT:    [[CMP12:%.*]] = icmp slt i32 4, [[ADD11]]
-// CHECK2-NEXT:    br i1 [[CMP12]], label [[COND_TRUE13:%.*]], label [[COND_FALSE14:%.*]]
-// CHECK2:       cond.true13:
-// CHECK2-NEXT:    br label [[COND_END16:%.*]]
-// CHECK2:       cond.false14:
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP10]], 5
-// CHECK2-NEXT:    br label [[COND_END16]]
-// CHECK2:       cond.end16:
-// CHECK2-NEXT:    [[COND17:%.*]] = phi i32 [ 4, [[COND_TRUE13]] ], [ [[ADD15]], [[COND_FALSE14]] ]
-// CHECK2-NEXT:    [[CMP18:%.*]] = icmp slt i32 [[TMP8]], [[COND17]]
-// CHECK2-NEXT:    br i1 [[CMP18]], label [[FOR_BODY19:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body19:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[TMP11]], 3
-// CHECK2-NEXT:    [[ADD21:%.*]] = add nsw i32 7, [[MUL20]]
-// CHECK2-NEXT:    store i32 [[ADD21]], ptr [[J]], align 4
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP12]], i32 noundef [[TMP13]])
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
+// CHECK2-NEXT:    [[CMP13:%.*]] = icmp slt i32 [[ADD12]], 4
+// CHECK2-NEXT:    br label [[LAND_END]]
+// CHECK2:       land.end:
+// CHECK2-NEXT:    [[TMP11:%.*]] = phi i1 [ false, [[FOR_COND8]] ], [ [[CMP13]], [[LAND_RHS]] ]
+// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP9]], [[TMP11]]
+// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body14:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP12]], [[TMP13]]
+// CHECK2-NEXT:    store i32 [[ADD15]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[MUL16:%.*]] = mul nsw i32 [[TMP14]], 3
+// CHECK2-NEXT:    [[ADD17:%.*]] = add nsw i32 7, [[MUL16]]
+// CHECK2-NEXT:    store i32 [[ADD17]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP15]], i32 noundef [[TMP16]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND10]], !llvm.loop [[LOOP12:![0-9]+]]
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP17]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP12:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC22:%.*]]
-// CHECK2:       for.inc22:
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC23:%.*]] = add nsw i32 [[TMP15]], 1
-// CHECK2-NEXT:    store i32 [[INC23]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_INC18:%.*]]
+// CHECK2:       for.inc18:
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC19:%.*]] = add nsw i32 [[TMP18]], 1
+// CHECK2-NEXT:    store i32 [[INC19]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP13:![0-9]+]]
-// CHECK2:       for.end24:
-// CHECK2-NEXT:    br label [[FOR_INC25:%.*]]
-// CHECK2:       for.inc25:
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD26:%.*]] = add nsw i32 [[TMP16]], 5
-// CHECK2-NEXT:    store i32 [[ADD26]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2:       for.end20:
+// CHECK2-NEXT:    br label [[FOR_INC21:%.*]]
+// CHECK2:       for.inc21:
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD22:%.*]] = add nsw i32 [[TMP19]], 5
+// CHECK2-NEXT:    store i32 [[ADD22]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP14:![0-9]+]]
-// CHECK2:       for.end27:
-// CHECK2-NEXT:    br label [[FOR_INC28:%.*]]
-// CHECK2:       for.inc28:
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD29:%.*]] = add nsw i32 [[TMP17]], 5
-// CHECK2-NEXT:    store i32 [[ADD29]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2:       for.end23:
+// CHECK2-NEXT:    br label [[FOR_INC24:%.*]]
+// CHECK2:       for.inc24:
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD25:%.*]] = add nsw i32 [[TMP20]], 5
+// CHECK2-NEXT:    store i32 [[ADD25]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP15:![0-9]+]]
-// CHECK2:       for.end30:
+// CHECK2:       for.end26:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -1718,14 +1678,14 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP7]], 4
-// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END22:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
 // CHECK2:       for.body:
 // CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND3:%.*]]
 // CHECK2:       for.cond3:
 // CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP8]], 5
-// CHECK2-NEXT:    br i1 [[CMP4]], label [[FOR_BODY5:%.*]], label [[FOR_END19:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP4]], label [[FOR_BODY5:%.*]], label [[FOR_END21:%.*]]
 // CHECK2:       for.body5:
 // CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
@@ -1740,57 +1700,61 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond9:
 // CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
 // CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i32 [[TMP12]], 5
-// CHECK2-NEXT:    br i1 [[CMP10]], label [[FOR_BODY11:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body11:
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP13]], [[TMP14]]
-// CHECK2-NEXT:    store i32 [[ADD12]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[MUL13:%.*]] = mul nsw i32 [[TMP15]], 3
-// CHECK2-NEXT:    [[ADD14:%.*]] = add nsw i32 7, [[MUL13]]
-// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[J]], align 4
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP15:%.*]] = icmp slt i32 [[TMP16]], 4
-// CHECK2-NEXT:    br i1 [[CMP15]], label [[LAND_LHS_TRUE:%.*]], label [[IF_END:%.*]]
-// CHECK2:       land.lhs.true:
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[CMP16:%.*]] = icmp slt i32 [[TMP17]], 4
-// CHECK2-NEXT:    br i1 [[CMP16]], label [[IF_THEN:%.*]], label [[IF_END]]
-// CHECK2:       if.then:
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[J]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP18]], i32 noundef [[TMP19]])
-// CHECK2-NEXT:    br label [[IF_END]]
-// CHECK2:       if.end:
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP13]], [[TMP14]]
+// CHECK2-NEXT:    [[CMP12:%.*]] = icmp slt i32 [[ADD11]], 4
+// CHECK2-NEXT:    br i1 [[CMP12]], label [[LAND_RHS:%.*]], label [[LAND_END:%.*]]
+// CHECK2:       land.rhs:
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP15]], [[TMP16]]
+// CHECK2-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[ADD13]], 4
+// CHECK2-NEXT:    br label [[LAND_END]]
+// CHECK2:       land.end:
+// CHECK2-NEXT:    [[TMP17:%.*]] = phi i1 [ false, [[FOR_COND9]] ], [ [[CMP14]], [[LAND_RHS]] ]
+// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP10]], [[TMP17]]
+// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY15:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body15:
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP18]], [[TMP19]]
+// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[MUL17:%.*]] = mul nsw i32 [[TMP20]], 3
+// CHECK2-NEXT:    [[ADD18:%.*]] = add nsw i32 7, [[MUL17]]
+// CHECK2-NEXT:    store i32 [[ADD18]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP21]], i32 noundef [[TMP22]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP20]], 1
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP23]], 1
 // CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND9]], !llvm.loop [[LOOP16:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC17:%.*]]
-// CHECK2:       for.inc17:
-// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC18:%.*]] = add nsw i32 [[TMP21]], 1
-// CHECK2-NEXT:    store i32 [[INC18]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_INC19:%.*]]
+// CHECK2:       for.inc19:
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC20:%.*]] = add nsw i32 [[TMP24]], 1
+// CHECK2-NEXT:    store i32 [[INC20]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND3]], !llvm.loop [[LOOP17:![0-9]+]]
-// CHECK2:       for.end19:
-// CHECK2-NEXT:    br label [[FOR_INC20:%.*]]
-// CHECK2:       for.inc20:
-// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD21:%.*]] = add nsw i32 [[TMP22]], 5
-// CHECK2-NEXT:    store i32 [[ADD21]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2:       for.end21:
+// CHECK2-NEXT:    br label [[FOR_INC22:%.*]]
+// CHECK2:       for.inc22:
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP25]], 5
+// CHECK2-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP18:![0-9]+]]
-// CHECK2:       for.end22:
+// CHECK2:       for.end24:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
-// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP23]], 1
-// CHECK2-NEXT:    store i32 [[ADD23]], ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[ADD25:%.*]] = add nsw i32 [[TMP26]], 1
+// CHECK2-NEXT:    store i32 [[ADD25]], ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -1865,14 +1829,14 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP9]], 4
-// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY:%.*]], label [[FOR_END27:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY:%.*]], label [[FOR_END29:%.*]]
 // CHECK2:       for.body:
 // CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND8:%.*]]
 // CHECK2:       for.cond8:
 // CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP10]], 5
-// CHECK2-NEXT:    br i1 [[CMP9]], label [[FOR_BODY10:%.*]], label [[FOR_END24:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP9]], label [[FOR_BODY10:%.*]], label [[FOR_END26:%.*]]
 // CHECK2:       for.body10:
 // CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
@@ -1887,57 +1851,61 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond14:
 // CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
 // CHECK2-NEXT:    [[CMP15:%.*]] = icmp slt i32 [[TMP14]], 5
-// CHECK2-NEXT:    br i1 [[CMP15]], label [[FOR_BODY16:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body16:
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD17:%.*]] = add nsw i32 [[TMP15]], [[TMP16]]
-// CHECK2-NEXT:    store i32 [[ADD17]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[MUL18:%.*]] = mul nsw i32 [[TMP17]], 3
-// CHECK2-NEXT:    [[ADD19:%.*]] = add nsw i32 7, [[MUL18]]
-// CHECK2-NEXT:    store i32 [[ADD19]], ptr [[J]], align 4
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP20:%.*]] = icmp slt i32 [[TMP18]], 4
-// CHECK2-NEXT:    br i1 [[CMP20]], label [[LAND_LHS_TRUE:%.*]], label [[IF_END:%.*]]
-// CHECK2:       land.lhs.true:
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[CMP21:%.*]] = icmp slt i32 [[TMP19]], 4
-// CHECK2-NEXT:    br i1 [[CMP21]], label [[IF_THEN:%.*]], label [[IF_END]]
-// CHECK2:       if.then:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[J]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP20]], i32 noundef [[TMP21]])
-// CHECK2-NEXT:    br label [[IF_END]]
-// CHECK2:       if.end:
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP15]], [[TMP16]]
+// CHECK2-NEXT:    [[CMP17:%.*]] = icmp slt i32 [[ADD16]], 4
+// CHECK2-NEXT:    br i1 [[CMP17]], label [[LAND_RHS:%.*]], label [[LAND_END:%.*]]
+// CHECK2:       land.rhs:
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD18:%.*]] = add nsw i32 [[TMP17]], [[TMP18]]
+// CHECK2-NEXT:    [[CMP19:%.*]] = icmp slt i32 [[ADD18]], 4
+// CHECK2-NEXT:    br label [[LAND_END]]
+// CHECK2:       land.end:
+// CHECK2-NEXT:    [[TMP19:%.*]] = phi i1 [ false, [[FOR_COND14]] ], [ [[CMP19]], [[LAND_RHS]] ]
+// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP15]], [[TMP19]]
+// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY20:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body20:
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD21:%.*]] = add nsw i32 [[TMP20]], [[TMP21]]
+// CHECK2-NEXT:    store i32 [[ADD21]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[MUL22:%.*]] = mul nsw i32 [[TMP22]], 3
+// CHECK2-NEXT:    [[ADD23:%.*]] = add nsw i32 7, [[MUL22]]
+// CHECK2-NEXT:    store i32 [[ADD23]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP23]], i32 noundef [[TMP24]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP22]], 1
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP25]], 1
 // CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND14]], !llvm.loop [[LOOP19:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC22:%.*]]
-// CHECK2:       for.inc22:
-// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC23:%.*]] = add nsw i32 [[TMP23]], 1
-// CHECK2-NEXT:    store i32 [[INC23]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_INC24:%.*]]
+// CHECK2:       for.inc24:
+// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC25:%.*]] = add nsw i32 [[TMP26]], 1
+// CHECK2-NEXT:    store i32 [[INC25]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP20:![0-9]+]]
-// CHECK2:       for.end24:
-// CHECK2-NEXT:    br label [[FOR_INC25:%.*]]
-// CHECK2:       for.inc25:
-// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD26:%.*]] = add nsw i32 [[TMP24]], 5
-// CHECK2-NEXT:    store i32 [[ADD26]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2:       for.end26:
+// CHECK2-NEXT:    br label [[FOR_INC27:%.*]]
+// CHECK2:       for.inc27:
+// CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD28:%.*]] = add nsw i32 [[TMP27]], 5
+// CHECK2-NEXT:    store i32 [[ADD28]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP21:![0-9]+]]
-// CHECK2:       for.end27:
+// CHECK2:       for.end29:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
-// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[ADD28:%.*]] = add nsw i32 [[TMP25]], 1
-// CHECK2-NEXT:    store i32 [[ADD28]], ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[ADD30:%.*]] = add nsw i32 [[TMP28]], 1
+// CHECK2-NEXT:    store i32 [[ADD30]], ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -1953,6 +1921,7 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[_TMP1:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[_TMP2:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
@@ -1965,12 +1934,12 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK2-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
-// CHECK2-NEXT:    store i32 4, ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    store i32 19, ptr [[DOTOMP_UB]], align 4
 // CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
 // CHECK2-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
 // CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// CHECK2-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], 19
 // CHECK2-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK2:       cond.true:
 // CHECK2-NEXT:    br label [[COND_END:%.*]]
@@ -1978,7 +1947,7 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
 // CHECK2-NEXT:    br label [[COND_END]]
 // CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[TMP2]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 19, [[COND_TRUE]] ], [ [[TMP2]], [[COND_FALSE]] ]
 // CHECK2-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
 // CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
 // CHECK2-NEXT:    store i32 [[TMP3]], ptr [[DOTOMP_IV]], align 4
@@ -1986,60 +1955,65 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       omp.inner.for.cond:
 // CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// CHECK2-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], [[TMP5]]
-// CHECK2-NEXT:    br i1 [[CMP2]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK2-NEXT:    [[CMP3:%.*]] = icmp sle i32 [[TMP4]], [[TMP5]]
+// CHECK2-NEXT:    br i1 [[CMP3]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
 // CHECK2:       omp.inner.for.body:
 // CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP6]], 5
+// CHECK2-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP6]], 20
 // CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[DIV]], 5
 // CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[DIV3:%.*]] = sdiv i32 [[TMP8]], 5
-// CHECK2-NEXT:    [[MUL4:%.*]] = mul nsw i32 [[DIV3]], 5
-// CHECK2-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP7]], [[MUL4]]
-// CHECK2-NEXT:    [[MUL5:%.*]] = mul nsw i32 [[SUB]], 1
-// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL5]]
-// CHECK2-NEXT:    store i32 [[ADD6]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
-// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL8:%.*]] = mul nsw i32 [[TMP11]], 3
-// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL8]]
-// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i32 [[TMP12]], 4
-// CHECK2-NEXT:    br i1 [[CMP10]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK2:       if.then:
-// CHECK2-NEXT:    store i32 7, ptr [[J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND:%.*]]
-// CHECK2:       for.cond:
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
-// CHECK2-NEXT:    [[CMP11:%.*]] = icmp slt i32 [[TMP13]], 17
-// CHECK2-NEXT:    br i1 [[CMP11]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[J]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP14]], i32 noundef [[TMP15]])
-// CHECK2-NEXT:    br label [[FOR_INC:%.*]]
-// CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[J]], align 4
-// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP16]], 3
-// CHECK2-NEXT:    store i32 [[ADD12]], ptr [[J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP22:![0-9]+]]
-// CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[IF_END]]
-// CHECK2:       if.end:
+// CHECK2-NEXT:    [[DIV4:%.*]] = sdiv i32 [[TMP8]], 20
+// CHECK2-NEXT:    [[MUL5:%.*]] = mul nsw i32 [[DIV4]], 20
+// CHECK2-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP7]], [[MUL5]]
+// CHECK2-NEXT:    [[DIV6:%.*]] = sdiv i32 [[SUB]], 4
+// CHECK2-NEXT:    [[MUL7:%.*]] = mul nsw i32 [[DIV6]], 1
+// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i32 0, [[MUL7]]
+// CHECK2-NEXT:    store i32 [[ADD8]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[DIV9:%.*]] = sdiv i32 [[TMP10]], 20
+// CHECK2-NEXT:    [[MUL10:%.*]] = mul nsw i32 [[DIV9]], 20
+// CHECK2-NEXT:    [[SUB11:%.*]] = sub nsw i32 [[TMP9]], [[MUL10]]
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[DIV12:%.*]] = sdiv i32 [[TMP12]], 20
+// CHECK2-NEXT:    [[MUL13:%.*]] = mul nsw i32 [[DIV12]], 20
+// CHECK2-NEXT:    [[SUB14:%.*]] = sub nsw i32 [[TMP11]], [[MUL13]]
+// CHECK2-NEXT:    [[DIV15:%.*]] = sdiv i32 [[SUB14]], 4
+// CHECK2-NEXT:    [[MUL16:%.*]] = mul nsw i32 [[DIV15]], 4
+// CHECK2-NEXT:    [[SUB17:%.*]] = sub nsw i32 [[SUB11]], [[MUL16]]
+// CHECK2-NEXT:    [[MUL18:%.*]] = mul nsw i32 [[SUB17]], 3
+// CHECK2-NEXT:    [[ADD19:%.*]] = add nsw i32 7, [[MUL18]]
+// CHECK2-NEXT:    store i32 [[ADD19]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP13]], [[TMP14]]
+// CHECK2-NEXT:    store i32 [[ADD20]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL21:%.*]] = mul nsw i32 [[TMP15]], 3
+// CHECK2-NEXT:    [[ADD22:%.*]] = add nsw i32 7, [[MUL21]]
+// CHECK2-NEXT:    store i32 [[ADD22]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP16]], [[TMP17]]
+// CHECK2-NEXT:    [[CMP24:%.*]] = icmp slt i32 [[ADD23]], 4
+// CHECK2-NEXT:    br i1 [[CMP24]], label [[OMP_TILE_PRED_THEN:%.*]], label [[OMP_TILE_PRED_END:%.*]]
+// CHECK2:       omp_tile.pred.then:
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP18]], i32 noundef [[TMP19]])
+// CHECK2-NEXT:    br label [[OMP_TILE_PRED_END]]
+// CHECK2:       omp_tile.pred.end:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP17]], 1
-// CHECK2-NEXT:    store i32 [[ADD13]], ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[ADD25:%.*]] = add nsw i32 [[TMP20]], 1
+// CHECK2-NEXT:    store i32 [[ADD25]], ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -2110,38 +2084,37 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP8]], 5
-// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body:
 // CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
-// CHECK2-NEXT:    store i32 [[ADD3]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL4:%.*]] = mul nsw i32 [[TMP11]], 3
-// CHECK2-NEXT:    [[ADD5:%.*]] = add nsw i32 7, [[MUL4]]
-// CHECK2-NEXT:    store i32 [[ADD5]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP6:%.*]] = icmp slt i32 [[TMP12]], 4
-// CHECK2-NEXT:    br i1 [[CMP6]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK2:       if.then:
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP13]])
-// CHECK2-NEXT:    br label [[IF_END]]
-// CHECK2:       if.end:
+// CHECK2-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[ADD3]], 4
+// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP2]], [[CMP4]]
+// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body:
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD5:%.*]] = add nsw i32 [[TMP11]], [[TMP12]]
+// CHECK2-NEXT:    store i32 [[ADD5]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL6:%.*]] = mul nsw i32 [[TMP13]], 3
+// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL6]]
+// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP14]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP15]], 1
 // CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP23:![0-9]+]]
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP22:![0-9]+]]
 // CHECK2:       for.end:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 [[TMP15]], 1
-// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP16]], 1
+// CHECK2-NEXT:    store i32 [[ADD8]], ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -2155,8 +2128,9 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:  entry:
 // CHECK2-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
 // CHECK2-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
@@ -2164,78 +2138,64 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END15:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK2:       for.cond1:
-// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], 0
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP2]], 0
 // CHECK2-NEXT:    br i1 [[CMP2]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK2:       cond.true:
 // CHECK2-NEXT:    br label [[COND_END:%.*]]
 // CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[A_ADDR]], align 4
 // CHECK2-NEXT:    br label [[COND_END]]
 // CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP5]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[COND]]
-// CHECK2-NEXT:    [[CMP3:%.*]] = icmp slt i32 4, [[ADD]]
-// CHECK2-NEXT:    br i1 [[CMP3]], label [[COND_TRUE4:%.*]], label [[COND_FALSE5:%.*]]
-// CHECK2:       cond.true4:
-// CHECK2-NEXT:    br label [[COND_END12:%.*]]
-// CHECK2:       cond.false5:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP3]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[TMP1]], [[COND]]
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
+// CHECK2-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[ADD]], 4
+// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP3]], [[CMP4]]
+// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY5:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body5:
 // CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    [[CMP6:%.*]] = icmp sle i32 [[TMP7]], 0
-// CHECK2-NEXT:    br i1 [[CMP6]], label [[COND_TRUE7:%.*]], label [[COND_FALSE8:%.*]]
-// CHECK2:       cond.true7:
-// CHECK2-NEXT:    br label [[COND_END9:%.*]]
-// CHECK2:       cond.false8:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    br label [[COND_END9]]
-// CHECK2:       cond.end9:
-// CHECK2-NEXT:    [[COND10:%.*]] = phi i32 [ 1, [[COND_TRUE7]] ], [ [[TMP8]], [[COND_FALSE8]] ]
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP6]], [[COND10]]
-// CHECK2-NEXT:    br label [[COND_END12]]
-// CHECK2:       cond.end12:
-// CHECK2-NEXT:    [[COND13:%.*]] = phi i32 [ 4, [[COND_TRUE4]] ], [ [[ADD11]], [[COND_END9]] ]
-// CHECK2-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP2]], [[COND13]]
-// CHECK2-NEXT:    br i1 [[CMP14]], label [[FOR_BODY15:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body15:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP9]], 3
-// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP10]])
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP6]], [[TMP7]]
+// CHECK2-NEXT:    store i32 [[ADD6]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP8]], 3
+// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP9]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP26:![0-9]+]]
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP25:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC17:%.*]]
-// CHECK2:       for.inc17:
+// CHECK2-NEXT:    br label [[FOR_INC8:%.*]]
+// CHECK2:       for.inc8:
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    [[CMP9:%.*]] = icmp sle i32 [[TMP11]], 0
+// CHECK2-NEXT:    br i1 [[CMP9]], label [[COND_TRUE10:%.*]], label [[COND_FALSE11:%.*]]
+// CHECK2:       cond.true10:
+// CHECK2-NEXT:    br label [[COND_END12:%.*]]
+// CHECK2:       cond.false11:
 // CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    [[CMP18:%.*]] = icmp sle i32 [[TMP12]], 0
-// CHECK2-NEXT:    br i1 [[CMP18]], label [[COND_TRUE19:%.*]], label [[COND_FALSE20:%.*]]
-// CHECK2:       cond.true19:
-// CHECK2-NEXT:    br label [[COND_END21:%.*]]
-// CHECK2:       cond.false20:
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    br label [[COND_END21]]
-// CHECK2:       cond.end21:
-// CHECK2-NEXT:    [[COND22:%.*]] = phi i32 [ 1, [[COND_TRUE19]] ], [ [[TMP13]], [[COND_FALSE20]] ]
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP14]], [[COND22]]
-// CHECK2-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP27:![0-9]+]]
-// CHECK2:       for.end24:
+// CHECK2-NEXT:    br label [[COND_END12]]
+// CHECK2:       cond.end12:
+// CHECK2-NEXT:    [[COND13:%.*]] = phi i32 [ 1, [[COND_TRUE10]] ], [ [[TMP12]], [[COND_FALSE11]] ]
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD14:%.*]] = add nsw i32 [[TMP13]], [[COND13]]
+// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP26:![0-9]+]]
+// CHECK2:       for.end15:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -2248,8 +2208,9 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK2-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
 // CHECK2-NEXT:    store ptr [[ARRAY]], ptr [[__RANGE2]], align 8
@@ -2283,37 +2244,31 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK2:       for.cond7:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
-// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
-// CHECK2-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK2:       cond.true:
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
-// CHECK2-NEXT:    br label [[COND_END:%.*]]
-// CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
-// CHECK2-NEXT:    br label [[COND_END]]
-// CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
-// CHECK2-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body14:
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK2-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP11]], 1
+// CHECK2-NEXT:    [[CMP11:%.*]] = icmp slt i64 [[ADD9]], [[ADD10]]
+// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP8]], [[CMP11]]
+// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY12:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body12:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i64 [[TMP12]], [[TMP13]]
+// CHECK2-NEXT:    store i64 [[ADD13]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
-// CHECK2-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
-// CHECK2-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD_PTR14:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
+// CHECK2-NEXT:    store ptr [[ADD_PTR14]], ptr [[__BEGIN2]], align 8
 // CHECK2-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
 // CHECK2-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
 // CHECK2-NEXT:    store double [[TMP17]], ptr [[V]], align 8
@@ -2321,18 +2276,18 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    call void (...) @body(double noundef [[TMP18]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP19]], 1
-// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP28:![0-9]+]]
+// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP27:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC16:%.*]]
-// CHECK2:       for.inc16:
+// CHECK2-NEXT:    br label [[FOR_INC15:%.*]]
+// CHECK2:       for.inc15:
 // CHECK2-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP20]], 5
-// CHECK2-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP29:![0-9]+]]
-// CHECK2:       for.end18:
+// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i64 [[TMP20]], 5
+// CHECK2-NEXT:    store i64 [[ADD16]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP28:![0-9]+]]
+// CHECK2:       for.end17:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -2352,8 +2307,9 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[START_ADDR]], align 4
@@ -2377,53 +2333,47 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK2-NEXT:    [[ADD5:%.*]] = add i32 [[TMP6]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP5]], [[ADD5]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK2:       for.cond6:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[ADD7:%.*]] = add i32 [[TMP9]], 1
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD8:%.*]] = add i32 [[TMP10]], 5
-// CHECK2-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
-// CHECK2-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
-// CHECK2:       cond.true:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[ADD10:%.*]] = add i32 [[TMP11]], 1
-// CHECK2-NEXT:    br label [[COND_END:%.*]]
-// CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD11:%.*]] = add i32 [[TMP12]], 5
-// CHECK2-NEXT:    br label [[COND_END]]
-// CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP8]], [[COND]]
-// CHECK2-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body13:
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP7]], 5
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD8:%.*]] = add i32 [[TMP8]], [[TMP9]]
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD9:%.*]] = add i32 [[TMP10]], 1
+// CHECK2-NEXT:    [[CMP10:%.*]] = icmp ult i32 [[ADD8]], [[ADD9]]
+// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP7]], [[CMP10]]
+// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY11:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body11:
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD12:%.*]] = add i32 [[TMP11]], [[TMP12]]
+// CHECK2-NEXT:    store i32 [[ADD12]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
 // CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP14]], 3
-// CHECK2-NEXT:    [[ADD14:%.*]] = add i32 [[TMP13]], [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[ADD13:%.*]] = add i32 [[TMP13]], [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD13]], ptr [[I]], align 4
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
 // CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP15]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP16]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP30:![0-9]+]]
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP29:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC15:%.*]]
-// CHECK2:       for.inc15:
+// CHECK2-NEXT:    br label [[FOR_INC14:%.*]]
+// CHECK2:       for.inc14:
 // CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD16:%.*]] = add i32 [[TMP17]], 5
-// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP31:![0-9]+]]
-// CHECK2:       for.end17:
+// CHECK2-NEXT:    [[ADD15:%.*]] = add i32 [[TMP17]], 5
+// CHECK2-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP30:![0-9]+]]
+// CHECK2:       for.end16:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -2452,16 +2402,15 @@ extern "C" void foo10(data_t data) {
 // CHECK1: [[LOOP17]] = distinct !{[[LOOP17]], [[META5]]}
 // CHECK1: [[LOOP18]] = distinct !{[[LOOP18]], [[META5]]}
 // CHECK1: [[LOOP19]] = distinct !{[[LOOP19]], [[META5]]}
-// CHECK1: [[LOOP20]] = distinct !{[[LOOP20]], [[META5]]}
+// CHECK1: [[LOOP22]] = distinct !{[[LOOP22]], [[META5]]}
 // CHECK1: [[LOOP23]] = distinct !{[[LOOP23]], [[META5]]}
 // CHECK1: [[LOOP24]] = distinct !{[[LOOP24]], [[META5]]}
 // CHECK1: [[LOOP25]] = distinct !{[[LOOP25]], [[META5]]}
-// CHECK1: [[LOOP26]] = distinct !{[[LOOP26]], [[META5]]}
-// CHECK1: [[META27]] = !{i64 8}
+// CHECK1: [[META26]] = !{i64 8}
+// CHECK1: [[LOOP27]] = distinct !{[[LOOP27]], [[META5]]}
 // CHECK1: [[LOOP28]] = distinct !{[[LOOP28]], [[META5]]}
 // CHECK1: [[LOOP29]] = distinct !{[[LOOP29]], [[META5]]}
 // CHECK1: [[LOOP30]] = distinct !{[[LOOP30]], [[META5]]}
-// CHECK1: [[LOOP31]] = distinct !{[[LOOP31]], [[META5]]}
 //.
 // CHECK2: [[META2]] = !{}
 // CHECK2: [[META3]] = !{i64 4}
@@ -2484,11 +2433,10 @@ extern "C" void foo10(data_t data) {
 // CHECK2: [[LOOP20]] = distinct !{[[LOOP20]], [[META5]]}
 // CHECK2: [[LOOP21]] = distinct !{[[LOOP21]], [[META5]]}
 // CHECK2: [[LOOP22]] = distinct !{[[LOOP22]], [[META5]]}
-// CHECK2: [[LOOP23]] = distinct !{[[LOOP23]], [[META5]]}
+// CHECK2: [[LOOP25]] = distinct !{[[LOOP25]], [[META5]]}
 // CHECK2: [[LOOP26]] = distinct !{[[LOOP26]], [[META5]]}
 // CHECK2: [[LOOP27]] = distinct !{[[LOOP27]], [[META5]]}
 // CHECK2: [[LOOP28]] = distinct !{[[LOOP28]], [[META5]]}
 // CHECK2: [[LOOP29]] = distinct !{[[LOOP29]], [[META5]]}
 // CHECK2: [[LOOP30]] = distinct !{[[LOOP30]], [[META5]]}
-// CHECK2: [[LOOP31]] = distinct !{[[LOOP31]], [[META5]]}
 //.
diff --git a/clang/test/OpenMP/tile_codegen_for_dependent.cpp b/clang/test/OpenMP/tile_codegen_for_dependent.cpp
index aadc8e58bebfc..808f0e71dc3b0 100644
--- a/clang/test/OpenMP/tile_codegen_for_dependent.cpp
+++ b/clang/test/OpenMP/tile_codegen_for_dependent.cpp
@@ -114,35 +114,34 @@ extern "C" void body(...) {}
 // IR:       [[FOR_COND]]:
 // IR-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    [[CMP16:%.*]] = icmp ult i32 [[TMP21]], 4
-// IR-NEXT:    br i1 [[CMP16]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY]]:
 // IR-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I11]], align 4
 // IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    [[ADD17:%.*]] = add i32 [[TMP22]], [[TMP23]]
-// IR-NEXT:    store i32 [[ADD17]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[MUL18:%.*]] = mul i32 [[TMP25]], [[TMP26]]
-// IR-NEXT:    [[ADD19:%.*]] = add i32 [[TMP24]], [[MUL18]]
-// IR-NEXT:    store i32 [[ADD19]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[ADD20:%.*]] = add i32 [[TMP28]], 1
-// IR-NEXT:    [[CMP21:%.*]] = icmp ult i32 [[TMP27]], [[ADD20]]
-// IR-NEXT:    br i1 [[CMP21]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
-// IR:       [[IF_THEN]]:
-// IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void (...) @body(i32 noundef [[TMP29]], i32 noundef [[TMP30]], i32 noundef [[TMP31]], i32 noundef [[TMP32]])
-// IR-NEXT:    br label %[[IF_END]]
-// IR:       [[IF_END]]:
+// IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    [[ADD18:%.*]] = add i32 [[TMP24]], 1
+// IR-NEXT:    [[CMP19:%.*]] = icmp ult i32 [[ADD17]], [[ADD18]]
+// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP16]], [[CMP19]]
+// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY]]:
+// IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I11]], align 4
+// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD20:%.*]] = add i32 [[TMP25]], [[TMP26]]
+// IR-NEXT:    store i32 [[ADD20]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[MUL21:%.*]] = mul i32 [[TMP28]], [[TMP29]]
+// IR-NEXT:    [[ADD22:%.*]] = add i32 [[TMP27]], [[MUL21]]
+// IR-NEXT:    store i32 [[ADD22]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void (...) @body(i32 noundef [[TMP30]], i32 noundef [[TMP31]], i32 noundef [[TMP32]], i32 noundef [[TMP33]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add i32 [[TMP33]], 1
+// IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add i32 [[TMP34]], 1
 // IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP2:![0-9]+]]
 // IR:       [[FOR_END]]:
@@ -150,9 +149,9 @@ extern "C" void body(...) {}
 // IR:       [[OMP_BODY_CONTINUE]]:
 // IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.*]]
 // IR:       [[OMP_INNER_FOR_INC]]:
-// IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[ADD22:%.*]] = add i32 [[TMP34]], 1
-// IR-NEXT:    store i32 [[ADD22]], ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[ADD23:%.*]] = add i32 [[TMP35]], 1
+// IR-NEXT:    store i32 [[ADD23]], ptr [[DOTOMP_IV]], align 4
 // IR-NEXT:    br label %[[OMP_INNER_FOR_COND]]
 // IR:       [[OMP_INNER_FOR_END]]:
 // IR-NEXT:    br label %[[OMP_LOOP_EXIT:.*]]
diff --git a/clang/test/OpenMP/tile_codegen_tile_for.cpp b/clang/test/OpenMP/tile_codegen_tile_for.cpp
index 8153f76b7e1a9..9ef96834fcfc0 100644
--- a/clang/test/OpenMP/tile_codegen_tile_for.cpp
+++ b/clang/test/OpenMP/tile_codegen_tile_for.cpp
@@ -129,75 +129,73 @@ extern "C" void body(...) {}
 // IR:       [[FOR_COND]]:
 // IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
 // IR-NEXT:    [[CMP22:%.*]] = icmp ult i32 [[TMP23]], 3
-// IR-NEXT:    br i1 [[CMP22]], label %[[FOR_BODY:.*]], label %[[FOR_END40:.*]]
-// IR:       [[FOR_BODY]]:
 // IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
 // IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
 // IR-NEXT:    [[ADD23:%.*]] = add i32 [[TMP24]], [[TMP25]]
-// IR-NEXT:    store i32 [[ADD23]], ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[MUL24:%.*]] = mul i32 [[TMP26]], 4
-// IR-NEXT:    [[ADD25:%.*]] = add i32 0, [[MUL24]]
-// IR-NEXT:    store i32 [[ADD25]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    [[ADD26:%.*]] = add i32 [[TMP28]], 1
-// IR-NEXT:    [[CMP27:%.*]] = icmp ult i32 [[TMP27]], [[ADD26]]
-// IR-NEXT:    br i1 [[CMP27]], label %[[IF_THEN:.*]], label %[[IF_END37:.*]]
-// IR:       [[IF_THEN]]:
+// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    [[ADD24:%.*]] = add i32 [[TMP26]], 1
+// IR-NEXT:    [[CMP25:%.*]] = icmp ult i32 [[ADD23]], [[ADD24]]
+// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP22]], [[CMP25]]
+// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label %[[FOR_BODY:.*]], label %[[FOR_END41:.*]]
+// IR:       [[FOR_BODY]]:
+// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
+// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD26:%.*]] = add i32 [[TMP27]], [[TMP28]]
+// IR-NEXT:    store i32 [[ADD26]], ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[MUL27:%.*]] = mul i32 [[TMP29]], 4
+// IR-NEXT:    [[ADD28:%.*]] = add i32 0, [[MUL27]]
+// IR-NEXT:    store i32 [[ADD28]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND28:.*]]
-// IR:       [[FOR_COND28]]:
-// IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[CMP29:%.*]] = icmp ult i32 [[TMP29]], 4
-// IR-NEXT:    br i1 [[CMP29]], label %[[FOR_BODY30:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY30]]:
-// IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD31:%.*]] = add i32 [[TMP30]], [[TMP31]]
-// IR-NEXT:    store i32 [[ADD31]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[MUL32:%.*]] = mul i32 [[TMP33]], [[TMP34]]
-// IR-NEXT:    [[ADD33:%.*]] = add i32 [[TMP32]], [[MUL32]]
-// IR-NEXT:    store i32 [[ADD33]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[ADD34:%.*]] = add i32 [[TMP36]], 1
-// IR-NEXT:    [[CMP35:%.*]] = icmp ult i32 [[TMP35]], [[ADD34]]
-// IR-NEXT:    br i1 [[CMP35]], label %[[IF_THEN36:.*]], label %[[IF_END:.*]]
-// IR:       [[IF_THEN36]]:
-// IR-NEXT:    [[TMP37:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// IR-NEXT:    [[TMP38:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// IR-NEXT:    [[TMP39:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// IR-NEXT:    [[TMP40:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void (...) @body(i32 noundef [[TMP37]], i32 noundef [[TMP38]], i32 noundef [[TMP39]], i32 noundef [[TMP40]])
-// IR-NEXT:    br label %[[IF_END]]
-// IR:       [[IF_END]]:
+// IR-NEXT:    br label %[[FOR_COND29:.*]]
+// IR:       [[FOR_COND29]]:
+// IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[CMP30:%.*]] = icmp ult i32 [[TMP30]], 4
+// IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD31:%.*]] = add i32 [[TMP31]], [[TMP32]]
+// IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    [[ADD32:%.*]] = add i32 [[TMP33]], 1
+// IR-NEXT:    [[CMP33:%.*]] = icmp ult i32 [[ADD31]], [[ADD32]]
+// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND34:%.*]] = and i1 [[CMP30]], [[CMP33]]
+// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND34]], label %[[FOR_BODY35:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY35]]:
+// IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD36:%.*]] = add i32 [[TMP34]], [[TMP35]]
+// IR-NEXT:    store i32 [[ADD36]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[MUL37:%.*]] = mul i32 [[TMP37]], [[TMP38]]
+// IR-NEXT:    [[ADD38:%.*]] = add i32 [[TMP36]], [[MUL37]]
+// IR-NEXT:    store i32 [[ADD38]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP39:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// IR-NEXT:    [[TMP40:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// IR-NEXT:    [[TMP41:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// IR-NEXT:    [[TMP42:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void (...) @body(i32 noundef [[TMP39]], i32 noundef [[TMP40]], i32 noundef [[TMP41]], i32 noundef [[TMP42]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP41:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add i32 [[TMP41]], 1
+// IR-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add i32 [[TMP43]], 1
 // IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND28]], !llvm.loop [[LOOP2:![0-9]+]]
+// IR-NEXT:    br label %[[FOR_COND29]], !llvm.loop [[LOOP2:![0-9]+]]
 // IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[IF_END37]]
-// IR:       [[IF_END37]]:
-// IR-NEXT:    br label %[[FOR_INC38:.*]]
-// IR:       [[FOR_INC38]]:
-// IR-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[INC39:%.*]] = add i32 [[TMP42]], 1
-// IR-NEXT:    store i32 [[INC39]], ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_INC39:.*]]
+// IR:       [[FOR_INC39]]:
+// IR-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[INC40:%.*]] = add i32 [[TMP44]], 1
+// IR-NEXT:    store i32 [[INC40]], ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP4:![0-9]+]]
-// IR:       [[FOR_END40]]:
+// IR:       [[FOR_END41]]:
 // IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.*]]
 // IR:       [[OMP_BODY_CONTINUE]]:
 // IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.*]]
 // IR:       [[OMP_INNER_FOR_INC]]:
-// IR-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[ADD41:%.*]] = add i32 [[TMP43]], 1
-// IR-NEXT:    store i32 [[ADD41]], ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[ADD42:%.*]] = add i32 [[TMP45]], 1
+// IR-NEXT:    store i32 [[ADD42]], ptr [[DOTOMP_IV]], align 4
 // IR-NEXT:    br label %[[OMP_INNER_FOR_COND]]
 // IR:       [[OMP_INNER_FOR_END]]:
 // IR-NEXT:    br label %[[OMP_LOOP_EXIT:.*]]
diff --git a/clang/test/OpenMP/tile_messages.cpp b/clang/test/OpenMP/tile_messages.cpp
index 8915d1bb3c546..e1f5155c924e5 100644
--- a/clang/test/OpenMP/tile_messages.cpp
+++ b/clang/test/OpenMP/tile_messages.cpp
@@ -101,7 +101,7 @@ void func() {
     for (int j = 0; j < i; ++j)
       ;
 
-  // expected-error@* {{expected 3 for loops after '#pragma omp for', but found only 2}}
+  // expected-error at +5 {{expected 3 for loops after '#pragma omp for', but found only 2}}
   // expected-note at +1 {{as specified in 'collapse' clause}}
   #pragma omp for collapse(3)
   #pragma omp tile sizes(5)
diff --git a/clang/test/OpenMP/tile_rect_codegen_ir.cpp b/clang/test/OpenMP/tile_rect_codegen_ir.cpp
index 808f1cf0a3d0c..f2b64ced7c46b 100644
--- a/clang/test/OpenMP/tile_rect_codegen_ir.cpp
+++ b/clang/test/OpenMP/tile_rect_codegen_ir.cpp
@@ -1,3 +1,4 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
 // Regression tests for rectangular #pragma omp tile lowering:
 // - Inner loop counter .tile.cnt.* with bound == tile size (constant).
 // - Validity guard (icmp + branch) when remainder iterations exist or trip count
@@ -13,8 +14,9 @@ extern "C" void body(int);
 // IR-SAME: ) #[[ATTR0:[0-9]+]] {
 // IR-NEXT:  [[ENTRY:.*:]]
 // IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    store i32 0, ptr [[I]], align 4
 // IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND:.*]]
@@ -23,43 +25,39 @@ extern "C" void body(int);
 // IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 6
 // IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END9:.*]]
 // IR:       [[FOR_BODY]]:
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1:.*]]
 // IR:       [[FOR_COND1]]:
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 4
-// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 6, [[ADD]]
-// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
-// IR:       [[COND_TRUE]]:
-// IR-NEXT:    br label %[[COND_END:.*]]
-// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 4
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
+// IR-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[ADD]], 6
+// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP2]], [[CMP3]]
+// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label %[[FOR_BODY4:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY4]]:
 // IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 4
-// IR-NEXT:    br label %[[COND_END]]
-// IR:       [[COND_END]]:
-// IR-NEXT:    [[COND:%.*]] = phi i32 [ 6, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
-// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
-// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY5]]:
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD5:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
+// IR-NEXT:    store i32 [[ADD5]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 1
 // IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
 // IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void @body(i32 noundef [[TMP6]])
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void @body(i32 noundef [[TMP7]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP7]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP8]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP2:![0-9]+]]
 // IR:       [[FOR_END]]:
 // IR-NEXT:    br label %[[FOR_INC7:.*]]
 // IR:       [[FOR_INC7]]:
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP8]], 4
+// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP9]], 4
 // IR-NEXT:    store i32 [[ADD8]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP4:![0-9]+]]
 // IR:       [[FOR_END9]]:
@@ -77,56 +75,48 @@ void remainder_6_tile_4(void) {
 // IR-SAME: ) #[[ATTR0]] {
 // IR-NEXT:  [[ENTRY:.*:]]
 // IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    store i32 0, ptr [[I]], align 4
 // IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND:.*]]
 // IR:       [[FOR_COND]]:
 // IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 10
-// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END9:.*]]
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END7:.*]]
 // IR:       [[FOR_BODY]]:
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1:.*]]
 // IR:       [[FOR_COND1]]:
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 5
-// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 10, [[ADD]]
-// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
-// IR:       [[COND_TRUE]]:
-// IR-NEXT:    br label %[[COND_END:.*]]
-// IR:       [[COND_FALSE]]:
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 5
-// IR-NEXT:    br label %[[COND_END]]
-// IR:       [[COND_END]]:
-// IR-NEXT:    [[COND:%.*]] = phi i32 [ 10, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
-// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
-// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY5]]:
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
-// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void @body(i32 noundef [[TMP6]])
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 5
+// IR-NEXT:    br i1 [[CMP2]], label %[[FOR_BODY3:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY3]]:
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
+// IR-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP4]], 1
+// IR-NEXT:    [[ADD4:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD4]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void @body(i32 noundef [[TMP5]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP7]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP6]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP5:![0-9]+]]
 // IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC7:.*]]
-// IR:       [[FOR_INC7]]:
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP8]], 5
-// IR-NEXT:    store i32 [[ADD8]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_INC5:.*]]
+// IR:       [[FOR_INC5]]:
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP7]], 5
+// IR-NEXT:    store i32 [[ADD6]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
-// IR:       [[FOR_END9]]:
+// IR:       [[FOR_END7]]:
 // IR-NEXT:    ret void
 //
 void full_tiles_10_5(void) {
@@ -141,8 +131,9 @@ void full_tiles_10_5(void) {
 // IR-NEXT:  [[ENTRY:.*:]]
 // IR-NEXT:    [[TS_ADDR:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    store i32 [[TS]], ptr [[TS_ADDR]], align 4
 // IR-NEXT:    store i32 0, ptr [[I]], align 4
 // IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
@@ -150,78 +141,64 @@ void full_tiles_10_5(void) {
 // IR:       [[FOR_COND]]:
 // IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 12
-// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END24:.*]]
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END15:.*]]
 // IR:       [[FOR_BODY]]:
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1:.*]]
 // IR:       [[FOR_COND1]]:
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TS_ADDR]], align 4
-// IR-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], 0
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TS_ADDR]], align 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP2]], 0
 // IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
 // IR:       [[COND_TRUE]]:
 // IR-NEXT:    br label %[[COND_END:.*]]
 // IR:       [[COND_FALSE]]:
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TS_ADDR]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TS_ADDR]], align 4
 // IR-NEXT:    br label %[[COND_END]]
 // IR:       [[COND_END]]:
-// IR-NEXT:    [[COND:%.*]] = phi i32 [ 1, %[[COND_TRUE]] ], [ [[TMP5]], %[[COND_FALSE]] ]
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[COND]]
-// IR-NEXT:    [[CMP3:%.*]] = icmp slt i32 12, [[ADD]]
-// IR-NEXT:    br i1 [[CMP3]], label %[[COND_TRUE4:.*]], label %[[COND_FALSE5:.*]]
-// IR:       [[COND_TRUE4]]:
-// IR-NEXT:    br label %[[COND_END12:.*]]
-// IR:       [[COND_FALSE5]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ 1, %[[COND_TRUE]] ], [ [[TMP3]], %[[COND_FALSE]] ]
+// IR-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[TMP1]], [[COND]]
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
+// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[ADD]], 12
+// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP3]], [[CMP4]]
+// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label %[[FOR_BODY5:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY5]]:
 // IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TS_ADDR]], align 4
-// IR-NEXT:    [[CMP6:%.*]] = icmp sle i32 [[TMP7]], 0
-// IR-NEXT:    br i1 [[CMP6]], label %[[COND_TRUE7:.*]], label %[[COND_FALSE8:.*]]
-// IR:       [[COND_TRUE7]]:
-// IR-NEXT:    br label %[[COND_END9:.*]]
-// IR:       [[COND_FALSE8]]:
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TS_ADDR]], align 4
-// IR-NEXT:    br label %[[COND_END9]]
-// IR:       [[COND_END9]]:
-// IR-NEXT:    [[COND10:%.*]] = phi i32 [ 1, %[[COND_TRUE7]] ], [ [[TMP8]], %[[COND_FALSE8]] ]
-// IR-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP6]], [[COND10]]
-// IR-NEXT:    br label %[[COND_END12]]
-// IR:       [[COND_END12]]:
-// IR-NEXT:    [[COND13:%.*]] = phi i32 [ 12, %[[COND_TRUE4]] ], [ [[ADD11]], %[[COND_END9]] ]
-// IR-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP2]], [[COND13]]
-// IR-NEXT:    br i1 [[CMP14]], label %[[FOR_BODY15:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY15]]:
-// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP9]], 1
-// IR-NEXT:    [[ADD16:%.*]] = add nsw i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD16]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void @body(i32 noundef [[TMP10]])
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP6]], [[TMP7]]
+// IR-NEXT:    store i32 [[ADD6]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP8]], 1
+// IR-NEXT:    [[ADD7:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void @body(i32 noundef [[TMP9]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP7:![0-9]+]]
 // IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC17:.*]]
-// IR:       [[FOR_INC17]]:
+// IR-NEXT:    br label %[[FOR_INC8:.*]]
+// IR:       [[FOR_INC8]]:
+// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TS_ADDR]], align 4
+// IR-NEXT:    [[CMP9:%.*]] = icmp sle i32 [[TMP11]], 0
+// IR-NEXT:    br i1 [[CMP9]], label %[[COND_TRUE10:.*]], label %[[COND_FALSE11:.*]]
+// IR:       [[COND_TRUE10]]:
+// IR-NEXT:    br label %[[COND_END12:.*]]
+// IR:       [[COND_FALSE11]]:
 // IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TS_ADDR]], align 4
-// IR-NEXT:    [[CMP18:%.*]] = icmp sle i32 [[TMP12]], 0
-// IR-NEXT:    br i1 [[CMP18]], label %[[COND_TRUE19:.*]], label %[[COND_FALSE20:.*]]
-// IR:       [[COND_TRUE19]]:
-// IR-NEXT:    br label %[[COND_END21:.*]]
-// IR:       [[COND_FALSE20]]:
-// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TS_ADDR]], align 4
-// IR-NEXT:    br label %[[COND_END21]]
-// IR:       [[COND_END21]]:
-// IR-NEXT:    [[COND22:%.*]] = phi i32 [ 1, %[[COND_TRUE19]] ], [ [[TMP13]], %[[COND_FALSE20]] ]
-// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP14]], [[COND22]]
-// IR-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[COND_END12]]
+// IR:       [[COND_END12]]:
+// IR-NEXT:    [[COND13:%.*]] = phi i32 [ 1, %[[COND_TRUE10]] ], [ [[TMP12]], %[[COND_FALSE11]] ]
+// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD14:%.*]] = add nsw i32 [[TMP13]], [[COND13]]
+// IR-NEXT:    store i32 [[ADD14]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
-// IR:       [[FOR_END24]]:
+// IR:       [[FOR_END15]]:
 // IR-NEXT:    ret void
 //
 void var_tile(int ts) {
@@ -235,11 +212,13 @@ void var_tile(int ts) {
 // IR-SAME: ) #[[ATTR0]] {
 // IR-NEXT:  [[ENTRY:.*:]]
 // IR-NEXT:    [[I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[J:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
 // IR-NEXT:    store i32 0, ptr [[I]], align 4
 // IR-NEXT:    store i32 0, ptr [[J]], align 4
 // IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
@@ -247,95 +226,91 @@ void var_tile(int ts) {
 // IR:       [[FOR_COND]]:
 // IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 3
-// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END31:.*]]
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END27:.*]]
 // IR:       [[FOR_BODY]]:
 // IR-NEXT:    store i32 0, ptr [[DOTFLOOR_1_IV_J]], align 4
 // IR-NEXT:    br label %[[FOR_COND1:.*]]
 // IR:       [[FOR_COND1]]:
 // IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 3
-// IR-NEXT:    br i1 [[CMP2]], label %[[FOR_BODY3:.*]], label %[[FOR_END28:.*]]
+// IR-NEXT:    br i1 [[CMP2]], label %[[FOR_BODY3:.*]], label %[[FOR_END24:.*]]
 // IR:       [[FOR_BODY3]]:
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    store i32 [[TMP2]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND4:.*]]
 // IR:       [[FOR_COND4]]:
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], 2
-// IR-NEXT:    [[CMP5:%.*]] = icmp slt i32 3, [[ADD]]
-// IR-NEXT:    br i1 [[CMP5]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
-// IR:       [[COND_TRUE]]:
-// IR-NEXT:    br label %[[COND_END:.*]]
-// IR:       [[COND_FALSE]]:
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP5]], 2
-// IR-NEXT:    br label %[[COND_END]]
-// IR:       [[COND_END]]:
-// IR-NEXT:    [[COND:%.*]] = phi i32 [ 3, %[[COND_TRUE]] ], [ [[ADD6]], %[[COND_FALSE]] ]
-// IR-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP3]], [[COND]]
-// IR-NEXT:    br i1 [[CMP7]], label %[[FOR_BODY8:.*]], label %[[FOR_END25:.*]]
-// IR:       [[FOR_BODY8]]:
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 1
-// IR-NEXT:    [[ADD9:%.*]] = add nsw i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// IR-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_1_IV_J]], align 4
-// IR-NEXT:    br label %[[FOR_COND10:.*]]
-// IR:       [[FOR_COND10]]:
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP2]], 2
+// IR-NEXT:    br i1 [[CMP5]], label %[[FOR_BODY6:.*]], label %[[FOR_END21:.*]]
+// IR:       [[FOR_BODY6]]:
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[TMP4]]
+// IR-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
+// IR-NEXT:    [[ADD7:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// IR-NEXT:    br label %[[FOR_COND8:.*]]
+// IR:       [[FOR_COND8]]:
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// IR-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP6]], 2
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP7]], [[TMP8]]
+// IR-NEXT:    [[CMP11:%.*]] = icmp slt i32 [[ADD10]], 3
+// IR-NEXT:    br i1 [[CMP11]], label %[[LAND_RHS:.*]], label %[[LAND_END:.*]]
+// IR:       [[LAND_RHS]]:
 // IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// IR-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP9]], 2
-// IR-NEXT:    [[CMP12:%.*]] = icmp slt i32 3, [[ADD11]]
-// IR-NEXT:    br i1 [[CMP12]], label %[[COND_TRUE13:.*]], label %[[COND_FALSE14:.*]]
-// IR:       [[COND_TRUE13]]:
-// IR-NEXT:    br label %[[COND_END16:.*]]
-// IR:       [[COND_FALSE14]]:
-// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// IR-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP10]], 2
-// IR-NEXT:    br label %[[COND_END16]]
-// IR:       [[COND_END16]]:
-// IR-NEXT:    [[COND17:%.*]] = phi i32 [ 3, %[[COND_TRUE13]] ], [ [[ADD15]], %[[COND_FALSE14]] ]
-// IR-NEXT:    [[CMP18:%.*]] = icmp slt i32 [[TMP8]], [[COND17]]
-// IR-NEXT:    br i1 [[CMP18]], label %[[FOR_BODY19:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY19]]:
-// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// IR-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[TMP11]], 1
-// IR-NEXT:    [[ADD21:%.*]] = add nsw i32 0, [[MUL20]]
-// IR-NEXT:    store i32 [[ADD21]], ptr [[J]], align 4
-// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
-// IR-NEXT:    [[ADD22:%.*]] = add nsw i32 [[TMP12]], [[TMP13]]
-// IR-NEXT:    call void @body(i32 noundef [[ADD22]])
+// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// IR-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
+// IR-NEXT:    [[CMP13:%.*]] = icmp slt i32 [[ADD12]], 3
+// IR-NEXT:    br label %[[LAND_END]]
+// IR:       [[LAND_END]]:
+// IR-NEXT:    [[TMP11:%.*]] = phi i1 [ false, %[[FOR_COND8]] ], [ [[CMP13]], %[[LAND_RHS]] ]
+// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP9]], [[TMP11]]
+// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label %[[FOR_BODY14:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY14]]:
+// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// IR-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP12]], [[TMP13]]
+// IR-NEXT:    store i32 [[ADD15]], ptr [[DOTTILE_1_IV_J]], align 4
+// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// IR-NEXT:    [[MUL16:%.*]] = mul nsw i32 [[TMP14]], 1
+// IR-NEXT:    [[ADD17:%.*]] = add nsw i32 0, [[MUL16]]
+// IR-NEXT:    store i32 [[ADD17]], ptr [[J]], align 4
+// IR-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    [[TMP16:%.*]] = load i32, ptr [[J]], align 4
+// IR-NEXT:    [[ADD18:%.*]] = add nsw i32 [[TMP15]], [[TMP16]]
+// IR-NEXT:    call void @body(i32 noundef [[ADD18]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
-// IR-NEXT:    br label %[[FOR_COND10]], !llvm.loop [[LOOP9:![0-9]+]]
+// IR-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP17]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
+// IR-NEXT:    br label %[[FOR_COND8]], !llvm.loop [[LOOP9:![0-9]+]]
 // IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC23:.*]]
-// IR:       [[FOR_INC23]]:
-// IR-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[INC24:%.*]] = add nsw i32 [[TMP15]], 1
-// IR-NEXT:    store i32 [[INC24]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_INC19:.*]]
+// IR:       [[FOR_INC19]]:
+// IR-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[INC20:%.*]] = add nsw i32 [[TMP18]], 1
+// IR-NEXT:    store i32 [[INC20]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND4]], !llvm.loop [[LOOP10:![0-9]+]]
-// IR:       [[FOR_END25]]:
-// IR-NEXT:    br label %[[FOR_INC26:.*]]
-// IR:       [[FOR_INC26]]:
-// IR-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// IR-NEXT:    [[ADD27:%.*]] = add nsw i32 [[TMP16]], 2
-// IR-NEXT:    store i32 [[ADD27]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// IR:       [[FOR_END21]]:
+// IR-NEXT:    br label %[[FOR_INC22:.*]]
+// IR:       [[FOR_INC22]]:
+// IR-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// IR-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP19]], 2
+// IR-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP11:![0-9]+]]
-// IR:       [[FOR_END28]]:
-// IR-NEXT:    br label %[[FOR_INC29:.*]]
-// IR:       [[FOR_INC29]]:
-// IR-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD30:%.*]] = add nsw i32 [[TMP17]], 2
-// IR-NEXT:    store i32 [[ADD30]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR:       [[FOR_END24]]:
+// IR-NEXT:    br label %[[FOR_INC25:.*]]
+// IR:       [[FOR_INC25]]:
+// IR-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD26:%.*]] = add nsw i32 [[TMP20]], 2
+// IR-NEXT:    store i32 [[ADD26]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP12:![0-9]+]]
-// IR:       [[FOR_END31]]:
+// IR:       [[FOR_END27]]:
 // IR-NEXT:    ret void
 //
 void two_dim(void) {
@@ -351,8 +326,9 @@ void two_dim(void) {
 // IR-NEXT:  [[ENTRY:.*:]]
 // IR-NEXT:    [[N_ADDR:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[J:%.*]] = alloca i32, align 4
 // IR-NEXT:    store i32 [[N]], ptr [[N_ADDR]], align 4
 // IR-NEXT:    store i32 0, ptr [[I]], align 4
@@ -363,60 +339,56 @@ void two_dim(void) {
 // IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 6
 // IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END16:.*]]
 // IR:       [[FOR_BODY]]:
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1:.*]]
 // IR:       [[FOR_COND1]]:
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 4
-// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 6, [[ADD]]
-// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
-// IR:       [[COND_TRUE]]:
-// IR-NEXT:    br label %[[COND_END:.*]]
-// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 4
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
+// IR-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[ADD]], 6
+// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP2]], [[CMP3]]
+// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label %[[FOR_BODY4:.*]], label %[[FOR_END13:.*]]
+// IR:       [[FOR_BODY4]]:
 // IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 4
-// IR-NEXT:    br label %[[COND_END]]
-// IR:       [[COND_END]]:
-// IR-NEXT:    [[COND:%.*]] = phi i32 [ 6, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
-// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
-// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END13:.*]]
-// IR:       [[FOR_BODY5]]:
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD5:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
+// IR-NEXT:    store i32 [[ADD5]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 1
 // IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
 // IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
 // IR-NEXT:    store i32 0, ptr [[J]], align 4
 // IR-NEXT:    br label %[[FOR_COND7:.*]]
 // IR:       [[FOR_COND7]]:
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[J]], align 4
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[N_ADDR]], align 4
-// IR-NEXT:    [[CMP8:%.*]] = icmp slt i32 [[TMP6]], [[TMP7]]
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[J]], align 4
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[N_ADDR]], align 4
+// IR-NEXT:    [[CMP8:%.*]] = icmp slt i32 [[TMP7]], [[TMP8]]
 // IR-NEXT:    br i1 [[CMP8]], label %[[FOR_BODY9:.*]], label %[[FOR_END:.*]]
 // IR:       [[FOR_BODY9]]:
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[J]], align 4
-// IR-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP8]], [[TMP9]]
+// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[J]], align 4
+// IR-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
 // IR-NEXT:    call void @body(i32 noundef [[ADD10]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[J]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
+// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[J]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
 // IR-NEXT:    store i32 [[INC]], ptr [[J]], align 4
 // IR-NEXT:    br label %[[FOR_COND7]], !llvm.loop [[LOOP13:![0-9]+]]
 // IR:       [[FOR_END]]:
 // IR-NEXT:    br label %[[FOR_INC11:.*]]
 // IR:       [[FOR_INC11]]:
-// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[INC12:%.*]] = add nsw i32 [[TMP11]], 1
-// IR-NEXT:    store i32 [[INC12]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[INC12:%.*]] = add nsw i32 [[TMP12]], 1
+// IR-NEXT:    store i32 [[INC12]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP14:![0-9]+]]
 // IR:       [[FOR_END13]]:
 // IR-NEXT:    br label %[[FOR_INC14:.*]]
 // IR:       [[FOR_INC14]]:
-// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP12]], 4
+// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP13]], 4
 // IR-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP15:![0-9]+]]
 // IR:       [[FOR_END16]]:
@@ -435,56 +407,48 @@ void nested_inner_j(int n) {
 // IR-SAME: ) #[[ATTR0]] {
 // IR-NEXT:  [[ENTRY:.*:]]
 // IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    store i32 0, ptr [[I]], align 4
 // IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND:.*]]
 // IR:       [[FOR_COND]]:
 // IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 12
-// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END9:.*]]
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END7:.*]]
 // IR:       [[FOR_BODY]]:
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1:.*]]
 // IR:       [[FOR_COND1]]:
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 4
-// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 12, [[ADD]]
-// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
-// IR:       [[COND_TRUE]]:
-// IR-NEXT:    br label %[[COND_END:.*]]
-// IR:       [[COND_FALSE]]:
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 4
-// IR-NEXT:    br label %[[COND_END]]
-// IR:       [[COND_END]]:
-// IR-NEXT:    [[COND:%.*]] = phi i32 [ 12, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
-// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
-// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY5]]:
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
-// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void @body(i32 noundef [[TMP6]])
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 4
+// IR-NEXT:    br i1 [[CMP2]], label %[[FOR_BODY3:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY3]]:
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
+// IR-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP4]], 1
+// IR-NEXT:    [[ADD4:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD4]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void @body(i32 noundef [[TMP5]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP7]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP6]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP16:![0-9]+]]
 // IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC7:.*]]
-// IR:       [[FOR_INC7]]:
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP8]], 4
-// IR-NEXT:    store i32 [[ADD8]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_INC5:.*]]
+// IR:       [[FOR_INC5]]:
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP7]], 4
+// IR-NEXT:    store i32 [[ADD6]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP17:![0-9]+]]
-// IR:       [[FOR_END9]]:
+// IR:       [[FOR_END7]]:
 // IR-NEXT:    ret void
 //
 void elided_predicate(void) {
diff --git a/clang/test/OpenMP/unroll_codegen_tile_for.cpp b/clang/test/OpenMP/unroll_codegen_tile_for.cpp
index 6f13f6b67c8cd..cd28ff73a11ec 100644
--- a/clang/test/OpenMP/unroll_codegen_tile_for.cpp
+++ b/clang/test/OpenMP/unroll_codegen_tile_for.cpp
@@ -123,75 +123,74 @@ extern "C" void body(...) {}
 // IR:       [[FOR_COND]]:
 // IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
 // IR-NEXT:    [[CMP22:%.*]] = icmp ult i32 [[TMP23]], 4
-// IR-NEXT:    br i1 [[CMP22]], label %[[FOR_BODY:.*]], label %[[FOR_END38:.*]]
-// IR:       [[FOR_BODY]]:
 // IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I17]], align 4
 // IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
 // IR-NEXT:    [[ADD23:%.*]] = add i32 [[TMP24]], [[TMP25]]
-// IR-NEXT:    store i32 [[ADD23]], ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    [[MUL24:%.*]] = mul i32 [[TMP26]], 2
-// IR-NEXT:    [[ADD25:%.*]] = add i32 0, [[MUL24]]
-// IR-NEXT:    store i32 [[ADD25]], ptr [[DOTUNROLLED_IV_I]], align 4
-// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    [[ADD26:%.*]] = add i32 [[TMP28]], 1
-// IR-NEXT:    [[CMP27:%.*]] = icmp ult i32 [[TMP27]], [[ADD26]]
-// IR-NEXT:    br i1 [[CMP27]], label %[[IF_THEN:.*]], label %[[IF_END:.*]]
-// IR:       [[IF_THEN]]:
-// IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTUNROLLED_IV_I]], align 4
-// IR-NEXT:    store i32 [[TMP29]], ptr [[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND28:.*]]
-// IR:       [[FOR_COND28]]:
-// IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTUNROLLED_IV_I]], align 4
-// IR-NEXT:    [[ADD29:%.*]] = add i32 [[TMP31]], 2
-// IR-NEXT:    [[CMP30:%.*]] = icmp ult i32 [[TMP30]], [[ADD29]]
-// IR-NEXT:    br i1 [[CMP30]], label %[[LAND_RHS:.*]], label %[[LAND_END:.*]]
+// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    [[ADD24:%.*]] = add i32 [[TMP26]], 1
+// IR-NEXT:    [[CMP25:%.*]] = icmp ult i32 [[ADD23]], [[ADD24]]
+// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP22]], [[CMP25]]
+// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label %[[FOR_BODY:.*]], label %[[FOR_END39:.*]]
+// IR:       [[FOR_BODY]]:
+// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I17]], align 4
+// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    [[ADD26:%.*]] = add i32 [[TMP27]], [[TMP28]]
+// IR-NEXT:    store i32 [[ADD26]], ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    [[MUL27:%.*]] = mul i32 [[TMP29]], 2
+// IR-NEXT:    [[ADD28:%.*]] = add i32 0, [[MUL27]]
+// IR-NEXT:    store i32 [[ADD28]], ptr [[DOTUNROLLED_IV_I]], align 4
+// IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTUNROLLED_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP30]], ptr [[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND29:.*]]
+// IR:       [[FOR_COND29]]:
+// IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTUNROLLED_IV_I]], align 4
+// IR-NEXT:    [[ADD30:%.*]] = add i32 [[TMP32]], 2
+// IR-NEXT:    [[CMP31:%.*]] = icmp ult i32 [[TMP31]], [[ADD30]]
+// IR-NEXT:    br i1 [[CMP31]], label %[[LAND_RHS:.*]], label %[[LAND_END:.*]]
 // IR:       [[LAND_RHS]]:
-// IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[ADD31:%.*]] = add i32 [[TMP33]], 1
-// IR-NEXT:    [[CMP32:%.*]] = icmp ult i32 [[TMP32]], [[ADD31]]
+// IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    [[ADD32:%.*]] = add i32 [[TMP34]], 1
+// IR-NEXT:    [[CMP33:%.*]] = icmp ult i32 [[TMP33]], [[ADD32]]
 // IR-NEXT:    br label %[[LAND_END]]
 // IR:       [[LAND_END]]:
-// IR-NEXT:    [[TMP34:%.*]] = phi i1 [ false, %[[FOR_COND28]] ], [ [[CMP32]], %[[LAND_RHS]] ]
-// IR-NEXT:    br i1 [[TMP34]], label %[[FOR_BODY33:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY33]]:
-// IR-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[MUL34:%.*]] = mul i32 [[TMP36]], [[TMP37]]
-// IR-NEXT:    [[ADD35:%.*]] = add i32 [[TMP35]], [[MUL34]]
-// IR-NEXT:    store i32 [[ADD35]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP38:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// IR-NEXT:    [[TMP39:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// IR-NEXT:    [[TMP40:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// IR-NEXT:    [[TMP41:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void (...) @body(i32 noundef [[TMP38]], i32 noundef [[TMP39]], i32 noundef [[TMP40]], i32 noundef [[TMP41]])
+// IR-NEXT:    [[TMP35:%.*]] = phi i1 [ false, %[[FOR_COND29]] ], [ [[CMP33]], %[[LAND_RHS]] ]
+// IR-NEXT:    br i1 [[TMP35]], label %[[FOR_BODY34:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY34]]:
+// IR-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// IR-NEXT:    [[MUL35:%.*]] = mul i32 [[TMP37]], [[TMP38]]
+// IR-NEXT:    [[ADD36:%.*]] = add i32 [[TMP36]], [[MUL35]]
+// IR-NEXT:    store i32 [[ADD36]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP39:%.*]] = load i32, ptr [[START_ADDR]], align 4
+// IR-NEXT:    [[TMP40:%.*]] = load i32, ptr [[END_ADDR]], align 4
+// IR-NEXT:    [[TMP41:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
+// IR-NEXT:    [[TMP42:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void (...) @body(i32 noundef [[TMP39]], i32 noundef [[TMP40]], i32 noundef [[TMP41]], i32 noundef [[TMP42]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add i32 [[TMP42]], 1
+// IR-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add i32 [[TMP43]], 1
 // IR-NEXT:    store i32 [[INC]], ptr [[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND28]], !llvm.loop [[LOOP2:![0-9]+]]
+// IR-NEXT:    br label %[[FOR_COND29]], !llvm.loop [[LOOP2:![0-9]+]]
 // IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[IF_END]]
-// IR:       [[IF_END]]:
-// IR-NEXT:    br label %[[FOR_INC36:.*]]
-// IR:       [[FOR_INC36]]:
-// IR-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    [[INC37:%.*]] = add i32 [[TMP43]], 1
-// IR-NEXT:    store i32 [[INC37]], ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_INC37:.*]]
+// IR:       [[FOR_INC37]]:
+// IR-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    [[INC38:%.*]] = add i32 [[TMP44]], 1
+// IR-NEXT:    store i32 [[INC38]], ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP5:![0-9]+]]
-// IR:       [[FOR_END38]]:
+// IR:       [[FOR_END39]]:
 // IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.*]]
 // IR:       [[OMP_BODY_CONTINUE]]:
 // IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.*]]
 // IR:       [[OMP_INNER_FOR_INC]]:
-// IR-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[ADD39:%.*]] = add i32 [[TMP44]], 1
-// IR-NEXT:    store i32 [[ADD39]], ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[ADD40:%.*]] = add i32 [[TMP45]], 1
+// IR-NEXT:    store i32 [[ADD40]], ptr [[DOTOMP_IV]], align 4
 // IR-NEXT:    br label %[[OMP_INNER_FOR_COND]]
 // IR:       [[OMP_INNER_FOR_END]]:
 // IR-NEXT:    br label %[[OMP_LOOP_EXIT:.*]]
diff --git a/openmp/runtime/test/transform/tile/foreach.cpp b/openmp/runtime/test/transform/tile/foreach.cpp
index 17031a4acf187..608a1c896fcb4 100644
--- a/openmp/runtime/test/transform/tile/foreach.cpp
+++ b/openmp/runtime/test/transform/tile/foreach.cpp
@@ -182,10 +182,6 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: v=0 w=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 1
 // CHECK-NEXT: [A] iterator move assign
@@ -195,10 +191,6 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: v=1 w=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -217,14 +209,6 @@ int main() {
 // CHECK-NEXT: [A] iterator advance: 0 += 3
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [B] iterator advance: 0 += 0
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 0
-// CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 1
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 1
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -234,22 +218,10 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: v=2 w=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 3
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [B] iterator advance: 0 += 2
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 2
-// CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
diff --git a/openmp/runtime/test/transform/tile/intfor.c b/openmp/runtime/test/transform/tile/intfor.c
index e980165277b79..c5ceddd83da70 100644
--- a/openmp/runtime/test/transform/tile/intfor.c
+++ b/openmp/runtime/test/transform/tile/intfor.c
@@ -81,8 +81,6 @@ int main() {
 // CHECK-NEXT: i=7 j=19
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
@@ -93,8 +91,6 @@ int main() {
 // CHECK-NEXT: i=10 j=19
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
@@ -105,8 +101,6 @@ int main() {
 // CHECK-NEXT: i=13 j=19
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
@@ -130,22 +124,10 @@ int main() {
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
@@ -160,28 +142,14 @@ int main() {
 // CHECK-NEXT: i=16 j=19
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
diff --git a/openmp/runtime/test/transform/tile/iterfor.cpp b/openmp/runtime/test/transform/tile/iterfor.cpp
index d07b10b3e841a..4b9bf6f41ada6 100644
--- a/openmp/runtime/test/transform/tile/iterfor.cpp
+++ b/openmp/runtime/test/transform/tile/iterfor.cpp
@@ -190,9 +190,6 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=0 j=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 1
 // CHECK-NEXT: [A] iterator move assign
@@ -202,9 +199,6 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=1 j=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -223,12 +217,6 @@ int main() {
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 3
 // CHECK-NEXT: [A] iterator move assign
-// CHECK-NEXT: [B] iterator advance: 0 += 0
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 1
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -238,18 +226,9 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=2 j=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 3
 // CHECK-NEXT: [A] iterator move assign
-// CHECK-NEXT: [B] iterator advance: 0 += 2
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
diff --git a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-foreach.cpp b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-foreach.cpp
index 966ff7f84f548..a007a369453d4 100644
--- a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-foreach.cpp
+++ b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-foreach.cpp
@@ -182,10 +182,6 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=0 v=0 w=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 1
 // CHECK-NEXT: [A] iterator move assign
@@ -195,10 +191,6 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=0 v=1 w=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -217,14 +209,6 @@ int main() {
 // CHECK-NEXT: [A] iterator advance: 0 += 3
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [B] iterator advance: 0 += 0
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 0
-// CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 1
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 1
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -234,22 +218,10 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=0 v=2 w=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 3
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [B] iterator advance: 0 += 2
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 2
-// CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 0
 // CHECK-NEXT: [A] iterator move assign
@@ -287,10 +259,6 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=1 v=0 w=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 1
 // CHECK-NEXT: [A] iterator move assign
@@ -300,10 +268,6 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=1 v=1 w=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -322,14 +286,6 @@ int main() {
 // CHECK-NEXT: [A] iterator advance: 0 += 3
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [B] iterator advance: 0 += 0
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 0
-// CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 1
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 1
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -339,22 +295,10 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=1 v=2 w=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 3
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [B] iterator advance: 0 += 2
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 2
-// CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 0
 // CHECK-NEXT: [A] iterator move assign
@@ -392,10 +336,6 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=2 v=0 w=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 1
 // CHECK-NEXT: [A] iterator move assign
@@ -405,10 +345,6 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=2 v=1 w=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -427,14 +363,6 @@ int main() {
 // CHECK-NEXT: [A] iterator advance: 0 += 3
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [B] iterator advance: 0 += 0
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 0
-// CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 1
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 1
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
@@ -444,22 +372,10 @@ int main() {
 // CHECK-NEXT: [B] iterator deref: 2
 // CHECK-NEXT: i=2 v=2 w=2
 // CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 3
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [B] iterator advance: 0 += 2
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 2
-// CHECK-NEXT: [B] iterator dtor
-// CHECK-NEXT: [B] iterator advance: 0 += 3
-// CHECK-NEXT: [B] iterator move assign
-// CHECK-NEXT: [B] iterator deref: 3
-// CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
diff --git a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile.cpp b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile.cpp
new file mode 100644
index 0000000000000..c7f7e143f7983
--- /dev/null
+++ b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile.cpp
@@ -0,0 +1,44 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+
+// After tiling `i`, the logical loops are `floor_i`, `tile_i`, and `j`, so
+// `collapse(3)` must reach through the tile guard to include `j`. CodeGen must
+// then keep that guard around the collapsed body once, instead of emitting `j`
+// again as a normal nested loop.
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdio>
+
+int main() {
+  printf("do\n");
+#pragma omp parallel for collapse(3) num_threads(1)
+#pragma omp tile sizes(4)
+  for (int i = 0; i < 6; ++i)
+    for (int j = 0; j < 2; ++j)
+      printf("i=%d j=%d\n", i, j);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+
+// Full tile (.floor.i=0): i=0..3
+// CHECK-NEXT: i=0 j=0
+// CHECK-NEXT: i=0 j=1
+// CHECK-NEXT: i=1 j=0
+// CHECK-NEXT: i=1 j=1
+// CHECK-NEXT: i=2 j=0
+// CHECK-NEXT: i=2 j=1
+// CHECK-NEXT: i=3 j=0
+// CHECK-NEXT: i=3 j=1
+
+// Partial tile (.floor.i=4): i=4,5 in bounds; i=6,7 skipped by the predicate
+// CHECK-NEXT: i=4 j=0
+// CHECK-NEXT: i=4 j=1
+// CHECK-NEXT: i=5 j=0
+// CHECK-NEXT: i=5 j=1
+// CHECK-NEXT: done

>From c42d747d0677fd0059f0bcff09699e965bae91e2 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 15 Jul 2026 10:05:50 -0400
Subject: [PATCH 15/19] preserve vectorized-form through reinterpretation hint

---
 clang/include/clang/Basic/Attr.td  |   2 +
 clang/lib/AST/StmtOpenMP.cpp       |  66 +++---
 clang/lib/CodeGen/CGStmt.cpp       |  48 +----
 clang/lib/CodeGen/CGStmtOpenMP.cpp |  56 ++---
 clang/lib/Sema/SemaOpenMP.cpp      | 325 ++++++++++++++++-------------
 5 files changed, 234 insertions(+), 263 deletions(-)

diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 22f5e0fd33433..7d90c40c5ecf9 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -4881,6 +4881,8 @@ def OMPInvariantPredicateBound : StmtAttr {
   let Spellings = [];
   let SemaHandler = 0;
   let Subjects = SubjectList<[Stmt], ErrorDiag>;
+  let Args = [ExprArgument<"RectCond">, ExprArgument<"Predicate">,
+              ExprArgument<"TileSize">];
   let Documentation = [InternalOnly];
 }
 
diff --git a/clang/lib/AST/StmtOpenMP.cpp b/clang/lib/AST/StmtOpenMP.cpp
index 94eeddc01d1a3..26000198518a8 100644
--- a/clang/lib/AST/StmtOpenMP.cpp
+++ b/clang/lib/AST/StmtOpenMP.cpp
@@ -18,8 +18,8 @@
 using namespace clang;
 using namespace llvm::omp;
 
-// True if the attributed statement carries the internal `omp tile` intra-tile
-// body-guard marker.
+/// Returns true if the attributed statement carries the internal `omp tile`
+/// intra-tile reinterpretation hint (see OMPInvariantPredicateBoundAttr).
 static bool hasOMPInvariantPredicateBound(const AttributedStmt *AS) {
   for (const Attr *A : AS->getAttrs())
     if (isa<OMPInvariantPredicateBoundAttr>(A))
@@ -27,17 +27,26 @@ static bool hasOMPInvariantPredicateBound(const AttributedStmt *AS) {
   return false;
 }
 
-// Like `Stmt::IgnoreContainers`, but also treats the `omp tile` body-guard
-static Stmt *IgnoreContainersAndOMPTileBodyGuard(Stmt *S) {
+/// Peeks through an intra-tile hint wrapper to return the underlying loop;
+/// otherwise returns \p S unchanged. Used by the loop walker once the hint has
+/// already been delivered to the analysis callback and we need to look through
+/// the wrapper to reach the loop body.
+static Stmt *ignoreOMPIntraTileHint(Stmt *S) {
+  if (auto *AS = dyn_cast_or_null<AttributedStmt>(S))
+    if (hasOMPInvariantPredicateBound(AS))
+      return AS->getSubStmt();
+  return S;
+}
+
+/// Like `Stmt::IgnoreContainers`, but stops at (and preserves) an intra-tile
+/// hint wrapper so that the enclosing loop walker can deliver the hint to
+/// loop-associated analysis (e.g. `collapse`). Non-hint attributed statements
+/// and single-statement compounds are still skipped.
+static Stmt *ignoreContainersKeepingIntraTileHint(Stmt *S) {
   while (true) {
     if (auto *AS = dyn_cast_or_null<AttributedStmt>(S)) {
-      if (hasOMPInvariantPredicateBound(AS)) {
-        if (auto *If = dyn_cast<IfStmt>(AS->getSubStmt());
-            If && !If->getElse()) {
-          S = If->getThen();
-          continue;
-        }
-      }
+      if (hasOMPInvariantPredicateBound(AS))
+        break;
       S = AS->getSubStmt();
       continue;
     }
@@ -114,7 +123,7 @@ Stmt *
 OMPLoopBasedDirective::tryToFindNextInnerLoop(Stmt *CurStmt,
                                               bool TryImperfectlyNestedLoops) {
   Stmt *OrigStmt = CurStmt;
-  CurStmt = IgnoreContainersAndOMPTileBodyGuard(CurStmt);
+  CurStmt = ignoreContainersKeepingIntraTileHint(CurStmt);
   // Additional work for imperfectly nested loops, introduced in OpenMP 5.0.
   if (TryImperfectlyNestedLoops) {
     if (auto *CS = dyn_cast<CompoundStmt>(CurStmt)) {
@@ -128,14 +137,17 @@ OMPLoopBasedDirective::tryToFindNextInnerLoop(Stmt *CurStmt,
         for (Stmt *S : CS->body()) {
           if (!S)
             continue;
-          if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(S))
-            S = CanonLoop->getLoopStmt();
-          // Check for the `omp tile` body-guard marker.
-          if (auto *AS = dyn_cast<AttributedStmt>(S);
-              AS && hasOMPInvariantPredicateBound(AS))
-            S = IgnoreContainersAndOMPTileBodyGuard(S);
-          if (isa<ForStmt>(S) || isa<CXXForRangeStmt>(S) ||
-              (isa<OMPLoopBasedDirective>(S) && !isa<OMPLoopDirective>(S))) {
+          // Peek past an OMPCanonicalLoop wrapper and/or an intra-tile hint to
+          // check whether this child is loop-like; keep the original (wrapped)
+          // node in CurStmt so the hint still reaches the loop-analysis
+          // callback.
+          Stmt *Inner = S;
+          if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(Inner))
+            Inner = CanonLoop->getLoopStmt();
+          Inner = ignoreOMPIntraTileHint(Inner);
+          if (isa<ForStmt>(Inner) || isa<CXXForRangeStmt>(Inner) ||
+              (isa<OMPLoopBasedDirective>(Inner) &&
+               !isa<OMPLoopDirective>(Inner))) {
             // Only single loop construct is allowed.
             if (CurStmt) {
               CurStmt = OrigStmt;
@@ -167,7 +179,7 @@ bool OMPLoopBasedDirective::doForAllLoops(
     llvm::function_ref<bool(unsigned, Stmt *)> Callback,
     llvm::function_ref<void(OMPLoopTransformationDirective *)>
         OnTransformationCallback) {
-  CurStmt = IgnoreContainersAndOMPTileBodyGuard(CurStmt);
+  CurStmt = ignoreContainersKeepingIntraTileHint(CurStmt);
   for (unsigned Cnt = 0; Cnt < NumLoops; ++Cnt) {
     while (true) {
       auto *Dir = dyn_cast<OMPLoopTransformationDirective>(CurStmt);
@@ -197,19 +209,23 @@ bool OMPLoopBasedDirective::doForAllLoops(
     }
     if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(CurStmt))
       CurStmt = CanonLoop->getLoopStmt();
+    // Deliver the (possibly hint-wrapped) loop node to the callback so it can
+    // read the intra-tile reinterpretation hint, then look through the wrapper
+    // to reach the loop body.
     if (Callback(Cnt, CurStmt))
       return false;
+    Stmt *LoopStmt = ignoreOMPIntraTileHint(CurStmt);
     // Move on to the next nested for loop, or to the loop body.
     // OpenMP [2.8.1, simd construct, Restrictions]
     // All loops associated with the construct must be perfectly nested; that
     // is, there must be no intervening code nor any OpenMP directive between
     // any two loops.
-    if (auto *For = dyn_cast<ForStmt>(CurStmt)) {
+    if (auto *For = dyn_cast<ForStmt>(LoopStmt)) {
       CurStmt = For->getBody();
     } else {
-      assert(isa<CXXForRangeStmt>(CurStmt) &&
+      assert(isa<CXXForRangeStmt>(LoopStmt) &&
              "Expected canonical for or range-based for loops.");
-      CurStmt = cast<CXXForRangeStmt>(CurStmt)->getBody();
+      CurStmt = cast<CXXForRangeStmt>(LoopStmt)->getBody();
     }
     CurStmt = OMPLoopBasedDirective::tryToFindNextInnerLoop(
         CurStmt, TryImperfectlyNestedLoops);
@@ -223,6 +239,8 @@ void OMPLoopBasedDirective::doForAllLoopsBodies(
   bool Res = OMPLoopBasedDirective::doForAllLoops(
       CurStmt, TryImperfectlyNestedLoops, NumLoops,
       [Callback](unsigned Cnt, Stmt *Loop) {
+        // Look through an intra-tile hint wrapper to reach the loop itself.
+        Loop = ignoreOMPIntraTileHint(Loop);
         Stmt *Body = nullptr;
         if (auto *For = dyn_cast<ForStmt>(Loop)) {
           Body = For->getBody();
diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index da7a1b561b207..27e74d966eca1 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -1271,35 +1271,9 @@ void CodeGenFunction::EmitDoStmt(const DoStmt &S,
     ConvergenceTokenStack.pop_back();
 }
 
-// Helper function to find and return the omp-tile body-guard marker on the
-// for-body's last item.
-static std::pair<const AttributedStmt *, const IfStmt *>
-findOMPTileBodyGuard(const Stmt *Body) {
-  const auto *CS = dyn_cast_or_null<CompoundStmt>(Body);
-  if (!CS || CS->body().empty())
-    return {nullptr, nullptr};
-  const auto *AS = dyn_cast<AttributedStmt>(CS->body_back());
-  if (!AS)
-    return {nullptr, nullptr};
-  bool IsMarker = false;
-  for (const Attr *A : AS->getAttrs()) {
-    if (!isa<OMPInvariantPredicateBoundAttr>(A))
-      continue;
-    IsMarker = true;
-    break;
-  }
-  if (!IsMarker)
-    return {nullptr, nullptr};
-  const auto *If = dyn_cast<IfStmt>(AS->getSubStmt());
-  if (!If || If->getElse())
-    return {nullptr, nullptr};
-  return {AS, If};
-}
-
 void CodeGenFunction::EmitForStmt(const ForStmt &S,
                                   ArrayRef<const Attr *> ForAttrs) {
   JumpDest LoopExit = getJumpDestInCurrentScope("for.end");
-  const auto [MarkerAS, MarkerIf] = findOMPTileBodyGuard(S.getBody());
 
   std::optional<LexicalScope> ForScope;
   if (getLangOpts().C99 || getLangOpts().CPlusPlus)
@@ -1367,14 +1341,6 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S,
     // compares unequal to 0.  The condition must be a scalar type.
     llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond());
 
-    // If the for-body has a marker, evaluate the predicate and AND it into the
-    // loop condition.
-    if (MarkerIf) {
-      llvm::Value *PredVal = EvaluateExprAsBool(MarkerIf->getCond());
-      BoolCondVal = Builder.CreateAnd(BoolCondVal, PredVal,
-                                      "omp_tile.invariant_pred_bound");
-    }
-
     MaybeEmitDeferredVarDeclInit(S.getConditionVariable());
 
     llvm::MDNode *Weights =
@@ -1411,19 +1377,7 @@ void CodeGenFunction::EmitForStmt(const ForStmt &S,
     // Create a separate cleanup scope for the body, in case it is not
     // a compound statement.
     RunCleanupsScope BodyScope(*this);
-    if (MarkerAS) {
-      // If the for-body has a marker, walk the body and emit the then-block of
-      // the marker's predicate.
-      const auto *CS = cast<CompoundStmt>(S.getBody());
-      for (const Stmt *Sub : CS->body()) {
-        if (Sub == MarkerAS)
-          EmitStmt(MarkerIf->getThen());
-        else
-          EmitStmt(Sub);
-      }
-    } else {
-      EmitStmt(S.getBody());
-    }
+    EmitStmt(S.getBody());
   }
 
   // The last block in the loop's body (which unconditionally branches to the
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 93805f7b54432..509d4f3d35815 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -2231,32 +2231,15 @@ class OMPTransformDirectiveScopeRAII {
 };
 } // namespace
 
-// Helper function to look through the statement and return the special marked
-// `omp tile` guard `if`, or nullptr if this is not that shape.
-static const IfStmt *getOMPTileBodyGuardIf(const Stmt *S) {
-  while (S) {
-    if (const auto *AS = dyn_cast<AttributedStmt>(S)) {
-      bool IsMarker = llvm::any_of(AS->getAttrs(), [](const Attr *A) {
-        return isa<OMPInvariantPredicateBoundAttr>(A);
-      });
-      if (!IsMarker) {
-        S = AS->getSubStmt();
-        continue;
-      }
-      const auto *If = dyn_cast<IfStmt>(AS->getSubStmt());
-      if (!If || If->getElse())
-        return nullptr;
-      return If;
-    }
-    if (const auto *CS = dyn_cast<CompoundStmt>(S)) {
-      if (CS->size() != 1)
-        return nullptr;
-      S = CS->body_back();
-      continue;
-    }
-    return nullptr;
-  }
-  return nullptr;
+/// Helper function to look through the droppable intra-tile hint wrapper
+/// (OMPInvariantPredicateBoundAttr) to the loop statement it annotates.
+static const Stmt *lookThroughOMPIntraTileHint(const Stmt *S) {
+  if (const auto *AS = dyn_cast_or_null<AttributedStmt>(S))
+    if (llvm::any_of(AS->getAttrs(), [](const Attr *A) {
+          return isa<OMPInvariantPredicateBoundAttr>(A);
+        }))
+      return AS->getSubStmt();
+  return S;
 }
 
 static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop,
@@ -2275,23 +2258,10 @@ static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop,
     return;
   }
 
-  // If this is the marked `omp tile` guard, emit its predicate as a runtime
-  // body guard and recurse into `Then` so consumed loops are still walked.
-  if (const IfStmt *Guard = getOMPTileBodyGuardIf(S)) {
-    assert(!Guard->getInit() && !Guard->getConditionVariable() &&
-           "omp tile body-guard must be a plain `if (P) Then`");
-    llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_tile.pred.then");
-    llvm::BasicBlock *EndBlock = CGF.createBasicBlock("omp_tile.pred.end");
-    CGF.EmitBranchOnBoolExpr(
-        Guard->getCond(), ThenBlock, EndBlock, /*TrueCount=*/0,
-        Stmt::getLikelihood(Guard->getThen(), Guard->getElse()));
-    CGF.EmitBlock(ThenBlock);
-    emitBody(CGF, Guard->getThen(), NextLoop, MaxLevel, Level);
-    CGF.EmitBranch(EndBlock);
-    CGF.EmitBlock(EndBlock);
-    return;
-  }
-  if (SimplifiedS == NextLoop) {
+  // The loop walker may hand back the intra-tile hint wrapper as `NextLoop`;
+  // match against the loop it annotates. The tile overshoot guard is emitted
+  // separately via EmitOMPLoopBody's finals-conditions handling.
+  if (SimplifiedS == lookThroughOMPIntraTileHint(NextLoop)) {
     if (auto *Dir = dyn_cast<OMPLoopTransformationDirective>(SimplifiedS))
       SimplifiedS = Dir->getTransformedStmt();
     if (const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(SimplifiedS))
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 20b8c99a1def2..701ba2d9f127d 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -8018,6 +8018,9 @@ struct LoopIterationSpace final {
   /// check that the number of iterations for this particular counter must be
   /// finished.
   Expr *FinalCondition = nullptr;
+  /// True if this is a reinterpreted intra-tile loop whose lower bound is a
+  /// live floor dependence.
+  bool ReinterpretTileLB = false;
 };
 
 /// Scan an AST subtree, checking that no decls in the CollapsedLoopVarDecls
@@ -9577,6 +9580,23 @@ static bool checkOpenMPIterationSpace(
     const llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopVarDecls,
     llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopInductionVars) {
   bool SupportsNonRectangular = !isOpenMPLoopTransformationDirective(DKind);
+  // See the tile reinterpretation design note in ActOnOpenMPTileDirective.
+  // If the loop carries the hint, analyze its rectangular form instead.
+  Expr *TileRectCond = nullptr;
+  Expr *TileBodyPredicate = nullptr;
+  Expr *TileTripCount = nullptr;
+  if (auto *AS = dyn_cast_or_null<AttributedStmt>(S)) {
+    for (const Attr *A : AS->getAttrs()) {
+      if (const auto *H = dyn_cast<OMPInvariantPredicateBoundAttr>(A)) {
+        TileRectCond = H->getRectCond();
+        TileBodyPredicate = H->getPredicate();
+        TileTripCount = H->getTileSize();
+        break;
+      }
+    }
+    if (TileRectCond)
+      S = AS->getSubStmt();
+  }
   // OpenMP [2.9.1, Canonical Loop Form]
   //   for (init-expr; test-expr; incr-expr) structured-block
   //   for (range-decl: range-expr) structured-block
@@ -9626,6 +9646,11 @@ static bool checkOpenMPIterationSpace(
 
   bool HasErrors = false;
 
+  // Condition used for iteration-space analysis: the rectangular hint when
+  // reinterpreting an intra-tile loop, otherwise the loop's own condition.
+  Expr *EffectiveCond =
+      TileRectCond ? TileRectCond : (For ? For->getCond() : CXXFor->getCond());
+
   // Check loop variable's type.
   if (ValueDecl *LCDecl = ISC.getLoopDecl()) {
     // OpenMP [2.6, Canonical Loop Form]
@@ -9658,7 +9683,7 @@ static bool checkOpenMPIterationSpace(
            "DSA for non-loop vars");
 
     // Check test-expr.
-    HasErrors |= ISC.checkAndSetCond(For ? For->getCond() : CXXFor->getCond());
+    HasErrors |= ISC.checkAndSetCond(EffectiveCond);
 
     // Check incr-expr.
     HasErrors |= ISC.checkAndSetInc(For ? For->getInc() : CXXFor->getInc());
@@ -9668,8 +9693,18 @@ static bool checkOpenMPIterationSpace(
     return HasErrors;
 
   // Build the loop's iteration space representation.
-  ResultIterSpaces[CurrentNestedLoopCount].PreCond = ISC.buildPreCond(
-      DSA.getCurScope(), For ? For->getCond() : CXXFor->getCond(), Captures);
+  ResultIterSpaces[CurrentNestedLoopCount].PreCond =
+      ISC.buildPreCond(DSA.getCurScope(), EffectiveCond, Captures);
+  // Reinterpreted intra-tile: precondition is trivially true (.floor.iv
+  // unreadable here); overshoot handled by Predicate.
+  if (TileRectCond)
+    ResultIterSpaces[CurrentNestedLoopCount].PreCond =
+        SemaRef
+            .PerformImplicitConversion(
+                SemaRef.ActOnIntegerConstant(SourceLocation(), 1).get(),
+                SemaRef.Context.BoolTy, AssignmentAction::Casting,
+                /*AllowExplicit=*/true)
+            .get();
   ResultIterSpaces[CurrentNestedLoopCount].NumIterations =
       ISC.buildNumIterations(DSA.getCurScope(), ResultIterSpaces,
                              (isOpenMPWorksharingDirective(DKind) ||
@@ -9678,6 +9713,9 @@ static bool checkOpenMPIterationSpace(
                               isOpenMPDistributeDirective(DKind) ||
                               isOpenMPLoopTransformationDirective(DKind)),
                              Captures);
+  // Reinterpreted intra-tile: trip count is the constant TileSize.
+  if (TileTripCount)
+    ResultIterSpaces[CurrentNestedLoopCount].NumIterations = TileTripCount;
   ResultIterSpaces[CurrentNestedLoopCount].CounterVar =
       ISC.buildCounterVar(Captures, DSA);
   ResultIterSpaces[CurrentNestedLoopCount].PrivateCounterVar =
@@ -9697,6 +9735,13 @@ static bool checkOpenMPIterationSpace(
       ISC.buildMinMaxValues(DSA.getCurScope(), Captures);
   ResultIterSpaces[CurrentNestedLoopCount].FinalCondition =
       ISC.buildFinalCondition(DSA.getCurScope());
+  // Reinterpreted intra-tile: body guard is the overshoot Predicate.
+  if (TileBodyPredicate)
+    ResultIterSpaces[CurrentNestedLoopCount].FinalCondition = TileBodyPredicate;
+  // Lower bound is the live .floor.iv; wire as a floor dependence for collapse
+  // (see design note in ActOnOpenMPTileDirective).
+  if (TileRectCond)
+    ResultIterSpaces[CurrentNestedLoopCount].ReinterpretTileLB = true;
   ResultIterSpaces[CurrentNestedLoopCount].IsNonRectangularLB =
       ISC.doesInitDependOnLC();
   ResultIterSpaces[CurrentNestedLoopCount].IsNonRectangularUB =
@@ -10621,6 +10666,23 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
       SourceLocation UpdLoc = IS.IncSrcRange.getBegin();
       ExprResult Iter;
 
+      // Wire the reinterpreted lower bound to its floor dependence (see design
+      // note in ActOnOpenMPTileDirective).
+      if (IS.ReinterpretTileLB && IS.CounterInit) {
+        if (const auto *LBRef =
+                dyn_cast<DeclRefExpr>(IS.CounterInit->IgnoreParenImpCasts())) {
+          const Decl *FloorDecl = LBRef->getDecl()->getCanonicalDecl();
+          for (unsigned K = 0; K < Cnt; ++K) {
+            const auto *CV = dyn_cast_or_null<DeclRefExpr>(Built.Counters[K]);
+            if (CV && CV->getDecl()->getCanonicalDecl() == FloorDecl) {
+              IS.IsNonRectangularLB = true;
+              IS.LoopDependentIdx = K + 1;
+              break;
+            }
+          }
+        }
+      }
+
       // Compute prod
       ExprResult Prod = SemaRef.ActOnIntegerConstant(SourceLocation(), 1).get();
       for (unsigned int K = Cnt + 1; K < NestedLoopCount; ++K)
@@ -10693,11 +10755,13 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
       Built.Finals[Cnt] = Final.get();
       Built.DependentCounters[Cnt] = nullptr;
       Built.DependentInits[Cnt] = nullptr;
-      Built.FinalsConditions[Cnt] = nullptr;
+      // Transfer the body-guard condition: the loop condition for
+      // non-rectangular loops, the overshoot predicate for reinterpreted tiles,
+      // null otherwise.
+      Built.FinalsConditions[Cnt] = IS.FinalCondition;
       if (IS.IsNonRectangularLB || IS.IsNonRectangularUB) {
         Built.DependentCounters[Cnt] = Built.Counters[IS.LoopDependentIdx - 1];
         Built.DependentInits[Cnt] = Built.Inits[IS.LoopDependentIdx - 1];
-        Built.FinalsConditions[Cnt] = IS.FinalCondition;
       }
     }
   }
@@ -15081,10 +15145,8 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
   // Create iteration variables for the generated loops.
   SmallVector<VarDecl *, 4> FloorIndVars;
   SmallVector<VarDecl *, 4> TileIndVars;
-  SmallVector<VarDecl *, 4> TileCntVars;
   FloorIndVars.resize(NumLoops);
   TileIndVars.resize(NumLoops);
-  TileCntVars.resize(NumLoops);
   for (unsigned I = 0; I < NumLoops; ++I) {
     OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
 
@@ -15104,110 +15166,27 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
       FloorIndVars[I] = FloorCntDecl;
     }
 
-    // Logical iteration variable for the tile loop. Retains the meaning of
-    // the original logical iteration number (floor_iv + tile_cnt) so that
-    // LoopHelper.Updates can derive the original loop variable unchanged.
+    // Iteration variable for the tile (i.e. inner) loop.
     {
-      std::string TileIVName =
+      std::string TileCntName =
           (Twine(".tile_") + llvm::utostr(I) + ".iv." + OrigVarName).str();
 
-      auto *TileIVDecl = cast<VarDecl>(IterVarRef->getDecl());
-      TileIVDecl->setDeclName(&SemaRef.PP.getIdentifierTable().get(TileIVName));
-      TileIndVars[I] = TileIVDecl;
-    }
-
-    // Loop counter for the rectangular tile loop [0, TileSize).
-    {
-      std::string TileCntName =
-          (Twine(".tile.cnt.") + llvm::utostr(I) + ".iv." + OrigVarName).str();
-      VarDecl *TileCntDecl =
-          buildVarDecl(SemaRef, {}, CntTy, TileCntName, nullptr, OrigCntVar);
-      TileCntVars[I] = TileCntDecl;
+      // Reuse the iteration variable created by checkOpenMPLoop. It is also
+      // used by the expressions to derive the original iteration variable's
+      // value from the logical iteration number.
+      auto *TileCntDecl = cast<VarDecl>(IterVarRef->getDecl());
+      TileCntDecl->setDeclName(
+          &SemaRef.PP.getIdentifierTable().get(TileCntName));
+      TileIndVars[I] = TileCntDecl;
     }
 
     addLoopPreInits(Context, LoopHelper, LoopStmts[I], OriginalInits[I],
                     PreInits);
-
-    // Declare the logical tile IV in PreInits so it is in scope for the
-    // entire loop nest (it will be assigned in each tile loop body).
-    {
-      Decl *TileIVDeclPtr = TileIndVars[I];
-      PreInits.push_back(new (Context) DeclStmt(
-          DeclGroupRef::Create(Context, &TileIVDeclPtr, 1), {}, {}));
-    }
   }
 
   // Once the original iteration values are set, append the innermost body.
   Stmt *Inner = Body;
 
-  // Build a combined validity predicate that guards the innermost body.
-  // For each tiled dimension, check that the logical iteration number
-  // (.tile.iv) is within the original trip count. This is required because the
-  // tile loop now has rectangular (constant) bounds and may overshoot on the
-  // remainder tile. The predicate is: .tile.iv.0 < N0 && .tile.iv.1 < N1 ...
-  //
-  // Optimization: if every dimension's trip count is a compile-time constant
-  // that is evenly divisible by the corresponding tile size (also a constant),
-  // then the remainder tile is empty and the predicate is trivially true.
-  //
-  bool PredicateNeeded = false;
-  for (unsigned I = 0; I < NumLoops; ++I) {
-    Expr *TSExpr = SizesClause->getSizesRefs()[I];
-    Expr *NExpr = LoopHelpers[I].NumIterations;
-    Expr::EvalResult TSResult;
-    Expr::EvalResult NResult;
-    if (TSExpr->EvaluateAsInt(TSResult, Context) &&
-        NExpr->EvaluateAsInt(NResult, Context)) {
-      llvm::APSInt TileVal = TSResult.Val.getInt();
-      llvm::APSInt TripVal = NResult.Val.getInt();
-      if (TileVal.isStrictlyPositive() && TripVal.srem(TileVal).isZero())
-        continue;
-    }
-    PredicateNeeded = true;
-    break;
-  }
-
-  if (PredicateNeeded) {
-    Expr *CombinedPred = nullptr;
-    for (unsigned I = 0; I < NumLoops; ++I) {
-      auto *OrigCntVar = cast<DeclRefExpr>(LoopHelpers[I].Counters[0]);
-      QualType IVTy = LoopHelpers[I].NumIterations->getType();
-      Expr *FloorRef = buildDeclRefExpr(SemaRef, FloorIndVars[I], IVTy,
-                                        OrigCntVar->getExprLoc());
-      Expr *TileCntRef = buildDeclRefExpr(SemaRef, TileCntVars[I], IVTy,
-                                          OrigCntVar->getExprLoc());
-      ExprResult Sum = SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(),
-                                          BO_Add, FloorRef, TileCntRef);
-      if (!Sum.isUsable())
-        return StmtError();
-      ExprResult DimPred =
-          SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(), BO_LT,
-                             Sum.get(), LoopHelpers[I].NumIterations);
-      if (!DimPred.isUsable())
-        return StmtError();
-      if (CombinedPred) {
-        ExprResult Combined =
-            SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(), BO_LAnd,
-                               CombinedPred, DimPred.get());
-        if (!Combined.isUsable())
-          return StmtError();
-        CombinedPred = Combined.get();
-      } else {
-        CombinedPred = DimPred.get();
-      }
-    }
-    // Use the source body's location for the synthetic if-statement and
-    // AttributedStmt so any later diagnostic points at the original body
-    // location.
-    SourceLocation SyntheticLoc =
-        Body ? Body->getBeginLoc() : Inner->getBeginLoc();
-    Stmt *PredIf = IfStmt::Create(
-        Context, SyntheticLoc, IfStatementKind::Ordinary, nullptr, nullptr,
-        CombinedPred, SyntheticLoc, SyntheticLoc, Inner);
-    auto *Marker = OMPInvariantPredicateBoundAttr::CreateImplicit(Context);
-    Inner = AttributedStmt::Create(Context, SyntheticLoc, {Marker}, PredIf);
-  }
-
   auto MakeDimTileSize = [&SemaRef = this->SemaRef, &CopyTransformer, &Context,
                           SizesClause, CurScope](int I) -> Expr * {
     Expr *DimTileSizeExpr = SizesClause->getSizesRefs()[I];
@@ -15215,7 +15194,7 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     if (DimTileSizeExpr->containsErrors())
       return nullptr;
 
-    if (DimTileSizeExpr->isIntegerConstantExpr(Context))
+    if (isa<ConstantExpr>(DimTileSizeExpr))
       return AssertSuccess(CopyTransformer.TransformExpr(DimTileSizeExpr));
 
     // When the tile size is not a constant but a variable, it is possible to
@@ -15251,11 +15230,26 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
   };
 
   // Create tile loops from the inside to the outside.
-  // Each tile loop uses .tile.cnt as its counter with rectangular bounds
-  // [0, TileSize), and computes .tile.iv = .floor.iv + .tile.cnt to set
-  // the logical iteration number for LoopHelper.Updates.
-  // The min-bounded path instead uses .tile.iv directly, initialized to
-  // .floor.iv and bounded by min(.floor.iv + DimTileSize, NumIterations).
+  //
+  // Each intra-tile loop is emitted in its natural min-bounded form, which has
+  // no per-iteration body predicate and vectorizes well when run directly:
+  //   for (.tile.iv = .floor.iv; .tile.iv < min(.floor.iv + T, N); ++.tile.iv)
+  //
+  // A loop-associated directive that *consumes* this loop (e.g. `collapse`)
+  // instead needs a constant, floor-independent trip count to linearize the
+  // nest, which the min() bound cannot give it. Rather than change the
+  // emitted loop, we attach a droppable OMPInvariantPredicateBoundAttr hint
+  // below carrying an equivalent rectangular reinterpretation:
+  //   RectCond  : .tile.iv < .floor.iv + T   -- rectangular bound, analyzed by
+  //               checkOpenMPIterationSpace in place of the stored condition
+  //   Predicate : .tile.iv < N               -- remainder-tile overshoot,
+  //               applied as a body guard instead of shortening the trip count
+  //   TileSize  : T                          -- constant per-tile trip count
+  // Because the reinterpreted lower bound is still `.floor.iv`, checkOpenMPLoop
+  // wires it as a live dependence on the matching floor counter, so a
+  // collapsed `.tile.iv` re-reads the floor's current value instead of a
+  // stale preheader snapshot. Collapsing through stacked tiles (tile-of-tile)
+  // is not supported: the inner floor is not itself a collapsed counter.
   for (int I = NumLoops - 1; I >= 0; --I) {
     OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
     Expr *NumIterations = LoopHelper.NumIterations;
@@ -15266,68 +15260,96 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     // Commonly used variables. One of the constraints of an AST is that every
     // node object must appear at most once, hence we define a lambda that
     // creates a new AST node at every use.
-    auto MakeTileCntRef = [&SemaRef = this->SemaRef, &TileCntVars, I, IVTy,
-                           OrigCntVar]() {
-      return buildDeclRefExpr(SemaRef, TileCntVars[I], IVTy,
-                              OrigCntVar->getExprLoc());
-    };
     auto MakeTileIVRef = [&SemaRef = this->SemaRef, &TileIndVars, I, IVTy,
                           OrigCntVar]() {
       return buildDeclRefExpr(SemaRef, TileIndVars[I], IVTy,
                               OrigCntVar->getExprLoc());
     };
 
-    Expr *DimTileSize = MakeDimTileSize(I);
-    if (!DimTileSize)
-      return StmtError();
-
-    StmtResult InitStmt;
-    ExprResult CondExpr;
-    ExprResult IncrStmt;
-    Stmt *TileIVAssignStmt = nullptr;
-
-    // For init-statement: auto .tile.cnt = 0
+    // For init-statement: auto .tile.iv = .floor.iv
     SemaRef.AddInitializerToDecl(
-        TileCntVars[I],
-        SemaRef.ActOnIntegerConstant(LoopHelper.Init->getExprLoc(), 0).get(),
+        TileIndVars[I],
+        SemaRef
+            .DefaultLvalueConversion(
+                makeFloorIVRef(SemaRef, FloorIndVars, I, IVTy, OrigCntVar))
+            .get(),
         /*DirectInit=*/false);
-    Decl *CounterDecl = TileCntVars[I];
-    InitStmt = new (Context)
+    Decl *CounterDecl = TileIndVars[I];
+    StmtResult InitStmt = new (Context)
         DeclStmt(DeclGroupRef::Create(Context, &CounterDecl, 1),
                  OrigCntVar->getBeginLoc(), OrigCntVar->getEndLoc());
+    if (!InitStmt.isUsable())
+      return StmtError();
 
-    // For cond-expression: .tile.cnt < DimTileSize  (rectangular bound)
-    CondExpr = SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(),
-                                  BO_LT, MakeTileCntRef(), DimTileSize);
+    // For cond-expression:
+    //   .tile.iv < min(.floor.iv + DimTileSize, NumIterations)
+    Expr *DimTileSize = MakeDimTileSize(I);
+    if (!DimTileSize)
+      return StmtError();
+    ExprResult EndOfTile = SemaRef.BuildBinOp(
+        CurScope, LoopHelper.Cond->getExprLoc(), BO_Add,
+        makeFloorIVRef(SemaRef, FloorIndVars, I, IVTy, OrigCntVar),
+        DimTileSize);
+    if (!EndOfTile.isUsable())
+      return StmtError();
+    ExprResult IsPartialTile =
+        SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
+                           NumIterations, EndOfTile.get());
+    if (!IsPartialTile.isUsable())
+      return StmtError();
+    ExprResult MinTileAndIterSpace = SemaRef.ActOnConditionalOp(
+        LoopHelper.Cond->getBeginLoc(), LoopHelper.Cond->getEndLoc(),
+        IsPartialTile.get(), NumIterations, EndOfTile.get());
+    if (!MinTileAndIterSpace.isUsable())
+      return StmtError();
+    ExprResult CondExpr =
+        SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
+                           MakeTileIVRef(), MinTileAndIterSpace.get());
+    if (!CondExpr.isUsable())
+      return StmtError();
 
-    // For incr-statement: ++.tile.cnt
-    IncrStmt = SemaRef.BuildUnaryOp(CurScope, LoopHelper.Inc->getExprLoc(),
-                                    UO_PreInc, MakeTileCntRef());
+    // For incr-statement: ++.tile.iv
+    ExprResult IncrStmt = SemaRef.BuildUnaryOp(
+        CurScope, LoopHelper.Inc->getExprLoc(), UO_PreInc, MakeTileIVRef());
+    if (!IncrStmt.isUsable())
+      return StmtError();
 
-    // Compute the logical iteration number:
-    //   .tile.iv = .floor.iv + .tile.cnt
-    ExprResult FloorPlusCnt = SemaRef.BuildBinOp(
-        CurScope, OrigCntVar->getExprLoc(), BO_Add,
+    // Build OMPInvariantPredicateBoundAttr hint (RectCond, Predicate,
+    // TileSize).
+    Expr *RectDimTileSize = MakeDimTileSize(I);
+    if (!RectDimTileSize)
+      return StmtError();
+    ExprResult RectEndOfTile = SemaRef.BuildBinOp(
+        CurScope, LoopHelper.Cond->getExprLoc(), BO_Add,
         makeFloorIVRef(SemaRef, FloorIndVars, I, IVTy, OrigCntVar),
-        MakeTileCntRef());
-    if (!FloorPlusCnt.isUsable())
+        RectDimTileSize);
+    if (!RectEndOfTile.isUsable())
       return StmtError();
-    ExprResult TileIVAssign =
-        SemaRef.BuildBinOp(CurScope, OrigCntVar->getExprLoc(), BO_Assign,
-                           MakeTileIVRef(), FloorPlusCnt.get());
-    if (!TileIVAssign.isUsable())
+    ExprResult RectCond =
+        SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
+                           MakeTileIVRef(), RectEndOfTile.get());
+    ExprResult Predicate =
+        SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
+                           MakeTileIVRef(), NumIterations);
+    Expr *TileSizeExpr = MakeDimTileSize(I);
+    if (!TileSizeExpr)
       return StmtError();
-    TileIVAssignStmt = TileIVAssign.get();
-
-    if (!InitStmt.isUsable() || !CondExpr.isUsable() || !IncrStmt.isUsable())
+    ExprResult TileSize = SemaRef.PerformImplicitConversion(
+        TileSizeExpr, IVTy, AssignmentAction::Converting,
+        /*AllowExplicit=*/true);
+    if (!RectCond.isUsable() || !Predicate.isUsable() || !TileSize.isUsable())
       return StmtError();
 
+    // Statements to set the original iteration variable's value from the
+    // logical iteration number.
     // Generated for loop is:
     // \code
-    // for (auto .tile.cnt = 0; .tile.cnt < DimTileSize; ++.tile.cnt) {
-    //   .tile.iv = .floor.iv + .tile.cnt;
-    //   Original_counter_update;  // derives orig var from .tile.iv
-    //   Inner;                    // predicated body or inner tile loops
+    // Original_for_init;
+    // for (auto .tile.iv = .floor.iv;
+    //      .tile.iv < min(.floor.iv + DimTileSize, NumIterations);
+    //      ++.tile.iv) {
+    //   Original_Body;
+    //   Original_counter_update;
     // }
     // \endcode
     // FIXME: If the innermost body is an loop itself, inserting these
@@ -15335,8 +15357,6 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     // for applying tiling again). If this is the case, sink the expressions
     // further into the inner loop.
     SmallVector<Stmt *, 4> BodyParts;
-    if (TileIVAssignStmt)
-      BodyParts.push_back(TileIVAssignStmt);
     BodyParts.append(LoopHelper.Updates.begin(), LoopHelper.Updates.end());
     if (auto *SourceCXXFor = dyn_cast<CXXForRangeStmt>(LoopStmt))
       BodyParts.push_back(SourceCXXFor->getLoopVarStmt());
@@ -15347,6 +15367,13 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
         ForStmt(Context, InitStmt.get(), CondExpr.get(), nullptr,
                 IncrStmt.get(), Inner, LoopHelper.Init->getBeginLoc(),
                 LoopHelper.Init->getBeginLoc(), LoopHelper.Inc->getEndLoc());
+
+    // Attach the droppable reinterpretation attribute to the intra-tile loop.
+    auto *Hint = OMPInvariantPredicateBoundAttr::CreateImplicit(
+        Context, RectCond.get(), Predicate.get(), TileSize.get(),
+        Inner->getSourceRange());
+    Inner =
+        AttributedStmt::Create(Context, Inner->getBeginLoc(), {Hint}, Inner);
   }
 
   // Create floor loops from the inside to the outside.

>From 7933f9d84e4cccce35c67db56a9cc002a9cb3a25 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 15 Jul 2026 10:12:35 -0400
Subject: [PATCH 16/19] tests-updated

---
 clang/test/OpenMP/interchange_codegen.cpp     | 3444 +++++++++--------
 .../tile_canonical_required_codegen.cpp       |   86 -
 clang/test/OpenMP/tile_codegen.cpp            | 1980 +++++-----
 .../OpenMP/tile_codegen_for_dependent.cpp     |   53 +-
 clang/test/OpenMP/tile_codegen_tile_for.cpp   |  118 +-
 .../tile_collapse_reinterpret_codegen.cpp     |  108 +
 clang/test/OpenMP/tile_rect_codegen.cpp       |   15 +-
 clang/test/OpenMP/tile_rect_codegen_ir.cpp    |  510 +--
 clang/test/OpenMP/tile_vectorize_codegen.cpp  |   14 +
 clang/test/OpenMP/unroll_codegen_tile_for.cpp |   89 +-
 .../runtime/test/transform/tile/foreach.cpp   |    8 -
 openmp/runtime/test/transform/tile/intfor.c   |   58 +-
 .../runtime/test/transform/tile/iterfor.cpp   |    6 -
 .../tile/parallel-wsloop-collapse-foreach.cpp |   24 -
 ...llel-wsloop-collapse-stacked-tile-deep.cpp |   32 +
 .../parallel-wsloop-collapse-stacked-tile.cpp |   31 +
 ...rallel-wsloop-collapse-through-tile-2d.cpp |   56 +
 ...l-wsloop-collapse-through-tile-compose.cpp |   51 +
 ...-wsloop-collapse-through-tile-iterator.cpp |   74 +
 .../parallel-wsloop-collapse-through-tile.cpp |   10 +-
 20 files changed, 3600 insertions(+), 3167 deletions(-)
 delete mode 100644 clang/test/OpenMP/tile_canonical_required_codegen.cpp
 create mode 100644 clang/test/OpenMP/tile_collapse_reinterpret_codegen.cpp
 create mode 100644 clang/test/OpenMP/tile_vectorize_codegen.cpp
 create mode 100644 openmp/runtime/test/transform/tile/parallel-wsloop-collapse-stacked-tile-deep.cpp
 create mode 100644 openmp/runtime/test/transform/tile/parallel-wsloop-collapse-stacked-tile.cpp
 create mode 100644 openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile-2d.cpp
 create mode 100644 openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile-compose.cpp
 create mode 100644 openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile-iterator.cpp

diff --git a/clang/test/OpenMP/interchange_codegen.cpp b/clang/test/OpenMP/interchange_codegen.cpp
index 67ec7e34c5dfd..cc6e644ac70ac 100644
--- a/clang/test/OpenMP/interchange_codegen.cpp
+++ b/clang/test/OpenMP/interchange_codegen.cpp
@@ -811,21 +811,23 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTNEW_STEP10:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_14:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_17:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_19:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_K:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_K:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[_TMP35:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[I35:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[J36:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV_K37:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_K38:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[I39:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[J40:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV_K41:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_K42:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
@@ -863,454 +865,455 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
 // CHECK1-NEXT:    [[ADD15:%.*]] = add i32 [[TMP15]], 1
 // CHECK1-NEXT:    store i32 [[ADD15]], ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[SUB17:%.*]] = sub i32 [[TMP16]], [[TMP17]]
-// CHECK1-NEXT:    [[SUB18:%.*]] = sub i32 [[SUB17]], 1
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[ADD19:%.*]] = add i32 [[SUB18]], [[TMP18]]
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[DIV20:%.*]] = udiv i32 [[ADD19]], [[TMP19]]
-// CHECK1-NEXT:    [[CONV:%.*]] = zext i32 [[DIV20]] to i64
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB21:%.*]] = sub i32 [[TMP20]], [[TMP21]]
-// CHECK1-NEXT:    [[SUB22:%.*]] = sub i32 [[SUB21]], 1
-// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD23:%.*]] = add i32 [[SUB22]], [[TMP22]]
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV24:%.*]] = udiv i32 [[ADD23]], [[TMP23]]
-// CHECK1-NEXT:    [[CONV25:%.*]] = zext i32 [[DIV24]] to i64
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV25]]
-// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB26:%.*]] = sub i32 [[TMP24]], -31
-// CHECK1-NEXT:    [[DIV27:%.*]] = udiv i32 [[SUB26]], 32
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[_TMP2]], align 4
+// CHECK1-NEXT:    store i32 [[TMP16]], ptr [[DOTCAPTURE_EXPR_16]], align 4
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[_TMP2]], align 4
+// CHECK1-NEXT:    [[ADD18:%.*]] = add i32 [[TMP17]], 32
+// CHECK1-NEXT:    store i32 [[ADD18]], ptr [[DOTCAPTURE_EXPR_17]], align 4
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[SUB20:%.*]] = sub i32 [[TMP18]], [[TMP19]]
+// CHECK1-NEXT:    [[SUB21:%.*]] = sub i32 [[SUB20]], 1
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[ADD22:%.*]] = add i32 [[SUB21]], [[TMP20]]
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[DIV23:%.*]] = udiv i32 [[ADD22]], [[TMP21]]
+// CHECK1-NEXT:    [[CONV:%.*]] = zext i32 [[DIV23]] to i64
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB24:%.*]] = sub i32 [[TMP22]], [[TMP23]]
+// CHECK1-NEXT:    [[SUB25:%.*]] = sub i32 [[SUB24]], 1
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD26:%.*]] = add i32 [[SUB25]], [[TMP24]]
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV27:%.*]] = udiv i32 [[ADD26]], [[TMP25]]
 // CHECK1-NEXT:    [[CONV28:%.*]] = zext i32 [[DIV27]] to i64
-// CHECK1-NEXT:    [[MUL29:%.*]] = mul nsw i64 [[MUL]], [[CONV28]]
-// CHECK1-NEXT:    [[MUL30:%.*]] = mul nsw i64 [[MUL29]], 32
-// CHECK1-NEXT:    [[SUB31:%.*]] = sub nsw i64 [[MUL30]], 1
-// CHECK1-NEXT:    store i64 [[SUB31]], ptr [[DOTCAPTURE_EXPR_16]], align 8
-// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    store i32 [[TMP25]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    store i32 [[TMP26]], ptr [[J]], align 4
-// CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_K]], align 4
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_K]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV28]]
+// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB29:%.*]] = sub i32 [[TMP26]], -31
+// CHECK1-NEXT:    [[DIV30:%.*]] = udiv i32 [[SUB29]], 32
+// CHECK1-NEXT:    [[CONV31:%.*]] = zext i32 [[DIV30]] to i64
+// CHECK1-NEXT:    [[MUL32:%.*]] = mul nsw i64 [[MUL]], [[CONV31]]
+// CHECK1-NEXT:    [[MUL33:%.*]] = mul nsw i64 [[MUL32]], 32
+// CHECK1-NEXT:    [[SUB34:%.*]] = sub nsw i64 [[MUL33]], 1
+// CHECK1-NEXT:    store i64 [[SUB34]], ptr [[DOTCAPTURE_EXPR_19]], align 8
 // CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP27]], [[TMP28]]
+// CHECK1-NEXT:    store i32 [[TMP27]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    store i32 [[TMP28]], ptr [[J]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_K]], align 4
+// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K]], align 4
+// CHECK1-NEXT:    store i32 [[TMP29]], ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[_TMP35]], align 4
+// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP30]], [[TMP31]]
 // CHECK1-NEXT:    br i1 [[CMP]], label [[LAND_LHS_TRUE:%.*]], label [[OMP_PRECOND_END:%.*]]
 // CHECK1:       land.lhs.true:
-// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[CMP32:%.*]] = icmp slt i32 [[TMP29]], [[TMP30]]
-// CHECK1-NEXT:    br i1 [[CMP32]], label [[LAND_LHS_TRUE33:%.*]], label [[OMP_PRECOND_END]]
-// CHECK1:       land.lhs.true33:
-// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[CMP34:%.*]] = icmp ult i32 0, [[TMP31]]
-// CHECK1-NEXT:    br i1 [[CMP34]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
+// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[CMP36:%.*]] = icmp slt i32 [[TMP32]], [[TMP33]]
+// CHECK1-NEXT:    br i1 [[CMP36]], label [[LAND_LHS_TRUE37:%.*]], label [[OMP_PRECOND_END]]
+// CHECK1:       land.lhs.true37:
+// CHECK1-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[CMP38:%.*]] = icmp ult i32 0, [[TMP34]]
+// CHECK1-NEXT:    br i1 [[CMP38]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
 // CHECK1:       omp.precond.then:
 // CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
-// CHECK1-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
-// CHECK1-NEXT:    store i64 [[TMP32]], ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_19]], align 8
+// CHECK1-NEXT:    store i64 [[TMP35]], ptr [[DOTOMP_UB]], align 8
 // CHECK1-NEXT:    store i64 1, ptr [[DOTOMP_STRIDE]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
 // CHECK1-NEXT:    call void @__kmpc_for_static_init_8(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i64 1, i64 1)
-// CHECK1-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
-// CHECK1-NEXT:    [[CMP39:%.*]] = icmp sgt i64 [[TMP33]], [[TMP34]]
-// CHECK1-NEXT:    br i1 [[CMP39]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_19]], align 8
+// CHECK1-NEXT:    [[CMP43:%.*]] = icmp sgt i64 [[TMP36]], [[TMP37]]
+// CHECK1-NEXT:    br i1 [[CMP43]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK1:       cond.true:
-// CHECK1-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK1-NEXT:    [[TMP38:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_19]], align 8
 // CHECK1-NEXT:    br label [[COND_END:%.*]]
 // CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[TMP39:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
 // CHECK1-NEXT:    br label [[COND_END]]
 // CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i64 [ [[TMP35]], [[COND_TRUE]] ], [ [[TMP36]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[COND:%.*]] = phi i64 [ [[TMP38]], [[COND_TRUE]] ], [ [[TMP39]], [[COND_FALSE]] ]
 // CHECK1-NEXT:    store i64 [[COND]], ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
-// CHECK1-NEXT:    store i64 [[TMP37]], ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP40:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
+// CHECK1-NEXT:    store i64 [[TMP40]], ptr [[DOTOMP_IV]], align 8
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK1:       omp.inner.for.cond:
-// CHECK1-NEXT:    [[TMP38:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP39:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[CMP40:%.*]] = icmp sle i64 [[TMP38]], [[TMP39]]
-// CHECK1-NEXT:    br i1 [[CMP40]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
-// CHECK1:       omp.inner.for.body:
-// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[CONV41:%.*]] = sext i32 [[TMP40]] to i64
 // CHECK1-NEXT:    [[TMP41:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB42:%.*]] = sub i32 [[TMP42]], [[TMP43]]
-// CHECK1-NEXT:    [[SUB43:%.*]] = sub i32 [[SUB42]], 1
-// CHECK1-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD44:%.*]] = add i32 [[SUB43]], [[TMP44]]
-// CHECK1-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV45:%.*]] = udiv i32 [[ADD44]], [[TMP45]]
-// CHECK1-NEXT:    [[MUL46:%.*]] = mul i32 1, [[DIV45]]
-// CHECK1-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB47:%.*]] = sub i32 [[TMP46]], -31
-// CHECK1-NEXT:    [[DIV48:%.*]] = udiv i32 [[SUB47]], 32
-// CHECK1-NEXT:    [[MUL49:%.*]] = mul i32 [[MUL46]], [[DIV48]]
-// CHECK1-NEXT:    [[MUL50:%.*]] = mul i32 [[MUL49]], 32
-// CHECK1-NEXT:    [[CONV51:%.*]] = zext i32 [[MUL50]] to i64
-// CHECK1-NEXT:    [[DIV52:%.*]] = sdiv i64 [[TMP41]], [[CONV51]]
-// CHECK1-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[CONV53:%.*]] = sext i32 [[TMP47]] to i64
-// CHECK1-NEXT:    [[MUL54:%.*]] = mul nsw i64 [[DIV52]], [[CONV53]]
-// CHECK1-NEXT:    [[ADD55:%.*]] = add nsw i64 [[CONV41]], [[MUL54]]
-// CHECK1-NEXT:    [[CONV56:%.*]] = trunc i64 [[ADD55]] to i32
-// CHECK1-NEXT:    store i32 [[CONV56]], ptr [[I35]], align 4
-// CHECK1-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP48]] to i64
-// CHECK1-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP50:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB58:%.*]] = sub i32 [[TMP51]], [[TMP52]]
-// CHECK1-NEXT:    [[SUB59:%.*]] = sub i32 [[SUB58]], 1
-// CHECK1-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD60:%.*]] = add i32 [[SUB59]], [[TMP53]]
-// CHECK1-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV61:%.*]] = udiv i32 [[ADD60]], [[TMP54]]
-// CHECK1-NEXT:    [[MUL62:%.*]] = mul i32 1, [[DIV61]]
-// CHECK1-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB63:%.*]] = sub i32 [[TMP55]], -31
-// CHECK1-NEXT:    [[DIV64:%.*]] = udiv i32 [[SUB63]], 32
-// CHECK1-NEXT:    [[MUL65:%.*]] = mul i32 [[MUL62]], [[DIV64]]
-// CHECK1-NEXT:    [[MUL66:%.*]] = mul i32 [[MUL65]], 32
-// CHECK1-NEXT:    [[CONV67:%.*]] = zext i32 [[MUL66]] to i64
-// CHECK1-NEXT:    [[DIV68:%.*]] = sdiv i64 [[TMP50]], [[CONV67]]
-// CHECK1-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB69:%.*]] = sub i32 [[TMP56]], [[TMP57]]
-// CHECK1-NEXT:    [[SUB70:%.*]] = sub i32 [[SUB69]], 1
-// CHECK1-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD71:%.*]] = add i32 [[SUB70]], [[TMP58]]
-// CHECK1-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV72:%.*]] = udiv i32 [[ADD71]], [[TMP59]]
-// CHECK1-NEXT:    [[MUL73:%.*]] = mul i32 1, [[DIV72]]
-// CHECK1-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB74:%.*]] = sub i32 [[TMP60]], -31
-// CHECK1-NEXT:    [[DIV75:%.*]] = udiv i32 [[SUB74]], 32
-// CHECK1-NEXT:    [[MUL76:%.*]] = mul i32 [[MUL73]], [[DIV75]]
-// CHECK1-NEXT:    [[MUL77:%.*]] = mul i32 [[MUL76]], 32
-// CHECK1-NEXT:    [[CONV78:%.*]] = zext i32 [[MUL77]] to i64
-// CHECK1-NEXT:    [[MUL79:%.*]] = mul nsw i64 [[DIV68]], [[CONV78]]
-// CHECK1-NEXT:    [[SUB80:%.*]] = sub nsw i64 [[TMP49]], [[MUL79]]
-// CHECK1-NEXT:    [[TMP61:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB81:%.*]] = sub i32 [[TMP61]], -31
-// CHECK1-NEXT:    [[DIV82:%.*]] = udiv i32 [[SUB81]], 32
-// CHECK1-NEXT:    [[MUL83:%.*]] = mul i32 1, [[DIV82]]
-// CHECK1-NEXT:    [[MUL84:%.*]] = mul i32 [[MUL83]], 32
-// CHECK1-NEXT:    [[CONV85:%.*]] = zext i32 [[MUL84]] to i64
-// CHECK1-NEXT:    [[DIV86:%.*]] = sdiv i64 [[SUB80]], [[CONV85]]
+// CHECK1-NEXT:    [[TMP42:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[CMP44:%.*]] = icmp sle i64 [[TMP41]], [[TMP42]]
+// CHECK1-NEXT:    br i1 [[CMP44]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK1:       omp.inner.for.body:
+// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[CONV45:%.*]] = sext i32 [[TMP43]] to i64
+// CHECK1-NEXT:    [[TMP44:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB46:%.*]] = sub i32 [[TMP45]], [[TMP46]]
+// CHECK1-NEXT:    [[SUB47:%.*]] = sub i32 [[SUB46]], 1
+// CHECK1-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD48:%.*]] = add i32 [[SUB47]], [[TMP47]]
+// CHECK1-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV49:%.*]] = udiv i32 [[ADD48]], [[TMP48]]
+// CHECK1-NEXT:    [[MUL50:%.*]] = mul i32 1, [[DIV49]]
+// CHECK1-NEXT:    [[TMP49:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB51:%.*]] = sub i32 [[TMP49]], -31
+// CHECK1-NEXT:    [[DIV52:%.*]] = udiv i32 [[SUB51]], 32
+// CHECK1-NEXT:    [[MUL53:%.*]] = mul i32 [[MUL50]], [[DIV52]]
+// CHECK1-NEXT:    [[MUL54:%.*]] = mul i32 [[MUL53]], 32
+// CHECK1-NEXT:    [[CONV55:%.*]] = zext i32 [[MUL54]] to i64
+// CHECK1-NEXT:    [[DIV56:%.*]] = sdiv i64 [[TMP44]], [[CONV55]]
+// CHECK1-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP50]] to i64
+// CHECK1-NEXT:    [[MUL58:%.*]] = mul nsw i64 [[DIV56]], [[CONV57]]
+// CHECK1-NEXT:    [[ADD59:%.*]] = add nsw i64 [[CONV45]], [[MUL58]]
+// CHECK1-NEXT:    [[CONV60:%.*]] = trunc i64 [[ADD59]] to i32
+// CHECK1-NEXT:    store i32 [[CONV60]], ptr [[I39]], align 4
+// CHECK1-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[CONV61:%.*]] = sext i32 [[TMP51]] to i64
+// CHECK1-NEXT:    [[TMP52:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP53:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB62:%.*]] = sub i32 [[TMP54]], [[TMP55]]
+// CHECK1-NEXT:    [[SUB63:%.*]] = sub i32 [[SUB62]], 1
+// CHECK1-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD64:%.*]] = add i32 [[SUB63]], [[TMP56]]
+// CHECK1-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV65:%.*]] = udiv i32 [[ADD64]], [[TMP57]]
+// CHECK1-NEXT:    [[MUL66:%.*]] = mul i32 1, [[DIV65]]
+// CHECK1-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB67:%.*]] = sub i32 [[TMP58]], -31
+// CHECK1-NEXT:    [[DIV68:%.*]] = udiv i32 [[SUB67]], 32
+// CHECK1-NEXT:    [[MUL69:%.*]] = mul i32 [[MUL66]], [[DIV68]]
+// CHECK1-NEXT:    [[MUL70:%.*]] = mul i32 [[MUL69]], 32
+// CHECK1-NEXT:    [[CONV71:%.*]] = zext i32 [[MUL70]] to i64
+// CHECK1-NEXT:    [[DIV72:%.*]] = sdiv i64 [[TMP53]], [[CONV71]]
+// CHECK1-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB73:%.*]] = sub i32 [[TMP59]], [[TMP60]]
+// CHECK1-NEXT:    [[SUB74:%.*]] = sub i32 [[SUB73]], 1
+// CHECK1-NEXT:    [[TMP61:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD75:%.*]] = add i32 [[SUB74]], [[TMP61]]
 // CHECK1-NEXT:    [[TMP62:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[CONV87:%.*]] = sext i32 [[TMP62]] to i64
-// CHECK1-NEXT:    [[MUL88:%.*]] = mul nsw i64 [[DIV86]], [[CONV87]]
-// CHECK1-NEXT:    [[ADD89:%.*]] = add nsw i64 [[CONV57]], [[MUL88]]
-// CHECK1-NEXT:    [[CONV90:%.*]] = trunc i64 [[ADD89]] to i32
-// CHECK1-NEXT:    store i32 [[CONV90]], ptr [[J36]], align 4
-// CHECK1-NEXT:    [[TMP63:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP64:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB91:%.*]] = sub i32 [[TMP65]], [[TMP66]]
-// CHECK1-NEXT:    [[SUB92:%.*]] = sub i32 [[SUB91]], 1
-// CHECK1-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD93:%.*]] = add i32 [[SUB92]], [[TMP67]]
-// CHECK1-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV94:%.*]] = udiv i32 [[ADD93]], [[TMP68]]
-// CHECK1-NEXT:    [[MUL95:%.*]] = mul i32 1, [[DIV94]]
-// CHECK1-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB96:%.*]] = sub i32 [[TMP69]], -31
-// CHECK1-NEXT:    [[DIV97:%.*]] = udiv i32 [[SUB96]], 32
-// CHECK1-NEXT:    [[MUL98:%.*]] = mul i32 [[MUL95]], [[DIV97]]
-// CHECK1-NEXT:    [[MUL99:%.*]] = mul i32 [[MUL98]], 32
-// CHECK1-NEXT:    [[CONV100:%.*]] = zext i32 [[MUL99]] to i64
-// CHECK1-NEXT:    [[DIV101:%.*]] = sdiv i64 [[TMP64]], [[CONV100]]
-// CHECK1-NEXT:    [[TMP70:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB102:%.*]] = sub i32 [[TMP70]], [[TMP71]]
-// CHECK1-NEXT:    [[SUB103:%.*]] = sub i32 [[SUB102]], 1
-// CHECK1-NEXT:    [[TMP72:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD104:%.*]] = add i32 [[SUB103]], [[TMP72]]
-// CHECK1-NEXT:    [[TMP73:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV105:%.*]] = udiv i32 [[ADD104]], [[TMP73]]
-// CHECK1-NEXT:    [[MUL106:%.*]] = mul i32 1, [[DIV105]]
-// CHECK1-NEXT:    [[TMP74:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB107:%.*]] = sub i32 [[TMP74]], -31
-// CHECK1-NEXT:    [[DIV108:%.*]] = udiv i32 [[SUB107]], 32
-// CHECK1-NEXT:    [[MUL109:%.*]] = mul i32 [[MUL106]], [[DIV108]]
-// CHECK1-NEXT:    [[MUL110:%.*]] = mul i32 [[MUL109]], 32
-// CHECK1-NEXT:    [[CONV111:%.*]] = zext i32 [[MUL110]] to i64
-// CHECK1-NEXT:    [[MUL112:%.*]] = mul nsw i64 [[DIV101]], [[CONV111]]
-// CHECK1-NEXT:    [[SUB113:%.*]] = sub nsw i64 [[TMP63]], [[MUL112]]
-// CHECK1-NEXT:    [[TMP75:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP76:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP78:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB114:%.*]] = sub i32 [[TMP77]], [[TMP78]]
-// CHECK1-NEXT:    [[SUB115:%.*]] = sub i32 [[SUB114]], 1
-// CHECK1-NEXT:    [[TMP79:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD116:%.*]] = add i32 [[SUB115]], [[TMP79]]
-// CHECK1-NEXT:    [[TMP80:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV117:%.*]] = udiv i32 [[ADD116]], [[TMP80]]
-// CHECK1-NEXT:    [[MUL118:%.*]] = mul i32 1, [[DIV117]]
-// CHECK1-NEXT:    [[TMP81:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB119:%.*]] = sub i32 [[TMP81]], -31
-// CHECK1-NEXT:    [[DIV120:%.*]] = udiv i32 [[SUB119]], 32
-// CHECK1-NEXT:    [[MUL121:%.*]] = mul i32 [[MUL118]], [[DIV120]]
-// CHECK1-NEXT:    [[MUL122:%.*]] = mul i32 [[MUL121]], 32
-// CHECK1-NEXT:    [[CONV123:%.*]] = zext i32 [[MUL122]] to i64
-// CHECK1-NEXT:    [[DIV124:%.*]] = sdiv i64 [[TMP76]], [[CONV123]]
-// CHECK1-NEXT:    [[TMP82:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB125:%.*]] = sub i32 [[TMP82]], [[TMP83]]
-// CHECK1-NEXT:    [[SUB126:%.*]] = sub i32 [[SUB125]], 1
-// CHECK1-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD127:%.*]] = add i32 [[SUB126]], [[TMP84]]
-// CHECK1-NEXT:    [[TMP85:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV128:%.*]] = udiv i32 [[ADD127]], [[TMP85]]
-// CHECK1-NEXT:    [[MUL129:%.*]] = mul i32 1, [[DIV128]]
-// CHECK1-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB130:%.*]] = sub i32 [[TMP86]], -31
-// CHECK1-NEXT:    [[DIV131:%.*]] = udiv i32 [[SUB130]], 32
-// CHECK1-NEXT:    [[MUL132:%.*]] = mul i32 [[MUL129]], [[DIV131]]
-// CHECK1-NEXT:    [[MUL133:%.*]] = mul i32 [[MUL132]], 32
-// CHECK1-NEXT:    [[CONV134:%.*]] = zext i32 [[MUL133]] to i64
-// CHECK1-NEXT:    [[MUL135:%.*]] = mul nsw i64 [[DIV124]], [[CONV134]]
-// CHECK1-NEXT:    [[SUB136:%.*]] = sub nsw i64 [[TMP75]], [[MUL135]]
-// CHECK1-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB137:%.*]] = sub i32 [[TMP87]], -31
-// CHECK1-NEXT:    [[DIV138:%.*]] = udiv i32 [[SUB137]], 32
-// CHECK1-NEXT:    [[MUL139:%.*]] = mul i32 1, [[DIV138]]
-// CHECK1-NEXT:    [[MUL140:%.*]] = mul i32 [[MUL139]], 32
-// CHECK1-NEXT:    [[CONV141:%.*]] = zext i32 [[MUL140]] to i64
-// CHECK1-NEXT:    [[DIV142:%.*]] = sdiv i64 [[SUB136]], [[CONV141]]
-// CHECK1-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB143:%.*]] = sub i32 [[TMP88]], -31
-// CHECK1-NEXT:    [[DIV144:%.*]] = udiv i32 [[SUB143]], 32
-// CHECK1-NEXT:    [[MUL145:%.*]] = mul i32 1, [[DIV144]]
-// CHECK1-NEXT:    [[MUL146:%.*]] = mul i32 [[MUL145]], 32
-// CHECK1-NEXT:    [[CONV147:%.*]] = zext i32 [[MUL146]] to i64
-// CHECK1-NEXT:    [[MUL148:%.*]] = mul nsw i64 [[DIV142]], [[CONV147]]
-// CHECK1-NEXT:    [[SUB149:%.*]] = sub nsw i64 [[SUB113]], [[MUL148]]
-// CHECK1-NEXT:    [[DIV150:%.*]] = sdiv i64 [[SUB149]], 32
-// CHECK1-NEXT:    [[MUL151:%.*]] = mul nsw i64 [[DIV150]], 32
-// CHECK1-NEXT:    [[ADD152:%.*]] = add nsw i64 0, [[MUL151]]
-// CHECK1-NEXT:    [[CONV153:%.*]] = trunc i64 [[ADD152]] to i32
-// CHECK1-NEXT:    store i32 [[CONV153]], ptr [[DOTFLOOR_0_IV_K37]], align 4
-// CHECK1-NEXT:    [[TMP89:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP90:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB154:%.*]] = sub i32 [[TMP91]], [[TMP92]]
-// CHECK1-NEXT:    [[SUB155:%.*]] = sub i32 [[SUB154]], 1
-// CHECK1-NEXT:    [[TMP93:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD156:%.*]] = add i32 [[SUB155]], [[TMP93]]
-// CHECK1-NEXT:    [[TMP94:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV157:%.*]] = udiv i32 [[ADD156]], [[TMP94]]
-// CHECK1-NEXT:    [[MUL158:%.*]] = mul i32 1, [[DIV157]]
-// CHECK1-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB159:%.*]] = sub i32 [[TMP95]], -31
-// CHECK1-NEXT:    [[DIV160:%.*]] = udiv i32 [[SUB159]], 32
-// CHECK1-NEXT:    [[MUL161:%.*]] = mul i32 [[MUL158]], [[DIV160]]
-// CHECK1-NEXT:    [[MUL162:%.*]] = mul i32 [[MUL161]], 32
-// CHECK1-NEXT:    [[CONV163:%.*]] = zext i32 [[MUL162]] to i64
-// CHECK1-NEXT:    [[DIV164:%.*]] = sdiv i64 [[TMP90]], [[CONV163]]
-// CHECK1-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP97:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB165:%.*]] = sub i32 [[TMP96]], [[TMP97]]
-// CHECK1-NEXT:    [[SUB166:%.*]] = sub i32 [[SUB165]], 1
+// CHECK1-NEXT:    [[DIV76:%.*]] = udiv i32 [[ADD75]], [[TMP62]]
+// CHECK1-NEXT:    [[MUL77:%.*]] = mul i32 1, [[DIV76]]
+// CHECK1-NEXT:    [[TMP63:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB78:%.*]] = sub i32 [[TMP63]], -31
+// CHECK1-NEXT:    [[DIV79:%.*]] = udiv i32 [[SUB78]], 32
+// CHECK1-NEXT:    [[MUL80:%.*]] = mul i32 [[MUL77]], [[DIV79]]
+// CHECK1-NEXT:    [[MUL81:%.*]] = mul i32 [[MUL80]], 32
+// CHECK1-NEXT:    [[CONV82:%.*]] = zext i32 [[MUL81]] to i64
+// CHECK1-NEXT:    [[MUL83:%.*]] = mul nsw i64 [[DIV72]], [[CONV82]]
+// CHECK1-NEXT:    [[SUB84:%.*]] = sub nsw i64 [[TMP52]], [[MUL83]]
+// CHECK1-NEXT:    [[TMP64:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB85:%.*]] = sub i32 [[TMP64]], -31
+// CHECK1-NEXT:    [[DIV86:%.*]] = udiv i32 [[SUB85]], 32
+// CHECK1-NEXT:    [[MUL87:%.*]] = mul i32 1, [[DIV86]]
+// CHECK1-NEXT:    [[MUL88:%.*]] = mul i32 [[MUL87]], 32
+// CHECK1-NEXT:    [[CONV89:%.*]] = zext i32 [[MUL88]] to i64
+// CHECK1-NEXT:    [[DIV90:%.*]] = sdiv i64 [[SUB84]], [[CONV89]]
+// CHECK1-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[CONV91:%.*]] = sext i32 [[TMP65]] to i64
+// CHECK1-NEXT:    [[MUL92:%.*]] = mul nsw i64 [[DIV90]], [[CONV91]]
+// CHECK1-NEXT:    [[ADD93:%.*]] = add nsw i64 [[CONV61]], [[MUL92]]
+// CHECK1-NEXT:    [[CONV94:%.*]] = trunc i64 [[ADD93]] to i32
+// CHECK1-NEXT:    store i32 [[CONV94]], ptr [[J40]], align 4
+// CHECK1-NEXT:    [[TMP66:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP67:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB95:%.*]] = sub i32 [[TMP68]], [[TMP69]]
+// CHECK1-NEXT:    [[SUB96:%.*]] = sub i32 [[SUB95]], 1
+// CHECK1-NEXT:    [[TMP70:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD97:%.*]] = add i32 [[SUB96]], [[TMP70]]
+// CHECK1-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV98:%.*]] = udiv i32 [[ADD97]], [[TMP71]]
+// CHECK1-NEXT:    [[MUL99:%.*]] = mul i32 1, [[DIV98]]
+// CHECK1-NEXT:    [[TMP72:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB100:%.*]] = sub i32 [[TMP72]], -31
+// CHECK1-NEXT:    [[DIV101:%.*]] = udiv i32 [[SUB100]], 32
+// CHECK1-NEXT:    [[MUL102:%.*]] = mul i32 [[MUL99]], [[DIV101]]
+// CHECK1-NEXT:    [[MUL103:%.*]] = mul i32 [[MUL102]], 32
+// CHECK1-NEXT:    [[CONV104:%.*]] = zext i32 [[MUL103]] to i64
+// CHECK1-NEXT:    [[DIV105:%.*]] = sdiv i64 [[TMP67]], [[CONV104]]
+// CHECK1-NEXT:    [[TMP73:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP74:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB106:%.*]] = sub i32 [[TMP73]], [[TMP74]]
+// CHECK1-NEXT:    [[SUB107:%.*]] = sub i32 [[SUB106]], 1
+// CHECK1-NEXT:    [[TMP75:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD108:%.*]] = add i32 [[SUB107]], [[TMP75]]
+// CHECK1-NEXT:    [[TMP76:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV109:%.*]] = udiv i32 [[ADD108]], [[TMP76]]
+// CHECK1-NEXT:    [[MUL110:%.*]] = mul i32 1, [[DIV109]]
+// CHECK1-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB111:%.*]] = sub i32 [[TMP77]], -31
+// CHECK1-NEXT:    [[DIV112:%.*]] = udiv i32 [[SUB111]], 32
+// CHECK1-NEXT:    [[MUL113:%.*]] = mul i32 [[MUL110]], [[DIV112]]
+// CHECK1-NEXT:    [[MUL114:%.*]] = mul i32 [[MUL113]], 32
+// CHECK1-NEXT:    [[CONV115:%.*]] = zext i32 [[MUL114]] to i64
+// CHECK1-NEXT:    [[MUL116:%.*]] = mul nsw i64 [[DIV105]], [[CONV115]]
+// CHECK1-NEXT:    [[SUB117:%.*]] = sub nsw i64 [[TMP66]], [[MUL116]]
+// CHECK1-NEXT:    [[TMP78:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP79:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP80:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP81:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB118:%.*]] = sub i32 [[TMP80]], [[TMP81]]
+// CHECK1-NEXT:    [[SUB119:%.*]] = sub i32 [[SUB118]], 1
+// CHECK1-NEXT:    [[TMP82:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD120:%.*]] = add i32 [[SUB119]], [[TMP82]]
+// CHECK1-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV121:%.*]] = udiv i32 [[ADD120]], [[TMP83]]
+// CHECK1-NEXT:    [[MUL122:%.*]] = mul i32 1, [[DIV121]]
+// CHECK1-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB123:%.*]] = sub i32 [[TMP84]], -31
+// CHECK1-NEXT:    [[DIV124:%.*]] = udiv i32 [[SUB123]], 32
+// CHECK1-NEXT:    [[MUL125:%.*]] = mul i32 [[MUL122]], [[DIV124]]
+// CHECK1-NEXT:    [[MUL126:%.*]] = mul i32 [[MUL125]], 32
+// CHECK1-NEXT:    [[CONV127:%.*]] = zext i32 [[MUL126]] to i64
+// CHECK1-NEXT:    [[DIV128:%.*]] = sdiv i64 [[TMP79]], [[CONV127]]
+// CHECK1-NEXT:    [[TMP85:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB129:%.*]] = sub i32 [[TMP85]], [[TMP86]]
+// CHECK1-NEXT:    [[SUB130:%.*]] = sub i32 [[SUB129]], 1
+// CHECK1-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD131:%.*]] = add i32 [[SUB130]], [[TMP87]]
+// CHECK1-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV132:%.*]] = udiv i32 [[ADD131]], [[TMP88]]
+// CHECK1-NEXT:    [[MUL133:%.*]] = mul i32 1, [[DIV132]]
+// CHECK1-NEXT:    [[TMP89:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB134:%.*]] = sub i32 [[TMP89]], -31
+// CHECK1-NEXT:    [[DIV135:%.*]] = udiv i32 [[SUB134]], 32
+// CHECK1-NEXT:    [[MUL136:%.*]] = mul i32 [[MUL133]], [[DIV135]]
+// CHECK1-NEXT:    [[MUL137:%.*]] = mul i32 [[MUL136]], 32
+// CHECK1-NEXT:    [[CONV138:%.*]] = zext i32 [[MUL137]] to i64
+// CHECK1-NEXT:    [[MUL139:%.*]] = mul nsw i64 [[DIV128]], [[CONV138]]
+// CHECK1-NEXT:    [[SUB140:%.*]] = sub nsw i64 [[TMP78]], [[MUL139]]
+// CHECK1-NEXT:    [[TMP90:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB141:%.*]] = sub i32 [[TMP90]], -31
+// CHECK1-NEXT:    [[DIV142:%.*]] = udiv i32 [[SUB141]], 32
+// CHECK1-NEXT:    [[MUL143:%.*]] = mul i32 1, [[DIV142]]
+// CHECK1-NEXT:    [[MUL144:%.*]] = mul i32 [[MUL143]], 32
+// CHECK1-NEXT:    [[CONV145:%.*]] = zext i32 [[MUL144]] to i64
+// CHECK1-NEXT:    [[DIV146:%.*]] = sdiv i64 [[SUB140]], [[CONV145]]
+// CHECK1-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB147:%.*]] = sub i32 [[TMP91]], -31
+// CHECK1-NEXT:    [[DIV148:%.*]] = udiv i32 [[SUB147]], 32
+// CHECK1-NEXT:    [[MUL149:%.*]] = mul i32 1, [[DIV148]]
+// CHECK1-NEXT:    [[MUL150:%.*]] = mul i32 [[MUL149]], 32
+// CHECK1-NEXT:    [[CONV151:%.*]] = zext i32 [[MUL150]] to i64
+// CHECK1-NEXT:    [[MUL152:%.*]] = mul nsw i64 [[DIV146]], [[CONV151]]
+// CHECK1-NEXT:    [[SUB153:%.*]] = sub nsw i64 [[SUB117]], [[MUL152]]
+// CHECK1-NEXT:    [[DIV154:%.*]] = sdiv i64 [[SUB153]], 32
+// CHECK1-NEXT:    [[MUL155:%.*]] = mul nsw i64 [[DIV154]], 32
+// CHECK1-NEXT:    [[ADD156:%.*]] = add nsw i64 0, [[MUL155]]
+// CHECK1-NEXT:    [[CONV157:%.*]] = trunc i64 [[ADD156]] to i32
+// CHECK1-NEXT:    store i32 [[CONV157]], ptr [[DOTFLOOR_0_IV_K41]], align 4
+// CHECK1-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K41]], align 4
+// CHECK1-NEXT:    [[CONV158:%.*]] = zext i32 [[TMP92]] to i64
+// CHECK1-NEXT:    [[TMP93:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP94:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB159:%.*]] = sub i32 [[TMP95]], [[TMP96]]
+// CHECK1-NEXT:    [[SUB160:%.*]] = sub i32 [[SUB159]], 1
+// CHECK1-NEXT:    [[TMP97:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD161:%.*]] = add i32 [[SUB160]], [[TMP97]]
 // CHECK1-NEXT:    [[TMP98:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD167:%.*]] = add i32 [[SUB166]], [[TMP98]]
-// CHECK1-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV168:%.*]] = udiv i32 [[ADD167]], [[TMP99]]
-// CHECK1-NEXT:    [[MUL169:%.*]] = mul i32 1, [[DIV168]]
-// CHECK1-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB170:%.*]] = sub i32 [[TMP100]], -31
-// CHECK1-NEXT:    [[DIV171:%.*]] = udiv i32 [[SUB170]], 32
-// CHECK1-NEXT:    [[MUL172:%.*]] = mul i32 [[MUL169]], [[DIV171]]
-// CHECK1-NEXT:    [[MUL173:%.*]] = mul i32 [[MUL172]], 32
-// CHECK1-NEXT:    [[CONV174:%.*]] = zext i32 [[MUL173]] to i64
-// CHECK1-NEXT:    [[MUL175:%.*]] = mul nsw i64 [[DIV164]], [[CONV174]]
-// CHECK1-NEXT:    [[SUB176:%.*]] = sub nsw i64 [[TMP89]], [[MUL175]]
-// CHECK1-NEXT:    [[TMP101:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP102:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB177:%.*]] = sub i32 [[TMP103]], [[TMP104]]
-// CHECK1-NEXT:    [[SUB178:%.*]] = sub i32 [[SUB177]], 1
-// CHECK1-NEXT:    [[TMP105:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD179:%.*]] = add i32 [[SUB178]], [[TMP105]]
-// CHECK1-NEXT:    [[TMP106:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV180:%.*]] = udiv i32 [[ADD179]], [[TMP106]]
-// CHECK1-NEXT:    [[MUL181:%.*]] = mul i32 1, [[DIV180]]
-// CHECK1-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB182:%.*]] = sub i32 [[TMP107]], -31
-// CHECK1-NEXT:    [[DIV183:%.*]] = udiv i32 [[SUB182]], 32
-// CHECK1-NEXT:    [[MUL184:%.*]] = mul i32 [[MUL181]], [[DIV183]]
-// CHECK1-NEXT:    [[MUL185:%.*]] = mul i32 [[MUL184]], 32
-// CHECK1-NEXT:    [[CONV186:%.*]] = zext i32 [[MUL185]] to i64
-// CHECK1-NEXT:    [[DIV187:%.*]] = sdiv i64 [[TMP102]], [[CONV186]]
-// CHECK1-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB188:%.*]] = sub i32 [[TMP108]], [[TMP109]]
-// CHECK1-NEXT:    [[SUB189:%.*]] = sub i32 [[SUB188]], 1
+// CHECK1-NEXT:    [[DIV162:%.*]] = udiv i32 [[ADD161]], [[TMP98]]
+// CHECK1-NEXT:    [[MUL163:%.*]] = mul i32 1, [[DIV162]]
+// CHECK1-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB164:%.*]] = sub i32 [[TMP99]], -31
+// CHECK1-NEXT:    [[DIV165:%.*]] = udiv i32 [[SUB164]], 32
+// CHECK1-NEXT:    [[MUL166:%.*]] = mul i32 [[MUL163]], [[DIV165]]
+// CHECK1-NEXT:    [[MUL167:%.*]] = mul i32 [[MUL166]], 32
+// CHECK1-NEXT:    [[CONV168:%.*]] = zext i32 [[MUL167]] to i64
+// CHECK1-NEXT:    [[DIV169:%.*]] = sdiv i64 [[TMP94]], [[CONV168]]
+// CHECK1-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP101:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB170:%.*]] = sub i32 [[TMP100]], [[TMP101]]
+// CHECK1-NEXT:    [[SUB171:%.*]] = sub i32 [[SUB170]], 1
+// CHECK1-NEXT:    [[TMP102:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD172:%.*]] = add i32 [[SUB171]], [[TMP102]]
+// CHECK1-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV173:%.*]] = udiv i32 [[ADD172]], [[TMP103]]
+// CHECK1-NEXT:    [[MUL174:%.*]] = mul i32 1, [[DIV173]]
+// CHECK1-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB175:%.*]] = sub i32 [[TMP104]], -31
+// CHECK1-NEXT:    [[DIV176:%.*]] = udiv i32 [[SUB175]], 32
+// CHECK1-NEXT:    [[MUL177:%.*]] = mul i32 [[MUL174]], [[DIV176]]
+// CHECK1-NEXT:    [[MUL178:%.*]] = mul i32 [[MUL177]], 32
+// CHECK1-NEXT:    [[CONV179:%.*]] = zext i32 [[MUL178]] to i64
+// CHECK1-NEXT:    [[MUL180:%.*]] = mul nsw i64 [[DIV169]], [[CONV179]]
+// CHECK1-NEXT:    [[SUB181:%.*]] = sub nsw i64 [[TMP93]], [[MUL180]]
+// CHECK1-NEXT:    [[TMP105:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP106:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB182:%.*]] = sub i32 [[TMP107]], [[TMP108]]
+// CHECK1-NEXT:    [[SUB183:%.*]] = sub i32 [[SUB182]], 1
+// CHECK1-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD184:%.*]] = add i32 [[SUB183]], [[TMP109]]
 // CHECK1-NEXT:    [[TMP110:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD190:%.*]] = add i32 [[SUB189]], [[TMP110]]
-// CHECK1-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV191:%.*]] = udiv i32 [[ADD190]], [[TMP111]]
-// CHECK1-NEXT:    [[MUL192:%.*]] = mul i32 1, [[DIV191]]
-// CHECK1-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB193:%.*]] = sub i32 [[TMP112]], -31
-// CHECK1-NEXT:    [[DIV194:%.*]] = udiv i32 [[SUB193]], 32
-// CHECK1-NEXT:    [[MUL195:%.*]] = mul i32 [[MUL192]], [[DIV194]]
-// CHECK1-NEXT:    [[MUL196:%.*]] = mul i32 [[MUL195]], 32
-// CHECK1-NEXT:    [[CONV197:%.*]] = zext i32 [[MUL196]] to i64
-// CHECK1-NEXT:    [[MUL198:%.*]] = mul nsw i64 [[DIV187]], [[CONV197]]
-// CHECK1-NEXT:    [[SUB199:%.*]] = sub nsw i64 [[TMP101]], [[MUL198]]
-// CHECK1-NEXT:    [[TMP113:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB200:%.*]] = sub i32 [[TMP113]], -31
-// CHECK1-NEXT:    [[DIV201:%.*]] = udiv i32 [[SUB200]], 32
-// CHECK1-NEXT:    [[MUL202:%.*]] = mul i32 1, [[DIV201]]
-// CHECK1-NEXT:    [[MUL203:%.*]] = mul i32 [[MUL202]], 32
-// CHECK1-NEXT:    [[CONV204:%.*]] = zext i32 [[MUL203]] to i64
-// CHECK1-NEXT:    [[DIV205:%.*]] = sdiv i64 [[SUB199]], [[CONV204]]
-// CHECK1-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB206:%.*]] = sub i32 [[TMP114]], -31
-// CHECK1-NEXT:    [[DIV207:%.*]] = udiv i32 [[SUB206]], 32
-// CHECK1-NEXT:    [[MUL208:%.*]] = mul i32 1, [[DIV207]]
-// CHECK1-NEXT:    [[MUL209:%.*]] = mul i32 [[MUL208]], 32
-// CHECK1-NEXT:    [[CONV210:%.*]] = zext i32 [[MUL209]] to i64
-// CHECK1-NEXT:    [[MUL211:%.*]] = mul nsw i64 [[DIV205]], [[CONV210]]
-// CHECK1-NEXT:    [[SUB212:%.*]] = sub nsw i64 [[SUB176]], [[MUL211]]
-// CHECK1-NEXT:    [[TMP115:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP116:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB213:%.*]] = sub i32 [[TMP117]], [[TMP118]]
-// CHECK1-NEXT:    [[SUB214:%.*]] = sub i32 [[SUB213]], 1
-// CHECK1-NEXT:    [[TMP119:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD215:%.*]] = add i32 [[SUB214]], [[TMP119]]
-// CHECK1-NEXT:    [[TMP120:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV216:%.*]] = udiv i32 [[ADD215]], [[TMP120]]
-// CHECK1-NEXT:    [[MUL217:%.*]] = mul i32 1, [[DIV216]]
-// CHECK1-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB218:%.*]] = sub i32 [[TMP121]], -31
-// CHECK1-NEXT:    [[DIV219:%.*]] = udiv i32 [[SUB218]], 32
-// CHECK1-NEXT:    [[MUL220:%.*]] = mul i32 [[MUL217]], [[DIV219]]
-// CHECK1-NEXT:    [[MUL221:%.*]] = mul i32 [[MUL220]], 32
-// CHECK1-NEXT:    [[CONV222:%.*]] = zext i32 [[MUL221]] to i64
-// CHECK1-NEXT:    [[DIV223:%.*]] = sdiv i64 [[TMP116]], [[CONV222]]
-// CHECK1-NEXT:    [[TMP122:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP123:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB224:%.*]] = sub i32 [[TMP122]], [[TMP123]]
-// CHECK1-NEXT:    [[SUB225:%.*]] = sub i32 [[SUB224]], 1
+// CHECK1-NEXT:    [[DIV185:%.*]] = udiv i32 [[ADD184]], [[TMP110]]
+// CHECK1-NEXT:    [[MUL186:%.*]] = mul i32 1, [[DIV185]]
+// CHECK1-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB187:%.*]] = sub i32 [[TMP111]], -31
+// CHECK1-NEXT:    [[DIV188:%.*]] = udiv i32 [[SUB187]], 32
+// CHECK1-NEXT:    [[MUL189:%.*]] = mul i32 [[MUL186]], [[DIV188]]
+// CHECK1-NEXT:    [[MUL190:%.*]] = mul i32 [[MUL189]], 32
+// CHECK1-NEXT:    [[CONV191:%.*]] = zext i32 [[MUL190]] to i64
+// CHECK1-NEXT:    [[DIV192:%.*]] = sdiv i64 [[TMP106]], [[CONV191]]
+// CHECK1-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP113:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB193:%.*]] = sub i32 [[TMP112]], [[TMP113]]
+// CHECK1-NEXT:    [[SUB194:%.*]] = sub i32 [[SUB193]], 1
+// CHECK1-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD195:%.*]] = add i32 [[SUB194]], [[TMP114]]
+// CHECK1-NEXT:    [[TMP115:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV196:%.*]] = udiv i32 [[ADD195]], [[TMP115]]
+// CHECK1-NEXT:    [[MUL197:%.*]] = mul i32 1, [[DIV196]]
+// CHECK1-NEXT:    [[TMP116:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB198:%.*]] = sub i32 [[TMP116]], -31
+// CHECK1-NEXT:    [[DIV199:%.*]] = udiv i32 [[SUB198]], 32
+// CHECK1-NEXT:    [[MUL200:%.*]] = mul i32 [[MUL197]], [[DIV199]]
+// CHECK1-NEXT:    [[MUL201:%.*]] = mul i32 [[MUL200]], 32
+// CHECK1-NEXT:    [[CONV202:%.*]] = zext i32 [[MUL201]] to i64
+// CHECK1-NEXT:    [[MUL203:%.*]] = mul nsw i64 [[DIV192]], [[CONV202]]
+// CHECK1-NEXT:    [[SUB204:%.*]] = sub nsw i64 [[TMP105]], [[MUL203]]
+// CHECK1-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB205:%.*]] = sub i32 [[TMP117]], -31
+// CHECK1-NEXT:    [[DIV206:%.*]] = udiv i32 [[SUB205]], 32
+// CHECK1-NEXT:    [[MUL207:%.*]] = mul i32 1, [[DIV206]]
+// CHECK1-NEXT:    [[MUL208:%.*]] = mul i32 [[MUL207]], 32
+// CHECK1-NEXT:    [[CONV209:%.*]] = zext i32 [[MUL208]] to i64
+// CHECK1-NEXT:    [[DIV210:%.*]] = sdiv i64 [[SUB204]], [[CONV209]]
+// CHECK1-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB211:%.*]] = sub i32 [[TMP118]], -31
+// CHECK1-NEXT:    [[DIV212:%.*]] = udiv i32 [[SUB211]], 32
+// CHECK1-NEXT:    [[MUL213:%.*]] = mul i32 1, [[DIV212]]
+// CHECK1-NEXT:    [[MUL214:%.*]] = mul i32 [[MUL213]], 32
+// CHECK1-NEXT:    [[CONV215:%.*]] = zext i32 [[MUL214]] to i64
+// CHECK1-NEXT:    [[MUL216:%.*]] = mul nsw i64 [[DIV210]], [[CONV215]]
+// CHECK1-NEXT:    [[SUB217:%.*]] = sub nsw i64 [[SUB181]], [[MUL216]]
+// CHECK1-NEXT:    [[TMP119:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP120:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP122:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB218:%.*]] = sub i32 [[TMP121]], [[TMP122]]
+// CHECK1-NEXT:    [[SUB219:%.*]] = sub i32 [[SUB218]], 1
+// CHECK1-NEXT:    [[TMP123:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD220:%.*]] = add i32 [[SUB219]], [[TMP123]]
 // CHECK1-NEXT:    [[TMP124:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD226:%.*]] = add i32 [[SUB225]], [[TMP124]]
-// CHECK1-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV227:%.*]] = udiv i32 [[ADD226]], [[TMP125]]
-// CHECK1-NEXT:    [[MUL228:%.*]] = mul i32 1, [[DIV227]]
-// CHECK1-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB229:%.*]] = sub i32 [[TMP126]], -31
-// CHECK1-NEXT:    [[DIV230:%.*]] = udiv i32 [[SUB229]], 32
-// CHECK1-NEXT:    [[MUL231:%.*]] = mul i32 [[MUL228]], [[DIV230]]
-// CHECK1-NEXT:    [[MUL232:%.*]] = mul i32 [[MUL231]], 32
-// CHECK1-NEXT:    [[CONV233:%.*]] = zext i32 [[MUL232]] to i64
-// CHECK1-NEXT:    [[MUL234:%.*]] = mul nsw i64 [[DIV223]], [[CONV233]]
-// CHECK1-NEXT:    [[SUB235:%.*]] = sub nsw i64 [[TMP115]], [[MUL234]]
-// CHECK1-NEXT:    [[TMP127:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP128:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB236:%.*]] = sub i32 [[TMP129]], [[TMP130]]
-// CHECK1-NEXT:    [[SUB237:%.*]] = sub i32 [[SUB236]], 1
-// CHECK1-NEXT:    [[TMP131:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD238:%.*]] = add i32 [[SUB237]], [[TMP131]]
-// CHECK1-NEXT:    [[TMP132:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV239:%.*]] = udiv i32 [[ADD238]], [[TMP132]]
-// CHECK1-NEXT:    [[MUL240:%.*]] = mul i32 1, [[DIV239]]
-// CHECK1-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB241:%.*]] = sub i32 [[TMP133]], -31
-// CHECK1-NEXT:    [[DIV242:%.*]] = udiv i32 [[SUB241]], 32
-// CHECK1-NEXT:    [[MUL243:%.*]] = mul i32 [[MUL240]], [[DIV242]]
-// CHECK1-NEXT:    [[MUL244:%.*]] = mul i32 [[MUL243]], 32
-// CHECK1-NEXT:    [[CONV245:%.*]] = zext i32 [[MUL244]] to i64
-// CHECK1-NEXT:    [[DIV246:%.*]] = sdiv i64 [[TMP128]], [[CONV245]]
-// CHECK1-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB247:%.*]] = sub i32 [[TMP134]], [[TMP135]]
-// CHECK1-NEXT:    [[SUB248:%.*]] = sub i32 [[SUB247]], 1
+// CHECK1-NEXT:    [[DIV221:%.*]] = udiv i32 [[ADD220]], [[TMP124]]
+// CHECK1-NEXT:    [[MUL222:%.*]] = mul i32 1, [[DIV221]]
+// CHECK1-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB223:%.*]] = sub i32 [[TMP125]], -31
+// CHECK1-NEXT:    [[DIV224:%.*]] = udiv i32 [[SUB223]], 32
+// CHECK1-NEXT:    [[MUL225:%.*]] = mul i32 [[MUL222]], [[DIV224]]
+// CHECK1-NEXT:    [[MUL226:%.*]] = mul i32 [[MUL225]], 32
+// CHECK1-NEXT:    [[CONV227:%.*]] = zext i32 [[MUL226]] to i64
+// CHECK1-NEXT:    [[DIV228:%.*]] = sdiv i64 [[TMP120]], [[CONV227]]
+// CHECK1-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP127:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB229:%.*]] = sub i32 [[TMP126]], [[TMP127]]
+// CHECK1-NEXT:    [[SUB230:%.*]] = sub i32 [[SUB229]], 1
+// CHECK1-NEXT:    [[TMP128:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD231:%.*]] = add i32 [[SUB230]], [[TMP128]]
+// CHECK1-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV232:%.*]] = udiv i32 [[ADD231]], [[TMP129]]
+// CHECK1-NEXT:    [[MUL233:%.*]] = mul i32 1, [[DIV232]]
+// CHECK1-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB234:%.*]] = sub i32 [[TMP130]], -31
+// CHECK1-NEXT:    [[DIV235:%.*]] = udiv i32 [[SUB234]], 32
+// CHECK1-NEXT:    [[MUL236:%.*]] = mul i32 [[MUL233]], [[DIV235]]
+// CHECK1-NEXT:    [[MUL237:%.*]] = mul i32 [[MUL236]], 32
+// CHECK1-NEXT:    [[CONV238:%.*]] = zext i32 [[MUL237]] to i64
+// CHECK1-NEXT:    [[MUL239:%.*]] = mul nsw i64 [[DIV228]], [[CONV238]]
+// CHECK1-NEXT:    [[SUB240:%.*]] = sub nsw i64 [[TMP119]], [[MUL239]]
+// CHECK1-NEXT:    [[TMP131:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP132:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB241:%.*]] = sub i32 [[TMP133]], [[TMP134]]
+// CHECK1-NEXT:    [[SUB242:%.*]] = sub i32 [[SUB241]], 1
+// CHECK1-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD243:%.*]] = add i32 [[SUB242]], [[TMP135]]
 // CHECK1-NEXT:    [[TMP136:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD249:%.*]] = add i32 [[SUB248]], [[TMP136]]
-// CHECK1-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV250:%.*]] = udiv i32 [[ADD249]], [[TMP137]]
-// CHECK1-NEXT:    [[MUL251:%.*]] = mul i32 1, [[DIV250]]
-// CHECK1-NEXT:    [[TMP138:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB252:%.*]] = sub i32 [[TMP138]], -31
-// CHECK1-NEXT:    [[DIV253:%.*]] = udiv i32 [[SUB252]], 32
-// CHECK1-NEXT:    [[MUL254:%.*]] = mul i32 [[MUL251]], [[DIV253]]
-// CHECK1-NEXT:    [[MUL255:%.*]] = mul i32 [[MUL254]], 32
-// CHECK1-NEXT:    [[CONV256:%.*]] = zext i32 [[MUL255]] to i64
-// CHECK1-NEXT:    [[MUL257:%.*]] = mul nsw i64 [[DIV246]], [[CONV256]]
-// CHECK1-NEXT:    [[SUB258:%.*]] = sub nsw i64 [[TMP127]], [[MUL257]]
-// CHECK1-NEXT:    [[TMP139:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB259:%.*]] = sub i32 [[TMP139]], -31
-// CHECK1-NEXT:    [[DIV260:%.*]] = udiv i32 [[SUB259]], 32
-// CHECK1-NEXT:    [[MUL261:%.*]] = mul i32 1, [[DIV260]]
-// CHECK1-NEXT:    [[MUL262:%.*]] = mul i32 [[MUL261]], 32
-// CHECK1-NEXT:    [[CONV263:%.*]] = zext i32 [[MUL262]] to i64
-// CHECK1-NEXT:    [[DIV264:%.*]] = sdiv i64 [[SUB258]], [[CONV263]]
-// CHECK1-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB265:%.*]] = sub i32 [[TMP140]], -31
-// CHECK1-NEXT:    [[DIV266:%.*]] = udiv i32 [[SUB265]], 32
-// CHECK1-NEXT:    [[MUL267:%.*]] = mul i32 1, [[DIV266]]
-// CHECK1-NEXT:    [[MUL268:%.*]] = mul i32 [[MUL267]], 32
-// CHECK1-NEXT:    [[CONV269:%.*]] = zext i32 [[MUL268]] to i64
-// CHECK1-NEXT:    [[MUL270:%.*]] = mul nsw i64 [[DIV264]], [[CONV269]]
-// CHECK1-NEXT:    [[SUB271:%.*]] = sub nsw i64 [[SUB235]], [[MUL270]]
-// CHECK1-NEXT:    [[DIV272:%.*]] = sdiv i64 [[SUB271]], 32
-// CHECK1-NEXT:    [[MUL273:%.*]] = mul nsw i64 [[DIV272]], 32
-// CHECK1-NEXT:    [[SUB274:%.*]] = sub nsw i64 [[SUB212]], [[MUL273]]
-// CHECK1-NEXT:    [[MUL275:%.*]] = mul nsw i64 [[SUB274]], 1
-// CHECK1-NEXT:    [[ADD276:%.*]] = add nsw i64 0, [[MUL275]]
-// CHECK1-NEXT:    [[CONV277:%.*]] = trunc i64 [[ADD276]] to i32
-// CHECK1-NEXT:    store i32 [[CONV277]], ptr [[DOTTILE_CNT_0_IV_K38]], align 4
-// CHECK1-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
-// CHECK1-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
-// CHECK1-NEXT:    [[ADD278:%.*]] = add i32 [[TMP141]], [[TMP142]]
-// CHECK1-NEXT:    store i32 [[ADD278]], ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK1-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
-// CHECK1-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK1-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
-// CHECK1-NEXT:    [[MUL279:%.*]] = mul i32 [[TMP144]], [[TMP145]]
-// CHECK1-NEXT:    [[ADD280:%.*]] = add i32 [[TMP143]], [[MUL279]]
-// CHECK1-NEXT:    store i32 [[ADD280]], ptr [[K]], align 4
-// CHECK1-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
-// CHECK1-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
-// CHECK1-NEXT:    [[ADD281:%.*]] = add i32 [[TMP146]], [[TMP147]]
-// CHECK1-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK1-NEXT:    [[ADD282:%.*]] = add i32 [[TMP148]], 1
-// CHECK1-NEXT:    [[CMP283:%.*]] = icmp ult i32 [[ADD281]], [[ADD282]]
-// CHECK1-NEXT:    br i1 [[CMP283]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK1:       if.then:
-// CHECK1-NEXT:    [[TMP149:%.*]] = load i32, ptr [[I35]], align 4
-// CHECK1-NEXT:    [[TMP150:%.*]] = load i32, ptr [[J36]], align 4
-// CHECK1-NEXT:    [[TMP151:%.*]] = load i32, ptr [[K]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP149]], i32 noundef [[TMP150]], i32 noundef [[TMP151]])
-// CHECK1-NEXT:    br label [[IF_END]]
-// CHECK1:       if.end:
-// CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK1-NEXT:    [[DIV244:%.*]] = udiv i32 [[ADD243]], [[TMP136]]
+// CHECK1-NEXT:    [[MUL245:%.*]] = mul i32 1, [[DIV244]]
+// CHECK1-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB246:%.*]] = sub i32 [[TMP137]], -31
+// CHECK1-NEXT:    [[DIV247:%.*]] = udiv i32 [[SUB246]], 32
+// CHECK1-NEXT:    [[MUL248:%.*]] = mul i32 [[MUL245]], [[DIV247]]
+// CHECK1-NEXT:    [[MUL249:%.*]] = mul i32 [[MUL248]], 32
+// CHECK1-NEXT:    [[CONV250:%.*]] = zext i32 [[MUL249]] to i64
+// CHECK1-NEXT:    [[DIV251:%.*]] = sdiv i64 [[TMP132]], [[CONV250]]
+// CHECK1-NEXT:    [[TMP138:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP139:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB252:%.*]] = sub i32 [[TMP138]], [[TMP139]]
+// CHECK1-NEXT:    [[SUB253:%.*]] = sub i32 [[SUB252]], 1
+// CHECK1-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD254:%.*]] = add i32 [[SUB253]], [[TMP140]]
+// CHECK1-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV255:%.*]] = udiv i32 [[ADD254]], [[TMP141]]
+// CHECK1-NEXT:    [[MUL256:%.*]] = mul i32 1, [[DIV255]]
+// CHECK1-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB257:%.*]] = sub i32 [[TMP142]], -31
+// CHECK1-NEXT:    [[DIV258:%.*]] = udiv i32 [[SUB257]], 32
+// CHECK1-NEXT:    [[MUL259:%.*]] = mul i32 [[MUL256]], [[DIV258]]
+// CHECK1-NEXT:    [[MUL260:%.*]] = mul i32 [[MUL259]], 32
+// CHECK1-NEXT:    [[CONV261:%.*]] = zext i32 [[MUL260]] to i64
+// CHECK1-NEXT:    [[MUL262:%.*]] = mul nsw i64 [[DIV251]], [[CONV261]]
+// CHECK1-NEXT:    [[SUB263:%.*]] = sub nsw i64 [[TMP131]], [[MUL262]]
+// CHECK1-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB264:%.*]] = sub i32 [[TMP143]], -31
+// CHECK1-NEXT:    [[DIV265:%.*]] = udiv i32 [[SUB264]], 32
+// CHECK1-NEXT:    [[MUL266:%.*]] = mul i32 1, [[DIV265]]
+// CHECK1-NEXT:    [[MUL267:%.*]] = mul i32 [[MUL266]], 32
+// CHECK1-NEXT:    [[CONV268:%.*]] = zext i32 [[MUL267]] to i64
+// CHECK1-NEXT:    [[DIV269:%.*]] = sdiv i64 [[SUB263]], [[CONV268]]
+// CHECK1-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB270:%.*]] = sub i32 [[TMP144]], -31
+// CHECK1-NEXT:    [[DIV271:%.*]] = udiv i32 [[SUB270]], 32
+// CHECK1-NEXT:    [[MUL272:%.*]] = mul i32 1, [[DIV271]]
+// CHECK1-NEXT:    [[MUL273:%.*]] = mul i32 [[MUL272]], 32
+// CHECK1-NEXT:    [[CONV274:%.*]] = zext i32 [[MUL273]] to i64
+// CHECK1-NEXT:    [[MUL275:%.*]] = mul nsw i64 [[DIV269]], [[CONV274]]
+// CHECK1-NEXT:    [[SUB276:%.*]] = sub nsw i64 [[SUB240]], [[MUL275]]
+// CHECK1-NEXT:    [[DIV277:%.*]] = sdiv i64 [[SUB276]], 32
+// CHECK1-NEXT:    [[MUL278:%.*]] = mul nsw i64 [[DIV277]], 32
+// CHECK1-NEXT:    [[SUB279:%.*]] = sub nsw i64 [[SUB217]], [[MUL278]]
+// CHECK1-NEXT:    [[MUL280:%.*]] = mul nsw i64 [[SUB279]], 1
+// CHECK1-NEXT:    [[ADD281:%.*]] = add nsw i64 [[CONV158]], [[MUL280]]
+// CHECK1-NEXT:    [[CONV282:%.*]] = trunc i64 [[ADD281]] to i32
+// CHECK1-NEXT:    store i32 [[CONV282]], ptr [[DOTTILE_0_IV_K42]], align 4
+// CHECK1-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTTILE_0_IV_K42]], align 4
+// CHECK1-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// CHECK1-NEXT:    [[ADD283:%.*]] = add i32 [[TMP146]], 1
+// CHECK1-NEXT:    [[CMP284:%.*]] = icmp ult i32 [[TMP145]], [[ADD283]]
+// CHECK1-NEXT:    br i1 [[CMP284]], label [[OMP_BODY_NEXT:%.*]], label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK1:       omp.body.next:
+// CHECK1-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
+// CHECK1-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTTILE_0_IV_K42]], align 4
+// CHECK1-NEXT:    [[TMP149:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
+// CHECK1-NEXT:    [[MUL285:%.*]] = mul i32 [[TMP148]], [[TMP149]]
+// CHECK1-NEXT:    [[ADD286:%.*]] = add i32 [[TMP147]], [[MUL285]]
+// CHECK1-NEXT:    store i32 [[ADD286]], ptr [[K]], align 4
+// CHECK1-NEXT:    [[TMP150:%.*]] = load i32, ptr [[I39]], align 4
+// CHECK1-NEXT:    [[TMP151:%.*]] = load i32, ptr [[J40]], align 4
+// CHECK1-NEXT:    [[TMP152:%.*]] = load i32, ptr [[K]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP150]], i32 noundef [[TMP151]], i32 noundef [[TMP152]])
+// CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP152:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[ADD284:%.*]] = add nsw i64 [[TMP152]], 1
-// CHECK1-NEXT:    store i64 [[ADD284]], ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP153:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[ADD287:%.*]] = add nsw i64 [[TMP153]], 1
+// CHECK1-NEXT:    store i64 [[ADD287]], ptr [[DOTOMP_IV]], align 8
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -1344,21 +1347,23 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTNEW_STEP10:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_14:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_17:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_19:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_K:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_K:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[_TMP35:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[I35:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[J36:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTFLOOR_0_IV_K37:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_K38:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[I39:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[J40:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTFLOOR_0_IV_K41:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_K42:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
@@ -1396,454 +1401,455 @@ extern "C" void foo10() {
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
 // CHECK1-NEXT:    [[ADD15:%.*]] = add i32 [[TMP15]], 1
 // CHECK1-NEXT:    store i32 [[ADD15]], ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[SUB17:%.*]] = sub i32 [[TMP16]], [[TMP17]]
-// CHECK1-NEXT:    [[SUB18:%.*]] = sub i32 [[SUB17]], 1
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[ADD19:%.*]] = add i32 [[SUB18]], [[TMP18]]
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[DIV20:%.*]] = udiv i32 [[ADD19]], [[TMP19]]
-// CHECK1-NEXT:    [[CONV:%.*]] = zext i32 [[DIV20]] to i64
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB21:%.*]] = sub i32 [[TMP20]], [[TMP21]]
-// CHECK1-NEXT:    [[SUB22:%.*]] = sub i32 [[SUB21]], 1
-// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD23:%.*]] = add i32 [[SUB22]], [[TMP22]]
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV24:%.*]] = udiv i32 [[ADD23]], [[TMP23]]
-// CHECK1-NEXT:    [[CONV25:%.*]] = zext i32 [[DIV24]] to i64
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV25]]
-// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB26:%.*]] = sub i32 [[TMP24]], -63
-// CHECK1-NEXT:    [[DIV27:%.*]] = udiv i32 [[SUB26]], 64
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[_TMP2]], align 4
+// CHECK1-NEXT:    store i32 [[TMP16]], ptr [[DOTCAPTURE_EXPR_16]], align 4
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[_TMP2]], align 4
+// CHECK1-NEXT:    [[ADD18:%.*]] = add i32 [[TMP17]], 64
+// CHECK1-NEXT:    store i32 [[ADD18]], ptr [[DOTCAPTURE_EXPR_17]], align 4
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[SUB20:%.*]] = sub i32 [[TMP18]], [[TMP19]]
+// CHECK1-NEXT:    [[SUB21:%.*]] = sub i32 [[SUB20]], 1
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[ADD22:%.*]] = add i32 [[SUB21]], [[TMP20]]
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[DIV23:%.*]] = udiv i32 [[ADD22]], [[TMP21]]
+// CHECK1-NEXT:    [[CONV:%.*]] = zext i32 [[DIV23]] to i64
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB24:%.*]] = sub i32 [[TMP22]], [[TMP23]]
+// CHECK1-NEXT:    [[SUB25:%.*]] = sub i32 [[SUB24]], 1
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD26:%.*]] = add i32 [[SUB25]], [[TMP24]]
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV27:%.*]] = udiv i32 [[ADD26]], [[TMP25]]
 // CHECK1-NEXT:    [[CONV28:%.*]] = zext i32 [[DIV27]] to i64
-// CHECK1-NEXT:    [[MUL29:%.*]] = mul nsw i64 [[MUL]], [[CONV28]]
-// CHECK1-NEXT:    [[MUL30:%.*]] = mul nsw i64 [[MUL29]], 64
-// CHECK1-NEXT:    [[SUB31:%.*]] = sub nsw i64 [[MUL30]], 1
-// CHECK1-NEXT:    store i64 [[SUB31]], ptr [[DOTCAPTURE_EXPR_16]], align 8
-// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    store i32 [[TMP25]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    store i32 [[TMP26]], ptr [[J]], align 4
-// CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_K]], align 4
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_K]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV28]]
+// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB29:%.*]] = sub i32 [[TMP26]], -63
+// CHECK1-NEXT:    [[DIV30:%.*]] = udiv i32 [[SUB29]], 64
+// CHECK1-NEXT:    [[CONV31:%.*]] = zext i32 [[DIV30]] to i64
+// CHECK1-NEXT:    [[MUL32:%.*]] = mul nsw i64 [[MUL]], [[CONV31]]
+// CHECK1-NEXT:    [[MUL33:%.*]] = mul nsw i64 [[MUL32]], 64
+// CHECK1-NEXT:    [[SUB34:%.*]] = sub nsw i64 [[MUL33]], 1
+// CHECK1-NEXT:    store i64 [[SUB34]], ptr [[DOTCAPTURE_EXPR_19]], align 8
 // CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP27]], [[TMP28]]
+// CHECK1-NEXT:    store i32 [[TMP27]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    store i32 [[TMP28]], ptr [[J]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_K]], align 4
+// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K]], align 4
+// CHECK1-NEXT:    store i32 [[TMP29]], ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK1-NEXT:    store i32 0, ptr [[_TMP35]], align 4
+// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP30]], [[TMP31]]
 // CHECK1-NEXT:    br i1 [[CMP]], label [[LAND_LHS_TRUE:%.*]], label [[OMP_PRECOND_END:%.*]]
 // CHECK1:       land.lhs.true:
-// CHECK1-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[CMP32:%.*]] = icmp slt i32 [[TMP29]], [[TMP30]]
-// CHECK1-NEXT:    br i1 [[CMP32]], label [[LAND_LHS_TRUE33:%.*]], label [[OMP_PRECOND_END]]
-// CHECK1:       land.lhs.true33:
-// CHECK1-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[CMP34:%.*]] = icmp ult i32 0, [[TMP31]]
-// CHECK1-NEXT:    br i1 [[CMP34]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
+// CHECK1-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[CMP36:%.*]] = icmp slt i32 [[TMP32]], [[TMP33]]
+// CHECK1-NEXT:    br i1 [[CMP36]], label [[LAND_LHS_TRUE37:%.*]], label [[OMP_PRECOND_END]]
+// CHECK1:       land.lhs.true37:
+// CHECK1-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[CMP38:%.*]] = icmp ult i32 0, [[TMP34]]
+// CHECK1-NEXT:    br i1 [[CMP38]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
 // CHECK1:       omp.precond.then:
 // CHECK1-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
-// CHECK1-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
-// CHECK1-NEXT:    store i64 [[TMP32]], ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_19]], align 8
+// CHECK1-NEXT:    store i64 [[TMP35]], ptr [[DOTOMP_UB]], align 8
 // CHECK1-NEXT:    store i64 1, ptr [[DOTOMP_STRIDE]], align 8
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
 // CHECK1-NEXT:    call void @__kmpc_for_static_init_8(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i64 1, i64 1)
-// CHECK1-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
-// CHECK1-NEXT:    [[CMP39:%.*]] = icmp sgt i64 [[TMP33]], [[TMP34]]
-// CHECK1-NEXT:    br i1 [[CMP39]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_19]], align 8
+// CHECK1-NEXT:    [[CMP43:%.*]] = icmp sgt i64 [[TMP36]], [[TMP37]]
+// CHECK1-NEXT:    br i1 [[CMP43]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK1:       cond.true:
-// CHECK1-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK1-NEXT:    [[TMP38:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_19]], align 8
 // CHECK1-NEXT:    br label [[COND_END:%.*]]
 // CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[TMP39:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
 // CHECK1-NEXT:    br label [[COND_END]]
 // CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i64 [ [[TMP35]], [[COND_TRUE]] ], [ [[TMP36]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[COND:%.*]] = phi i64 [ [[TMP38]], [[COND_TRUE]] ], [ [[TMP39]], [[COND_FALSE]] ]
 // CHECK1-NEXT:    store i64 [[COND]], ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
-// CHECK1-NEXT:    store i64 [[TMP37]], ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP40:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
+// CHECK1-NEXT:    store i64 [[TMP40]], ptr [[DOTOMP_IV]], align 8
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK1:       omp.inner.for.cond:
-// CHECK1-NEXT:    [[TMP38:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP39:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK1-NEXT:    [[CMP40:%.*]] = icmp sle i64 [[TMP38]], [[TMP39]]
-// CHECK1-NEXT:    br i1 [[CMP40]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
-// CHECK1:       omp.inner.for.body:
-// CHECK1-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK1-NEXT:    [[CONV41:%.*]] = sext i32 [[TMP40]] to i64
 // CHECK1-NEXT:    [[TMP41:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB42:%.*]] = sub i32 [[TMP42]], [[TMP43]]
-// CHECK1-NEXT:    [[SUB43:%.*]] = sub i32 [[SUB42]], 1
-// CHECK1-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD44:%.*]] = add i32 [[SUB43]], [[TMP44]]
-// CHECK1-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV45:%.*]] = udiv i32 [[ADD44]], [[TMP45]]
-// CHECK1-NEXT:    [[MUL46:%.*]] = mul i32 1, [[DIV45]]
-// CHECK1-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB47:%.*]] = sub i32 [[TMP46]], -63
-// CHECK1-NEXT:    [[DIV48:%.*]] = udiv i32 [[SUB47]], 64
-// CHECK1-NEXT:    [[MUL49:%.*]] = mul i32 [[MUL46]], [[DIV48]]
-// CHECK1-NEXT:    [[MUL50:%.*]] = mul i32 [[MUL49]], 64
-// CHECK1-NEXT:    [[CONV51:%.*]] = zext i32 [[MUL50]] to i64
-// CHECK1-NEXT:    [[DIV52:%.*]] = sdiv i64 [[TMP41]], [[CONV51]]
-// CHECK1-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK1-NEXT:    [[CONV53:%.*]] = sext i32 [[TMP47]] to i64
-// CHECK1-NEXT:    [[MUL54:%.*]] = mul nsw i64 [[DIV52]], [[CONV53]]
-// CHECK1-NEXT:    [[ADD55:%.*]] = add nsw i64 [[CONV41]], [[MUL54]]
-// CHECK1-NEXT:    [[CONV56:%.*]] = trunc i64 [[ADD55]] to i32
-// CHECK1-NEXT:    store i32 [[CONV56]], ptr [[I35]], align 4
-// CHECK1-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP48]] to i64
-// CHECK1-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP50:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB58:%.*]] = sub i32 [[TMP51]], [[TMP52]]
-// CHECK1-NEXT:    [[SUB59:%.*]] = sub i32 [[SUB58]], 1
-// CHECK1-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD60:%.*]] = add i32 [[SUB59]], [[TMP53]]
-// CHECK1-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV61:%.*]] = udiv i32 [[ADD60]], [[TMP54]]
-// CHECK1-NEXT:    [[MUL62:%.*]] = mul i32 1, [[DIV61]]
-// CHECK1-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB63:%.*]] = sub i32 [[TMP55]], -63
-// CHECK1-NEXT:    [[DIV64:%.*]] = udiv i32 [[SUB63]], 64
-// CHECK1-NEXT:    [[MUL65:%.*]] = mul i32 [[MUL62]], [[DIV64]]
-// CHECK1-NEXT:    [[MUL66:%.*]] = mul i32 [[MUL65]], 64
-// CHECK1-NEXT:    [[CONV67:%.*]] = zext i32 [[MUL66]] to i64
-// CHECK1-NEXT:    [[DIV68:%.*]] = sdiv i64 [[TMP50]], [[CONV67]]
-// CHECK1-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB69:%.*]] = sub i32 [[TMP56]], [[TMP57]]
-// CHECK1-NEXT:    [[SUB70:%.*]] = sub i32 [[SUB69]], 1
-// CHECK1-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD71:%.*]] = add i32 [[SUB70]], [[TMP58]]
-// CHECK1-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV72:%.*]] = udiv i32 [[ADD71]], [[TMP59]]
-// CHECK1-NEXT:    [[MUL73:%.*]] = mul i32 1, [[DIV72]]
-// CHECK1-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB74:%.*]] = sub i32 [[TMP60]], -63
-// CHECK1-NEXT:    [[DIV75:%.*]] = udiv i32 [[SUB74]], 64
-// CHECK1-NEXT:    [[MUL76:%.*]] = mul i32 [[MUL73]], [[DIV75]]
-// CHECK1-NEXT:    [[MUL77:%.*]] = mul i32 [[MUL76]], 64
-// CHECK1-NEXT:    [[CONV78:%.*]] = zext i32 [[MUL77]] to i64
-// CHECK1-NEXT:    [[MUL79:%.*]] = mul nsw i64 [[DIV68]], [[CONV78]]
-// CHECK1-NEXT:    [[SUB80:%.*]] = sub nsw i64 [[TMP49]], [[MUL79]]
-// CHECK1-NEXT:    [[TMP61:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB81:%.*]] = sub i32 [[TMP61]], -63
-// CHECK1-NEXT:    [[DIV82:%.*]] = udiv i32 [[SUB81]], 64
-// CHECK1-NEXT:    [[MUL83:%.*]] = mul i32 1, [[DIV82]]
-// CHECK1-NEXT:    [[MUL84:%.*]] = mul i32 [[MUL83]], 64
-// CHECK1-NEXT:    [[CONV85:%.*]] = zext i32 [[MUL84]] to i64
-// CHECK1-NEXT:    [[DIV86:%.*]] = sdiv i64 [[SUB80]], [[CONV85]]
+// CHECK1-NEXT:    [[TMP42:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK1-NEXT:    [[CMP44:%.*]] = icmp sle i64 [[TMP41]], [[TMP42]]
+// CHECK1-NEXT:    br i1 [[CMP44]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK1:       omp.inner.for.body:
+// CHECK1-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[CONV45:%.*]] = sext i32 [[TMP43]] to i64
+// CHECK1-NEXT:    [[TMP44:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB46:%.*]] = sub i32 [[TMP45]], [[TMP46]]
+// CHECK1-NEXT:    [[SUB47:%.*]] = sub i32 [[SUB46]], 1
+// CHECK1-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD48:%.*]] = add i32 [[SUB47]], [[TMP47]]
+// CHECK1-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV49:%.*]] = udiv i32 [[ADD48]], [[TMP48]]
+// CHECK1-NEXT:    [[MUL50:%.*]] = mul i32 1, [[DIV49]]
+// CHECK1-NEXT:    [[TMP49:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB51:%.*]] = sub i32 [[TMP49]], -63
+// CHECK1-NEXT:    [[DIV52:%.*]] = udiv i32 [[SUB51]], 64
+// CHECK1-NEXT:    [[MUL53:%.*]] = mul i32 [[MUL50]], [[DIV52]]
+// CHECK1-NEXT:    [[MUL54:%.*]] = mul i32 [[MUL53]], 64
+// CHECK1-NEXT:    [[CONV55:%.*]] = zext i32 [[MUL54]] to i64
+// CHECK1-NEXT:    [[DIV56:%.*]] = sdiv i64 [[TMP44]], [[CONV55]]
+// CHECK1-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK1-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP50]] to i64
+// CHECK1-NEXT:    [[MUL58:%.*]] = mul nsw i64 [[DIV56]], [[CONV57]]
+// CHECK1-NEXT:    [[ADD59:%.*]] = add nsw i64 [[CONV45]], [[MUL58]]
+// CHECK1-NEXT:    [[CONV60:%.*]] = trunc i64 [[ADD59]] to i32
+// CHECK1-NEXT:    store i32 [[CONV60]], ptr [[I39]], align 4
+// CHECK1-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[CONV61:%.*]] = sext i32 [[TMP51]] to i64
+// CHECK1-NEXT:    [[TMP52:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP53:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB62:%.*]] = sub i32 [[TMP54]], [[TMP55]]
+// CHECK1-NEXT:    [[SUB63:%.*]] = sub i32 [[SUB62]], 1
+// CHECK1-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD64:%.*]] = add i32 [[SUB63]], [[TMP56]]
+// CHECK1-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV65:%.*]] = udiv i32 [[ADD64]], [[TMP57]]
+// CHECK1-NEXT:    [[MUL66:%.*]] = mul i32 1, [[DIV65]]
+// CHECK1-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB67:%.*]] = sub i32 [[TMP58]], -63
+// CHECK1-NEXT:    [[DIV68:%.*]] = udiv i32 [[SUB67]], 64
+// CHECK1-NEXT:    [[MUL69:%.*]] = mul i32 [[MUL66]], [[DIV68]]
+// CHECK1-NEXT:    [[MUL70:%.*]] = mul i32 [[MUL69]], 64
+// CHECK1-NEXT:    [[CONV71:%.*]] = zext i32 [[MUL70]] to i64
+// CHECK1-NEXT:    [[DIV72:%.*]] = sdiv i64 [[TMP53]], [[CONV71]]
+// CHECK1-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB73:%.*]] = sub i32 [[TMP59]], [[TMP60]]
+// CHECK1-NEXT:    [[SUB74:%.*]] = sub i32 [[SUB73]], 1
+// CHECK1-NEXT:    [[TMP61:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD75:%.*]] = add i32 [[SUB74]], [[TMP61]]
 // CHECK1-NEXT:    [[TMP62:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[CONV87:%.*]] = sext i32 [[TMP62]] to i64
-// CHECK1-NEXT:    [[MUL88:%.*]] = mul nsw i64 [[DIV86]], [[CONV87]]
-// CHECK1-NEXT:    [[ADD89:%.*]] = add nsw i64 [[CONV57]], [[MUL88]]
-// CHECK1-NEXT:    [[CONV90:%.*]] = trunc i64 [[ADD89]] to i32
-// CHECK1-NEXT:    store i32 [[CONV90]], ptr [[J36]], align 4
-// CHECK1-NEXT:    [[TMP63:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP64:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB91:%.*]] = sub i32 [[TMP65]], [[TMP66]]
-// CHECK1-NEXT:    [[SUB92:%.*]] = sub i32 [[SUB91]], 1
-// CHECK1-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD93:%.*]] = add i32 [[SUB92]], [[TMP67]]
-// CHECK1-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV94:%.*]] = udiv i32 [[ADD93]], [[TMP68]]
-// CHECK1-NEXT:    [[MUL95:%.*]] = mul i32 1, [[DIV94]]
-// CHECK1-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB96:%.*]] = sub i32 [[TMP69]], -63
-// CHECK1-NEXT:    [[DIV97:%.*]] = udiv i32 [[SUB96]], 64
-// CHECK1-NEXT:    [[MUL98:%.*]] = mul i32 [[MUL95]], [[DIV97]]
-// CHECK1-NEXT:    [[MUL99:%.*]] = mul i32 [[MUL98]], 64
-// CHECK1-NEXT:    [[CONV100:%.*]] = zext i32 [[MUL99]] to i64
-// CHECK1-NEXT:    [[DIV101:%.*]] = sdiv i64 [[TMP64]], [[CONV100]]
-// CHECK1-NEXT:    [[TMP70:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB102:%.*]] = sub i32 [[TMP70]], [[TMP71]]
-// CHECK1-NEXT:    [[SUB103:%.*]] = sub i32 [[SUB102]], 1
-// CHECK1-NEXT:    [[TMP72:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD104:%.*]] = add i32 [[SUB103]], [[TMP72]]
-// CHECK1-NEXT:    [[TMP73:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV105:%.*]] = udiv i32 [[ADD104]], [[TMP73]]
-// CHECK1-NEXT:    [[MUL106:%.*]] = mul i32 1, [[DIV105]]
-// CHECK1-NEXT:    [[TMP74:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB107:%.*]] = sub i32 [[TMP74]], -63
-// CHECK1-NEXT:    [[DIV108:%.*]] = udiv i32 [[SUB107]], 64
-// CHECK1-NEXT:    [[MUL109:%.*]] = mul i32 [[MUL106]], [[DIV108]]
-// CHECK1-NEXT:    [[MUL110:%.*]] = mul i32 [[MUL109]], 64
-// CHECK1-NEXT:    [[CONV111:%.*]] = zext i32 [[MUL110]] to i64
-// CHECK1-NEXT:    [[MUL112:%.*]] = mul nsw i64 [[DIV101]], [[CONV111]]
-// CHECK1-NEXT:    [[SUB113:%.*]] = sub nsw i64 [[TMP63]], [[MUL112]]
-// CHECK1-NEXT:    [[TMP75:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP76:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP78:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB114:%.*]] = sub i32 [[TMP77]], [[TMP78]]
-// CHECK1-NEXT:    [[SUB115:%.*]] = sub i32 [[SUB114]], 1
-// CHECK1-NEXT:    [[TMP79:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD116:%.*]] = add i32 [[SUB115]], [[TMP79]]
-// CHECK1-NEXT:    [[TMP80:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV117:%.*]] = udiv i32 [[ADD116]], [[TMP80]]
-// CHECK1-NEXT:    [[MUL118:%.*]] = mul i32 1, [[DIV117]]
-// CHECK1-NEXT:    [[TMP81:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB119:%.*]] = sub i32 [[TMP81]], -63
-// CHECK1-NEXT:    [[DIV120:%.*]] = udiv i32 [[SUB119]], 64
-// CHECK1-NEXT:    [[MUL121:%.*]] = mul i32 [[MUL118]], [[DIV120]]
-// CHECK1-NEXT:    [[MUL122:%.*]] = mul i32 [[MUL121]], 64
-// CHECK1-NEXT:    [[CONV123:%.*]] = zext i32 [[MUL122]] to i64
-// CHECK1-NEXT:    [[DIV124:%.*]] = sdiv i64 [[TMP76]], [[CONV123]]
-// CHECK1-NEXT:    [[TMP82:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB125:%.*]] = sub i32 [[TMP82]], [[TMP83]]
-// CHECK1-NEXT:    [[SUB126:%.*]] = sub i32 [[SUB125]], 1
-// CHECK1-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD127:%.*]] = add i32 [[SUB126]], [[TMP84]]
-// CHECK1-NEXT:    [[TMP85:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV128:%.*]] = udiv i32 [[ADD127]], [[TMP85]]
-// CHECK1-NEXT:    [[MUL129:%.*]] = mul i32 1, [[DIV128]]
-// CHECK1-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB130:%.*]] = sub i32 [[TMP86]], -63
-// CHECK1-NEXT:    [[DIV131:%.*]] = udiv i32 [[SUB130]], 64
-// CHECK1-NEXT:    [[MUL132:%.*]] = mul i32 [[MUL129]], [[DIV131]]
-// CHECK1-NEXT:    [[MUL133:%.*]] = mul i32 [[MUL132]], 64
-// CHECK1-NEXT:    [[CONV134:%.*]] = zext i32 [[MUL133]] to i64
-// CHECK1-NEXT:    [[MUL135:%.*]] = mul nsw i64 [[DIV124]], [[CONV134]]
-// CHECK1-NEXT:    [[SUB136:%.*]] = sub nsw i64 [[TMP75]], [[MUL135]]
-// CHECK1-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB137:%.*]] = sub i32 [[TMP87]], -63
-// CHECK1-NEXT:    [[DIV138:%.*]] = udiv i32 [[SUB137]], 64
-// CHECK1-NEXT:    [[MUL139:%.*]] = mul i32 1, [[DIV138]]
-// CHECK1-NEXT:    [[MUL140:%.*]] = mul i32 [[MUL139]], 64
-// CHECK1-NEXT:    [[CONV141:%.*]] = zext i32 [[MUL140]] to i64
-// CHECK1-NEXT:    [[DIV142:%.*]] = sdiv i64 [[SUB136]], [[CONV141]]
-// CHECK1-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB143:%.*]] = sub i32 [[TMP88]], -63
-// CHECK1-NEXT:    [[DIV144:%.*]] = udiv i32 [[SUB143]], 64
-// CHECK1-NEXT:    [[MUL145:%.*]] = mul i32 1, [[DIV144]]
-// CHECK1-NEXT:    [[MUL146:%.*]] = mul i32 [[MUL145]], 64
-// CHECK1-NEXT:    [[CONV147:%.*]] = zext i32 [[MUL146]] to i64
-// CHECK1-NEXT:    [[MUL148:%.*]] = mul nsw i64 [[DIV142]], [[CONV147]]
-// CHECK1-NEXT:    [[SUB149:%.*]] = sub nsw i64 [[SUB113]], [[MUL148]]
-// CHECK1-NEXT:    [[DIV150:%.*]] = sdiv i64 [[SUB149]], 64
-// CHECK1-NEXT:    [[MUL151:%.*]] = mul nsw i64 [[DIV150]], 64
-// CHECK1-NEXT:    [[ADD152:%.*]] = add nsw i64 0, [[MUL151]]
-// CHECK1-NEXT:    [[CONV153:%.*]] = trunc i64 [[ADD152]] to i32
-// CHECK1-NEXT:    store i32 [[CONV153]], ptr [[DOTFLOOR_0_IV_K37]], align 4
-// CHECK1-NEXT:    [[TMP89:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP90:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB154:%.*]] = sub i32 [[TMP91]], [[TMP92]]
-// CHECK1-NEXT:    [[SUB155:%.*]] = sub i32 [[SUB154]], 1
-// CHECK1-NEXT:    [[TMP93:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD156:%.*]] = add i32 [[SUB155]], [[TMP93]]
-// CHECK1-NEXT:    [[TMP94:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV157:%.*]] = udiv i32 [[ADD156]], [[TMP94]]
-// CHECK1-NEXT:    [[MUL158:%.*]] = mul i32 1, [[DIV157]]
-// CHECK1-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB159:%.*]] = sub i32 [[TMP95]], -63
-// CHECK1-NEXT:    [[DIV160:%.*]] = udiv i32 [[SUB159]], 64
-// CHECK1-NEXT:    [[MUL161:%.*]] = mul i32 [[MUL158]], [[DIV160]]
-// CHECK1-NEXT:    [[MUL162:%.*]] = mul i32 [[MUL161]], 64
-// CHECK1-NEXT:    [[CONV163:%.*]] = zext i32 [[MUL162]] to i64
-// CHECK1-NEXT:    [[DIV164:%.*]] = sdiv i64 [[TMP90]], [[CONV163]]
-// CHECK1-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP97:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB165:%.*]] = sub i32 [[TMP96]], [[TMP97]]
-// CHECK1-NEXT:    [[SUB166:%.*]] = sub i32 [[SUB165]], 1
+// CHECK1-NEXT:    [[DIV76:%.*]] = udiv i32 [[ADD75]], [[TMP62]]
+// CHECK1-NEXT:    [[MUL77:%.*]] = mul i32 1, [[DIV76]]
+// CHECK1-NEXT:    [[TMP63:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB78:%.*]] = sub i32 [[TMP63]], -63
+// CHECK1-NEXT:    [[DIV79:%.*]] = udiv i32 [[SUB78]], 64
+// CHECK1-NEXT:    [[MUL80:%.*]] = mul i32 [[MUL77]], [[DIV79]]
+// CHECK1-NEXT:    [[MUL81:%.*]] = mul i32 [[MUL80]], 64
+// CHECK1-NEXT:    [[CONV82:%.*]] = zext i32 [[MUL81]] to i64
+// CHECK1-NEXT:    [[MUL83:%.*]] = mul nsw i64 [[DIV72]], [[CONV82]]
+// CHECK1-NEXT:    [[SUB84:%.*]] = sub nsw i64 [[TMP52]], [[MUL83]]
+// CHECK1-NEXT:    [[TMP64:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB85:%.*]] = sub i32 [[TMP64]], -63
+// CHECK1-NEXT:    [[DIV86:%.*]] = udiv i32 [[SUB85]], 64
+// CHECK1-NEXT:    [[MUL87:%.*]] = mul i32 1, [[DIV86]]
+// CHECK1-NEXT:    [[MUL88:%.*]] = mul i32 [[MUL87]], 64
+// CHECK1-NEXT:    [[CONV89:%.*]] = zext i32 [[MUL88]] to i64
+// CHECK1-NEXT:    [[DIV90:%.*]] = sdiv i64 [[SUB84]], [[CONV89]]
+// CHECK1-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[CONV91:%.*]] = sext i32 [[TMP65]] to i64
+// CHECK1-NEXT:    [[MUL92:%.*]] = mul nsw i64 [[DIV90]], [[CONV91]]
+// CHECK1-NEXT:    [[ADD93:%.*]] = add nsw i64 [[CONV61]], [[MUL92]]
+// CHECK1-NEXT:    [[CONV94:%.*]] = trunc i64 [[ADD93]] to i32
+// CHECK1-NEXT:    store i32 [[CONV94]], ptr [[J40]], align 4
+// CHECK1-NEXT:    [[TMP66:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP67:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB95:%.*]] = sub i32 [[TMP68]], [[TMP69]]
+// CHECK1-NEXT:    [[SUB96:%.*]] = sub i32 [[SUB95]], 1
+// CHECK1-NEXT:    [[TMP70:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD97:%.*]] = add i32 [[SUB96]], [[TMP70]]
+// CHECK1-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV98:%.*]] = udiv i32 [[ADD97]], [[TMP71]]
+// CHECK1-NEXT:    [[MUL99:%.*]] = mul i32 1, [[DIV98]]
+// CHECK1-NEXT:    [[TMP72:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB100:%.*]] = sub i32 [[TMP72]], -63
+// CHECK1-NEXT:    [[DIV101:%.*]] = udiv i32 [[SUB100]], 64
+// CHECK1-NEXT:    [[MUL102:%.*]] = mul i32 [[MUL99]], [[DIV101]]
+// CHECK1-NEXT:    [[MUL103:%.*]] = mul i32 [[MUL102]], 64
+// CHECK1-NEXT:    [[CONV104:%.*]] = zext i32 [[MUL103]] to i64
+// CHECK1-NEXT:    [[DIV105:%.*]] = sdiv i64 [[TMP67]], [[CONV104]]
+// CHECK1-NEXT:    [[TMP73:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP74:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB106:%.*]] = sub i32 [[TMP73]], [[TMP74]]
+// CHECK1-NEXT:    [[SUB107:%.*]] = sub i32 [[SUB106]], 1
+// CHECK1-NEXT:    [[TMP75:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD108:%.*]] = add i32 [[SUB107]], [[TMP75]]
+// CHECK1-NEXT:    [[TMP76:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV109:%.*]] = udiv i32 [[ADD108]], [[TMP76]]
+// CHECK1-NEXT:    [[MUL110:%.*]] = mul i32 1, [[DIV109]]
+// CHECK1-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB111:%.*]] = sub i32 [[TMP77]], -63
+// CHECK1-NEXT:    [[DIV112:%.*]] = udiv i32 [[SUB111]], 64
+// CHECK1-NEXT:    [[MUL113:%.*]] = mul i32 [[MUL110]], [[DIV112]]
+// CHECK1-NEXT:    [[MUL114:%.*]] = mul i32 [[MUL113]], 64
+// CHECK1-NEXT:    [[CONV115:%.*]] = zext i32 [[MUL114]] to i64
+// CHECK1-NEXT:    [[MUL116:%.*]] = mul nsw i64 [[DIV105]], [[CONV115]]
+// CHECK1-NEXT:    [[SUB117:%.*]] = sub nsw i64 [[TMP66]], [[MUL116]]
+// CHECK1-NEXT:    [[TMP78:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP79:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP80:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP81:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB118:%.*]] = sub i32 [[TMP80]], [[TMP81]]
+// CHECK1-NEXT:    [[SUB119:%.*]] = sub i32 [[SUB118]], 1
+// CHECK1-NEXT:    [[TMP82:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD120:%.*]] = add i32 [[SUB119]], [[TMP82]]
+// CHECK1-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV121:%.*]] = udiv i32 [[ADD120]], [[TMP83]]
+// CHECK1-NEXT:    [[MUL122:%.*]] = mul i32 1, [[DIV121]]
+// CHECK1-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB123:%.*]] = sub i32 [[TMP84]], -63
+// CHECK1-NEXT:    [[DIV124:%.*]] = udiv i32 [[SUB123]], 64
+// CHECK1-NEXT:    [[MUL125:%.*]] = mul i32 [[MUL122]], [[DIV124]]
+// CHECK1-NEXT:    [[MUL126:%.*]] = mul i32 [[MUL125]], 64
+// CHECK1-NEXT:    [[CONV127:%.*]] = zext i32 [[MUL126]] to i64
+// CHECK1-NEXT:    [[DIV128:%.*]] = sdiv i64 [[TMP79]], [[CONV127]]
+// CHECK1-NEXT:    [[TMP85:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB129:%.*]] = sub i32 [[TMP85]], [[TMP86]]
+// CHECK1-NEXT:    [[SUB130:%.*]] = sub i32 [[SUB129]], 1
+// CHECK1-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD131:%.*]] = add i32 [[SUB130]], [[TMP87]]
+// CHECK1-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV132:%.*]] = udiv i32 [[ADD131]], [[TMP88]]
+// CHECK1-NEXT:    [[MUL133:%.*]] = mul i32 1, [[DIV132]]
+// CHECK1-NEXT:    [[TMP89:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB134:%.*]] = sub i32 [[TMP89]], -63
+// CHECK1-NEXT:    [[DIV135:%.*]] = udiv i32 [[SUB134]], 64
+// CHECK1-NEXT:    [[MUL136:%.*]] = mul i32 [[MUL133]], [[DIV135]]
+// CHECK1-NEXT:    [[MUL137:%.*]] = mul i32 [[MUL136]], 64
+// CHECK1-NEXT:    [[CONV138:%.*]] = zext i32 [[MUL137]] to i64
+// CHECK1-NEXT:    [[MUL139:%.*]] = mul nsw i64 [[DIV128]], [[CONV138]]
+// CHECK1-NEXT:    [[SUB140:%.*]] = sub nsw i64 [[TMP78]], [[MUL139]]
+// CHECK1-NEXT:    [[TMP90:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB141:%.*]] = sub i32 [[TMP90]], -63
+// CHECK1-NEXT:    [[DIV142:%.*]] = udiv i32 [[SUB141]], 64
+// CHECK1-NEXT:    [[MUL143:%.*]] = mul i32 1, [[DIV142]]
+// CHECK1-NEXT:    [[MUL144:%.*]] = mul i32 [[MUL143]], 64
+// CHECK1-NEXT:    [[CONV145:%.*]] = zext i32 [[MUL144]] to i64
+// CHECK1-NEXT:    [[DIV146:%.*]] = sdiv i64 [[SUB140]], [[CONV145]]
+// CHECK1-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB147:%.*]] = sub i32 [[TMP91]], -63
+// CHECK1-NEXT:    [[DIV148:%.*]] = udiv i32 [[SUB147]], 64
+// CHECK1-NEXT:    [[MUL149:%.*]] = mul i32 1, [[DIV148]]
+// CHECK1-NEXT:    [[MUL150:%.*]] = mul i32 [[MUL149]], 64
+// CHECK1-NEXT:    [[CONV151:%.*]] = zext i32 [[MUL150]] to i64
+// CHECK1-NEXT:    [[MUL152:%.*]] = mul nsw i64 [[DIV146]], [[CONV151]]
+// CHECK1-NEXT:    [[SUB153:%.*]] = sub nsw i64 [[SUB117]], [[MUL152]]
+// CHECK1-NEXT:    [[DIV154:%.*]] = sdiv i64 [[SUB153]], 64
+// CHECK1-NEXT:    [[MUL155:%.*]] = mul nsw i64 [[DIV154]], 64
+// CHECK1-NEXT:    [[ADD156:%.*]] = add nsw i64 0, [[MUL155]]
+// CHECK1-NEXT:    [[CONV157:%.*]] = trunc i64 [[ADD156]] to i32
+// CHECK1-NEXT:    store i32 [[CONV157]], ptr [[DOTFLOOR_0_IV_K41]], align 4
+// CHECK1-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K41]], align 4
+// CHECK1-NEXT:    [[CONV158:%.*]] = zext i32 [[TMP92]] to i64
+// CHECK1-NEXT:    [[TMP93:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP94:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB159:%.*]] = sub i32 [[TMP95]], [[TMP96]]
+// CHECK1-NEXT:    [[SUB160:%.*]] = sub i32 [[SUB159]], 1
+// CHECK1-NEXT:    [[TMP97:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD161:%.*]] = add i32 [[SUB160]], [[TMP97]]
 // CHECK1-NEXT:    [[TMP98:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD167:%.*]] = add i32 [[SUB166]], [[TMP98]]
-// CHECK1-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV168:%.*]] = udiv i32 [[ADD167]], [[TMP99]]
-// CHECK1-NEXT:    [[MUL169:%.*]] = mul i32 1, [[DIV168]]
-// CHECK1-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB170:%.*]] = sub i32 [[TMP100]], -63
-// CHECK1-NEXT:    [[DIV171:%.*]] = udiv i32 [[SUB170]], 64
-// CHECK1-NEXT:    [[MUL172:%.*]] = mul i32 [[MUL169]], [[DIV171]]
-// CHECK1-NEXT:    [[MUL173:%.*]] = mul i32 [[MUL172]], 64
-// CHECK1-NEXT:    [[CONV174:%.*]] = zext i32 [[MUL173]] to i64
-// CHECK1-NEXT:    [[MUL175:%.*]] = mul nsw i64 [[DIV164]], [[CONV174]]
-// CHECK1-NEXT:    [[SUB176:%.*]] = sub nsw i64 [[TMP89]], [[MUL175]]
-// CHECK1-NEXT:    [[TMP101:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP102:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB177:%.*]] = sub i32 [[TMP103]], [[TMP104]]
-// CHECK1-NEXT:    [[SUB178:%.*]] = sub i32 [[SUB177]], 1
-// CHECK1-NEXT:    [[TMP105:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD179:%.*]] = add i32 [[SUB178]], [[TMP105]]
-// CHECK1-NEXT:    [[TMP106:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV180:%.*]] = udiv i32 [[ADD179]], [[TMP106]]
-// CHECK1-NEXT:    [[MUL181:%.*]] = mul i32 1, [[DIV180]]
-// CHECK1-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB182:%.*]] = sub i32 [[TMP107]], -63
-// CHECK1-NEXT:    [[DIV183:%.*]] = udiv i32 [[SUB182]], 64
-// CHECK1-NEXT:    [[MUL184:%.*]] = mul i32 [[MUL181]], [[DIV183]]
-// CHECK1-NEXT:    [[MUL185:%.*]] = mul i32 [[MUL184]], 64
-// CHECK1-NEXT:    [[CONV186:%.*]] = zext i32 [[MUL185]] to i64
-// CHECK1-NEXT:    [[DIV187:%.*]] = sdiv i64 [[TMP102]], [[CONV186]]
-// CHECK1-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB188:%.*]] = sub i32 [[TMP108]], [[TMP109]]
-// CHECK1-NEXT:    [[SUB189:%.*]] = sub i32 [[SUB188]], 1
+// CHECK1-NEXT:    [[DIV162:%.*]] = udiv i32 [[ADD161]], [[TMP98]]
+// CHECK1-NEXT:    [[MUL163:%.*]] = mul i32 1, [[DIV162]]
+// CHECK1-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB164:%.*]] = sub i32 [[TMP99]], -63
+// CHECK1-NEXT:    [[DIV165:%.*]] = udiv i32 [[SUB164]], 64
+// CHECK1-NEXT:    [[MUL166:%.*]] = mul i32 [[MUL163]], [[DIV165]]
+// CHECK1-NEXT:    [[MUL167:%.*]] = mul i32 [[MUL166]], 64
+// CHECK1-NEXT:    [[CONV168:%.*]] = zext i32 [[MUL167]] to i64
+// CHECK1-NEXT:    [[DIV169:%.*]] = sdiv i64 [[TMP94]], [[CONV168]]
+// CHECK1-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP101:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB170:%.*]] = sub i32 [[TMP100]], [[TMP101]]
+// CHECK1-NEXT:    [[SUB171:%.*]] = sub i32 [[SUB170]], 1
+// CHECK1-NEXT:    [[TMP102:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD172:%.*]] = add i32 [[SUB171]], [[TMP102]]
+// CHECK1-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV173:%.*]] = udiv i32 [[ADD172]], [[TMP103]]
+// CHECK1-NEXT:    [[MUL174:%.*]] = mul i32 1, [[DIV173]]
+// CHECK1-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB175:%.*]] = sub i32 [[TMP104]], -63
+// CHECK1-NEXT:    [[DIV176:%.*]] = udiv i32 [[SUB175]], 64
+// CHECK1-NEXT:    [[MUL177:%.*]] = mul i32 [[MUL174]], [[DIV176]]
+// CHECK1-NEXT:    [[MUL178:%.*]] = mul i32 [[MUL177]], 64
+// CHECK1-NEXT:    [[CONV179:%.*]] = zext i32 [[MUL178]] to i64
+// CHECK1-NEXT:    [[MUL180:%.*]] = mul nsw i64 [[DIV169]], [[CONV179]]
+// CHECK1-NEXT:    [[SUB181:%.*]] = sub nsw i64 [[TMP93]], [[MUL180]]
+// CHECK1-NEXT:    [[TMP105:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP106:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB182:%.*]] = sub i32 [[TMP107]], [[TMP108]]
+// CHECK1-NEXT:    [[SUB183:%.*]] = sub i32 [[SUB182]], 1
+// CHECK1-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD184:%.*]] = add i32 [[SUB183]], [[TMP109]]
 // CHECK1-NEXT:    [[TMP110:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD190:%.*]] = add i32 [[SUB189]], [[TMP110]]
-// CHECK1-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV191:%.*]] = udiv i32 [[ADD190]], [[TMP111]]
-// CHECK1-NEXT:    [[MUL192:%.*]] = mul i32 1, [[DIV191]]
-// CHECK1-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB193:%.*]] = sub i32 [[TMP112]], -63
-// CHECK1-NEXT:    [[DIV194:%.*]] = udiv i32 [[SUB193]], 64
-// CHECK1-NEXT:    [[MUL195:%.*]] = mul i32 [[MUL192]], [[DIV194]]
-// CHECK1-NEXT:    [[MUL196:%.*]] = mul i32 [[MUL195]], 64
-// CHECK1-NEXT:    [[CONV197:%.*]] = zext i32 [[MUL196]] to i64
-// CHECK1-NEXT:    [[MUL198:%.*]] = mul nsw i64 [[DIV187]], [[CONV197]]
-// CHECK1-NEXT:    [[SUB199:%.*]] = sub nsw i64 [[TMP101]], [[MUL198]]
-// CHECK1-NEXT:    [[TMP113:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB200:%.*]] = sub i32 [[TMP113]], -63
-// CHECK1-NEXT:    [[DIV201:%.*]] = udiv i32 [[SUB200]], 64
-// CHECK1-NEXT:    [[MUL202:%.*]] = mul i32 1, [[DIV201]]
-// CHECK1-NEXT:    [[MUL203:%.*]] = mul i32 [[MUL202]], 64
-// CHECK1-NEXT:    [[CONV204:%.*]] = zext i32 [[MUL203]] to i64
-// CHECK1-NEXT:    [[DIV205:%.*]] = sdiv i64 [[SUB199]], [[CONV204]]
-// CHECK1-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB206:%.*]] = sub i32 [[TMP114]], -63
-// CHECK1-NEXT:    [[DIV207:%.*]] = udiv i32 [[SUB206]], 64
-// CHECK1-NEXT:    [[MUL208:%.*]] = mul i32 1, [[DIV207]]
-// CHECK1-NEXT:    [[MUL209:%.*]] = mul i32 [[MUL208]], 64
-// CHECK1-NEXT:    [[CONV210:%.*]] = zext i32 [[MUL209]] to i64
-// CHECK1-NEXT:    [[MUL211:%.*]] = mul nsw i64 [[DIV205]], [[CONV210]]
-// CHECK1-NEXT:    [[SUB212:%.*]] = sub nsw i64 [[SUB176]], [[MUL211]]
-// CHECK1-NEXT:    [[TMP115:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP116:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB213:%.*]] = sub i32 [[TMP117]], [[TMP118]]
-// CHECK1-NEXT:    [[SUB214:%.*]] = sub i32 [[SUB213]], 1
-// CHECK1-NEXT:    [[TMP119:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD215:%.*]] = add i32 [[SUB214]], [[TMP119]]
-// CHECK1-NEXT:    [[TMP120:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV216:%.*]] = udiv i32 [[ADD215]], [[TMP120]]
-// CHECK1-NEXT:    [[MUL217:%.*]] = mul i32 1, [[DIV216]]
-// CHECK1-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB218:%.*]] = sub i32 [[TMP121]], -63
-// CHECK1-NEXT:    [[DIV219:%.*]] = udiv i32 [[SUB218]], 64
-// CHECK1-NEXT:    [[MUL220:%.*]] = mul i32 [[MUL217]], [[DIV219]]
-// CHECK1-NEXT:    [[MUL221:%.*]] = mul i32 [[MUL220]], 64
-// CHECK1-NEXT:    [[CONV222:%.*]] = zext i32 [[MUL221]] to i64
-// CHECK1-NEXT:    [[DIV223:%.*]] = sdiv i64 [[TMP116]], [[CONV222]]
-// CHECK1-NEXT:    [[TMP122:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP123:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB224:%.*]] = sub i32 [[TMP122]], [[TMP123]]
-// CHECK1-NEXT:    [[SUB225:%.*]] = sub i32 [[SUB224]], 1
+// CHECK1-NEXT:    [[DIV185:%.*]] = udiv i32 [[ADD184]], [[TMP110]]
+// CHECK1-NEXT:    [[MUL186:%.*]] = mul i32 1, [[DIV185]]
+// CHECK1-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB187:%.*]] = sub i32 [[TMP111]], -63
+// CHECK1-NEXT:    [[DIV188:%.*]] = udiv i32 [[SUB187]], 64
+// CHECK1-NEXT:    [[MUL189:%.*]] = mul i32 [[MUL186]], [[DIV188]]
+// CHECK1-NEXT:    [[MUL190:%.*]] = mul i32 [[MUL189]], 64
+// CHECK1-NEXT:    [[CONV191:%.*]] = zext i32 [[MUL190]] to i64
+// CHECK1-NEXT:    [[DIV192:%.*]] = sdiv i64 [[TMP106]], [[CONV191]]
+// CHECK1-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP113:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB193:%.*]] = sub i32 [[TMP112]], [[TMP113]]
+// CHECK1-NEXT:    [[SUB194:%.*]] = sub i32 [[SUB193]], 1
+// CHECK1-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD195:%.*]] = add i32 [[SUB194]], [[TMP114]]
+// CHECK1-NEXT:    [[TMP115:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV196:%.*]] = udiv i32 [[ADD195]], [[TMP115]]
+// CHECK1-NEXT:    [[MUL197:%.*]] = mul i32 1, [[DIV196]]
+// CHECK1-NEXT:    [[TMP116:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB198:%.*]] = sub i32 [[TMP116]], -63
+// CHECK1-NEXT:    [[DIV199:%.*]] = udiv i32 [[SUB198]], 64
+// CHECK1-NEXT:    [[MUL200:%.*]] = mul i32 [[MUL197]], [[DIV199]]
+// CHECK1-NEXT:    [[MUL201:%.*]] = mul i32 [[MUL200]], 64
+// CHECK1-NEXT:    [[CONV202:%.*]] = zext i32 [[MUL201]] to i64
+// CHECK1-NEXT:    [[MUL203:%.*]] = mul nsw i64 [[DIV192]], [[CONV202]]
+// CHECK1-NEXT:    [[SUB204:%.*]] = sub nsw i64 [[TMP105]], [[MUL203]]
+// CHECK1-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB205:%.*]] = sub i32 [[TMP117]], -63
+// CHECK1-NEXT:    [[DIV206:%.*]] = udiv i32 [[SUB205]], 64
+// CHECK1-NEXT:    [[MUL207:%.*]] = mul i32 1, [[DIV206]]
+// CHECK1-NEXT:    [[MUL208:%.*]] = mul i32 [[MUL207]], 64
+// CHECK1-NEXT:    [[CONV209:%.*]] = zext i32 [[MUL208]] to i64
+// CHECK1-NEXT:    [[DIV210:%.*]] = sdiv i64 [[SUB204]], [[CONV209]]
+// CHECK1-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB211:%.*]] = sub i32 [[TMP118]], -63
+// CHECK1-NEXT:    [[DIV212:%.*]] = udiv i32 [[SUB211]], 64
+// CHECK1-NEXT:    [[MUL213:%.*]] = mul i32 1, [[DIV212]]
+// CHECK1-NEXT:    [[MUL214:%.*]] = mul i32 [[MUL213]], 64
+// CHECK1-NEXT:    [[CONV215:%.*]] = zext i32 [[MUL214]] to i64
+// CHECK1-NEXT:    [[MUL216:%.*]] = mul nsw i64 [[DIV210]], [[CONV215]]
+// CHECK1-NEXT:    [[SUB217:%.*]] = sub nsw i64 [[SUB181]], [[MUL216]]
+// CHECK1-NEXT:    [[TMP119:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP120:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP122:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB218:%.*]] = sub i32 [[TMP121]], [[TMP122]]
+// CHECK1-NEXT:    [[SUB219:%.*]] = sub i32 [[SUB218]], 1
+// CHECK1-NEXT:    [[TMP123:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD220:%.*]] = add i32 [[SUB219]], [[TMP123]]
 // CHECK1-NEXT:    [[TMP124:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD226:%.*]] = add i32 [[SUB225]], [[TMP124]]
-// CHECK1-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV227:%.*]] = udiv i32 [[ADD226]], [[TMP125]]
-// CHECK1-NEXT:    [[MUL228:%.*]] = mul i32 1, [[DIV227]]
-// CHECK1-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB229:%.*]] = sub i32 [[TMP126]], -63
-// CHECK1-NEXT:    [[DIV230:%.*]] = udiv i32 [[SUB229]], 64
-// CHECK1-NEXT:    [[MUL231:%.*]] = mul i32 [[MUL228]], [[DIV230]]
-// CHECK1-NEXT:    [[MUL232:%.*]] = mul i32 [[MUL231]], 64
-// CHECK1-NEXT:    [[CONV233:%.*]] = zext i32 [[MUL232]] to i64
-// CHECK1-NEXT:    [[MUL234:%.*]] = mul nsw i64 [[DIV223]], [[CONV233]]
-// CHECK1-NEXT:    [[SUB235:%.*]] = sub nsw i64 [[TMP115]], [[MUL234]]
-// CHECK1-NEXT:    [[TMP127:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP128:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB236:%.*]] = sub i32 [[TMP129]], [[TMP130]]
-// CHECK1-NEXT:    [[SUB237:%.*]] = sub i32 [[SUB236]], 1
-// CHECK1-NEXT:    [[TMP131:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD238:%.*]] = add i32 [[SUB237]], [[TMP131]]
-// CHECK1-NEXT:    [[TMP132:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV239:%.*]] = udiv i32 [[ADD238]], [[TMP132]]
-// CHECK1-NEXT:    [[MUL240:%.*]] = mul i32 1, [[DIV239]]
-// CHECK1-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB241:%.*]] = sub i32 [[TMP133]], -63
-// CHECK1-NEXT:    [[DIV242:%.*]] = udiv i32 [[SUB241]], 64
-// CHECK1-NEXT:    [[MUL243:%.*]] = mul i32 [[MUL240]], [[DIV242]]
-// CHECK1-NEXT:    [[MUL244:%.*]] = mul i32 [[MUL243]], 64
-// CHECK1-NEXT:    [[CONV245:%.*]] = zext i32 [[MUL244]] to i64
-// CHECK1-NEXT:    [[DIV246:%.*]] = sdiv i64 [[TMP128]], [[CONV245]]
-// CHECK1-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK1-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK1-NEXT:    [[SUB247:%.*]] = sub i32 [[TMP134]], [[TMP135]]
-// CHECK1-NEXT:    [[SUB248:%.*]] = sub i32 [[SUB247]], 1
+// CHECK1-NEXT:    [[DIV221:%.*]] = udiv i32 [[ADD220]], [[TMP124]]
+// CHECK1-NEXT:    [[MUL222:%.*]] = mul i32 1, [[DIV221]]
+// CHECK1-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB223:%.*]] = sub i32 [[TMP125]], -63
+// CHECK1-NEXT:    [[DIV224:%.*]] = udiv i32 [[SUB223]], 64
+// CHECK1-NEXT:    [[MUL225:%.*]] = mul i32 [[MUL222]], [[DIV224]]
+// CHECK1-NEXT:    [[MUL226:%.*]] = mul i32 [[MUL225]], 64
+// CHECK1-NEXT:    [[CONV227:%.*]] = zext i32 [[MUL226]] to i64
+// CHECK1-NEXT:    [[DIV228:%.*]] = sdiv i64 [[TMP120]], [[CONV227]]
+// CHECK1-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP127:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB229:%.*]] = sub i32 [[TMP126]], [[TMP127]]
+// CHECK1-NEXT:    [[SUB230:%.*]] = sub i32 [[SUB229]], 1
+// CHECK1-NEXT:    [[TMP128:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD231:%.*]] = add i32 [[SUB230]], [[TMP128]]
+// CHECK1-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV232:%.*]] = udiv i32 [[ADD231]], [[TMP129]]
+// CHECK1-NEXT:    [[MUL233:%.*]] = mul i32 1, [[DIV232]]
+// CHECK1-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB234:%.*]] = sub i32 [[TMP130]], -63
+// CHECK1-NEXT:    [[DIV235:%.*]] = udiv i32 [[SUB234]], 64
+// CHECK1-NEXT:    [[MUL236:%.*]] = mul i32 [[MUL233]], [[DIV235]]
+// CHECK1-NEXT:    [[MUL237:%.*]] = mul i32 [[MUL236]], 64
+// CHECK1-NEXT:    [[CONV238:%.*]] = zext i32 [[MUL237]] to i64
+// CHECK1-NEXT:    [[MUL239:%.*]] = mul nsw i64 [[DIV228]], [[CONV238]]
+// CHECK1-NEXT:    [[SUB240:%.*]] = sub nsw i64 [[TMP119]], [[MUL239]]
+// CHECK1-NEXT:    [[TMP131:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP132:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB241:%.*]] = sub i32 [[TMP133]], [[TMP134]]
+// CHECK1-NEXT:    [[SUB242:%.*]] = sub i32 [[SUB241]], 1
+// CHECK1-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD243:%.*]] = add i32 [[SUB242]], [[TMP135]]
 // CHECK1-NEXT:    [[TMP136:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[ADD249:%.*]] = add i32 [[SUB248]], [[TMP136]]
-// CHECK1-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK1-NEXT:    [[DIV250:%.*]] = udiv i32 [[ADD249]], [[TMP137]]
-// CHECK1-NEXT:    [[MUL251:%.*]] = mul i32 1, [[DIV250]]
-// CHECK1-NEXT:    [[TMP138:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB252:%.*]] = sub i32 [[TMP138]], -63
-// CHECK1-NEXT:    [[DIV253:%.*]] = udiv i32 [[SUB252]], 64
-// CHECK1-NEXT:    [[MUL254:%.*]] = mul i32 [[MUL251]], [[DIV253]]
-// CHECK1-NEXT:    [[MUL255:%.*]] = mul i32 [[MUL254]], 64
-// CHECK1-NEXT:    [[CONV256:%.*]] = zext i32 [[MUL255]] to i64
-// CHECK1-NEXT:    [[MUL257:%.*]] = mul nsw i64 [[DIV246]], [[CONV256]]
-// CHECK1-NEXT:    [[SUB258:%.*]] = sub nsw i64 [[TMP127]], [[MUL257]]
-// CHECK1-NEXT:    [[TMP139:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB259:%.*]] = sub i32 [[TMP139]], -63
-// CHECK1-NEXT:    [[DIV260:%.*]] = udiv i32 [[SUB259]], 64
-// CHECK1-NEXT:    [[MUL261:%.*]] = mul i32 1, [[DIV260]]
-// CHECK1-NEXT:    [[MUL262:%.*]] = mul i32 [[MUL261]], 64
-// CHECK1-NEXT:    [[CONV263:%.*]] = zext i32 [[MUL262]] to i64
-// CHECK1-NEXT:    [[DIV264:%.*]] = sdiv i64 [[SUB258]], [[CONV263]]
-// CHECK1-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK1-NEXT:    [[SUB265:%.*]] = sub i32 [[TMP140]], -63
-// CHECK1-NEXT:    [[DIV266:%.*]] = udiv i32 [[SUB265]], 64
-// CHECK1-NEXT:    [[MUL267:%.*]] = mul i32 1, [[DIV266]]
-// CHECK1-NEXT:    [[MUL268:%.*]] = mul i32 [[MUL267]], 64
-// CHECK1-NEXT:    [[CONV269:%.*]] = zext i32 [[MUL268]] to i64
-// CHECK1-NEXT:    [[MUL270:%.*]] = mul nsw i64 [[DIV264]], [[CONV269]]
-// CHECK1-NEXT:    [[SUB271:%.*]] = sub nsw i64 [[SUB235]], [[MUL270]]
-// CHECK1-NEXT:    [[DIV272:%.*]] = sdiv i64 [[SUB271]], 64
-// CHECK1-NEXT:    [[MUL273:%.*]] = mul nsw i64 [[DIV272]], 64
-// CHECK1-NEXT:    [[SUB274:%.*]] = sub nsw i64 [[SUB212]], [[MUL273]]
-// CHECK1-NEXT:    [[MUL275:%.*]] = mul nsw i64 [[SUB274]], 1
-// CHECK1-NEXT:    [[ADD276:%.*]] = add nsw i64 0, [[MUL275]]
-// CHECK1-NEXT:    [[CONV277:%.*]] = trunc i64 [[ADD276]] to i32
-// CHECK1-NEXT:    store i32 [[CONV277]], ptr [[DOTTILE_CNT_0_IV_K38]], align 4
-// CHECK1-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
-// CHECK1-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
-// CHECK1-NEXT:    [[ADD278:%.*]] = add i32 [[TMP141]], [[TMP142]]
-// CHECK1-NEXT:    store i32 [[ADD278]], ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK1-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
-// CHECK1-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK1-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
-// CHECK1-NEXT:    [[MUL279:%.*]] = mul i32 [[TMP144]], [[TMP145]]
-// CHECK1-NEXT:    [[ADD280:%.*]] = add i32 [[TMP143]], [[MUL279]]
-// CHECK1-NEXT:    store i32 [[ADD280]], ptr [[K]], align 4
-// CHECK1-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
-// CHECK1-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
-// CHECK1-NEXT:    [[ADD281:%.*]] = add i32 [[TMP146]], [[TMP147]]
-// CHECK1-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK1-NEXT:    [[ADD282:%.*]] = add i32 [[TMP148]], 1
-// CHECK1-NEXT:    [[CMP283:%.*]] = icmp ult i32 [[ADD281]], [[ADD282]]
-// CHECK1-NEXT:    br i1 [[CMP283]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK1:       if.then:
-// CHECK1-NEXT:    [[TMP149:%.*]] = load i32, ptr [[I35]], align 4
-// CHECK1-NEXT:    [[TMP150:%.*]] = load i32, ptr [[J36]], align 4
-// CHECK1-NEXT:    [[TMP151:%.*]] = load i32, ptr [[K]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP149]], i32 noundef [[TMP150]], i32 noundef [[TMP151]])
-// CHECK1-NEXT:    br label [[IF_END]]
-// CHECK1:       if.end:
-// CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK1-NEXT:    [[DIV244:%.*]] = udiv i32 [[ADD243]], [[TMP136]]
+// CHECK1-NEXT:    [[MUL245:%.*]] = mul i32 1, [[DIV244]]
+// CHECK1-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB246:%.*]] = sub i32 [[TMP137]], -63
+// CHECK1-NEXT:    [[DIV247:%.*]] = udiv i32 [[SUB246]], 64
+// CHECK1-NEXT:    [[MUL248:%.*]] = mul i32 [[MUL245]], [[DIV247]]
+// CHECK1-NEXT:    [[MUL249:%.*]] = mul i32 [[MUL248]], 64
+// CHECK1-NEXT:    [[CONV250:%.*]] = zext i32 [[MUL249]] to i64
+// CHECK1-NEXT:    [[DIV251:%.*]] = sdiv i64 [[TMP132]], [[CONV250]]
+// CHECK1-NEXT:    [[TMP138:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK1-NEXT:    [[TMP139:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK1-NEXT:    [[SUB252:%.*]] = sub i32 [[TMP138]], [[TMP139]]
+// CHECK1-NEXT:    [[SUB253:%.*]] = sub i32 [[SUB252]], 1
+// CHECK1-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[ADD254:%.*]] = add i32 [[SUB253]], [[TMP140]]
+// CHECK1-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK1-NEXT:    [[DIV255:%.*]] = udiv i32 [[ADD254]], [[TMP141]]
+// CHECK1-NEXT:    [[MUL256:%.*]] = mul i32 1, [[DIV255]]
+// CHECK1-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB257:%.*]] = sub i32 [[TMP142]], -63
+// CHECK1-NEXT:    [[DIV258:%.*]] = udiv i32 [[SUB257]], 64
+// CHECK1-NEXT:    [[MUL259:%.*]] = mul i32 [[MUL256]], [[DIV258]]
+// CHECK1-NEXT:    [[MUL260:%.*]] = mul i32 [[MUL259]], 64
+// CHECK1-NEXT:    [[CONV261:%.*]] = zext i32 [[MUL260]] to i64
+// CHECK1-NEXT:    [[MUL262:%.*]] = mul nsw i64 [[DIV251]], [[CONV261]]
+// CHECK1-NEXT:    [[SUB263:%.*]] = sub nsw i64 [[TMP131]], [[MUL262]]
+// CHECK1-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB264:%.*]] = sub i32 [[TMP143]], -63
+// CHECK1-NEXT:    [[DIV265:%.*]] = udiv i32 [[SUB264]], 64
+// CHECK1-NEXT:    [[MUL266:%.*]] = mul i32 1, [[DIV265]]
+// CHECK1-NEXT:    [[MUL267:%.*]] = mul i32 [[MUL266]], 64
+// CHECK1-NEXT:    [[CONV268:%.*]] = zext i32 [[MUL267]] to i64
+// CHECK1-NEXT:    [[DIV269:%.*]] = sdiv i64 [[SUB263]], [[CONV268]]
+// CHECK1-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK1-NEXT:    [[SUB270:%.*]] = sub i32 [[TMP144]], -63
+// CHECK1-NEXT:    [[DIV271:%.*]] = udiv i32 [[SUB270]], 64
+// CHECK1-NEXT:    [[MUL272:%.*]] = mul i32 1, [[DIV271]]
+// CHECK1-NEXT:    [[MUL273:%.*]] = mul i32 [[MUL272]], 64
+// CHECK1-NEXT:    [[CONV274:%.*]] = zext i32 [[MUL273]] to i64
+// CHECK1-NEXT:    [[MUL275:%.*]] = mul nsw i64 [[DIV269]], [[CONV274]]
+// CHECK1-NEXT:    [[SUB276:%.*]] = sub nsw i64 [[SUB240]], [[MUL275]]
+// CHECK1-NEXT:    [[DIV277:%.*]] = sdiv i64 [[SUB276]], 64
+// CHECK1-NEXT:    [[MUL278:%.*]] = mul nsw i64 [[DIV277]], 64
+// CHECK1-NEXT:    [[SUB279:%.*]] = sub nsw i64 [[SUB217]], [[MUL278]]
+// CHECK1-NEXT:    [[MUL280:%.*]] = mul nsw i64 [[SUB279]], 1
+// CHECK1-NEXT:    [[ADD281:%.*]] = add nsw i64 [[CONV158]], [[MUL280]]
+// CHECK1-NEXT:    [[CONV282:%.*]] = trunc i64 [[ADD281]] to i32
+// CHECK1-NEXT:    store i32 [[CONV282]], ptr [[DOTTILE_0_IV_K42]], align 4
+// CHECK1-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTTILE_0_IV_K42]], align 4
+// CHECK1-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// CHECK1-NEXT:    [[ADD283:%.*]] = add i32 [[TMP146]], 1
+// CHECK1-NEXT:    [[CMP284:%.*]] = icmp ult i32 [[TMP145]], [[ADD283]]
+// CHECK1-NEXT:    br i1 [[CMP284]], label [[OMP_BODY_NEXT:%.*]], label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK1:       omp.body.next:
+// CHECK1-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
+// CHECK1-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTTILE_0_IV_K42]], align 4
+// CHECK1-NEXT:    [[TMP149:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
+// CHECK1-NEXT:    [[MUL285:%.*]] = mul i32 [[TMP148]], [[TMP149]]
+// CHECK1-NEXT:    [[ADD286:%.*]] = add i32 [[TMP147]], [[MUL285]]
+// CHECK1-NEXT:    store i32 [[ADD286]], ptr [[K]], align 4
+// CHECK1-NEXT:    [[TMP150:%.*]] = load i32, ptr [[I39]], align 4
+// CHECK1-NEXT:    [[TMP151:%.*]] = load i32, ptr [[J40]], align 4
+// CHECK1-NEXT:    [[TMP152:%.*]] = load i32, ptr [[K]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP150]], i32 noundef [[TMP151]], i32 noundef [[TMP152]])
+// CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP152:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK1-NEXT:    [[ADD284:%.*]] = add nsw i64 [[TMP152]], 1
-// CHECK1-NEXT:    store i64 [[ADD284]], ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[TMP153:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK1-NEXT:    [[ADD287:%.*]] = add nsw i64 [[TMP153]], 1
+// CHECK1-NEXT:    store i64 [[ADD287]], ptr [[DOTOMP_IV]], align 8
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -3572,21 +3578,23 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTNEW_STEP10:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_14:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_17:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_19:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_K:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_K:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[_TMP35:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[I35:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[J36:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV_K37:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_K38:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[I39:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[J40:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV_K41:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_K42:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
@@ -3624,454 +3632,455 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
 // CHECK2-NEXT:    [[ADD15:%.*]] = add i32 [[TMP15]], 1
 // CHECK2-NEXT:    store i32 [[ADD15]], ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[SUB17:%.*]] = sub i32 [[TMP16]], [[TMP17]]
-// CHECK2-NEXT:    [[SUB18:%.*]] = sub i32 [[SUB17]], 1
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[ADD19:%.*]] = add i32 [[SUB18]], [[TMP18]]
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[DIV20:%.*]] = udiv i32 [[ADD19]], [[TMP19]]
-// CHECK2-NEXT:    [[CONV:%.*]] = zext i32 [[DIV20]] to i64
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB21:%.*]] = sub i32 [[TMP20]], [[TMP21]]
-// CHECK2-NEXT:    [[SUB22:%.*]] = sub i32 [[SUB21]], 1
-// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD23:%.*]] = add i32 [[SUB22]], [[TMP22]]
-// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV24:%.*]] = udiv i32 [[ADD23]], [[TMP23]]
-// CHECK2-NEXT:    [[CONV25:%.*]] = zext i32 [[DIV24]] to i64
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV25]]
-// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB26:%.*]] = sub i32 [[TMP24]], -31
-// CHECK2-NEXT:    [[DIV27:%.*]] = udiv i32 [[SUB26]], 32
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[_TMP2]], align 4
+// CHECK2-NEXT:    store i32 [[TMP16]], ptr [[DOTCAPTURE_EXPR_16]], align 4
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[_TMP2]], align 4
+// CHECK2-NEXT:    [[ADD18:%.*]] = add i32 [[TMP17]], 32
+// CHECK2-NEXT:    store i32 [[ADD18]], ptr [[DOTCAPTURE_EXPR_17]], align 4
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[SUB20:%.*]] = sub i32 [[TMP18]], [[TMP19]]
+// CHECK2-NEXT:    [[SUB21:%.*]] = sub i32 [[SUB20]], 1
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[ADD22:%.*]] = add i32 [[SUB21]], [[TMP20]]
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[DIV23:%.*]] = udiv i32 [[ADD22]], [[TMP21]]
+// CHECK2-NEXT:    [[CONV:%.*]] = zext i32 [[DIV23]] to i64
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB24:%.*]] = sub i32 [[TMP22]], [[TMP23]]
+// CHECK2-NEXT:    [[SUB25:%.*]] = sub i32 [[SUB24]], 1
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD26:%.*]] = add i32 [[SUB25]], [[TMP24]]
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV27:%.*]] = udiv i32 [[ADD26]], [[TMP25]]
 // CHECK2-NEXT:    [[CONV28:%.*]] = zext i32 [[DIV27]] to i64
-// CHECK2-NEXT:    [[MUL29:%.*]] = mul nsw i64 [[MUL]], [[CONV28]]
-// CHECK2-NEXT:    [[MUL30:%.*]] = mul nsw i64 [[MUL29]], 32
-// CHECK2-NEXT:    [[SUB31:%.*]] = sub nsw i64 [[MUL30]], 1
-// CHECK2-NEXT:    store i64 [[SUB31]], ptr [[DOTCAPTURE_EXPR_16]], align 8
-// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    store i32 [[TMP25]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    store i32 [[TMP26]], ptr [[J]], align 4
-// CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_K]], align 4
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_K]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV28]]
+// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB29:%.*]] = sub i32 [[TMP26]], -31
+// CHECK2-NEXT:    [[DIV30:%.*]] = udiv i32 [[SUB29]], 32
+// CHECK2-NEXT:    [[CONV31:%.*]] = zext i32 [[DIV30]] to i64
+// CHECK2-NEXT:    [[MUL32:%.*]] = mul nsw i64 [[MUL]], [[CONV31]]
+// CHECK2-NEXT:    [[MUL33:%.*]] = mul nsw i64 [[MUL32]], 32
+// CHECK2-NEXT:    [[SUB34:%.*]] = sub nsw i64 [[MUL33]], 1
+// CHECK2-NEXT:    store i64 [[SUB34]], ptr [[DOTCAPTURE_EXPR_19]], align 8
 // CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP27]], [[TMP28]]
+// CHECK2-NEXT:    store i32 [[TMP27]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    store i32 [[TMP28]], ptr [[J]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_K]], align 4
+// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K]], align 4
+// CHECK2-NEXT:    store i32 [[TMP29]], ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[_TMP35]], align 4
+// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP30]], [[TMP31]]
 // CHECK2-NEXT:    br i1 [[CMP]], label [[LAND_LHS_TRUE:%.*]], label [[OMP_PRECOND_END:%.*]]
 // CHECK2:       land.lhs.true:
-// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[CMP32:%.*]] = icmp slt i32 [[TMP29]], [[TMP30]]
-// CHECK2-NEXT:    br i1 [[CMP32]], label [[LAND_LHS_TRUE33:%.*]], label [[OMP_PRECOND_END]]
-// CHECK2:       land.lhs.true33:
-// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[CMP34:%.*]] = icmp ult i32 0, [[TMP31]]
-// CHECK2-NEXT:    br i1 [[CMP34]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
+// CHECK2-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[CMP36:%.*]] = icmp slt i32 [[TMP32]], [[TMP33]]
+// CHECK2-NEXT:    br i1 [[CMP36]], label [[LAND_LHS_TRUE37:%.*]], label [[OMP_PRECOND_END]]
+// CHECK2:       land.lhs.true37:
+// CHECK2-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[CMP38:%.*]] = icmp ult i32 0, [[TMP34]]
+// CHECK2-NEXT:    br i1 [[CMP38]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
 // CHECK2:       omp.precond.then:
 // CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
-// CHECK2-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
-// CHECK2-NEXT:    store i64 [[TMP32]], ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_19]], align 8
+// CHECK2-NEXT:    store i64 [[TMP35]], ptr [[DOTOMP_UB]], align 8
 // CHECK2-NEXT:    store i64 1, ptr [[DOTOMP_STRIDE]], align 8
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
 // CHECK2-NEXT:    call void @__kmpc_for_static_init_8(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i64 1, i64 1)
-// CHECK2-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
-// CHECK2-NEXT:    [[CMP39:%.*]] = icmp sgt i64 [[TMP33]], [[TMP34]]
-// CHECK2-NEXT:    br i1 [[CMP39]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_19]], align 8
+// CHECK2-NEXT:    [[CMP43:%.*]] = icmp sgt i64 [[TMP36]], [[TMP37]]
+// CHECK2-NEXT:    br i1 [[CMP43]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK2:       cond.true:
-// CHECK2-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK2-NEXT:    [[TMP38:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_19]], align 8
 // CHECK2-NEXT:    br label [[COND_END:%.*]]
 // CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[TMP39:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
 // CHECK2-NEXT:    br label [[COND_END]]
 // CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i64 [ [[TMP35]], [[COND_TRUE]] ], [ [[TMP36]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[COND:%.*]] = phi i64 [ [[TMP38]], [[COND_TRUE]] ], [ [[TMP39]], [[COND_FALSE]] ]
 // CHECK2-NEXT:    store i64 [[COND]], ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
-// CHECK2-NEXT:    store i64 [[TMP37]], ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP40:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
+// CHECK2-NEXT:    store i64 [[TMP40]], ptr [[DOTOMP_IV]], align 8
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK2:       omp.inner.for.cond:
-// CHECK2-NEXT:    [[TMP38:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP39:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[CMP40:%.*]] = icmp sle i64 [[TMP38]], [[TMP39]]
-// CHECK2-NEXT:    br i1 [[CMP40]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
-// CHECK2:       omp.inner.for.body:
-// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[CONV41:%.*]] = sext i32 [[TMP40]] to i64
 // CHECK2-NEXT:    [[TMP41:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB42:%.*]] = sub i32 [[TMP42]], [[TMP43]]
-// CHECK2-NEXT:    [[SUB43:%.*]] = sub i32 [[SUB42]], 1
-// CHECK2-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD44:%.*]] = add i32 [[SUB43]], [[TMP44]]
-// CHECK2-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV45:%.*]] = udiv i32 [[ADD44]], [[TMP45]]
-// CHECK2-NEXT:    [[MUL46:%.*]] = mul i32 1, [[DIV45]]
-// CHECK2-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB47:%.*]] = sub i32 [[TMP46]], -31
-// CHECK2-NEXT:    [[DIV48:%.*]] = udiv i32 [[SUB47]], 32
-// CHECK2-NEXT:    [[MUL49:%.*]] = mul i32 [[MUL46]], [[DIV48]]
-// CHECK2-NEXT:    [[MUL50:%.*]] = mul i32 [[MUL49]], 32
-// CHECK2-NEXT:    [[CONV51:%.*]] = zext i32 [[MUL50]] to i64
-// CHECK2-NEXT:    [[DIV52:%.*]] = sdiv i64 [[TMP41]], [[CONV51]]
-// CHECK2-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[CONV53:%.*]] = sext i32 [[TMP47]] to i64
-// CHECK2-NEXT:    [[MUL54:%.*]] = mul nsw i64 [[DIV52]], [[CONV53]]
-// CHECK2-NEXT:    [[ADD55:%.*]] = add nsw i64 [[CONV41]], [[MUL54]]
-// CHECK2-NEXT:    [[CONV56:%.*]] = trunc i64 [[ADD55]] to i32
-// CHECK2-NEXT:    store i32 [[CONV56]], ptr [[I35]], align 4
-// CHECK2-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP48]] to i64
-// CHECK2-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP50:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB58:%.*]] = sub i32 [[TMP51]], [[TMP52]]
-// CHECK2-NEXT:    [[SUB59:%.*]] = sub i32 [[SUB58]], 1
-// CHECK2-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD60:%.*]] = add i32 [[SUB59]], [[TMP53]]
-// CHECK2-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV61:%.*]] = udiv i32 [[ADD60]], [[TMP54]]
-// CHECK2-NEXT:    [[MUL62:%.*]] = mul i32 1, [[DIV61]]
-// CHECK2-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB63:%.*]] = sub i32 [[TMP55]], -31
-// CHECK2-NEXT:    [[DIV64:%.*]] = udiv i32 [[SUB63]], 32
-// CHECK2-NEXT:    [[MUL65:%.*]] = mul i32 [[MUL62]], [[DIV64]]
-// CHECK2-NEXT:    [[MUL66:%.*]] = mul i32 [[MUL65]], 32
-// CHECK2-NEXT:    [[CONV67:%.*]] = zext i32 [[MUL66]] to i64
-// CHECK2-NEXT:    [[DIV68:%.*]] = sdiv i64 [[TMP50]], [[CONV67]]
-// CHECK2-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB69:%.*]] = sub i32 [[TMP56]], [[TMP57]]
-// CHECK2-NEXT:    [[SUB70:%.*]] = sub i32 [[SUB69]], 1
-// CHECK2-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD71:%.*]] = add i32 [[SUB70]], [[TMP58]]
-// CHECK2-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV72:%.*]] = udiv i32 [[ADD71]], [[TMP59]]
-// CHECK2-NEXT:    [[MUL73:%.*]] = mul i32 1, [[DIV72]]
-// CHECK2-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB74:%.*]] = sub i32 [[TMP60]], -31
-// CHECK2-NEXT:    [[DIV75:%.*]] = udiv i32 [[SUB74]], 32
-// CHECK2-NEXT:    [[MUL76:%.*]] = mul i32 [[MUL73]], [[DIV75]]
-// CHECK2-NEXT:    [[MUL77:%.*]] = mul i32 [[MUL76]], 32
-// CHECK2-NEXT:    [[CONV78:%.*]] = zext i32 [[MUL77]] to i64
-// CHECK2-NEXT:    [[MUL79:%.*]] = mul nsw i64 [[DIV68]], [[CONV78]]
-// CHECK2-NEXT:    [[SUB80:%.*]] = sub nsw i64 [[TMP49]], [[MUL79]]
-// CHECK2-NEXT:    [[TMP61:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB81:%.*]] = sub i32 [[TMP61]], -31
-// CHECK2-NEXT:    [[DIV82:%.*]] = udiv i32 [[SUB81]], 32
-// CHECK2-NEXT:    [[MUL83:%.*]] = mul i32 1, [[DIV82]]
-// CHECK2-NEXT:    [[MUL84:%.*]] = mul i32 [[MUL83]], 32
-// CHECK2-NEXT:    [[CONV85:%.*]] = zext i32 [[MUL84]] to i64
-// CHECK2-NEXT:    [[DIV86:%.*]] = sdiv i64 [[SUB80]], [[CONV85]]
+// CHECK2-NEXT:    [[TMP42:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[CMP44:%.*]] = icmp sle i64 [[TMP41]], [[TMP42]]
+// CHECK2-NEXT:    br i1 [[CMP44]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK2:       omp.inner.for.body:
+// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[CONV45:%.*]] = sext i32 [[TMP43]] to i64
+// CHECK2-NEXT:    [[TMP44:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB46:%.*]] = sub i32 [[TMP45]], [[TMP46]]
+// CHECK2-NEXT:    [[SUB47:%.*]] = sub i32 [[SUB46]], 1
+// CHECK2-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD48:%.*]] = add i32 [[SUB47]], [[TMP47]]
+// CHECK2-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV49:%.*]] = udiv i32 [[ADD48]], [[TMP48]]
+// CHECK2-NEXT:    [[MUL50:%.*]] = mul i32 1, [[DIV49]]
+// CHECK2-NEXT:    [[TMP49:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB51:%.*]] = sub i32 [[TMP49]], -31
+// CHECK2-NEXT:    [[DIV52:%.*]] = udiv i32 [[SUB51]], 32
+// CHECK2-NEXT:    [[MUL53:%.*]] = mul i32 [[MUL50]], [[DIV52]]
+// CHECK2-NEXT:    [[MUL54:%.*]] = mul i32 [[MUL53]], 32
+// CHECK2-NEXT:    [[CONV55:%.*]] = zext i32 [[MUL54]] to i64
+// CHECK2-NEXT:    [[DIV56:%.*]] = sdiv i64 [[TMP44]], [[CONV55]]
+// CHECK2-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP50]] to i64
+// CHECK2-NEXT:    [[MUL58:%.*]] = mul nsw i64 [[DIV56]], [[CONV57]]
+// CHECK2-NEXT:    [[ADD59:%.*]] = add nsw i64 [[CONV45]], [[MUL58]]
+// CHECK2-NEXT:    [[CONV60:%.*]] = trunc i64 [[ADD59]] to i32
+// CHECK2-NEXT:    store i32 [[CONV60]], ptr [[I39]], align 4
+// CHECK2-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[CONV61:%.*]] = sext i32 [[TMP51]] to i64
+// CHECK2-NEXT:    [[TMP52:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP53:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB62:%.*]] = sub i32 [[TMP54]], [[TMP55]]
+// CHECK2-NEXT:    [[SUB63:%.*]] = sub i32 [[SUB62]], 1
+// CHECK2-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD64:%.*]] = add i32 [[SUB63]], [[TMP56]]
+// CHECK2-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV65:%.*]] = udiv i32 [[ADD64]], [[TMP57]]
+// CHECK2-NEXT:    [[MUL66:%.*]] = mul i32 1, [[DIV65]]
+// CHECK2-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB67:%.*]] = sub i32 [[TMP58]], -31
+// CHECK2-NEXT:    [[DIV68:%.*]] = udiv i32 [[SUB67]], 32
+// CHECK2-NEXT:    [[MUL69:%.*]] = mul i32 [[MUL66]], [[DIV68]]
+// CHECK2-NEXT:    [[MUL70:%.*]] = mul i32 [[MUL69]], 32
+// CHECK2-NEXT:    [[CONV71:%.*]] = zext i32 [[MUL70]] to i64
+// CHECK2-NEXT:    [[DIV72:%.*]] = sdiv i64 [[TMP53]], [[CONV71]]
+// CHECK2-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB73:%.*]] = sub i32 [[TMP59]], [[TMP60]]
+// CHECK2-NEXT:    [[SUB74:%.*]] = sub i32 [[SUB73]], 1
+// CHECK2-NEXT:    [[TMP61:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD75:%.*]] = add i32 [[SUB74]], [[TMP61]]
 // CHECK2-NEXT:    [[TMP62:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[CONV87:%.*]] = sext i32 [[TMP62]] to i64
-// CHECK2-NEXT:    [[MUL88:%.*]] = mul nsw i64 [[DIV86]], [[CONV87]]
-// CHECK2-NEXT:    [[ADD89:%.*]] = add nsw i64 [[CONV57]], [[MUL88]]
-// CHECK2-NEXT:    [[CONV90:%.*]] = trunc i64 [[ADD89]] to i32
-// CHECK2-NEXT:    store i32 [[CONV90]], ptr [[J36]], align 4
-// CHECK2-NEXT:    [[TMP63:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP64:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB91:%.*]] = sub i32 [[TMP65]], [[TMP66]]
-// CHECK2-NEXT:    [[SUB92:%.*]] = sub i32 [[SUB91]], 1
-// CHECK2-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD93:%.*]] = add i32 [[SUB92]], [[TMP67]]
-// CHECK2-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV94:%.*]] = udiv i32 [[ADD93]], [[TMP68]]
-// CHECK2-NEXT:    [[MUL95:%.*]] = mul i32 1, [[DIV94]]
-// CHECK2-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB96:%.*]] = sub i32 [[TMP69]], -31
-// CHECK2-NEXT:    [[DIV97:%.*]] = udiv i32 [[SUB96]], 32
-// CHECK2-NEXT:    [[MUL98:%.*]] = mul i32 [[MUL95]], [[DIV97]]
-// CHECK2-NEXT:    [[MUL99:%.*]] = mul i32 [[MUL98]], 32
-// CHECK2-NEXT:    [[CONV100:%.*]] = zext i32 [[MUL99]] to i64
-// CHECK2-NEXT:    [[DIV101:%.*]] = sdiv i64 [[TMP64]], [[CONV100]]
-// CHECK2-NEXT:    [[TMP70:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB102:%.*]] = sub i32 [[TMP70]], [[TMP71]]
-// CHECK2-NEXT:    [[SUB103:%.*]] = sub i32 [[SUB102]], 1
-// CHECK2-NEXT:    [[TMP72:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD104:%.*]] = add i32 [[SUB103]], [[TMP72]]
-// CHECK2-NEXT:    [[TMP73:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV105:%.*]] = udiv i32 [[ADD104]], [[TMP73]]
-// CHECK2-NEXT:    [[MUL106:%.*]] = mul i32 1, [[DIV105]]
-// CHECK2-NEXT:    [[TMP74:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB107:%.*]] = sub i32 [[TMP74]], -31
-// CHECK2-NEXT:    [[DIV108:%.*]] = udiv i32 [[SUB107]], 32
-// CHECK2-NEXT:    [[MUL109:%.*]] = mul i32 [[MUL106]], [[DIV108]]
-// CHECK2-NEXT:    [[MUL110:%.*]] = mul i32 [[MUL109]], 32
-// CHECK2-NEXT:    [[CONV111:%.*]] = zext i32 [[MUL110]] to i64
-// CHECK2-NEXT:    [[MUL112:%.*]] = mul nsw i64 [[DIV101]], [[CONV111]]
-// CHECK2-NEXT:    [[SUB113:%.*]] = sub nsw i64 [[TMP63]], [[MUL112]]
-// CHECK2-NEXT:    [[TMP75:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP76:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP78:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB114:%.*]] = sub i32 [[TMP77]], [[TMP78]]
-// CHECK2-NEXT:    [[SUB115:%.*]] = sub i32 [[SUB114]], 1
-// CHECK2-NEXT:    [[TMP79:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD116:%.*]] = add i32 [[SUB115]], [[TMP79]]
-// CHECK2-NEXT:    [[TMP80:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV117:%.*]] = udiv i32 [[ADD116]], [[TMP80]]
-// CHECK2-NEXT:    [[MUL118:%.*]] = mul i32 1, [[DIV117]]
-// CHECK2-NEXT:    [[TMP81:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB119:%.*]] = sub i32 [[TMP81]], -31
-// CHECK2-NEXT:    [[DIV120:%.*]] = udiv i32 [[SUB119]], 32
-// CHECK2-NEXT:    [[MUL121:%.*]] = mul i32 [[MUL118]], [[DIV120]]
-// CHECK2-NEXT:    [[MUL122:%.*]] = mul i32 [[MUL121]], 32
-// CHECK2-NEXT:    [[CONV123:%.*]] = zext i32 [[MUL122]] to i64
-// CHECK2-NEXT:    [[DIV124:%.*]] = sdiv i64 [[TMP76]], [[CONV123]]
-// CHECK2-NEXT:    [[TMP82:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB125:%.*]] = sub i32 [[TMP82]], [[TMP83]]
-// CHECK2-NEXT:    [[SUB126:%.*]] = sub i32 [[SUB125]], 1
-// CHECK2-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD127:%.*]] = add i32 [[SUB126]], [[TMP84]]
-// CHECK2-NEXT:    [[TMP85:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV128:%.*]] = udiv i32 [[ADD127]], [[TMP85]]
-// CHECK2-NEXT:    [[MUL129:%.*]] = mul i32 1, [[DIV128]]
-// CHECK2-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB130:%.*]] = sub i32 [[TMP86]], -31
-// CHECK2-NEXT:    [[DIV131:%.*]] = udiv i32 [[SUB130]], 32
-// CHECK2-NEXT:    [[MUL132:%.*]] = mul i32 [[MUL129]], [[DIV131]]
-// CHECK2-NEXT:    [[MUL133:%.*]] = mul i32 [[MUL132]], 32
-// CHECK2-NEXT:    [[CONV134:%.*]] = zext i32 [[MUL133]] to i64
-// CHECK2-NEXT:    [[MUL135:%.*]] = mul nsw i64 [[DIV124]], [[CONV134]]
-// CHECK2-NEXT:    [[SUB136:%.*]] = sub nsw i64 [[TMP75]], [[MUL135]]
-// CHECK2-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB137:%.*]] = sub i32 [[TMP87]], -31
-// CHECK2-NEXT:    [[DIV138:%.*]] = udiv i32 [[SUB137]], 32
-// CHECK2-NEXT:    [[MUL139:%.*]] = mul i32 1, [[DIV138]]
-// CHECK2-NEXT:    [[MUL140:%.*]] = mul i32 [[MUL139]], 32
-// CHECK2-NEXT:    [[CONV141:%.*]] = zext i32 [[MUL140]] to i64
-// CHECK2-NEXT:    [[DIV142:%.*]] = sdiv i64 [[SUB136]], [[CONV141]]
-// CHECK2-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB143:%.*]] = sub i32 [[TMP88]], -31
-// CHECK2-NEXT:    [[DIV144:%.*]] = udiv i32 [[SUB143]], 32
-// CHECK2-NEXT:    [[MUL145:%.*]] = mul i32 1, [[DIV144]]
-// CHECK2-NEXT:    [[MUL146:%.*]] = mul i32 [[MUL145]], 32
-// CHECK2-NEXT:    [[CONV147:%.*]] = zext i32 [[MUL146]] to i64
-// CHECK2-NEXT:    [[MUL148:%.*]] = mul nsw i64 [[DIV142]], [[CONV147]]
-// CHECK2-NEXT:    [[SUB149:%.*]] = sub nsw i64 [[SUB113]], [[MUL148]]
-// CHECK2-NEXT:    [[DIV150:%.*]] = sdiv i64 [[SUB149]], 32
-// CHECK2-NEXT:    [[MUL151:%.*]] = mul nsw i64 [[DIV150]], 32
-// CHECK2-NEXT:    [[ADD152:%.*]] = add nsw i64 0, [[MUL151]]
-// CHECK2-NEXT:    [[CONV153:%.*]] = trunc i64 [[ADD152]] to i32
-// CHECK2-NEXT:    store i32 [[CONV153]], ptr [[DOTFLOOR_0_IV_K37]], align 4
-// CHECK2-NEXT:    [[TMP89:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP90:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB154:%.*]] = sub i32 [[TMP91]], [[TMP92]]
-// CHECK2-NEXT:    [[SUB155:%.*]] = sub i32 [[SUB154]], 1
-// CHECK2-NEXT:    [[TMP93:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD156:%.*]] = add i32 [[SUB155]], [[TMP93]]
-// CHECK2-NEXT:    [[TMP94:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV157:%.*]] = udiv i32 [[ADD156]], [[TMP94]]
-// CHECK2-NEXT:    [[MUL158:%.*]] = mul i32 1, [[DIV157]]
-// CHECK2-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB159:%.*]] = sub i32 [[TMP95]], -31
-// CHECK2-NEXT:    [[DIV160:%.*]] = udiv i32 [[SUB159]], 32
-// CHECK2-NEXT:    [[MUL161:%.*]] = mul i32 [[MUL158]], [[DIV160]]
-// CHECK2-NEXT:    [[MUL162:%.*]] = mul i32 [[MUL161]], 32
-// CHECK2-NEXT:    [[CONV163:%.*]] = zext i32 [[MUL162]] to i64
-// CHECK2-NEXT:    [[DIV164:%.*]] = sdiv i64 [[TMP90]], [[CONV163]]
-// CHECK2-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP97:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB165:%.*]] = sub i32 [[TMP96]], [[TMP97]]
-// CHECK2-NEXT:    [[SUB166:%.*]] = sub i32 [[SUB165]], 1
+// CHECK2-NEXT:    [[DIV76:%.*]] = udiv i32 [[ADD75]], [[TMP62]]
+// CHECK2-NEXT:    [[MUL77:%.*]] = mul i32 1, [[DIV76]]
+// CHECK2-NEXT:    [[TMP63:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB78:%.*]] = sub i32 [[TMP63]], -31
+// CHECK2-NEXT:    [[DIV79:%.*]] = udiv i32 [[SUB78]], 32
+// CHECK2-NEXT:    [[MUL80:%.*]] = mul i32 [[MUL77]], [[DIV79]]
+// CHECK2-NEXT:    [[MUL81:%.*]] = mul i32 [[MUL80]], 32
+// CHECK2-NEXT:    [[CONV82:%.*]] = zext i32 [[MUL81]] to i64
+// CHECK2-NEXT:    [[MUL83:%.*]] = mul nsw i64 [[DIV72]], [[CONV82]]
+// CHECK2-NEXT:    [[SUB84:%.*]] = sub nsw i64 [[TMP52]], [[MUL83]]
+// CHECK2-NEXT:    [[TMP64:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB85:%.*]] = sub i32 [[TMP64]], -31
+// CHECK2-NEXT:    [[DIV86:%.*]] = udiv i32 [[SUB85]], 32
+// CHECK2-NEXT:    [[MUL87:%.*]] = mul i32 1, [[DIV86]]
+// CHECK2-NEXT:    [[MUL88:%.*]] = mul i32 [[MUL87]], 32
+// CHECK2-NEXT:    [[CONV89:%.*]] = zext i32 [[MUL88]] to i64
+// CHECK2-NEXT:    [[DIV90:%.*]] = sdiv i64 [[SUB84]], [[CONV89]]
+// CHECK2-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[CONV91:%.*]] = sext i32 [[TMP65]] to i64
+// CHECK2-NEXT:    [[MUL92:%.*]] = mul nsw i64 [[DIV90]], [[CONV91]]
+// CHECK2-NEXT:    [[ADD93:%.*]] = add nsw i64 [[CONV61]], [[MUL92]]
+// CHECK2-NEXT:    [[CONV94:%.*]] = trunc i64 [[ADD93]] to i32
+// CHECK2-NEXT:    store i32 [[CONV94]], ptr [[J40]], align 4
+// CHECK2-NEXT:    [[TMP66:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP67:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB95:%.*]] = sub i32 [[TMP68]], [[TMP69]]
+// CHECK2-NEXT:    [[SUB96:%.*]] = sub i32 [[SUB95]], 1
+// CHECK2-NEXT:    [[TMP70:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD97:%.*]] = add i32 [[SUB96]], [[TMP70]]
+// CHECK2-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV98:%.*]] = udiv i32 [[ADD97]], [[TMP71]]
+// CHECK2-NEXT:    [[MUL99:%.*]] = mul i32 1, [[DIV98]]
+// CHECK2-NEXT:    [[TMP72:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB100:%.*]] = sub i32 [[TMP72]], -31
+// CHECK2-NEXT:    [[DIV101:%.*]] = udiv i32 [[SUB100]], 32
+// CHECK2-NEXT:    [[MUL102:%.*]] = mul i32 [[MUL99]], [[DIV101]]
+// CHECK2-NEXT:    [[MUL103:%.*]] = mul i32 [[MUL102]], 32
+// CHECK2-NEXT:    [[CONV104:%.*]] = zext i32 [[MUL103]] to i64
+// CHECK2-NEXT:    [[DIV105:%.*]] = sdiv i64 [[TMP67]], [[CONV104]]
+// CHECK2-NEXT:    [[TMP73:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP74:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB106:%.*]] = sub i32 [[TMP73]], [[TMP74]]
+// CHECK2-NEXT:    [[SUB107:%.*]] = sub i32 [[SUB106]], 1
+// CHECK2-NEXT:    [[TMP75:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD108:%.*]] = add i32 [[SUB107]], [[TMP75]]
+// CHECK2-NEXT:    [[TMP76:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV109:%.*]] = udiv i32 [[ADD108]], [[TMP76]]
+// CHECK2-NEXT:    [[MUL110:%.*]] = mul i32 1, [[DIV109]]
+// CHECK2-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB111:%.*]] = sub i32 [[TMP77]], -31
+// CHECK2-NEXT:    [[DIV112:%.*]] = udiv i32 [[SUB111]], 32
+// CHECK2-NEXT:    [[MUL113:%.*]] = mul i32 [[MUL110]], [[DIV112]]
+// CHECK2-NEXT:    [[MUL114:%.*]] = mul i32 [[MUL113]], 32
+// CHECK2-NEXT:    [[CONV115:%.*]] = zext i32 [[MUL114]] to i64
+// CHECK2-NEXT:    [[MUL116:%.*]] = mul nsw i64 [[DIV105]], [[CONV115]]
+// CHECK2-NEXT:    [[SUB117:%.*]] = sub nsw i64 [[TMP66]], [[MUL116]]
+// CHECK2-NEXT:    [[TMP78:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP79:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP80:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP81:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB118:%.*]] = sub i32 [[TMP80]], [[TMP81]]
+// CHECK2-NEXT:    [[SUB119:%.*]] = sub i32 [[SUB118]], 1
+// CHECK2-NEXT:    [[TMP82:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD120:%.*]] = add i32 [[SUB119]], [[TMP82]]
+// CHECK2-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV121:%.*]] = udiv i32 [[ADD120]], [[TMP83]]
+// CHECK2-NEXT:    [[MUL122:%.*]] = mul i32 1, [[DIV121]]
+// CHECK2-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB123:%.*]] = sub i32 [[TMP84]], -31
+// CHECK2-NEXT:    [[DIV124:%.*]] = udiv i32 [[SUB123]], 32
+// CHECK2-NEXT:    [[MUL125:%.*]] = mul i32 [[MUL122]], [[DIV124]]
+// CHECK2-NEXT:    [[MUL126:%.*]] = mul i32 [[MUL125]], 32
+// CHECK2-NEXT:    [[CONV127:%.*]] = zext i32 [[MUL126]] to i64
+// CHECK2-NEXT:    [[DIV128:%.*]] = sdiv i64 [[TMP79]], [[CONV127]]
+// CHECK2-NEXT:    [[TMP85:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB129:%.*]] = sub i32 [[TMP85]], [[TMP86]]
+// CHECK2-NEXT:    [[SUB130:%.*]] = sub i32 [[SUB129]], 1
+// CHECK2-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD131:%.*]] = add i32 [[SUB130]], [[TMP87]]
+// CHECK2-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV132:%.*]] = udiv i32 [[ADD131]], [[TMP88]]
+// CHECK2-NEXT:    [[MUL133:%.*]] = mul i32 1, [[DIV132]]
+// CHECK2-NEXT:    [[TMP89:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB134:%.*]] = sub i32 [[TMP89]], -31
+// CHECK2-NEXT:    [[DIV135:%.*]] = udiv i32 [[SUB134]], 32
+// CHECK2-NEXT:    [[MUL136:%.*]] = mul i32 [[MUL133]], [[DIV135]]
+// CHECK2-NEXT:    [[MUL137:%.*]] = mul i32 [[MUL136]], 32
+// CHECK2-NEXT:    [[CONV138:%.*]] = zext i32 [[MUL137]] to i64
+// CHECK2-NEXT:    [[MUL139:%.*]] = mul nsw i64 [[DIV128]], [[CONV138]]
+// CHECK2-NEXT:    [[SUB140:%.*]] = sub nsw i64 [[TMP78]], [[MUL139]]
+// CHECK2-NEXT:    [[TMP90:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB141:%.*]] = sub i32 [[TMP90]], -31
+// CHECK2-NEXT:    [[DIV142:%.*]] = udiv i32 [[SUB141]], 32
+// CHECK2-NEXT:    [[MUL143:%.*]] = mul i32 1, [[DIV142]]
+// CHECK2-NEXT:    [[MUL144:%.*]] = mul i32 [[MUL143]], 32
+// CHECK2-NEXT:    [[CONV145:%.*]] = zext i32 [[MUL144]] to i64
+// CHECK2-NEXT:    [[DIV146:%.*]] = sdiv i64 [[SUB140]], [[CONV145]]
+// CHECK2-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB147:%.*]] = sub i32 [[TMP91]], -31
+// CHECK2-NEXT:    [[DIV148:%.*]] = udiv i32 [[SUB147]], 32
+// CHECK2-NEXT:    [[MUL149:%.*]] = mul i32 1, [[DIV148]]
+// CHECK2-NEXT:    [[MUL150:%.*]] = mul i32 [[MUL149]], 32
+// CHECK2-NEXT:    [[CONV151:%.*]] = zext i32 [[MUL150]] to i64
+// CHECK2-NEXT:    [[MUL152:%.*]] = mul nsw i64 [[DIV146]], [[CONV151]]
+// CHECK2-NEXT:    [[SUB153:%.*]] = sub nsw i64 [[SUB117]], [[MUL152]]
+// CHECK2-NEXT:    [[DIV154:%.*]] = sdiv i64 [[SUB153]], 32
+// CHECK2-NEXT:    [[MUL155:%.*]] = mul nsw i64 [[DIV154]], 32
+// CHECK2-NEXT:    [[ADD156:%.*]] = add nsw i64 0, [[MUL155]]
+// CHECK2-NEXT:    [[CONV157:%.*]] = trunc i64 [[ADD156]] to i32
+// CHECK2-NEXT:    store i32 [[CONV157]], ptr [[DOTFLOOR_0_IV_K41]], align 4
+// CHECK2-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K41]], align 4
+// CHECK2-NEXT:    [[CONV158:%.*]] = zext i32 [[TMP92]] to i64
+// CHECK2-NEXT:    [[TMP93:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP94:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB159:%.*]] = sub i32 [[TMP95]], [[TMP96]]
+// CHECK2-NEXT:    [[SUB160:%.*]] = sub i32 [[SUB159]], 1
+// CHECK2-NEXT:    [[TMP97:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD161:%.*]] = add i32 [[SUB160]], [[TMP97]]
 // CHECK2-NEXT:    [[TMP98:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD167:%.*]] = add i32 [[SUB166]], [[TMP98]]
-// CHECK2-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV168:%.*]] = udiv i32 [[ADD167]], [[TMP99]]
-// CHECK2-NEXT:    [[MUL169:%.*]] = mul i32 1, [[DIV168]]
-// CHECK2-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB170:%.*]] = sub i32 [[TMP100]], -31
-// CHECK2-NEXT:    [[DIV171:%.*]] = udiv i32 [[SUB170]], 32
-// CHECK2-NEXT:    [[MUL172:%.*]] = mul i32 [[MUL169]], [[DIV171]]
-// CHECK2-NEXT:    [[MUL173:%.*]] = mul i32 [[MUL172]], 32
-// CHECK2-NEXT:    [[CONV174:%.*]] = zext i32 [[MUL173]] to i64
-// CHECK2-NEXT:    [[MUL175:%.*]] = mul nsw i64 [[DIV164]], [[CONV174]]
-// CHECK2-NEXT:    [[SUB176:%.*]] = sub nsw i64 [[TMP89]], [[MUL175]]
-// CHECK2-NEXT:    [[TMP101:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP102:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB177:%.*]] = sub i32 [[TMP103]], [[TMP104]]
-// CHECK2-NEXT:    [[SUB178:%.*]] = sub i32 [[SUB177]], 1
-// CHECK2-NEXT:    [[TMP105:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD179:%.*]] = add i32 [[SUB178]], [[TMP105]]
-// CHECK2-NEXT:    [[TMP106:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV180:%.*]] = udiv i32 [[ADD179]], [[TMP106]]
-// CHECK2-NEXT:    [[MUL181:%.*]] = mul i32 1, [[DIV180]]
-// CHECK2-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB182:%.*]] = sub i32 [[TMP107]], -31
-// CHECK2-NEXT:    [[DIV183:%.*]] = udiv i32 [[SUB182]], 32
-// CHECK2-NEXT:    [[MUL184:%.*]] = mul i32 [[MUL181]], [[DIV183]]
-// CHECK2-NEXT:    [[MUL185:%.*]] = mul i32 [[MUL184]], 32
-// CHECK2-NEXT:    [[CONV186:%.*]] = zext i32 [[MUL185]] to i64
-// CHECK2-NEXT:    [[DIV187:%.*]] = sdiv i64 [[TMP102]], [[CONV186]]
-// CHECK2-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB188:%.*]] = sub i32 [[TMP108]], [[TMP109]]
-// CHECK2-NEXT:    [[SUB189:%.*]] = sub i32 [[SUB188]], 1
+// CHECK2-NEXT:    [[DIV162:%.*]] = udiv i32 [[ADD161]], [[TMP98]]
+// CHECK2-NEXT:    [[MUL163:%.*]] = mul i32 1, [[DIV162]]
+// CHECK2-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB164:%.*]] = sub i32 [[TMP99]], -31
+// CHECK2-NEXT:    [[DIV165:%.*]] = udiv i32 [[SUB164]], 32
+// CHECK2-NEXT:    [[MUL166:%.*]] = mul i32 [[MUL163]], [[DIV165]]
+// CHECK2-NEXT:    [[MUL167:%.*]] = mul i32 [[MUL166]], 32
+// CHECK2-NEXT:    [[CONV168:%.*]] = zext i32 [[MUL167]] to i64
+// CHECK2-NEXT:    [[DIV169:%.*]] = sdiv i64 [[TMP94]], [[CONV168]]
+// CHECK2-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP101:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB170:%.*]] = sub i32 [[TMP100]], [[TMP101]]
+// CHECK2-NEXT:    [[SUB171:%.*]] = sub i32 [[SUB170]], 1
+// CHECK2-NEXT:    [[TMP102:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD172:%.*]] = add i32 [[SUB171]], [[TMP102]]
+// CHECK2-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV173:%.*]] = udiv i32 [[ADD172]], [[TMP103]]
+// CHECK2-NEXT:    [[MUL174:%.*]] = mul i32 1, [[DIV173]]
+// CHECK2-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB175:%.*]] = sub i32 [[TMP104]], -31
+// CHECK2-NEXT:    [[DIV176:%.*]] = udiv i32 [[SUB175]], 32
+// CHECK2-NEXT:    [[MUL177:%.*]] = mul i32 [[MUL174]], [[DIV176]]
+// CHECK2-NEXT:    [[MUL178:%.*]] = mul i32 [[MUL177]], 32
+// CHECK2-NEXT:    [[CONV179:%.*]] = zext i32 [[MUL178]] to i64
+// CHECK2-NEXT:    [[MUL180:%.*]] = mul nsw i64 [[DIV169]], [[CONV179]]
+// CHECK2-NEXT:    [[SUB181:%.*]] = sub nsw i64 [[TMP93]], [[MUL180]]
+// CHECK2-NEXT:    [[TMP105:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP106:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB182:%.*]] = sub i32 [[TMP107]], [[TMP108]]
+// CHECK2-NEXT:    [[SUB183:%.*]] = sub i32 [[SUB182]], 1
+// CHECK2-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD184:%.*]] = add i32 [[SUB183]], [[TMP109]]
 // CHECK2-NEXT:    [[TMP110:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD190:%.*]] = add i32 [[SUB189]], [[TMP110]]
-// CHECK2-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV191:%.*]] = udiv i32 [[ADD190]], [[TMP111]]
-// CHECK2-NEXT:    [[MUL192:%.*]] = mul i32 1, [[DIV191]]
-// CHECK2-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB193:%.*]] = sub i32 [[TMP112]], -31
-// CHECK2-NEXT:    [[DIV194:%.*]] = udiv i32 [[SUB193]], 32
-// CHECK2-NEXT:    [[MUL195:%.*]] = mul i32 [[MUL192]], [[DIV194]]
-// CHECK2-NEXT:    [[MUL196:%.*]] = mul i32 [[MUL195]], 32
-// CHECK2-NEXT:    [[CONV197:%.*]] = zext i32 [[MUL196]] to i64
-// CHECK2-NEXT:    [[MUL198:%.*]] = mul nsw i64 [[DIV187]], [[CONV197]]
-// CHECK2-NEXT:    [[SUB199:%.*]] = sub nsw i64 [[TMP101]], [[MUL198]]
-// CHECK2-NEXT:    [[TMP113:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB200:%.*]] = sub i32 [[TMP113]], -31
-// CHECK2-NEXT:    [[DIV201:%.*]] = udiv i32 [[SUB200]], 32
-// CHECK2-NEXT:    [[MUL202:%.*]] = mul i32 1, [[DIV201]]
-// CHECK2-NEXT:    [[MUL203:%.*]] = mul i32 [[MUL202]], 32
-// CHECK2-NEXT:    [[CONV204:%.*]] = zext i32 [[MUL203]] to i64
-// CHECK2-NEXT:    [[DIV205:%.*]] = sdiv i64 [[SUB199]], [[CONV204]]
-// CHECK2-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB206:%.*]] = sub i32 [[TMP114]], -31
-// CHECK2-NEXT:    [[DIV207:%.*]] = udiv i32 [[SUB206]], 32
-// CHECK2-NEXT:    [[MUL208:%.*]] = mul i32 1, [[DIV207]]
-// CHECK2-NEXT:    [[MUL209:%.*]] = mul i32 [[MUL208]], 32
-// CHECK2-NEXT:    [[CONV210:%.*]] = zext i32 [[MUL209]] to i64
-// CHECK2-NEXT:    [[MUL211:%.*]] = mul nsw i64 [[DIV205]], [[CONV210]]
-// CHECK2-NEXT:    [[SUB212:%.*]] = sub nsw i64 [[SUB176]], [[MUL211]]
-// CHECK2-NEXT:    [[TMP115:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP116:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB213:%.*]] = sub i32 [[TMP117]], [[TMP118]]
-// CHECK2-NEXT:    [[SUB214:%.*]] = sub i32 [[SUB213]], 1
-// CHECK2-NEXT:    [[TMP119:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD215:%.*]] = add i32 [[SUB214]], [[TMP119]]
-// CHECK2-NEXT:    [[TMP120:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV216:%.*]] = udiv i32 [[ADD215]], [[TMP120]]
-// CHECK2-NEXT:    [[MUL217:%.*]] = mul i32 1, [[DIV216]]
-// CHECK2-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB218:%.*]] = sub i32 [[TMP121]], -31
-// CHECK2-NEXT:    [[DIV219:%.*]] = udiv i32 [[SUB218]], 32
-// CHECK2-NEXT:    [[MUL220:%.*]] = mul i32 [[MUL217]], [[DIV219]]
-// CHECK2-NEXT:    [[MUL221:%.*]] = mul i32 [[MUL220]], 32
-// CHECK2-NEXT:    [[CONV222:%.*]] = zext i32 [[MUL221]] to i64
-// CHECK2-NEXT:    [[DIV223:%.*]] = sdiv i64 [[TMP116]], [[CONV222]]
-// CHECK2-NEXT:    [[TMP122:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP123:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB224:%.*]] = sub i32 [[TMP122]], [[TMP123]]
-// CHECK2-NEXT:    [[SUB225:%.*]] = sub i32 [[SUB224]], 1
+// CHECK2-NEXT:    [[DIV185:%.*]] = udiv i32 [[ADD184]], [[TMP110]]
+// CHECK2-NEXT:    [[MUL186:%.*]] = mul i32 1, [[DIV185]]
+// CHECK2-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB187:%.*]] = sub i32 [[TMP111]], -31
+// CHECK2-NEXT:    [[DIV188:%.*]] = udiv i32 [[SUB187]], 32
+// CHECK2-NEXT:    [[MUL189:%.*]] = mul i32 [[MUL186]], [[DIV188]]
+// CHECK2-NEXT:    [[MUL190:%.*]] = mul i32 [[MUL189]], 32
+// CHECK2-NEXT:    [[CONV191:%.*]] = zext i32 [[MUL190]] to i64
+// CHECK2-NEXT:    [[DIV192:%.*]] = sdiv i64 [[TMP106]], [[CONV191]]
+// CHECK2-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP113:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB193:%.*]] = sub i32 [[TMP112]], [[TMP113]]
+// CHECK2-NEXT:    [[SUB194:%.*]] = sub i32 [[SUB193]], 1
+// CHECK2-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD195:%.*]] = add i32 [[SUB194]], [[TMP114]]
+// CHECK2-NEXT:    [[TMP115:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV196:%.*]] = udiv i32 [[ADD195]], [[TMP115]]
+// CHECK2-NEXT:    [[MUL197:%.*]] = mul i32 1, [[DIV196]]
+// CHECK2-NEXT:    [[TMP116:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB198:%.*]] = sub i32 [[TMP116]], -31
+// CHECK2-NEXT:    [[DIV199:%.*]] = udiv i32 [[SUB198]], 32
+// CHECK2-NEXT:    [[MUL200:%.*]] = mul i32 [[MUL197]], [[DIV199]]
+// CHECK2-NEXT:    [[MUL201:%.*]] = mul i32 [[MUL200]], 32
+// CHECK2-NEXT:    [[CONV202:%.*]] = zext i32 [[MUL201]] to i64
+// CHECK2-NEXT:    [[MUL203:%.*]] = mul nsw i64 [[DIV192]], [[CONV202]]
+// CHECK2-NEXT:    [[SUB204:%.*]] = sub nsw i64 [[TMP105]], [[MUL203]]
+// CHECK2-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB205:%.*]] = sub i32 [[TMP117]], -31
+// CHECK2-NEXT:    [[DIV206:%.*]] = udiv i32 [[SUB205]], 32
+// CHECK2-NEXT:    [[MUL207:%.*]] = mul i32 1, [[DIV206]]
+// CHECK2-NEXT:    [[MUL208:%.*]] = mul i32 [[MUL207]], 32
+// CHECK2-NEXT:    [[CONV209:%.*]] = zext i32 [[MUL208]] to i64
+// CHECK2-NEXT:    [[DIV210:%.*]] = sdiv i64 [[SUB204]], [[CONV209]]
+// CHECK2-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB211:%.*]] = sub i32 [[TMP118]], -31
+// CHECK2-NEXT:    [[DIV212:%.*]] = udiv i32 [[SUB211]], 32
+// CHECK2-NEXT:    [[MUL213:%.*]] = mul i32 1, [[DIV212]]
+// CHECK2-NEXT:    [[MUL214:%.*]] = mul i32 [[MUL213]], 32
+// CHECK2-NEXT:    [[CONV215:%.*]] = zext i32 [[MUL214]] to i64
+// CHECK2-NEXT:    [[MUL216:%.*]] = mul nsw i64 [[DIV210]], [[CONV215]]
+// CHECK2-NEXT:    [[SUB217:%.*]] = sub nsw i64 [[SUB181]], [[MUL216]]
+// CHECK2-NEXT:    [[TMP119:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP120:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP122:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB218:%.*]] = sub i32 [[TMP121]], [[TMP122]]
+// CHECK2-NEXT:    [[SUB219:%.*]] = sub i32 [[SUB218]], 1
+// CHECK2-NEXT:    [[TMP123:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD220:%.*]] = add i32 [[SUB219]], [[TMP123]]
 // CHECK2-NEXT:    [[TMP124:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD226:%.*]] = add i32 [[SUB225]], [[TMP124]]
-// CHECK2-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV227:%.*]] = udiv i32 [[ADD226]], [[TMP125]]
-// CHECK2-NEXT:    [[MUL228:%.*]] = mul i32 1, [[DIV227]]
-// CHECK2-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB229:%.*]] = sub i32 [[TMP126]], -31
-// CHECK2-NEXT:    [[DIV230:%.*]] = udiv i32 [[SUB229]], 32
-// CHECK2-NEXT:    [[MUL231:%.*]] = mul i32 [[MUL228]], [[DIV230]]
-// CHECK2-NEXT:    [[MUL232:%.*]] = mul i32 [[MUL231]], 32
-// CHECK2-NEXT:    [[CONV233:%.*]] = zext i32 [[MUL232]] to i64
-// CHECK2-NEXT:    [[MUL234:%.*]] = mul nsw i64 [[DIV223]], [[CONV233]]
-// CHECK2-NEXT:    [[SUB235:%.*]] = sub nsw i64 [[TMP115]], [[MUL234]]
-// CHECK2-NEXT:    [[TMP127:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP128:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB236:%.*]] = sub i32 [[TMP129]], [[TMP130]]
-// CHECK2-NEXT:    [[SUB237:%.*]] = sub i32 [[SUB236]], 1
-// CHECK2-NEXT:    [[TMP131:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD238:%.*]] = add i32 [[SUB237]], [[TMP131]]
-// CHECK2-NEXT:    [[TMP132:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV239:%.*]] = udiv i32 [[ADD238]], [[TMP132]]
-// CHECK2-NEXT:    [[MUL240:%.*]] = mul i32 1, [[DIV239]]
-// CHECK2-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB241:%.*]] = sub i32 [[TMP133]], -31
-// CHECK2-NEXT:    [[DIV242:%.*]] = udiv i32 [[SUB241]], 32
-// CHECK2-NEXT:    [[MUL243:%.*]] = mul i32 [[MUL240]], [[DIV242]]
-// CHECK2-NEXT:    [[MUL244:%.*]] = mul i32 [[MUL243]], 32
-// CHECK2-NEXT:    [[CONV245:%.*]] = zext i32 [[MUL244]] to i64
-// CHECK2-NEXT:    [[DIV246:%.*]] = sdiv i64 [[TMP128]], [[CONV245]]
-// CHECK2-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB247:%.*]] = sub i32 [[TMP134]], [[TMP135]]
-// CHECK2-NEXT:    [[SUB248:%.*]] = sub i32 [[SUB247]], 1
+// CHECK2-NEXT:    [[DIV221:%.*]] = udiv i32 [[ADD220]], [[TMP124]]
+// CHECK2-NEXT:    [[MUL222:%.*]] = mul i32 1, [[DIV221]]
+// CHECK2-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB223:%.*]] = sub i32 [[TMP125]], -31
+// CHECK2-NEXT:    [[DIV224:%.*]] = udiv i32 [[SUB223]], 32
+// CHECK2-NEXT:    [[MUL225:%.*]] = mul i32 [[MUL222]], [[DIV224]]
+// CHECK2-NEXT:    [[MUL226:%.*]] = mul i32 [[MUL225]], 32
+// CHECK2-NEXT:    [[CONV227:%.*]] = zext i32 [[MUL226]] to i64
+// CHECK2-NEXT:    [[DIV228:%.*]] = sdiv i64 [[TMP120]], [[CONV227]]
+// CHECK2-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP127:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB229:%.*]] = sub i32 [[TMP126]], [[TMP127]]
+// CHECK2-NEXT:    [[SUB230:%.*]] = sub i32 [[SUB229]], 1
+// CHECK2-NEXT:    [[TMP128:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD231:%.*]] = add i32 [[SUB230]], [[TMP128]]
+// CHECK2-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV232:%.*]] = udiv i32 [[ADD231]], [[TMP129]]
+// CHECK2-NEXT:    [[MUL233:%.*]] = mul i32 1, [[DIV232]]
+// CHECK2-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB234:%.*]] = sub i32 [[TMP130]], -31
+// CHECK2-NEXT:    [[DIV235:%.*]] = udiv i32 [[SUB234]], 32
+// CHECK2-NEXT:    [[MUL236:%.*]] = mul i32 [[MUL233]], [[DIV235]]
+// CHECK2-NEXT:    [[MUL237:%.*]] = mul i32 [[MUL236]], 32
+// CHECK2-NEXT:    [[CONV238:%.*]] = zext i32 [[MUL237]] to i64
+// CHECK2-NEXT:    [[MUL239:%.*]] = mul nsw i64 [[DIV228]], [[CONV238]]
+// CHECK2-NEXT:    [[SUB240:%.*]] = sub nsw i64 [[TMP119]], [[MUL239]]
+// CHECK2-NEXT:    [[TMP131:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP132:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB241:%.*]] = sub i32 [[TMP133]], [[TMP134]]
+// CHECK2-NEXT:    [[SUB242:%.*]] = sub i32 [[SUB241]], 1
+// CHECK2-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD243:%.*]] = add i32 [[SUB242]], [[TMP135]]
 // CHECK2-NEXT:    [[TMP136:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD249:%.*]] = add i32 [[SUB248]], [[TMP136]]
-// CHECK2-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV250:%.*]] = udiv i32 [[ADD249]], [[TMP137]]
-// CHECK2-NEXT:    [[MUL251:%.*]] = mul i32 1, [[DIV250]]
-// CHECK2-NEXT:    [[TMP138:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB252:%.*]] = sub i32 [[TMP138]], -31
-// CHECK2-NEXT:    [[DIV253:%.*]] = udiv i32 [[SUB252]], 32
-// CHECK2-NEXT:    [[MUL254:%.*]] = mul i32 [[MUL251]], [[DIV253]]
-// CHECK2-NEXT:    [[MUL255:%.*]] = mul i32 [[MUL254]], 32
-// CHECK2-NEXT:    [[CONV256:%.*]] = zext i32 [[MUL255]] to i64
-// CHECK2-NEXT:    [[MUL257:%.*]] = mul nsw i64 [[DIV246]], [[CONV256]]
-// CHECK2-NEXT:    [[SUB258:%.*]] = sub nsw i64 [[TMP127]], [[MUL257]]
-// CHECK2-NEXT:    [[TMP139:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB259:%.*]] = sub i32 [[TMP139]], -31
-// CHECK2-NEXT:    [[DIV260:%.*]] = udiv i32 [[SUB259]], 32
-// CHECK2-NEXT:    [[MUL261:%.*]] = mul i32 1, [[DIV260]]
-// CHECK2-NEXT:    [[MUL262:%.*]] = mul i32 [[MUL261]], 32
-// CHECK2-NEXT:    [[CONV263:%.*]] = zext i32 [[MUL262]] to i64
-// CHECK2-NEXT:    [[DIV264:%.*]] = sdiv i64 [[SUB258]], [[CONV263]]
-// CHECK2-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB265:%.*]] = sub i32 [[TMP140]], -31
-// CHECK2-NEXT:    [[DIV266:%.*]] = udiv i32 [[SUB265]], 32
-// CHECK2-NEXT:    [[MUL267:%.*]] = mul i32 1, [[DIV266]]
-// CHECK2-NEXT:    [[MUL268:%.*]] = mul i32 [[MUL267]], 32
-// CHECK2-NEXT:    [[CONV269:%.*]] = zext i32 [[MUL268]] to i64
-// CHECK2-NEXT:    [[MUL270:%.*]] = mul nsw i64 [[DIV264]], [[CONV269]]
-// CHECK2-NEXT:    [[SUB271:%.*]] = sub nsw i64 [[SUB235]], [[MUL270]]
-// CHECK2-NEXT:    [[DIV272:%.*]] = sdiv i64 [[SUB271]], 32
-// CHECK2-NEXT:    [[MUL273:%.*]] = mul nsw i64 [[DIV272]], 32
-// CHECK2-NEXT:    [[SUB274:%.*]] = sub nsw i64 [[SUB212]], [[MUL273]]
-// CHECK2-NEXT:    [[MUL275:%.*]] = mul nsw i64 [[SUB274]], 1
-// CHECK2-NEXT:    [[ADD276:%.*]] = add nsw i64 0, [[MUL275]]
-// CHECK2-NEXT:    [[CONV277:%.*]] = trunc i64 [[ADD276]] to i32
-// CHECK2-NEXT:    store i32 [[CONV277]], ptr [[DOTTILE_CNT_0_IV_K38]], align 4
-// CHECK2-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
-// CHECK2-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
-// CHECK2-NEXT:    [[ADD278:%.*]] = add i32 [[TMP141]], [[TMP142]]
-// CHECK2-NEXT:    store i32 [[ADD278]], ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK2-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
-// CHECK2-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK2-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
-// CHECK2-NEXT:    [[MUL279:%.*]] = mul i32 [[TMP144]], [[TMP145]]
-// CHECK2-NEXT:    [[ADD280:%.*]] = add i32 [[TMP143]], [[MUL279]]
-// CHECK2-NEXT:    store i32 [[ADD280]], ptr [[K]], align 4
-// CHECK2-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
-// CHECK2-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
-// CHECK2-NEXT:    [[ADD281:%.*]] = add i32 [[TMP146]], [[TMP147]]
-// CHECK2-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK2-NEXT:    [[ADD282:%.*]] = add i32 [[TMP148]], 1
-// CHECK2-NEXT:    [[CMP283:%.*]] = icmp ult i32 [[ADD281]], [[ADD282]]
-// CHECK2-NEXT:    br i1 [[CMP283]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK2:       if.then:
-// CHECK2-NEXT:    [[TMP149:%.*]] = load i32, ptr [[I35]], align 4
-// CHECK2-NEXT:    [[TMP150:%.*]] = load i32, ptr [[J36]], align 4
-// CHECK2-NEXT:    [[TMP151:%.*]] = load i32, ptr [[K]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP149]], i32 noundef [[TMP150]], i32 noundef [[TMP151]])
-// CHECK2-NEXT:    br label [[IF_END]]
-// CHECK2:       if.end:
-// CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK2-NEXT:    [[DIV244:%.*]] = udiv i32 [[ADD243]], [[TMP136]]
+// CHECK2-NEXT:    [[MUL245:%.*]] = mul i32 1, [[DIV244]]
+// CHECK2-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB246:%.*]] = sub i32 [[TMP137]], -31
+// CHECK2-NEXT:    [[DIV247:%.*]] = udiv i32 [[SUB246]], 32
+// CHECK2-NEXT:    [[MUL248:%.*]] = mul i32 [[MUL245]], [[DIV247]]
+// CHECK2-NEXT:    [[MUL249:%.*]] = mul i32 [[MUL248]], 32
+// CHECK2-NEXT:    [[CONV250:%.*]] = zext i32 [[MUL249]] to i64
+// CHECK2-NEXT:    [[DIV251:%.*]] = sdiv i64 [[TMP132]], [[CONV250]]
+// CHECK2-NEXT:    [[TMP138:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP139:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB252:%.*]] = sub i32 [[TMP138]], [[TMP139]]
+// CHECK2-NEXT:    [[SUB253:%.*]] = sub i32 [[SUB252]], 1
+// CHECK2-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD254:%.*]] = add i32 [[SUB253]], [[TMP140]]
+// CHECK2-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV255:%.*]] = udiv i32 [[ADD254]], [[TMP141]]
+// CHECK2-NEXT:    [[MUL256:%.*]] = mul i32 1, [[DIV255]]
+// CHECK2-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB257:%.*]] = sub i32 [[TMP142]], -31
+// CHECK2-NEXT:    [[DIV258:%.*]] = udiv i32 [[SUB257]], 32
+// CHECK2-NEXT:    [[MUL259:%.*]] = mul i32 [[MUL256]], [[DIV258]]
+// CHECK2-NEXT:    [[MUL260:%.*]] = mul i32 [[MUL259]], 32
+// CHECK2-NEXT:    [[CONV261:%.*]] = zext i32 [[MUL260]] to i64
+// CHECK2-NEXT:    [[MUL262:%.*]] = mul nsw i64 [[DIV251]], [[CONV261]]
+// CHECK2-NEXT:    [[SUB263:%.*]] = sub nsw i64 [[TMP131]], [[MUL262]]
+// CHECK2-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB264:%.*]] = sub i32 [[TMP143]], -31
+// CHECK2-NEXT:    [[DIV265:%.*]] = udiv i32 [[SUB264]], 32
+// CHECK2-NEXT:    [[MUL266:%.*]] = mul i32 1, [[DIV265]]
+// CHECK2-NEXT:    [[MUL267:%.*]] = mul i32 [[MUL266]], 32
+// CHECK2-NEXT:    [[CONV268:%.*]] = zext i32 [[MUL267]] to i64
+// CHECK2-NEXT:    [[DIV269:%.*]] = sdiv i64 [[SUB263]], [[CONV268]]
+// CHECK2-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB270:%.*]] = sub i32 [[TMP144]], -31
+// CHECK2-NEXT:    [[DIV271:%.*]] = udiv i32 [[SUB270]], 32
+// CHECK2-NEXT:    [[MUL272:%.*]] = mul i32 1, [[DIV271]]
+// CHECK2-NEXT:    [[MUL273:%.*]] = mul i32 [[MUL272]], 32
+// CHECK2-NEXT:    [[CONV274:%.*]] = zext i32 [[MUL273]] to i64
+// CHECK2-NEXT:    [[MUL275:%.*]] = mul nsw i64 [[DIV269]], [[CONV274]]
+// CHECK2-NEXT:    [[SUB276:%.*]] = sub nsw i64 [[SUB240]], [[MUL275]]
+// CHECK2-NEXT:    [[DIV277:%.*]] = sdiv i64 [[SUB276]], 32
+// CHECK2-NEXT:    [[MUL278:%.*]] = mul nsw i64 [[DIV277]], 32
+// CHECK2-NEXT:    [[SUB279:%.*]] = sub nsw i64 [[SUB217]], [[MUL278]]
+// CHECK2-NEXT:    [[MUL280:%.*]] = mul nsw i64 [[SUB279]], 1
+// CHECK2-NEXT:    [[ADD281:%.*]] = add nsw i64 [[CONV158]], [[MUL280]]
+// CHECK2-NEXT:    [[CONV282:%.*]] = trunc i64 [[ADD281]] to i32
+// CHECK2-NEXT:    store i32 [[CONV282]], ptr [[DOTTILE_0_IV_K42]], align 4
+// CHECK2-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTTILE_0_IV_K42]], align 4
+// CHECK2-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// CHECK2-NEXT:    [[ADD283:%.*]] = add i32 [[TMP146]], 1
+// CHECK2-NEXT:    [[CMP284:%.*]] = icmp ult i32 [[TMP145]], [[ADD283]]
+// CHECK2-NEXT:    br i1 [[CMP284]], label [[OMP_BODY_NEXT:%.*]], label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK2:       omp.body.next:
+// CHECK2-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
+// CHECK2-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTTILE_0_IV_K42]], align 4
+// CHECK2-NEXT:    [[TMP149:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
+// CHECK2-NEXT:    [[MUL285:%.*]] = mul i32 [[TMP148]], [[TMP149]]
+// CHECK2-NEXT:    [[ADD286:%.*]] = add i32 [[TMP147]], [[MUL285]]
+// CHECK2-NEXT:    store i32 [[ADD286]], ptr [[K]], align 4
+// CHECK2-NEXT:    [[TMP150:%.*]] = load i32, ptr [[I39]], align 4
+// CHECK2-NEXT:    [[TMP151:%.*]] = load i32, ptr [[J40]], align 4
+// CHECK2-NEXT:    [[TMP152:%.*]] = load i32, ptr [[K]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP150]], i32 noundef [[TMP151]], i32 noundef [[TMP152]])
+// CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
-// CHECK2-NEXT:    [[TMP152:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[ADD284:%.*]] = add nsw i64 [[TMP152]], 1
-// CHECK2-NEXT:    store i64 [[ADD284]], ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP153:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[ADD287:%.*]] = add nsw i64 [[TMP153]], 1
+// CHECK2-NEXT:    store i64 [[ADD287]], ptr [[DOTOMP_IV]], align 8
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -4105,21 +4114,23 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_9:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTNEW_STEP10:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_14:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_16:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_17:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_19:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_K:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_K:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_K:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[_TMP35:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[I35:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[J36:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTFLOOR_0_IV_K37:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_K38:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[I39:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[J40:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTFLOOR_0_IV_K41:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_K42:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
@@ -4157,454 +4168,455 @@ extern "C" void foo10() {
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
 // CHECK2-NEXT:    [[ADD15:%.*]] = add i32 [[TMP15]], 1
 // CHECK2-NEXT:    store i32 [[ADD15]], ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[SUB17:%.*]] = sub i32 [[TMP16]], [[TMP17]]
-// CHECK2-NEXT:    [[SUB18:%.*]] = sub i32 [[SUB17]], 1
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[ADD19:%.*]] = add i32 [[SUB18]], [[TMP18]]
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[DIV20:%.*]] = udiv i32 [[ADD19]], [[TMP19]]
-// CHECK2-NEXT:    [[CONV:%.*]] = zext i32 [[DIV20]] to i64
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB21:%.*]] = sub i32 [[TMP20]], [[TMP21]]
-// CHECK2-NEXT:    [[SUB22:%.*]] = sub i32 [[SUB21]], 1
-// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD23:%.*]] = add i32 [[SUB22]], [[TMP22]]
-// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV24:%.*]] = udiv i32 [[ADD23]], [[TMP23]]
-// CHECK2-NEXT:    [[CONV25:%.*]] = zext i32 [[DIV24]] to i64
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV25]]
-// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB26:%.*]] = sub i32 [[TMP24]], -63
-// CHECK2-NEXT:    [[DIV27:%.*]] = udiv i32 [[SUB26]], 64
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[_TMP2]], align 4
+// CHECK2-NEXT:    store i32 [[TMP16]], ptr [[DOTCAPTURE_EXPR_16]], align 4
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[_TMP2]], align 4
+// CHECK2-NEXT:    [[ADD18:%.*]] = add i32 [[TMP17]], 64
+// CHECK2-NEXT:    store i32 [[ADD18]], ptr [[DOTCAPTURE_EXPR_17]], align 4
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[SUB20:%.*]] = sub i32 [[TMP18]], [[TMP19]]
+// CHECK2-NEXT:    [[SUB21:%.*]] = sub i32 [[SUB20]], 1
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[ADD22:%.*]] = add i32 [[SUB21]], [[TMP20]]
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[DIV23:%.*]] = udiv i32 [[ADD22]], [[TMP21]]
+// CHECK2-NEXT:    [[CONV:%.*]] = zext i32 [[DIV23]] to i64
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB24:%.*]] = sub i32 [[TMP22]], [[TMP23]]
+// CHECK2-NEXT:    [[SUB25:%.*]] = sub i32 [[SUB24]], 1
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD26:%.*]] = add i32 [[SUB25]], [[TMP24]]
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV27:%.*]] = udiv i32 [[ADD26]], [[TMP25]]
 // CHECK2-NEXT:    [[CONV28:%.*]] = zext i32 [[DIV27]] to i64
-// CHECK2-NEXT:    [[MUL29:%.*]] = mul nsw i64 [[MUL]], [[CONV28]]
-// CHECK2-NEXT:    [[MUL30:%.*]] = mul nsw i64 [[MUL29]], 64
-// CHECK2-NEXT:    [[SUB31:%.*]] = sub nsw i64 [[MUL30]], 1
-// CHECK2-NEXT:    store i64 [[SUB31]], ptr [[DOTCAPTURE_EXPR_16]], align 8
-// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    store i32 [[TMP25]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    store i32 [[TMP26]], ptr [[J]], align 4
-// CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_K]], align 4
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_K]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[CONV]], [[CONV28]]
+// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB29:%.*]] = sub i32 [[TMP26]], -63
+// CHECK2-NEXT:    [[DIV30:%.*]] = udiv i32 [[SUB29]], 64
+// CHECK2-NEXT:    [[CONV31:%.*]] = zext i32 [[DIV30]] to i64
+// CHECK2-NEXT:    [[MUL32:%.*]] = mul nsw i64 [[MUL]], [[CONV31]]
+// CHECK2-NEXT:    [[MUL33:%.*]] = mul nsw i64 [[MUL32]], 64
+// CHECK2-NEXT:    [[SUB34:%.*]] = sub nsw i64 [[MUL33]], 1
+// CHECK2-NEXT:    store i64 [[SUB34]], ptr [[DOTCAPTURE_EXPR_19]], align 8
 // CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
-// CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP27]], [[TMP28]]
+// CHECK2-NEXT:    store i32 [[TMP27]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    store i32 [[TMP28]], ptr [[J]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_K]], align 4
+// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K]], align 4
+// CHECK2-NEXT:    store i32 [[TMP29]], ptr [[DOTTILE_0_IV_K]], align 4
+// CHECK2-NEXT:    store i32 0, ptr [[_TMP35]], align 4
+// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_4]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP30]], [[TMP31]]
 // CHECK2-NEXT:    br i1 [[CMP]], label [[LAND_LHS_TRUE:%.*]], label [[OMP_PRECOND_END:%.*]]
 // CHECK2:       land.lhs.true:
-// CHECK2-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[CMP32:%.*]] = icmp slt i32 [[TMP29]], [[TMP30]]
-// CHECK2-NEXT:    br i1 [[CMP32]], label [[LAND_LHS_TRUE33:%.*]], label [[OMP_PRECOND_END]]
-// CHECK2:       land.lhs.true33:
-// CHECK2-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[CMP34:%.*]] = icmp ult i32 0, [[TMP31]]
-// CHECK2-NEXT:    br i1 [[CMP34]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
+// CHECK2-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[CMP36:%.*]] = icmp slt i32 [[TMP32]], [[TMP33]]
+// CHECK2-NEXT:    br i1 [[CMP36]], label [[LAND_LHS_TRUE37:%.*]], label [[OMP_PRECOND_END]]
+// CHECK2:       land.lhs.true37:
+// CHECK2-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[CMP38:%.*]] = icmp ult i32 0, [[TMP34]]
+// CHECK2-NEXT:    br i1 [[CMP38]], label [[OMP_PRECOND_THEN:%.*]], label [[OMP_PRECOND_END]]
 // CHECK2:       omp.precond.then:
 // CHECK2-NEXT:    store i64 0, ptr [[DOTOMP_LB]], align 8
-// CHECK2-NEXT:    [[TMP32:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
-// CHECK2-NEXT:    store i64 [[TMP32]], ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_19]], align 8
+// CHECK2-NEXT:    store i64 [[TMP35]], ptr [[DOTOMP_UB]], align 8
 // CHECK2-NEXT:    store i64 1, ptr [[DOTOMP_STRIDE]], align 8
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
 // CHECK2-NEXT:    call void @__kmpc_for_static_init_8(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i64 1, i64 1)
-// CHECK2-NEXT:    [[TMP33:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[TMP34:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
-// CHECK2-NEXT:    [[CMP39:%.*]] = icmp sgt i64 [[TMP33]], [[TMP34]]
-// CHECK2-NEXT:    br i1 [[CMP39]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_19]], align 8
+// CHECK2-NEXT:    [[CMP43:%.*]] = icmp sgt i64 [[TMP36]], [[TMP37]]
+// CHECK2-NEXT:    br i1 [[CMP43]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK2:       cond.true:
-// CHECK2-NEXT:    [[TMP35:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_16]], align 8
+// CHECK2-NEXT:    [[TMP38:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_19]], align 8
 // CHECK2-NEXT:    br label [[COND_END:%.*]]
 // CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP36:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[TMP39:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
 // CHECK2-NEXT:    br label [[COND_END]]
 // CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i64 [ [[TMP35]], [[COND_TRUE]] ], [ [[TMP36]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[COND:%.*]] = phi i64 [ [[TMP38]], [[COND_TRUE]] ], [ [[TMP39]], [[COND_FALSE]] ]
 // CHECK2-NEXT:    store i64 [[COND]], ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[TMP37:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
-// CHECK2-NEXT:    store i64 [[TMP37]], ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP40:%.*]] = load i64, ptr [[DOTOMP_LB]], align 8
+// CHECK2-NEXT:    store i64 [[TMP40]], ptr [[DOTOMP_IV]], align 8
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK2:       omp.inner.for.cond:
-// CHECK2-NEXT:    [[TMP38:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP39:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
-// CHECK2-NEXT:    [[CMP40:%.*]] = icmp sle i64 [[TMP38]], [[TMP39]]
-// CHECK2-NEXT:    br i1 [[CMP40]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
-// CHECK2:       omp.inner.for.body:
-// CHECK2-NEXT:    [[TMP40:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// CHECK2-NEXT:    [[CONV41:%.*]] = sext i32 [[TMP40]] to i64
 // CHECK2-NEXT:    [[TMP41:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP42:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB42:%.*]] = sub i32 [[TMP42]], [[TMP43]]
-// CHECK2-NEXT:    [[SUB43:%.*]] = sub i32 [[SUB42]], 1
-// CHECK2-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD44:%.*]] = add i32 [[SUB43]], [[TMP44]]
-// CHECK2-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV45:%.*]] = udiv i32 [[ADD44]], [[TMP45]]
-// CHECK2-NEXT:    [[MUL46:%.*]] = mul i32 1, [[DIV45]]
-// CHECK2-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB47:%.*]] = sub i32 [[TMP46]], -63
-// CHECK2-NEXT:    [[DIV48:%.*]] = udiv i32 [[SUB47]], 64
-// CHECK2-NEXT:    [[MUL49:%.*]] = mul i32 [[MUL46]], [[DIV48]]
-// CHECK2-NEXT:    [[MUL50:%.*]] = mul i32 [[MUL49]], 64
-// CHECK2-NEXT:    [[CONV51:%.*]] = zext i32 [[MUL50]] to i64
-// CHECK2-NEXT:    [[DIV52:%.*]] = sdiv i64 [[TMP41]], [[CONV51]]
-// CHECK2-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// CHECK2-NEXT:    [[CONV53:%.*]] = sext i32 [[TMP47]] to i64
-// CHECK2-NEXT:    [[MUL54:%.*]] = mul nsw i64 [[DIV52]], [[CONV53]]
-// CHECK2-NEXT:    [[ADD55:%.*]] = add nsw i64 [[CONV41]], [[MUL54]]
-// CHECK2-NEXT:    [[CONV56:%.*]] = trunc i64 [[ADD55]] to i32
-// CHECK2-NEXT:    store i32 [[CONV56]], ptr [[I35]], align 4
-// CHECK2-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP48]] to i64
-// CHECK2-NEXT:    [[TMP49:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP50:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP52:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB58:%.*]] = sub i32 [[TMP51]], [[TMP52]]
-// CHECK2-NEXT:    [[SUB59:%.*]] = sub i32 [[SUB58]], 1
-// CHECK2-NEXT:    [[TMP53:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD60:%.*]] = add i32 [[SUB59]], [[TMP53]]
-// CHECK2-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV61:%.*]] = udiv i32 [[ADD60]], [[TMP54]]
-// CHECK2-NEXT:    [[MUL62:%.*]] = mul i32 1, [[DIV61]]
-// CHECK2-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB63:%.*]] = sub i32 [[TMP55]], -63
-// CHECK2-NEXT:    [[DIV64:%.*]] = udiv i32 [[SUB63]], 64
-// CHECK2-NEXT:    [[MUL65:%.*]] = mul i32 [[MUL62]], [[DIV64]]
-// CHECK2-NEXT:    [[MUL66:%.*]] = mul i32 [[MUL65]], 64
-// CHECK2-NEXT:    [[CONV67:%.*]] = zext i32 [[MUL66]] to i64
-// CHECK2-NEXT:    [[DIV68:%.*]] = sdiv i64 [[TMP50]], [[CONV67]]
-// CHECK2-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB69:%.*]] = sub i32 [[TMP56]], [[TMP57]]
-// CHECK2-NEXT:    [[SUB70:%.*]] = sub i32 [[SUB69]], 1
-// CHECK2-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD71:%.*]] = add i32 [[SUB70]], [[TMP58]]
-// CHECK2-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV72:%.*]] = udiv i32 [[ADD71]], [[TMP59]]
-// CHECK2-NEXT:    [[MUL73:%.*]] = mul i32 1, [[DIV72]]
-// CHECK2-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB74:%.*]] = sub i32 [[TMP60]], -63
-// CHECK2-NEXT:    [[DIV75:%.*]] = udiv i32 [[SUB74]], 64
-// CHECK2-NEXT:    [[MUL76:%.*]] = mul i32 [[MUL73]], [[DIV75]]
-// CHECK2-NEXT:    [[MUL77:%.*]] = mul i32 [[MUL76]], 64
-// CHECK2-NEXT:    [[CONV78:%.*]] = zext i32 [[MUL77]] to i64
-// CHECK2-NEXT:    [[MUL79:%.*]] = mul nsw i64 [[DIV68]], [[CONV78]]
-// CHECK2-NEXT:    [[SUB80:%.*]] = sub nsw i64 [[TMP49]], [[MUL79]]
-// CHECK2-NEXT:    [[TMP61:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB81:%.*]] = sub i32 [[TMP61]], -63
-// CHECK2-NEXT:    [[DIV82:%.*]] = udiv i32 [[SUB81]], 64
-// CHECK2-NEXT:    [[MUL83:%.*]] = mul i32 1, [[DIV82]]
-// CHECK2-NEXT:    [[MUL84:%.*]] = mul i32 [[MUL83]], 64
-// CHECK2-NEXT:    [[CONV85:%.*]] = zext i32 [[MUL84]] to i64
-// CHECK2-NEXT:    [[DIV86:%.*]] = sdiv i64 [[SUB80]], [[CONV85]]
+// CHECK2-NEXT:    [[TMP42:%.*]] = load i64, ptr [[DOTOMP_UB]], align 8
+// CHECK2-NEXT:    [[CMP44:%.*]] = icmp sle i64 [[TMP41]], [[TMP42]]
+// CHECK2-NEXT:    br i1 [[CMP44]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK2:       omp.inner.for.body:
+// CHECK2-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[CONV45:%.*]] = sext i32 [[TMP43]] to i64
+// CHECK2-NEXT:    [[TMP44:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP46:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB46:%.*]] = sub i32 [[TMP45]], [[TMP46]]
+// CHECK2-NEXT:    [[SUB47:%.*]] = sub i32 [[SUB46]], 1
+// CHECK2-NEXT:    [[TMP47:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD48:%.*]] = add i32 [[SUB47]], [[TMP47]]
+// CHECK2-NEXT:    [[TMP48:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV49:%.*]] = udiv i32 [[ADD48]], [[TMP48]]
+// CHECK2-NEXT:    [[MUL50:%.*]] = mul i32 1, [[DIV49]]
+// CHECK2-NEXT:    [[TMP49:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB51:%.*]] = sub i32 [[TMP49]], -63
+// CHECK2-NEXT:    [[DIV52:%.*]] = udiv i32 [[SUB51]], 64
+// CHECK2-NEXT:    [[MUL53:%.*]] = mul i32 [[MUL50]], [[DIV52]]
+// CHECK2-NEXT:    [[MUL54:%.*]] = mul i32 [[MUL53]], 64
+// CHECK2-NEXT:    [[CONV55:%.*]] = zext i32 [[MUL54]] to i64
+// CHECK2-NEXT:    [[DIV56:%.*]] = sdiv i64 [[TMP44]], [[CONV55]]
+// CHECK2-NEXT:    [[TMP50:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
+// CHECK2-NEXT:    [[CONV57:%.*]] = sext i32 [[TMP50]] to i64
+// CHECK2-NEXT:    [[MUL58:%.*]] = mul nsw i64 [[DIV56]], [[CONV57]]
+// CHECK2-NEXT:    [[ADD59:%.*]] = add nsw i64 [[CONV45]], [[MUL58]]
+// CHECK2-NEXT:    [[CONV60:%.*]] = trunc i64 [[ADD59]] to i32
+// CHECK2-NEXT:    store i32 [[CONV60]], ptr [[I39]], align 4
+// CHECK2-NEXT:    [[TMP51:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[CONV61:%.*]] = sext i32 [[TMP51]] to i64
+// CHECK2-NEXT:    [[TMP52:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP53:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP54:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP55:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB62:%.*]] = sub i32 [[TMP54]], [[TMP55]]
+// CHECK2-NEXT:    [[SUB63:%.*]] = sub i32 [[SUB62]], 1
+// CHECK2-NEXT:    [[TMP56:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD64:%.*]] = add i32 [[SUB63]], [[TMP56]]
+// CHECK2-NEXT:    [[TMP57:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV65:%.*]] = udiv i32 [[ADD64]], [[TMP57]]
+// CHECK2-NEXT:    [[MUL66:%.*]] = mul i32 1, [[DIV65]]
+// CHECK2-NEXT:    [[TMP58:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB67:%.*]] = sub i32 [[TMP58]], -63
+// CHECK2-NEXT:    [[DIV68:%.*]] = udiv i32 [[SUB67]], 64
+// CHECK2-NEXT:    [[MUL69:%.*]] = mul i32 [[MUL66]], [[DIV68]]
+// CHECK2-NEXT:    [[MUL70:%.*]] = mul i32 [[MUL69]], 64
+// CHECK2-NEXT:    [[CONV71:%.*]] = zext i32 [[MUL70]] to i64
+// CHECK2-NEXT:    [[DIV72:%.*]] = sdiv i64 [[TMP53]], [[CONV71]]
+// CHECK2-NEXT:    [[TMP59:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP60:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB73:%.*]] = sub i32 [[TMP59]], [[TMP60]]
+// CHECK2-NEXT:    [[SUB74:%.*]] = sub i32 [[SUB73]], 1
+// CHECK2-NEXT:    [[TMP61:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD75:%.*]] = add i32 [[SUB74]], [[TMP61]]
 // CHECK2-NEXT:    [[TMP62:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[CONV87:%.*]] = sext i32 [[TMP62]] to i64
-// CHECK2-NEXT:    [[MUL88:%.*]] = mul nsw i64 [[DIV86]], [[CONV87]]
-// CHECK2-NEXT:    [[ADD89:%.*]] = add nsw i64 [[CONV57]], [[MUL88]]
-// CHECK2-NEXT:    [[CONV90:%.*]] = trunc i64 [[ADD89]] to i32
-// CHECK2-NEXT:    store i32 [[CONV90]], ptr [[J36]], align 4
-// CHECK2-NEXT:    [[TMP63:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP64:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP66:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB91:%.*]] = sub i32 [[TMP65]], [[TMP66]]
-// CHECK2-NEXT:    [[SUB92:%.*]] = sub i32 [[SUB91]], 1
-// CHECK2-NEXT:    [[TMP67:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD93:%.*]] = add i32 [[SUB92]], [[TMP67]]
-// CHECK2-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV94:%.*]] = udiv i32 [[ADD93]], [[TMP68]]
-// CHECK2-NEXT:    [[MUL95:%.*]] = mul i32 1, [[DIV94]]
-// CHECK2-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB96:%.*]] = sub i32 [[TMP69]], -63
-// CHECK2-NEXT:    [[DIV97:%.*]] = udiv i32 [[SUB96]], 64
-// CHECK2-NEXT:    [[MUL98:%.*]] = mul i32 [[MUL95]], [[DIV97]]
-// CHECK2-NEXT:    [[MUL99:%.*]] = mul i32 [[MUL98]], 64
-// CHECK2-NEXT:    [[CONV100:%.*]] = zext i32 [[MUL99]] to i64
-// CHECK2-NEXT:    [[DIV101:%.*]] = sdiv i64 [[TMP64]], [[CONV100]]
-// CHECK2-NEXT:    [[TMP70:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB102:%.*]] = sub i32 [[TMP70]], [[TMP71]]
-// CHECK2-NEXT:    [[SUB103:%.*]] = sub i32 [[SUB102]], 1
-// CHECK2-NEXT:    [[TMP72:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD104:%.*]] = add i32 [[SUB103]], [[TMP72]]
-// CHECK2-NEXT:    [[TMP73:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV105:%.*]] = udiv i32 [[ADD104]], [[TMP73]]
-// CHECK2-NEXT:    [[MUL106:%.*]] = mul i32 1, [[DIV105]]
-// CHECK2-NEXT:    [[TMP74:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB107:%.*]] = sub i32 [[TMP74]], -63
-// CHECK2-NEXT:    [[DIV108:%.*]] = udiv i32 [[SUB107]], 64
-// CHECK2-NEXT:    [[MUL109:%.*]] = mul i32 [[MUL106]], [[DIV108]]
-// CHECK2-NEXT:    [[MUL110:%.*]] = mul i32 [[MUL109]], 64
-// CHECK2-NEXT:    [[CONV111:%.*]] = zext i32 [[MUL110]] to i64
-// CHECK2-NEXT:    [[MUL112:%.*]] = mul nsw i64 [[DIV101]], [[CONV111]]
-// CHECK2-NEXT:    [[SUB113:%.*]] = sub nsw i64 [[TMP63]], [[MUL112]]
-// CHECK2-NEXT:    [[TMP75:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP76:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP78:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB114:%.*]] = sub i32 [[TMP77]], [[TMP78]]
-// CHECK2-NEXT:    [[SUB115:%.*]] = sub i32 [[SUB114]], 1
-// CHECK2-NEXT:    [[TMP79:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD116:%.*]] = add i32 [[SUB115]], [[TMP79]]
-// CHECK2-NEXT:    [[TMP80:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV117:%.*]] = udiv i32 [[ADD116]], [[TMP80]]
-// CHECK2-NEXT:    [[MUL118:%.*]] = mul i32 1, [[DIV117]]
-// CHECK2-NEXT:    [[TMP81:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB119:%.*]] = sub i32 [[TMP81]], -63
-// CHECK2-NEXT:    [[DIV120:%.*]] = udiv i32 [[SUB119]], 64
-// CHECK2-NEXT:    [[MUL121:%.*]] = mul i32 [[MUL118]], [[DIV120]]
-// CHECK2-NEXT:    [[MUL122:%.*]] = mul i32 [[MUL121]], 64
-// CHECK2-NEXT:    [[CONV123:%.*]] = zext i32 [[MUL122]] to i64
-// CHECK2-NEXT:    [[DIV124:%.*]] = sdiv i64 [[TMP76]], [[CONV123]]
-// CHECK2-NEXT:    [[TMP82:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB125:%.*]] = sub i32 [[TMP82]], [[TMP83]]
-// CHECK2-NEXT:    [[SUB126:%.*]] = sub i32 [[SUB125]], 1
-// CHECK2-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD127:%.*]] = add i32 [[SUB126]], [[TMP84]]
-// CHECK2-NEXT:    [[TMP85:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV128:%.*]] = udiv i32 [[ADD127]], [[TMP85]]
-// CHECK2-NEXT:    [[MUL129:%.*]] = mul i32 1, [[DIV128]]
-// CHECK2-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB130:%.*]] = sub i32 [[TMP86]], -63
-// CHECK2-NEXT:    [[DIV131:%.*]] = udiv i32 [[SUB130]], 64
-// CHECK2-NEXT:    [[MUL132:%.*]] = mul i32 [[MUL129]], [[DIV131]]
-// CHECK2-NEXT:    [[MUL133:%.*]] = mul i32 [[MUL132]], 64
-// CHECK2-NEXT:    [[CONV134:%.*]] = zext i32 [[MUL133]] to i64
-// CHECK2-NEXT:    [[MUL135:%.*]] = mul nsw i64 [[DIV124]], [[CONV134]]
-// CHECK2-NEXT:    [[SUB136:%.*]] = sub nsw i64 [[TMP75]], [[MUL135]]
-// CHECK2-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB137:%.*]] = sub i32 [[TMP87]], -63
-// CHECK2-NEXT:    [[DIV138:%.*]] = udiv i32 [[SUB137]], 64
-// CHECK2-NEXT:    [[MUL139:%.*]] = mul i32 1, [[DIV138]]
-// CHECK2-NEXT:    [[MUL140:%.*]] = mul i32 [[MUL139]], 64
-// CHECK2-NEXT:    [[CONV141:%.*]] = zext i32 [[MUL140]] to i64
-// CHECK2-NEXT:    [[DIV142:%.*]] = sdiv i64 [[SUB136]], [[CONV141]]
-// CHECK2-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB143:%.*]] = sub i32 [[TMP88]], -63
-// CHECK2-NEXT:    [[DIV144:%.*]] = udiv i32 [[SUB143]], 64
-// CHECK2-NEXT:    [[MUL145:%.*]] = mul i32 1, [[DIV144]]
-// CHECK2-NEXT:    [[MUL146:%.*]] = mul i32 [[MUL145]], 64
-// CHECK2-NEXT:    [[CONV147:%.*]] = zext i32 [[MUL146]] to i64
-// CHECK2-NEXT:    [[MUL148:%.*]] = mul nsw i64 [[DIV142]], [[CONV147]]
-// CHECK2-NEXT:    [[SUB149:%.*]] = sub nsw i64 [[SUB113]], [[MUL148]]
-// CHECK2-NEXT:    [[DIV150:%.*]] = sdiv i64 [[SUB149]], 64
-// CHECK2-NEXT:    [[MUL151:%.*]] = mul nsw i64 [[DIV150]], 64
-// CHECK2-NEXT:    [[ADD152:%.*]] = add nsw i64 0, [[MUL151]]
-// CHECK2-NEXT:    [[CONV153:%.*]] = trunc i64 [[ADD152]] to i32
-// CHECK2-NEXT:    store i32 [[CONV153]], ptr [[DOTFLOOR_0_IV_K37]], align 4
-// CHECK2-NEXT:    [[TMP89:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP90:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB154:%.*]] = sub i32 [[TMP91]], [[TMP92]]
-// CHECK2-NEXT:    [[SUB155:%.*]] = sub i32 [[SUB154]], 1
-// CHECK2-NEXT:    [[TMP93:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD156:%.*]] = add i32 [[SUB155]], [[TMP93]]
-// CHECK2-NEXT:    [[TMP94:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV157:%.*]] = udiv i32 [[ADD156]], [[TMP94]]
-// CHECK2-NEXT:    [[MUL158:%.*]] = mul i32 1, [[DIV157]]
-// CHECK2-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB159:%.*]] = sub i32 [[TMP95]], -63
-// CHECK2-NEXT:    [[DIV160:%.*]] = udiv i32 [[SUB159]], 64
-// CHECK2-NEXT:    [[MUL161:%.*]] = mul i32 [[MUL158]], [[DIV160]]
-// CHECK2-NEXT:    [[MUL162:%.*]] = mul i32 [[MUL161]], 64
-// CHECK2-NEXT:    [[CONV163:%.*]] = zext i32 [[MUL162]] to i64
-// CHECK2-NEXT:    [[DIV164:%.*]] = sdiv i64 [[TMP90]], [[CONV163]]
-// CHECK2-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP97:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB165:%.*]] = sub i32 [[TMP96]], [[TMP97]]
-// CHECK2-NEXT:    [[SUB166:%.*]] = sub i32 [[SUB165]], 1
+// CHECK2-NEXT:    [[DIV76:%.*]] = udiv i32 [[ADD75]], [[TMP62]]
+// CHECK2-NEXT:    [[MUL77:%.*]] = mul i32 1, [[DIV76]]
+// CHECK2-NEXT:    [[TMP63:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB78:%.*]] = sub i32 [[TMP63]], -63
+// CHECK2-NEXT:    [[DIV79:%.*]] = udiv i32 [[SUB78]], 64
+// CHECK2-NEXT:    [[MUL80:%.*]] = mul i32 [[MUL77]], [[DIV79]]
+// CHECK2-NEXT:    [[MUL81:%.*]] = mul i32 [[MUL80]], 64
+// CHECK2-NEXT:    [[CONV82:%.*]] = zext i32 [[MUL81]] to i64
+// CHECK2-NEXT:    [[MUL83:%.*]] = mul nsw i64 [[DIV72]], [[CONV82]]
+// CHECK2-NEXT:    [[SUB84:%.*]] = sub nsw i64 [[TMP52]], [[MUL83]]
+// CHECK2-NEXT:    [[TMP64:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB85:%.*]] = sub i32 [[TMP64]], -63
+// CHECK2-NEXT:    [[DIV86:%.*]] = udiv i32 [[SUB85]], 64
+// CHECK2-NEXT:    [[MUL87:%.*]] = mul i32 1, [[DIV86]]
+// CHECK2-NEXT:    [[MUL88:%.*]] = mul i32 [[MUL87]], 64
+// CHECK2-NEXT:    [[CONV89:%.*]] = zext i32 [[MUL88]] to i64
+// CHECK2-NEXT:    [[DIV90:%.*]] = sdiv i64 [[SUB84]], [[CONV89]]
+// CHECK2-NEXT:    [[TMP65:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[CONV91:%.*]] = sext i32 [[TMP65]] to i64
+// CHECK2-NEXT:    [[MUL92:%.*]] = mul nsw i64 [[DIV90]], [[CONV91]]
+// CHECK2-NEXT:    [[ADD93:%.*]] = add nsw i64 [[CONV61]], [[MUL92]]
+// CHECK2-NEXT:    [[CONV94:%.*]] = trunc i64 [[ADD93]] to i32
+// CHECK2-NEXT:    store i32 [[CONV94]], ptr [[J40]], align 4
+// CHECK2-NEXT:    [[TMP66:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP67:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP68:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP69:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB95:%.*]] = sub i32 [[TMP68]], [[TMP69]]
+// CHECK2-NEXT:    [[SUB96:%.*]] = sub i32 [[SUB95]], 1
+// CHECK2-NEXT:    [[TMP70:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD97:%.*]] = add i32 [[SUB96]], [[TMP70]]
+// CHECK2-NEXT:    [[TMP71:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV98:%.*]] = udiv i32 [[ADD97]], [[TMP71]]
+// CHECK2-NEXT:    [[MUL99:%.*]] = mul i32 1, [[DIV98]]
+// CHECK2-NEXT:    [[TMP72:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB100:%.*]] = sub i32 [[TMP72]], -63
+// CHECK2-NEXT:    [[DIV101:%.*]] = udiv i32 [[SUB100]], 64
+// CHECK2-NEXT:    [[MUL102:%.*]] = mul i32 [[MUL99]], [[DIV101]]
+// CHECK2-NEXT:    [[MUL103:%.*]] = mul i32 [[MUL102]], 64
+// CHECK2-NEXT:    [[CONV104:%.*]] = zext i32 [[MUL103]] to i64
+// CHECK2-NEXT:    [[DIV105:%.*]] = sdiv i64 [[TMP67]], [[CONV104]]
+// CHECK2-NEXT:    [[TMP73:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP74:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB106:%.*]] = sub i32 [[TMP73]], [[TMP74]]
+// CHECK2-NEXT:    [[SUB107:%.*]] = sub i32 [[SUB106]], 1
+// CHECK2-NEXT:    [[TMP75:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD108:%.*]] = add i32 [[SUB107]], [[TMP75]]
+// CHECK2-NEXT:    [[TMP76:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV109:%.*]] = udiv i32 [[ADD108]], [[TMP76]]
+// CHECK2-NEXT:    [[MUL110:%.*]] = mul i32 1, [[DIV109]]
+// CHECK2-NEXT:    [[TMP77:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB111:%.*]] = sub i32 [[TMP77]], -63
+// CHECK2-NEXT:    [[DIV112:%.*]] = udiv i32 [[SUB111]], 64
+// CHECK2-NEXT:    [[MUL113:%.*]] = mul i32 [[MUL110]], [[DIV112]]
+// CHECK2-NEXT:    [[MUL114:%.*]] = mul i32 [[MUL113]], 64
+// CHECK2-NEXT:    [[CONV115:%.*]] = zext i32 [[MUL114]] to i64
+// CHECK2-NEXT:    [[MUL116:%.*]] = mul nsw i64 [[DIV105]], [[CONV115]]
+// CHECK2-NEXT:    [[SUB117:%.*]] = sub nsw i64 [[TMP66]], [[MUL116]]
+// CHECK2-NEXT:    [[TMP78:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP79:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP80:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP81:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB118:%.*]] = sub i32 [[TMP80]], [[TMP81]]
+// CHECK2-NEXT:    [[SUB119:%.*]] = sub i32 [[SUB118]], 1
+// CHECK2-NEXT:    [[TMP82:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD120:%.*]] = add i32 [[SUB119]], [[TMP82]]
+// CHECK2-NEXT:    [[TMP83:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV121:%.*]] = udiv i32 [[ADD120]], [[TMP83]]
+// CHECK2-NEXT:    [[MUL122:%.*]] = mul i32 1, [[DIV121]]
+// CHECK2-NEXT:    [[TMP84:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB123:%.*]] = sub i32 [[TMP84]], -63
+// CHECK2-NEXT:    [[DIV124:%.*]] = udiv i32 [[SUB123]], 64
+// CHECK2-NEXT:    [[MUL125:%.*]] = mul i32 [[MUL122]], [[DIV124]]
+// CHECK2-NEXT:    [[MUL126:%.*]] = mul i32 [[MUL125]], 64
+// CHECK2-NEXT:    [[CONV127:%.*]] = zext i32 [[MUL126]] to i64
+// CHECK2-NEXT:    [[DIV128:%.*]] = sdiv i64 [[TMP79]], [[CONV127]]
+// CHECK2-NEXT:    [[TMP85:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP86:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB129:%.*]] = sub i32 [[TMP85]], [[TMP86]]
+// CHECK2-NEXT:    [[SUB130:%.*]] = sub i32 [[SUB129]], 1
+// CHECK2-NEXT:    [[TMP87:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD131:%.*]] = add i32 [[SUB130]], [[TMP87]]
+// CHECK2-NEXT:    [[TMP88:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV132:%.*]] = udiv i32 [[ADD131]], [[TMP88]]
+// CHECK2-NEXT:    [[MUL133:%.*]] = mul i32 1, [[DIV132]]
+// CHECK2-NEXT:    [[TMP89:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB134:%.*]] = sub i32 [[TMP89]], -63
+// CHECK2-NEXT:    [[DIV135:%.*]] = udiv i32 [[SUB134]], 64
+// CHECK2-NEXT:    [[MUL136:%.*]] = mul i32 [[MUL133]], [[DIV135]]
+// CHECK2-NEXT:    [[MUL137:%.*]] = mul i32 [[MUL136]], 64
+// CHECK2-NEXT:    [[CONV138:%.*]] = zext i32 [[MUL137]] to i64
+// CHECK2-NEXT:    [[MUL139:%.*]] = mul nsw i64 [[DIV128]], [[CONV138]]
+// CHECK2-NEXT:    [[SUB140:%.*]] = sub nsw i64 [[TMP78]], [[MUL139]]
+// CHECK2-NEXT:    [[TMP90:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB141:%.*]] = sub i32 [[TMP90]], -63
+// CHECK2-NEXT:    [[DIV142:%.*]] = udiv i32 [[SUB141]], 64
+// CHECK2-NEXT:    [[MUL143:%.*]] = mul i32 1, [[DIV142]]
+// CHECK2-NEXT:    [[MUL144:%.*]] = mul i32 [[MUL143]], 64
+// CHECK2-NEXT:    [[CONV145:%.*]] = zext i32 [[MUL144]] to i64
+// CHECK2-NEXT:    [[DIV146:%.*]] = sdiv i64 [[SUB140]], [[CONV145]]
+// CHECK2-NEXT:    [[TMP91:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB147:%.*]] = sub i32 [[TMP91]], -63
+// CHECK2-NEXT:    [[DIV148:%.*]] = udiv i32 [[SUB147]], 64
+// CHECK2-NEXT:    [[MUL149:%.*]] = mul i32 1, [[DIV148]]
+// CHECK2-NEXT:    [[MUL150:%.*]] = mul i32 [[MUL149]], 64
+// CHECK2-NEXT:    [[CONV151:%.*]] = zext i32 [[MUL150]] to i64
+// CHECK2-NEXT:    [[MUL152:%.*]] = mul nsw i64 [[DIV146]], [[CONV151]]
+// CHECK2-NEXT:    [[SUB153:%.*]] = sub nsw i64 [[SUB117]], [[MUL152]]
+// CHECK2-NEXT:    [[DIV154:%.*]] = sdiv i64 [[SUB153]], 64
+// CHECK2-NEXT:    [[MUL155:%.*]] = mul nsw i64 [[DIV154]], 64
+// CHECK2-NEXT:    [[ADD156:%.*]] = add nsw i64 0, [[MUL155]]
+// CHECK2-NEXT:    [[CONV157:%.*]] = trunc i64 [[ADD156]] to i32
+// CHECK2-NEXT:    store i32 [[CONV157]], ptr [[DOTFLOOR_0_IV_K41]], align 4
+// CHECK2-NEXT:    [[TMP92:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K41]], align 4
+// CHECK2-NEXT:    [[CONV158:%.*]] = zext i32 [[TMP92]] to i64
+// CHECK2-NEXT:    [[TMP93:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP94:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP95:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP96:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB159:%.*]] = sub i32 [[TMP95]], [[TMP96]]
+// CHECK2-NEXT:    [[SUB160:%.*]] = sub i32 [[SUB159]], 1
+// CHECK2-NEXT:    [[TMP97:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD161:%.*]] = add i32 [[SUB160]], [[TMP97]]
 // CHECK2-NEXT:    [[TMP98:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD167:%.*]] = add i32 [[SUB166]], [[TMP98]]
-// CHECK2-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV168:%.*]] = udiv i32 [[ADD167]], [[TMP99]]
-// CHECK2-NEXT:    [[MUL169:%.*]] = mul i32 1, [[DIV168]]
-// CHECK2-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB170:%.*]] = sub i32 [[TMP100]], -63
-// CHECK2-NEXT:    [[DIV171:%.*]] = udiv i32 [[SUB170]], 64
-// CHECK2-NEXT:    [[MUL172:%.*]] = mul i32 [[MUL169]], [[DIV171]]
-// CHECK2-NEXT:    [[MUL173:%.*]] = mul i32 [[MUL172]], 64
-// CHECK2-NEXT:    [[CONV174:%.*]] = zext i32 [[MUL173]] to i64
-// CHECK2-NEXT:    [[MUL175:%.*]] = mul nsw i64 [[DIV164]], [[CONV174]]
-// CHECK2-NEXT:    [[SUB176:%.*]] = sub nsw i64 [[TMP89]], [[MUL175]]
-// CHECK2-NEXT:    [[TMP101:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP102:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB177:%.*]] = sub i32 [[TMP103]], [[TMP104]]
-// CHECK2-NEXT:    [[SUB178:%.*]] = sub i32 [[SUB177]], 1
-// CHECK2-NEXT:    [[TMP105:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD179:%.*]] = add i32 [[SUB178]], [[TMP105]]
-// CHECK2-NEXT:    [[TMP106:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV180:%.*]] = udiv i32 [[ADD179]], [[TMP106]]
-// CHECK2-NEXT:    [[MUL181:%.*]] = mul i32 1, [[DIV180]]
-// CHECK2-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB182:%.*]] = sub i32 [[TMP107]], -63
-// CHECK2-NEXT:    [[DIV183:%.*]] = udiv i32 [[SUB182]], 64
-// CHECK2-NEXT:    [[MUL184:%.*]] = mul i32 [[MUL181]], [[DIV183]]
-// CHECK2-NEXT:    [[MUL185:%.*]] = mul i32 [[MUL184]], 64
-// CHECK2-NEXT:    [[CONV186:%.*]] = zext i32 [[MUL185]] to i64
-// CHECK2-NEXT:    [[DIV187:%.*]] = sdiv i64 [[TMP102]], [[CONV186]]
-// CHECK2-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB188:%.*]] = sub i32 [[TMP108]], [[TMP109]]
-// CHECK2-NEXT:    [[SUB189:%.*]] = sub i32 [[SUB188]], 1
+// CHECK2-NEXT:    [[DIV162:%.*]] = udiv i32 [[ADD161]], [[TMP98]]
+// CHECK2-NEXT:    [[MUL163:%.*]] = mul i32 1, [[DIV162]]
+// CHECK2-NEXT:    [[TMP99:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB164:%.*]] = sub i32 [[TMP99]], -63
+// CHECK2-NEXT:    [[DIV165:%.*]] = udiv i32 [[SUB164]], 64
+// CHECK2-NEXT:    [[MUL166:%.*]] = mul i32 [[MUL163]], [[DIV165]]
+// CHECK2-NEXT:    [[MUL167:%.*]] = mul i32 [[MUL166]], 64
+// CHECK2-NEXT:    [[CONV168:%.*]] = zext i32 [[MUL167]] to i64
+// CHECK2-NEXT:    [[DIV169:%.*]] = sdiv i64 [[TMP94]], [[CONV168]]
+// CHECK2-NEXT:    [[TMP100:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP101:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB170:%.*]] = sub i32 [[TMP100]], [[TMP101]]
+// CHECK2-NEXT:    [[SUB171:%.*]] = sub i32 [[SUB170]], 1
+// CHECK2-NEXT:    [[TMP102:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD172:%.*]] = add i32 [[SUB171]], [[TMP102]]
+// CHECK2-NEXT:    [[TMP103:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV173:%.*]] = udiv i32 [[ADD172]], [[TMP103]]
+// CHECK2-NEXT:    [[MUL174:%.*]] = mul i32 1, [[DIV173]]
+// CHECK2-NEXT:    [[TMP104:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB175:%.*]] = sub i32 [[TMP104]], -63
+// CHECK2-NEXT:    [[DIV176:%.*]] = udiv i32 [[SUB175]], 64
+// CHECK2-NEXT:    [[MUL177:%.*]] = mul i32 [[MUL174]], [[DIV176]]
+// CHECK2-NEXT:    [[MUL178:%.*]] = mul i32 [[MUL177]], 64
+// CHECK2-NEXT:    [[CONV179:%.*]] = zext i32 [[MUL178]] to i64
+// CHECK2-NEXT:    [[MUL180:%.*]] = mul nsw i64 [[DIV169]], [[CONV179]]
+// CHECK2-NEXT:    [[SUB181:%.*]] = sub nsw i64 [[TMP93]], [[MUL180]]
+// CHECK2-NEXT:    [[TMP105:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP106:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP107:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP108:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB182:%.*]] = sub i32 [[TMP107]], [[TMP108]]
+// CHECK2-NEXT:    [[SUB183:%.*]] = sub i32 [[SUB182]], 1
+// CHECK2-NEXT:    [[TMP109:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD184:%.*]] = add i32 [[SUB183]], [[TMP109]]
 // CHECK2-NEXT:    [[TMP110:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD190:%.*]] = add i32 [[SUB189]], [[TMP110]]
-// CHECK2-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV191:%.*]] = udiv i32 [[ADD190]], [[TMP111]]
-// CHECK2-NEXT:    [[MUL192:%.*]] = mul i32 1, [[DIV191]]
-// CHECK2-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB193:%.*]] = sub i32 [[TMP112]], -63
-// CHECK2-NEXT:    [[DIV194:%.*]] = udiv i32 [[SUB193]], 64
-// CHECK2-NEXT:    [[MUL195:%.*]] = mul i32 [[MUL192]], [[DIV194]]
-// CHECK2-NEXT:    [[MUL196:%.*]] = mul i32 [[MUL195]], 64
-// CHECK2-NEXT:    [[CONV197:%.*]] = zext i32 [[MUL196]] to i64
-// CHECK2-NEXT:    [[MUL198:%.*]] = mul nsw i64 [[DIV187]], [[CONV197]]
-// CHECK2-NEXT:    [[SUB199:%.*]] = sub nsw i64 [[TMP101]], [[MUL198]]
-// CHECK2-NEXT:    [[TMP113:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB200:%.*]] = sub i32 [[TMP113]], -63
-// CHECK2-NEXT:    [[DIV201:%.*]] = udiv i32 [[SUB200]], 64
-// CHECK2-NEXT:    [[MUL202:%.*]] = mul i32 1, [[DIV201]]
-// CHECK2-NEXT:    [[MUL203:%.*]] = mul i32 [[MUL202]], 64
-// CHECK2-NEXT:    [[CONV204:%.*]] = zext i32 [[MUL203]] to i64
-// CHECK2-NEXT:    [[DIV205:%.*]] = sdiv i64 [[SUB199]], [[CONV204]]
-// CHECK2-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB206:%.*]] = sub i32 [[TMP114]], -63
-// CHECK2-NEXT:    [[DIV207:%.*]] = udiv i32 [[SUB206]], 64
-// CHECK2-NEXT:    [[MUL208:%.*]] = mul i32 1, [[DIV207]]
-// CHECK2-NEXT:    [[MUL209:%.*]] = mul i32 [[MUL208]], 64
-// CHECK2-NEXT:    [[CONV210:%.*]] = zext i32 [[MUL209]] to i64
-// CHECK2-NEXT:    [[MUL211:%.*]] = mul nsw i64 [[DIV205]], [[CONV210]]
-// CHECK2-NEXT:    [[SUB212:%.*]] = sub nsw i64 [[SUB176]], [[MUL211]]
-// CHECK2-NEXT:    [[TMP115:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP116:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB213:%.*]] = sub i32 [[TMP117]], [[TMP118]]
-// CHECK2-NEXT:    [[SUB214:%.*]] = sub i32 [[SUB213]], 1
-// CHECK2-NEXT:    [[TMP119:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD215:%.*]] = add i32 [[SUB214]], [[TMP119]]
-// CHECK2-NEXT:    [[TMP120:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV216:%.*]] = udiv i32 [[ADD215]], [[TMP120]]
-// CHECK2-NEXT:    [[MUL217:%.*]] = mul i32 1, [[DIV216]]
-// CHECK2-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB218:%.*]] = sub i32 [[TMP121]], -63
-// CHECK2-NEXT:    [[DIV219:%.*]] = udiv i32 [[SUB218]], 64
-// CHECK2-NEXT:    [[MUL220:%.*]] = mul i32 [[MUL217]], [[DIV219]]
-// CHECK2-NEXT:    [[MUL221:%.*]] = mul i32 [[MUL220]], 64
-// CHECK2-NEXT:    [[CONV222:%.*]] = zext i32 [[MUL221]] to i64
-// CHECK2-NEXT:    [[DIV223:%.*]] = sdiv i64 [[TMP116]], [[CONV222]]
-// CHECK2-NEXT:    [[TMP122:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP123:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB224:%.*]] = sub i32 [[TMP122]], [[TMP123]]
-// CHECK2-NEXT:    [[SUB225:%.*]] = sub i32 [[SUB224]], 1
+// CHECK2-NEXT:    [[DIV185:%.*]] = udiv i32 [[ADD184]], [[TMP110]]
+// CHECK2-NEXT:    [[MUL186:%.*]] = mul i32 1, [[DIV185]]
+// CHECK2-NEXT:    [[TMP111:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB187:%.*]] = sub i32 [[TMP111]], -63
+// CHECK2-NEXT:    [[DIV188:%.*]] = udiv i32 [[SUB187]], 64
+// CHECK2-NEXT:    [[MUL189:%.*]] = mul i32 [[MUL186]], [[DIV188]]
+// CHECK2-NEXT:    [[MUL190:%.*]] = mul i32 [[MUL189]], 64
+// CHECK2-NEXT:    [[CONV191:%.*]] = zext i32 [[MUL190]] to i64
+// CHECK2-NEXT:    [[DIV192:%.*]] = sdiv i64 [[TMP106]], [[CONV191]]
+// CHECK2-NEXT:    [[TMP112:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP113:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB193:%.*]] = sub i32 [[TMP112]], [[TMP113]]
+// CHECK2-NEXT:    [[SUB194:%.*]] = sub i32 [[SUB193]], 1
+// CHECK2-NEXT:    [[TMP114:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD195:%.*]] = add i32 [[SUB194]], [[TMP114]]
+// CHECK2-NEXT:    [[TMP115:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV196:%.*]] = udiv i32 [[ADD195]], [[TMP115]]
+// CHECK2-NEXT:    [[MUL197:%.*]] = mul i32 1, [[DIV196]]
+// CHECK2-NEXT:    [[TMP116:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB198:%.*]] = sub i32 [[TMP116]], -63
+// CHECK2-NEXT:    [[DIV199:%.*]] = udiv i32 [[SUB198]], 64
+// CHECK2-NEXT:    [[MUL200:%.*]] = mul i32 [[MUL197]], [[DIV199]]
+// CHECK2-NEXT:    [[MUL201:%.*]] = mul i32 [[MUL200]], 64
+// CHECK2-NEXT:    [[CONV202:%.*]] = zext i32 [[MUL201]] to i64
+// CHECK2-NEXT:    [[MUL203:%.*]] = mul nsw i64 [[DIV192]], [[CONV202]]
+// CHECK2-NEXT:    [[SUB204:%.*]] = sub nsw i64 [[TMP105]], [[MUL203]]
+// CHECK2-NEXT:    [[TMP117:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB205:%.*]] = sub i32 [[TMP117]], -63
+// CHECK2-NEXT:    [[DIV206:%.*]] = udiv i32 [[SUB205]], 64
+// CHECK2-NEXT:    [[MUL207:%.*]] = mul i32 1, [[DIV206]]
+// CHECK2-NEXT:    [[MUL208:%.*]] = mul i32 [[MUL207]], 64
+// CHECK2-NEXT:    [[CONV209:%.*]] = zext i32 [[MUL208]] to i64
+// CHECK2-NEXT:    [[DIV210:%.*]] = sdiv i64 [[SUB204]], [[CONV209]]
+// CHECK2-NEXT:    [[TMP118:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB211:%.*]] = sub i32 [[TMP118]], -63
+// CHECK2-NEXT:    [[DIV212:%.*]] = udiv i32 [[SUB211]], 64
+// CHECK2-NEXT:    [[MUL213:%.*]] = mul i32 1, [[DIV212]]
+// CHECK2-NEXT:    [[MUL214:%.*]] = mul i32 [[MUL213]], 64
+// CHECK2-NEXT:    [[CONV215:%.*]] = zext i32 [[MUL214]] to i64
+// CHECK2-NEXT:    [[MUL216:%.*]] = mul nsw i64 [[DIV210]], [[CONV215]]
+// CHECK2-NEXT:    [[SUB217:%.*]] = sub nsw i64 [[SUB181]], [[MUL216]]
+// CHECK2-NEXT:    [[TMP119:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP120:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP121:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP122:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB218:%.*]] = sub i32 [[TMP121]], [[TMP122]]
+// CHECK2-NEXT:    [[SUB219:%.*]] = sub i32 [[SUB218]], 1
+// CHECK2-NEXT:    [[TMP123:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD220:%.*]] = add i32 [[SUB219]], [[TMP123]]
 // CHECK2-NEXT:    [[TMP124:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD226:%.*]] = add i32 [[SUB225]], [[TMP124]]
-// CHECK2-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV227:%.*]] = udiv i32 [[ADD226]], [[TMP125]]
-// CHECK2-NEXT:    [[MUL228:%.*]] = mul i32 1, [[DIV227]]
-// CHECK2-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB229:%.*]] = sub i32 [[TMP126]], -63
-// CHECK2-NEXT:    [[DIV230:%.*]] = udiv i32 [[SUB229]], 64
-// CHECK2-NEXT:    [[MUL231:%.*]] = mul i32 [[MUL228]], [[DIV230]]
-// CHECK2-NEXT:    [[MUL232:%.*]] = mul i32 [[MUL231]], 64
-// CHECK2-NEXT:    [[CONV233:%.*]] = zext i32 [[MUL232]] to i64
-// CHECK2-NEXT:    [[MUL234:%.*]] = mul nsw i64 [[DIV223]], [[CONV233]]
-// CHECK2-NEXT:    [[SUB235:%.*]] = sub nsw i64 [[TMP115]], [[MUL234]]
-// CHECK2-NEXT:    [[TMP127:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP128:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB236:%.*]] = sub i32 [[TMP129]], [[TMP130]]
-// CHECK2-NEXT:    [[SUB237:%.*]] = sub i32 [[SUB236]], 1
-// CHECK2-NEXT:    [[TMP131:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD238:%.*]] = add i32 [[SUB237]], [[TMP131]]
-// CHECK2-NEXT:    [[TMP132:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV239:%.*]] = udiv i32 [[ADD238]], [[TMP132]]
-// CHECK2-NEXT:    [[MUL240:%.*]] = mul i32 1, [[DIV239]]
-// CHECK2-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB241:%.*]] = sub i32 [[TMP133]], -63
-// CHECK2-NEXT:    [[DIV242:%.*]] = udiv i32 [[SUB241]], 64
-// CHECK2-NEXT:    [[MUL243:%.*]] = mul i32 [[MUL240]], [[DIV242]]
-// CHECK2-NEXT:    [[MUL244:%.*]] = mul i32 [[MUL243]], 64
-// CHECK2-NEXT:    [[CONV245:%.*]] = zext i32 [[MUL244]] to i64
-// CHECK2-NEXT:    [[DIV246:%.*]] = sdiv i64 [[TMP128]], [[CONV245]]
-// CHECK2-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
-// CHECK2-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// CHECK2-NEXT:    [[SUB247:%.*]] = sub i32 [[TMP134]], [[TMP135]]
-// CHECK2-NEXT:    [[SUB248:%.*]] = sub i32 [[SUB247]], 1
+// CHECK2-NEXT:    [[DIV221:%.*]] = udiv i32 [[ADD220]], [[TMP124]]
+// CHECK2-NEXT:    [[MUL222:%.*]] = mul i32 1, [[DIV221]]
+// CHECK2-NEXT:    [[TMP125:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB223:%.*]] = sub i32 [[TMP125]], -63
+// CHECK2-NEXT:    [[DIV224:%.*]] = udiv i32 [[SUB223]], 64
+// CHECK2-NEXT:    [[MUL225:%.*]] = mul i32 [[MUL222]], [[DIV224]]
+// CHECK2-NEXT:    [[MUL226:%.*]] = mul i32 [[MUL225]], 64
+// CHECK2-NEXT:    [[CONV227:%.*]] = zext i32 [[MUL226]] to i64
+// CHECK2-NEXT:    [[DIV228:%.*]] = sdiv i64 [[TMP120]], [[CONV227]]
+// CHECK2-NEXT:    [[TMP126:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP127:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB229:%.*]] = sub i32 [[TMP126]], [[TMP127]]
+// CHECK2-NEXT:    [[SUB230:%.*]] = sub i32 [[SUB229]], 1
+// CHECK2-NEXT:    [[TMP128:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD231:%.*]] = add i32 [[SUB230]], [[TMP128]]
+// CHECK2-NEXT:    [[TMP129:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV232:%.*]] = udiv i32 [[ADD231]], [[TMP129]]
+// CHECK2-NEXT:    [[MUL233:%.*]] = mul i32 1, [[DIV232]]
+// CHECK2-NEXT:    [[TMP130:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB234:%.*]] = sub i32 [[TMP130]], -63
+// CHECK2-NEXT:    [[DIV235:%.*]] = udiv i32 [[SUB234]], 64
+// CHECK2-NEXT:    [[MUL236:%.*]] = mul i32 [[MUL233]], [[DIV235]]
+// CHECK2-NEXT:    [[MUL237:%.*]] = mul i32 [[MUL236]], 64
+// CHECK2-NEXT:    [[CONV238:%.*]] = zext i32 [[MUL237]] to i64
+// CHECK2-NEXT:    [[MUL239:%.*]] = mul nsw i64 [[DIV228]], [[CONV238]]
+// CHECK2-NEXT:    [[SUB240:%.*]] = sub nsw i64 [[TMP119]], [[MUL239]]
+// CHECK2-NEXT:    [[TMP131:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP132:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP133:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP134:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB241:%.*]] = sub i32 [[TMP133]], [[TMP134]]
+// CHECK2-NEXT:    [[SUB242:%.*]] = sub i32 [[SUB241]], 1
+// CHECK2-NEXT:    [[TMP135:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD243:%.*]] = add i32 [[SUB242]], [[TMP135]]
 // CHECK2-NEXT:    [[TMP136:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[ADD249:%.*]] = add i32 [[SUB248]], [[TMP136]]
-// CHECK2-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
-// CHECK2-NEXT:    [[DIV250:%.*]] = udiv i32 [[ADD249]], [[TMP137]]
-// CHECK2-NEXT:    [[MUL251:%.*]] = mul i32 1, [[DIV250]]
-// CHECK2-NEXT:    [[TMP138:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB252:%.*]] = sub i32 [[TMP138]], -63
-// CHECK2-NEXT:    [[DIV253:%.*]] = udiv i32 [[SUB252]], 64
-// CHECK2-NEXT:    [[MUL254:%.*]] = mul i32 [[MUL251]], [[DIV253]]
-// CHECK2-NEXT:    [[MUL255:%.*]] = mul i32 [[MUL254]], 64
-// CHECK2-NEXT:    [[CONV256:%.*]] = zext i32 [[MUL255]] to i64
-// CHECK2-NEXT:    [[MUL257:%.*]] = mul nsw i64 [[DIV246]], [[CONV256]]
-// CHECK2-NEXT:    [[SUB258:%.*]] = sub nsw i64 [[TMP127]], [[MUL257]]
-// CHECK2-NEXT:    [[TMP139:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB259:%.*]] = sub i32 [[TMP139]], -63
-// CHECK2-NEXT:    [[DIV260:%.*]] = udiv i32 [[SUB259]], 64
-// CHECK2-NEXT:    [[MUL261:%.*]] = mul i32 1, [[DIV260]]
-// CHECK2-NEXT:    [[MUL262:%.*]] = mul i32 [[MUL261]], 64
-// CHECK2-NEXT:    [[CONV263:%.*]] = zext i32 [[MUL262]] to i64
-// CHECK2-NEXT:    [[DIV264:%.*]] = sdiv i64 [[SUB258]], [[CONV263]]
-// CHECK2-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
-// CHECK2-NEXT:    [[SUB265:%.*]] = sub i32 [[TMP140]], -63
-// CHECK2-NEXT:    [[DIV266:%.*]] = udiv i32 [[SUB265]], 64
-// CHECK2-NEXT:    [[MUL267:%.*]] = mul i32 1, [[DIV266]]
-// CHECK2-NEXT:    [[MUL268:%.*]] = mul i32 [[MUL267]], 64
-// CHECK2-NEXT:    [[CONV269:%.*]] = zext i32 [[MUL268]] to i64
-// CHECK2-NEXT:    [[MUL270:%.*]] = mul nsw i64 [[DIV264]], [[CONV269]]
-// CHECK2-NEXT:    [[SUB271:%.*]] = sub nsw i64 [[SUB235]], [[MUL270]]
-// CHECK2-NEXT:    [[DIV272:%.*]] = sdiv i64 [[SUB271]], 64
-// CHECK2-NEXT:    [[MUL273:%.*]] = mul nsw i64 [[DIV272]], 64
-// CHECK2-NEXT:    [[SUB274:%.*]] = sub nsw i64 [[SUB212]], [[MUL273]]
-// CHECK2-NEXT:    [[MUL275:%.*]] = mul nsw i64 [[SUB274]], 1
-// CHECK2-NEXT:    [[ADD276:%.*]] = add nsw i64 0, [[MUL275]]
-// CHECK2-NEXT:    [[CONV277:%.*]] = trunc i64 [[ADD276]] to i32
-// CHECK2-NEXT:    store i32 [[CONV277]], ptr [[DOTTILE_CNT_0_IV_K38]], align 4
-// CHECK2-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
-// CHECK2-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
-// CHECK2-NEXT:    [[ADD278:%.*]] = add i32 [[TMP141]], [[TMP142]]
-// CHECK2-NEXT:    store i32 [[ADD278]], ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK2-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
-// CHECK2-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTTILE_0_IV_K]], align 4
-// CHECK2-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
-// CHECK2-NEXT:    [[MUL279:%.*]] = mul i32 [[TMP144]], [[TMP145]]
-// CHECK2-NEXT:    [[ADD280:%.*]] = add i32 [[TMP143]], [[MUL279]]
-// CHECK2-NEXT:    store i32 [[ADD280]], ptr [[K]], align 4
-// CHECK2-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_K37]], align 4
-// CHECK2-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_K38]], align 4
-// CHECK2-NEXT:    [[ADD281:%.*]] = add i32 [[TMP146]], [[TMP147]]
-// CHECK2-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// CHECK2-NEXT:    [[ADD282:%.*]] = add i32 [[TMP148]], 1
-// CHECK2-NEXT:    [[CMP283:%.*]] = icmp ult i32 [[ADD281]], [[ADD282]]
-// CHECK2-NEXT:    br i1 [[CMP283]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
-// CHECK2:       if.then:
-// CHECK2-NEXT:    [[TMP149:%.*]] = load i32, ptr [[I35]], align 4
-// CHECK2-NEXT:    [[TMP150:%.*]] = load i32, ptr [[J36]], align 4
-// CHECK2-NEXT:    [[TMP151:%.*]] = load i32, ptr [[K]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP149]], i32 noundef [[TMP150]], i32 noundef [[TMP151]])
-// CHECK2-NEXT:    br label [[IF_END]]
-// CHECK2:       if.end:
-// CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK2-NEXT:    [[DIV244:%.*]] = udiv i32 [[ADD243]], [[TMP136]]
+// CHECK2-NEXT:    [[MUL245:%.*]] = mul i32 1, [[DIV244]]
+// CHECK2-NEXT:    [[TMP137:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB246:%.*]] = sub i32 [[TMP137]], -63
+// CHECK2-NEXT:    [[DIV247:%.*]] = udiv i32 [[SUB246]], 64
+// CHECK2-NEXT:    [[MUL248:%.*]] = mul i32 [[MUL245]], [[DIV247]]
+// CHECK2-NEXT:    [[MUL249:%.*]] = mul i32 [[MUL248]], 64
+// CHECK2-NEXT:    [[CONV250:%.*]] = zext i32 [[MUL249]] to i64
+// CHECK2-NEXT:    [[DIV251:%.*]] = sdiv i64 [[TMP132]], [[CONV250]]
+// CHECK2-NEXT:    [[TMP138:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_6]], align 4
+// CHECK2-NEXT:    [[TMP139:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
+// CHECK2-NEXT:    [[SUB252:%.*]] = sub i32 [[TMP138]], [[TMP139]]
+// CHECK2-NEXT:    [[SUB253:%.*]] = sub i32 [[SUB252]], 1
+// CHECK2-NEXT:    [[TMP140:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[ADD254:%.*]] = add i32 [[SUB253]], [[TMP140]]
+// CHECK2-NEXT:    [[TMP141:%.*]] = load i32, ptr [[DOTNEW_STEP7]], align 4
+// CHECK2-NEXT:    [[DIV255:%.*]] = udiv i32 [[ADD254]], [[TMP141]]
+// CHECK2-NEXT:    [[MUL256:%.*]] = mul i32 1, [[DIV255]]
+// CHECK2-NEXT:    [[TMP142:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB257:%.*]] = sub i32 [[TMP142]], -63
+// CHECK2-NEXT:    [[DIV258:%.*]] = udiv i32 [[SUB257]], 64
+// CHECK2-NEXT:    [[MUL259:%.*]] = mul i32 [[MUL256]], [[DIV258]]
+// CHECK2-NEXT:    [[MUL260:%.*]] = mul i32 [[MUL259]], 64
+// CHECK2-NEXT:    [[CONV261:%.*]] = zext i32 [[MUL260]] to i64
+// CHECK2-NEXT:    [[MUL262:%.*]] = mul nsw i64 [[DIV251]], [[CONV261]]
+// CHECK2-NEXT:    [[SUB263:%.*]] = sub nsw i64 [[TMP131]], [[MUL262]]
+// CHECK2-NEXT:    [[TMP143:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB264:%.*]] = sub i32 [[TMP143]], -63
+// CHECK2-NEXT:    [[DIV265:%.*]] = udiv i32 [[SUB264]], 64
+// CHECK2-NEXT:    [[MUL266:%.*]] = mul i32 1, [[DIV265]]
+// CHECK2-NEXT:    [[MUL267:%.*]] = mul i32 [[MUL266]], 64
+// CHECK2-NEXT:    [[CONV268:%.*]] = zext i32 [[MUL267]] to i64
+// CHECK2-NEXT:    [[DIV269:%.*]] = sdiv i64 [[SUB263]], [[CONV268]]
+// CHECK2-NEXT:    [[TMP144:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_14]], align 4
+// CHECK2-NEXT:    [[SUB270:%.*]] = sub i32 [[TMP144]], -63
+// CHECK2-NEXT:    [[DIV271:%.*]] = udiv i32 [[SUB270]], 64
+// CHECK2-NEXT:    [[MUL272:%.*]] = mul i32 1, [[DIV271]]
+// CHECK2-NEXT:    [[MUL273:%.*]] = mul i32 [[MUL272]], 64
+// CHECK2-NEXT:    [[CONV274:%.*]] = zext i32 [[MUL273]] to i64
+// CHECK2-NEXT:    [[MUL275:%.*]] = mul nsw i64 [[DIV269]], [[CONV274]]
+// CHECK2-NEXT:    [[SUB276:%.*]] = sub nsw i64 [[SUB240]], [[MUL275]]
+// CHECK2-NEXT:    [[DIV277:%.*]] = sdiv i64 [[SUB276]], 64
+// CHECK2-NEXT:    [[MUL278:%.*]] = mul nsw i64 [[DIV277]], 64
+// CHECK2-NEXT:    [[SUB279:%.*]] = sub nsw i64 [[SUB217]], [[MUL278]]
+// CHECK2-NEXT:    [[MUL280:%.*]] = mul nsw i64 [[SUB279]], 1
+// CHECK2-NEXT:    [[ADD281:%.*]] = add nsw i64 [[CONV158]], [[MUL280]]
+// CHECK2-NEXT:    [[CONV282:%.*]] = trunc i64 [[ADD281]] to i32
+// CHECK2-NEXT:    store i32 [[CONV282]], ptr [[DOTTILE_0_IV_K42]], align 4
+// CHECK2-NEXT:    [[TMP145:%.*]] = load i32, ptr [[DOTTILE_0_IV_K42]], align 4
+// CHECK2-NEXT:    [[TMP146:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
+// CHECK2-NEXT:    [[ADD283:%.*]] = add i32 [[TMP146]], 1
+// CHECK2-NEXT:    [[CMP284:%.*]] = icmp ult i32 [[TMP145]], [[ADD283]]
+// CHECK2-NEXT:    br i1 [[CMP284]], label [[OMP_BODY_NEXT:%.*]], label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK2:       omp.body.next:
+// CHECK2-NEXT:    [[TMP147:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_8]], align 4
+// CHECK2-NEXT:    [[TMP148:%.*]] = load i32, ptr [[DOTTILE_0_IV_K42]], align 4
+// CHECK2-NEXT:    [[TMP149:%.*]] = load i32, ptr [[DOTNEW_STEP10]], align 4
+// CHECK2-NEXT:    [[MUL285:%.*]] = mul i32 [[TMP148]], [[TMP149]]
+// CHECK2-NEXT:    [[ADD286:%.*]] = add i32 [[TMP147]], [[MUL285]]
+// CHECK2-NEXT:    store i32 [[ADD286]], ptr [[K]], align 4
+// CHECK2-NEXT:    [[TMP150:%.*]] = load i32, ptr [[I39]], align 4
+// CHECK2-NEXT:    [[TMP151:%.*]] = load i32, ptr [[J40]], align 4
+// CHECK2-NEXT:    [[TMP152:%.*]] = load i32, ptr [[K]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP150]], i32 noundef [[TMP151]], i32 noundef [[TMP152]])
+// CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
-// CHECK2-NEXT:    [[TMP152:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
-// CHECK2-NEXT:    [[ADD284:%.*]] = add nsw i64 [[TMP152]], 1
-// CHECK2-NEXT:    store i64 [[ADD284]], ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[TMP153:%.*]] = load i64, ptr [[DOTOMP_IV]], align 8
+// CHECK2-NEXT:    [[ADD287:%.*]] = add nsw i64 [[TMP153]], 1
+// CHECK2-NEXT:    store i64 [[ADD287]], ptr [[DOTOMP_IV]], align 8
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
diff --git a/clang/test/OpenMP/tile_canonical_required_codegen.cpp b/clang/test/OpenMP/tile_canonical_required_codegen.cpp
deleted file mode 100644
index ceeaed4cb666a..0000000000000
--- a/clang/test/OpenMP/tile_canonical_required_codegen.cpp
+++ /dev/null
@@ -1,86 +0,0 @@
-// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fclang-abi-compat=latest -std=c++20 -fopenmp -ast-print %s | FileCheck %s --check-prefix=PRINT
-// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fclang-abi-compat=latest -std=c++20 -fopenmp -emit-llvm %s -o - | FileCheck %s --check-prefix=IR
-// expected-no-diagnostics
-
-// Positive tests for canonical intra-tile loop shape.
-
-extern "C" void body(...) {}
-
-// PRINT-LABEL: void collapse2_tile1d(
-// IR-LABEL: define {{.*}} @collapse2_tile1d(
-// collapse(2) consumes floor+tile; predicate stays as a body guard.
-// IR should have one flattened loop body and no nested floor/tile loop.
-// IR: %.tile.cnt.0.iv.i
-// IR: omp.inner.for.body:
-// IR: br i1 %{{.*}}, label %if.then, label %if.end
-// IR: if.then:
-// IR: call void (...) @body
-// IR-NEXT: br label %if.end
-// IR: if.end:
-extern "C" void collapse2_tile1d(int n) {
-#pragma omp parallel for collapse(2)
-#pragma omp tile sizes(2)
-  for (int i = 0; i < n; ++i)
-    body(i);
-}
-
-// PRINT-LABEL: void collapse3_tile1d_nested(
-// IR-LABEL: define {{.*}} @collapse3_tile1d_nested(
-// collapse(3) reaches through the tile guard to include `j`.
-// IR: br i1 %{{.*}}, label %omp_tile.pred.then, label %omp_tile.pred.end
-// IR: omp_tile.pred.then:
-// IR: call void (...) @body
-// IR-NEXT: br label %omp_tile.pred.end
-// IR: omp_tile.pred.end:
-extern "C" void collapse3_tile1d_nested(int n) {
-#pragma omp parallel for collapse(3)
-#pragma omp tile sizes(5)
-  for (int i = 0; i < n; ++i)
-    for (int j = 0; j < n; ++j)
-      body(i, j);
-}
-
-// PRINT-LABEL: void tile2d_inside_tile1d(
-// IR-LABEL: define {{.*}} @tile2d_inside_tile1d(
-// Nested tiling: both intra-tile loops stay canonical and hoist their predicates.
-// IR: %.tile.cnt.0.iv..floor_0.iv.i = alloca i32
-// IR: %.tile.cnt.0.iv.i = alloca i32
-// IR: %omp_tile.invariant_pred_bound = and i1
-// IR: %omp_tile.invariant_pred_bound{{[0-9]+}} = and i1
-extern "C" void tile2d_inside_tile1d(int n) {
-#pragma omp tile sizes(3)
-#pragma omp tile sizes(2)
-  for (int i = 0; i < n; ++i)
-    body(i);
-}
-
-// PRINT-LABEL: void collapse3_tile2d(
-// IR-LABEL: define {{.*}} @collapse3_tile2d(
-// 2-D tile + collapse(3): dim checks combine and are hoisted into the surviving loop.
-// IR: %.tile.cnt.0.iv.i
-// IR: %.tile.cnt.1.iv.j
-// IR: land.rhs:
-// IR: land.end:
-// IR: %omp_tile.invariant_pred_bound = and i1
-extern "C" void collapse3_tile2d(int n) {
-#pragma omp parallel for collapse(3)
-#pragma omp tile sizes(2, 3)
-  for (int i = 0; i < n; ++i)
-    for (int j = 0; j < n; ++j)
-      body(i, j);
-}
-
-// PRINT-LABEL: void tile1d_constant_div_no_predicate(
-// IR-LABEL: define {{.*}} @tile1d_constant_div_no_predicate(
-// Exact-divisible case: canonical tile-count loop, no predicate.
-// IR: %.tile.cnt.0.iv.i
-// IR: for.cond1:
-// IR: %[[TC:.*]] = icmp slt i32 %{{.*}}, 5
-// IR-NEXT: br i1 %[[TC]], label %for.body3, label %for.end
-// IR-NOT: omp_tile.invariant_pred_bound
-// IR-NOT: omp_tile.pred
-extern "C" void tile1d_constant_div_no_predicate() {
-#pragma omp tile sizes(5)
-  for (int i = 0; i < 10; ++i)
-    body(i);
-}
diff --git a/clang/test/OpenMP/tile_codegen.cpp b/clang/test/OpenMP/tile_codegen.cpp
index a76e46bddfb8a..6ec808d67e389 100644
--- a/clang/test/OpenMP/tile_codegen.cpp
+++ b/clang/test/OpenMP/tile_codegen.cpp
@@ -136,9 +136,8 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[I2:%.*]] = alloca ptr, align 8
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK1-NEXT:    [[I:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
@@ -152,41 +151,45 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
 // CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END12:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK1:       for.cond4:
-// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP1]], 5
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
-// CHECK1-NEXT:    [[CMP6:%.*]] = icmp slt i32 [[ADD]], 4
-// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP5]], [[CMP6]]
-// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY7:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body7:
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 5
+// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK1-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
 // CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
-// CHECK1-NEXT:    store i32 [[ADD8]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
+// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP4]], 5
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
+// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body8:
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
 // CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
-// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[TMP7]], align 4
-// CHECK1-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2]], !align [[META3]]
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP9]])
+// CHECK1-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
+// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[TMP6]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2]], !align [[META3]]
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP8]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP9]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP4:![0-9]+]]
 // CHECK1:       for.end:
 // CHECK1-NEXT:    br label [[FOR_INC10:%.*]]
 // CHECK1:       for.inc10:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP11]], 5
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP10]], 5
 // CHECK1-NEXT:    store i32 [[ADD11]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
 // CHECK1:       for.end12:
@@ -204,9 +207,8 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -235,48 +237,54 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK1-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP8]], [[ADD5]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK1:       for.cond6:
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP10]], 5
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD8:%.*]] = add i32 [[TMP11]], [[TMP12]]
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[ADD9:%.*]] = add i32 [[TMP13]], 1
-// CHECK1-NEXT:    [[CMP10:%.*]] = icmp ult i32 [[ADD8]], [[ADD9]]
-// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP7]], [[CMP10]]
-// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY11:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body11:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD12:%.*]] = add i32 [[TMP14]], [[TMP15]]
-// CHECK1-NEXT:    store i32 [[ADD12]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD7:%.*]] = add i32 [[TMP12]], 1
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD8:%.*]] = add i32 [[TMP13]], 5
+// CHECK1-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
+// CHECK1-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD10:%.*]] = add i32 [[TMP14]], 1
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD11:%.*]] = add i32 [[TMP15]], 5
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP11]], [[COND]]
+// CHECK1-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body13:
 // CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
 // CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
 // CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP17]], [[TMP18]]
-// CHECK1-NEXT:    [[ADD13:%.*]] = add i32 [[TMP16]], [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[ADD14:%.*]] = add i32 [[TMP16]], [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
 // CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP19]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP20]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP7:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC14:%.*]]
-// CHECK1:       for.inc14:
+// CHECK1-NEXT:    br label [[FOR_INC15:%.*]]
+// CHECK1:       for.inc15:
 // CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD15:%.*]] = add i32 [[TMP21]], 5
-// CHECK1-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD16:%.*]] = add i32 [[TMP21]], 5
+// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
-// CHECK1:       for.end16:
+// CHECK1:       for.end17:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -287,13 +295,11 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -304,90 +310,94 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END26:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END30:%.*]]
 // CHECK1:       for.body:
 // CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK1:       for.cond1:
 // CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 4
-// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END23:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END27:%.*]]
 // CHECK1:       for.body3:
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[TMP2]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK1:       for.cond4:
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP2]], 5
-// CHECK1-NEXT:    br i1 [[CMP5]], label [[FOR_BODY6:%.*]], label [[FOR_END20:%.*]]
-// CHECK1:       for.body6:
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[TMP4]]
-// CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
-// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND8:%.*]]
-// CHECK1:       for.cond8:
-// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP6]], 5
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP7]], [[TMP8]]
-// CHECK1-NEXT:    [[CMP11:%.*]] = icmp slt i32 [[ADD10]], 4
-// CHECK1-NEXT:    br i1 [[CMP11]], label [[LAND_RHS:%.*]], label [[LAND_END:%.*]]
-// CHECK1:       land.rhs:
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], 5
+// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK1-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP5]], 5
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP3]], [[COND]]
+// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END24:%.*]]
+// CHECK1:       for.body8:
+// CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
+// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND10:%.*]]
+// CHECK1:       for.cond10:
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
 // CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
-// CHECK1-NEXT:    [[CMP13:%.*]] = icmp slt i32 [[ADD12]], 4
-// CHECK1-NEXT:    br label [[LAND_END]]
-// CHECK1:       land.end:
-// CHECK1-NEXT:    [[TMP11:%.*]] = phi i1 [ false, [[FOR_COND8]] ], [ [[CMP13]], [[LAND_RHS]] ]
-// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP9]], [[TMP11]]
-// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body14:
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP12]], [[TMP13]]
-// CHECK1-NEXT:    store i32 [[ADD15]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[MUL16:%.*]] = mul nsw i32 [[TMP14]], 3
-// CHECK1-NEXT:    [[ADD17:%.*]] = add nsw i32 7, [[MUL16]]
-// CHECK1-NEXT:    store i32 [[ADD17]], ptr [[J]], align 4
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[J]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP15]], i32 noundef [[TMP16]])
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP9]], 5
+// CHECK1-NEXT:    [[CMP12:%.*]] = icmp slt i32 4, [[ADD11]]
+// CHECK1-NEXT:    br i1 [[CMP12]], label [[COND_TRUE13:%.*]], label [[COND_FALSE14:%.*]]
+// CHECK1:       cond.true13:
+// CHECK1-NEXT:    br label [[COND_END16:%.*]]
+// CHECK1:       cond.false14:
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP10]], 5
+// CHECK1-NEXT:    br label [[COND_END16]]
+// CHECK1:       cond.end16:
+// CHECK1-NEXT:    [[COND17:%.*]] = phi i32 [ 4, [[COND_TRUE13]] ], [ [[ADD15]], [[COND_FALSE14]] ]
+// CHECK1-NEXT:    [[CMP18:%.*]] = icmp slt i32 [[TMP8]], [[COND17]]
+// CHECK1-NEXT:    br i1 [[CMP18]], label [[FOR_BODY19:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body19:
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[TMP11]], 3
+// CHECK1-NEXT:    [[ADD21:%.*]] = add nsw i32 7, [[MUL20]]
+// CHECK1-NEXT:    store i32 [[ADD21]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP12]], i32 noundef [[TMP13]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP17]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP9:![0-9]+]]
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND10]], !llvm.loop [[LOOP9:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC18:%.*]]
-// CHECK1:       for.inc18:
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC19:%.*]] = add nsw i32 [[TMP18]], 1
-// CHECK1-NEXT:    store i32 [[INC19]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_INC22:%.*]]
+// CHECK1:       for.inc22:
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC23:%.*]] = add nsw i32 [[TMP15]], 1
+// CHECK1-NEXT:    store i32 [[INC23]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP10:![0-9]+]]
-// CHECK1:       for.end20:
-// CHECK1-NEXT:    br label [[FOR_INC21:%.*]]
-// CHECK1:       for.inc21:
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD22:%.*]] = add nsw i32 [[TMP19]], 5
-// CHECK1-NEXT:    store i32 [[ADD22]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1:       for.end24:
+// CHECK1-NEXT:    br label [[FOR_INC25:%.*]]
+// CHECK1:       for.inc25:
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD26:%.*]] = add nsw i32 [[TMP16]], 5
+// CHECK1-NEXT:    store i32 [[ADD26]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP11:![0-9]+]]
-// CHECK1:       for.end23:
-// CHECK1-NEXT:    br label [[FOR_INC24:%.*]]
-// CHECK1:       for.inc24:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD25:%.*]] = add nsw i32 [[TMP20]], 5
-// CHECK1-NEXT:    store i32 [[ADD25]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1:       for.end27:
+// CHECK1-NEXT:    br label [[FOR_INC28:%.*]]
+// CHECK1:       for.inc28:
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD29:%.*]] = add nsw i32 [[TMP17]], 5
+// CHECK1-NEXT:    store i32 [[ADD29]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP12:![0-9]+]]
-// CHECK1:       for.end26:
+// CHECK1:       for.end30:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -397,17 +407,15 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
 // CHECK1-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK1-NEXT:    store i32 7, ptr [[J]], align 4
@@ -445,83 +453,87 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP7]], 4
-// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END32:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[TMP8]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND3:%.*]]
 // CHECK1:       for.cond3:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP8]], 5
-// CHECK1-NEXT:    br i1 [[CMP4]], label [[FOR_BODY5:%.*]], label [[FOR_END21:%.*]]
-// CHECK1:       for.body5:
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
-// CHECK1-NEXT:    store i32 [[ADD6]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL7:%.*]] = mul nsw i32 [[TMP11]], 3
-// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i32 7, [[MUL7]]
-// CHECK1-NEXT:    store i32 [[ADD8]], ptr [[I]], align 4
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND9:%.*]]
-// CHECK1:       for.cond9:
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i32 [[TMP12]], 5
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP13]], [[TMP14]]
-// CHECK1-NEXT:    [[CMP12:%.*]] = icmp slt i32 [[ADD11]], 4
-// CHECK1-NEXT:    br i1 [[CMP12]], label [[LAND_RHS:%.*]], label [[LAND_END:%.*]]
-// CHECK1:       land.rhs:
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD4:%.*]] = add nsw i32 [[TMP10]], 5
+// CHECK1-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD4]]
+// CHECK1-NEXT:    br i1 [[CMP5]], label [[COND_TRUE6:%.*]], label [[COND_FALSE7:%.*]]
+// CHECK1:       cond.true6:
+// CHECK1-NEXT:    br label [[COND_END9:%.*]]
+// CHECK1:       cond.false7:
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP11]], 5
+// CHECK1-NEXT:    br label [[COND_END9]]
+// CHECK1:       cond.end9:
+// CHECK1-NEXT:    [[COND10:%.*]] = phi i32 [ 4, [[COND_TRUE6]] ], [ [[ADD8]], [[COND_FALSE7]] ]
+// CHECK1-NEXT:    [[CMP11:%.*]] = icmp slt i32 [[TMP9]], [[COND10]]
+// CHECK1-NEXT:    br i1 [[CMP11]], label [[FOR_BODY12:%.*]], label [[FOR_END29:%.*]]
+// CHECK1:       for.body12:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL13:%.*]] = mul nsw i32 [[TMP12]], 3
+// CHECK1-NEXT:    [[ADD14:%.*]] = add nsw i32 7, [[MUL13]]
+// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    store i32 [[TMP13]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND15:%.*]]
+// CHECK1:       for.cond15:
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP15]], [[TMP16]]
-// CHECK1-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[ADD13]], 4
-// CHECK1-NEXT:    br label [[LAND_END]]
-// CHECK1:       land.end:
-// CHECK1-NEXT:    [[TMP17:%.*]] = phi i1 [ false, [[FOR_COND9]] ], [ [[CMP14]], [[LAND_RHS]] ]
-// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP10]], [[TMP17]]
-// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY15:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body15:
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP18]], [[TMP19]]
-// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[MUL17:%.*]] = mul nsw i32 [[TMP20]], 3
-// CHECK1-NEXT:    [[ADD18:%.*]] = add nsw i32 7, [[MUL17]]
-// CHECK1-NEXT:    store i32 [[ADD18]], ptr [[J]], align 4
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[J]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP21]], i32 noundef [[TMP22]])
+// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP15]], 5
+// CHECK1-NEXT:    [[CMP17:%.*]] = icmp slt i32 4, [[ADD16]]
+// CHECK1-NEXT:    br i1 [[CMP17]], label [[COND_TRUE18:%.*]], label [[COND_FALSE19:%.*]]
+// CHECK1:       cond.true18:
+// CHECK1-NEXT:    br label [[COND_END21:%.*]]
+// CHECK1:       cond.false19:
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP16]], 5
+// CHECK1-NEXT:    br label [[COND_END21]]
+// CHECK1:       cond.end21:
+// CHECK1-NEXT:    [[COND22:%.*]] = phi i32 [ 4, [[COND_TRUE18]] ], [ [[ADD20]], [[COND_FALSE19]] ]
+// CHECK1-NEXT:    [[CMP23:%.*]] = icmp slt i32 [[TMP14]], [[COND22]]
+// CHECK1-NEXT:    br i1 [[CMP23]], label [[FOR_BODY24:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body24:
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[MUL25:%.*]] = mul nsw i32 [[TMP17]], 3
+// CHECK1-NEXT:    [[ADD26:%.*]] = add nsw i32 7, [[MUL25]]
+// CHECK1-NEXT:    store i32 [[ADD26]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP18]], i32 noundef [[TMP19]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP23]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND9]], !llvm.loop [[LOOP13:![0-9]+]]
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP20]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND15]], !llvm.loop [[LOOP13:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC19:%.*]]
-// CHECK1:       for.inc19:
-// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC20:%.*]] = add nsw i32 [[TMP24]], 1
-// CHECK1-NEXT:    store i32 [[INC20]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_INC27:%.*]]
+// CHECK1:       for.inc27:
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC28:%.*]] = add nsw i32 [[TMP21]], 1
+// CHECK1-NEXT:    store i32 [[INC28]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND3]], !llvm.loop [[LOOP14:![0-9]+]]
-// CHECK1:       for.end21:
-// CHECK1-NEXT:    br label [[FOR_INC22:%.*]]
-// CHECK1:       for.inc22:
-// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP25]], 5
-// CHECK1-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1:       for.end29:
+// CHECK1-NEXT:    br label [[FOR_INC30:%.*]]
+// CHECK1:       for.inc30:
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD31:%.*]] = add nsw i32 [[TMP22]], 5
+// CHECK1-NEXT:    store i32 [[ADD31]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP15:![0-9]+]]
-// CHECK1:       for.end24:
+// CHECK1:       for.end32:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[ADD25:%.*]] = add nsw i32 [[TMP26]], 1
-// CHECK1-NEXT:    store i32 [[ADD25]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[ADD33:%.*]] = add nsw i32 [[TMP23]], 1
+// CHECK1-NEXT:    store i32 [[ADD33]], ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -538,9 +550,7 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[_TMP1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
@@ -548,8 +558,8 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[K:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK1-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK1-NEXT:    store i32 7, ptr [[J]], align 4
@@ -596,83 +606,87 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP9]], 4
-// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY:%.*]], label [[FOR_END29:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP7]], label [[FOR_BODY:%.*]], label [[FOR_END37:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[TMP10]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND8:%.*]]
 // CHECK1:       for.cond8:
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP10]], 5
-// CHECK1-NEXT:    br i1 [[CMP9]], label [[FOR_BODY10:%.*]], label [[FOR_END26:%.*]]
-// CHECK1:       for.body10:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP11]], [[TMP12]]
-// CHECK1-NEXT:    store i32 [[ADD11]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL12:%.*]] = mul nsw i32 [[TMP13]], 3
-// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i32 7, [[MUL12]]
-// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[I]], align 4
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND14:%.*]]
-// CHECK1:       for.cond14:
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[CMP15:%.*]] = icmp slt i32 [[TMP14]], 5
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP15]], [[TMP16]]
-// CHECK1-NEXT:    [[CMP17:%.*]] = icmp slt i32 [[ADD16]], 4
-// CHECK1-NEXT:    br i1 [[CMP17]], label [[LAND_RHS:%.*]], label [[LAND_END:%.*]]
-// CHECK1:       land.rhs:
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i32 [[TMP12]], 5
+// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i32 4, [[ADD9]]
+// CHECK1-NEXT:    br i1 [[CMP10]], label [[COND_TRUE11:%.*]], label [[COND_FALSE12:%.*]]
+// CHECK1:       cond.true11:
+// CHECK1-NEXT:    br label [[COND_END14:%.*]]
+// CHECK1:       cond.false12:
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP13]], 5
+// CHECK1-NEXT:    br label [[COND_END14]]
+// CHECK1:       cond.end14:
+// CHECK1-NEXT:    [[COND15:%.*]] = phi i32 [ 4, [[COND_TRUE11]] ], [ [[ADD13]], [[COND_FALSE12]] ]
+// CHECK1-NEXT:    [[CMP16:%.*]] = icmp slt i32 [[TMP11]], [[COND15]]
+// CHECK1-NEXT:    br i1 [[CMP16]], label [[FOR_BODY17:%.*]], label [[FOR_END34:%.*]]
+// CHECK1:       for.body17:
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL18:%.*]] = mul nsw i32 [[TMP14]], 3
+// CHECK1-NEXT:    [[ADD19:%.*]] = add nsw i32 7, [[MUL18]]
+// CHECK1-NEXT:    store i32 [[ADD19]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    store i32 [[TMP15]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND20:%.*]]
+// CHECK1:       for.cond20:
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
 // CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD18:%.*]] = add nsw i32 [[TMP17]], [[TMP18]]
-// CHECK1-NEXT:    [[CMP19:%.*]] = icmp slt i32 [[ADD18]], 4
-// CHECK1-NEXT:    br label [[LAND_END]]
-// CHECK1:       land.end:
-// CHECK1-NEXT:    [[TMP19:%.*]] = phi i1 [ false, [[FOR_COND14]] ], [ [[CMP19]], [[LAND_RHS]] ]
-// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP15]], [[TMP19]]
-// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY20:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body20:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD21:%.*]] = add nsw i32 [[TMP20]], [[TMP21]]
-// CHECK1-NEXT:    store i32 [[ADD21]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK1-NEXT:    [[MUL22:%.*]] = mul nsw i32 [[TMP22]], 3
-// CHECK1-NEXT:    [[ADD23:%.*]] = add nsw i32 7, [[MUL22]]
-// CHECK1-NEXT:    store i32 [[ADD23]], ptr [[J]], align 4
-// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[J]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP23]], i32 noundef [[TMP24]])
+// CHECK1-NEXT:    [[ADD21:%.*]] = add nsw i32 [[TMP17]], 5
+// CHECK1-NEXT:    [[CMP22:%.*]] = icmp slt i32 4, [[ADD21]]
+// CHECK1-NEXT:    br i1 [[CMP22]], label [[COND_TRUE23:%.*]], label [[COND_FALSE24:%.*]]
+// CHECK1:       cond.true23:
+// CHECK1-NEXT:    br label [[COND_END26:%.*]]
+// CHECK1:       cond.false24:
+// CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD25:%.*]] = add nsw i32 [[TMP18]], 5
+// CHECK1-NEXT:    br label [[COND_END26]]
+// CHECK1:       cond.end26:
+// CHECK1-NEXT:    [[COND27:%.*]] = phi i32 [ 4, [[COND_TRUE23]] ], [ [[ADD25]], [[COND_FALSE24]] ]
+// CHECK1-NEXT:    [[CMP28:%.*]] = icmp slt i32 [[TMP16]], [[COND27]]
+// CHECK1-NEXT:    br i1 [[CMP28]], label [[FOR_BODY29:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body29:
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[MUL30:%.*]] = mul nsw i32 [[TMP19]], 3
+// CHECK1-NEXT:    [[ADD31:%.*]] = add nsw i32 7, [[MUL30]]
+// CHECK1-NEXT:    store i32 [[ADD31]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP21:%.*]] = load i32, ptr [[J]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP20]], i32 noundef [[TMP21]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP25]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK1-NEXT:    br label [[FOR_COND14]], !llvm.loop [[LOOP16:![0-9]+]]
+// CHECK1-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP22]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK1-NEXT:    br label [[FOR_COND20]], !llvm.loop [[LOOP16:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC24:%.*]]
-// CHECK1:       for.inc24:
-// CHECK1-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC25:%.*]] = add nsw i32 [[TMP26]], 1
-// CHECK1-NEXT:    store i32 [[INC25]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    br label [[FOR_INC32:%.*]]
+// CHECK1:       for.inc32:
+// CHECK1-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC33:%.*]] = add nsw i32 [[TMP23]], 1
+// CHECK1-NEXT:    store i32 [[INC33]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP17:![0-9]+]]
-// CHECK1:       for.end26:
-// CHECK1-NEXT:    br label [[FOR_INC27:%.*]]
-// CHECK1:       for.inc27:
-// CHECK1-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK1-NEXT:    [[ADD28:%.*]] = add nsw i32 [[TMP27]], 5
-// CHECK1-NEXT:    store i32 [[ADD28]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1:       for.end34:
+// CHECK1-NEXT:    br label [[FOR_INC35:%.*]]
+// CHECK1:       for.inc35:
+// CHECK1-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK1-NEXT:    [[ADD36:%.*]] = add nsw i32 [[TMP24]], 5
+// CHECK1-NEXT:    store i32 [[ADD36]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP18:![0-9]+]]
-// CHECK1:       for.end29:
+// CHECK1:       for.end37:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[ADD30:%.*]] = add nsw i32 [[TMP28]], 1
-// CHECK1-NEXT:    store i32 [[ADD30]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[ADD38:%.*]] = add nsw i32 [[TMP25]], 1
+// CHECK1-NEXT:    store i32 [[ADD38]], ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -690,91 +704,90 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[_TMP1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[_TMP2:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[J:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK1-NEXT:    store i32 7, ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP]], align 4
+// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP]], align 4
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], 5
+// CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTCAPTURE_EXPR_3]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK1-NEXT:    store i32 19, ptr [[DOTOMP_UB]], align 4
 // CHECK1-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
 // CHECK1-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
-// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// CHECK1-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], 19
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP3]], 19
 // CHECK1-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK1:       cond.true:
 // CHECK1-NEXT:    br label [[COND_END:%.*]]
 // CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
 // CHECK1-NEXT:    br label [[COND_END]]
 // CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 19, [[COND_TRUE]] ], [ [[TMP2]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 19, [[COND_TRUE]] ], [ [[TMP4]], [[COND_FALSE]] ]
 // CHECK1-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
-// CHECK1-NEXT:    store i32 [[TMP3]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
+// CHECK1-NEXT:    store i32 [[TMP5]], ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK1:       omp.inner.for.cond:
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// CHECK1-NEXT:    [[CMP3:%.*]] = icmp sle i32 [[TMP4]], [[TMP5]]
-// CHECK1-NEXT:    br i1 [[CMP3]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
-// CHECK1:       omp.inner.for.body:
 // CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP6]], 20
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[DIV]], 5
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK1-NEXT:    [[CMP4:%.*]] = icmp sle i32 [[TMP6]], [[TMP7]]
+// CHECK1-NEXT:    br i1 [[CMP4]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK1:       omp.inner.for.body:
 // CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[DIV4:%.*]] = sdiv i32 [[TMP8]], 20
-// CHECK1-NEXT:    [[MUL5:%.*]] = mul nsw i32 [[DIV4]], 20
-// CHECK1-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP7]], [[MUL5]]
-// CHECK1-NEXT:    [[DIV6:%.*]] = sdiv i32 [[SUB]], 4
-// CHECK1-NEXT:    [[MUL7:%.*]] = mul nsw i32 [[DIV6]], 1
-// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i32 0, [[MUL7]]
-// CHECK1-NEXT:    store i32 [[ADD8]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP8]], 20
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[DIV]], 5
+// CHECK1-NEXT:    [[ADD5:%.*]] = add nsw i32 0, [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD5]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[DIV9:%.*]] = sdiv i32 [[TMP10]], 20
-// CHECK1-NEXT:    [[MUL10:%.*]] = mul nsw i32 [[DIV9]], 20
-// CHECK1-NEXT:    [[SUB11:%.*]] = sub nsw i32 [[TMP9]], [[MUL10]]
 // CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[DIV6:%.*]] = sdiv i32 [[TMP11]], 20
+// CHECK1-NEXT:    [[MUL7:%.*]] = mul nsw i32 [[DIV6]], 20
+// CHECK1-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP10]], [[MUL7]]
+// CHECK1-NEXT:    [[DIV8:%.*]] = sdiv i32 [[SUB]], 4
+// CHECK1-NEXT:    [[MUL9:%.*]] = mul nsw i32 [[DIV8]], 1
+// CHECK1-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP9]], [[MUL9]]
+// CHECK1-NEXT:    store i32 [[ADD10]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[DIV12:%.*]] = sdiv i32 [[TMP12]], 20
-// CHECK1-NEXT:    [[MUL13:%.*]] = mul nsw i32 [[DIV12]], 20
-// CHECK1-NEXT:    [[SUB14:%.*]] = sub nsw i32 [[TMP11]], [[MUL13]]
-// CHECK1-NEXT:    [[DIV15:%.*]] = sdiv i32 [[SUB14]], 4
-// CHECK1-NEXT:    [[MUL16:%.*]] = mul nsw i32 [[DIV15]], 4
-// CHECK1-NEXT:    [[SUB17:%.*]] = sub nsw i32 [[SUB11]], [[MUL16]]
-// CHECK1-NEXT:    [[MUL18:%.*]] = mul nsw i32 [[SUB17]], 3
-// CHECK1-NEXT:    [[ADD19:%.*]] = add nsw i32 7, [[MUL18]]
-// CHECK1-NEXT:    store i32 [[ADD19]], ptr [[J]], align 4
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP13]], [[TMP14]]
-// CHECK1-NEXT:    store i32 [[ADD20]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL21:%.*]] = mul nsw i32 [[TMP15]], 3
-// CHECK1-NEXT:    [[ADD22:%.*]] = add nsw i32 7, [[MUL21]]
-// CHECK1-NEXT:    store i32 [[ADD22]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP16]], [[TMP17]]
-// CHECK1-NEXT:    [[CMP24:%.*]] = icmp slt i32 [[ADD23]], 4
-// CHECK1-NEXT:    br i1 [[CMP24]], label [[OMP_TILE_PRED_THEN:%.*]], label [[OMP_TILE_PRED_END:%.*]]
-// CHECK1:       omp_tile.pred.then:
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[DIV11:%.*]] = sdiv i32 [[TMP13]], 20
+// CHECK1-NEXT:    [[MUL12:%.*]] = mul nsw i32 [[DIV11]], 20
+// CHECK1-NEXT:    [[SUB13:%.*]] = sub nsw i32 [[TMP12]], [[MUL12]]
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[DIV14:%.*]] = sdiv i32 [[TMP15]], 20
+// CHECK1-NEXT:    [[MUL15:%.*]] = mul nsw i32 [[DIV14]], 20
+// CHECK1-NEXT:    [[SUB16:%.*]] = sub nsw i32 [[TMP14]], [[MUL15]]
+// CHECK1-NEXT:    [[DIV17:%.*]] = sdiv i32 [[SUB16]], 4
+// CHECK1-NEXT:    [[MUL18:%.*]] = mul nsw i32 [[DIV17]], 4
+// CHECK1-NEXT:    [[SUB19:%.*]] = sub nsw i32 [[SUB13]], [[MUL18]]
+// CHECK1-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[SUB19]], 3
+// CHECK1-NEXT:    [[ADD21:%.*]] = add nsw i32 7, [[MUL20]]
+// CHECK1-NEXT:    store i32 [[ADD21]], ptr [[J]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP22:%.*]] = icmp slt i32 [[TMP16]], 4
+// CHECK1-NEXT:    br i1 [[CMP22]], label [[OMP_BODY_NEXT:%.*]], label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK1:       omp.body.next:
+// CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL23:%.*]] = mul nsw i32 [[TMP17]], 3
+// CHECK1-NEXT:    [[ADD24:%.*]] = add nsw i32 7, [[MUL23]]
+// CHECK1-NEXT:    store i32 [[ADD24]], ptr [[I]], align 4
 // CHECK1-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    [[TMP19:%.*]] = load i32, ptr [[J]], align 4
 // CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP18]], i32 noundef [[TMP19]])
-// CHECK1-NEXT:    br label [[OMP_TILE_PRED_END]]
-// CHECK1:       omp_tile.pred.end:
-// CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
@@ -805,13 +818,12 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK1-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
 // CHECK1-NEXT:    store i32 7, ptr [[I]], align 4
@@ -846,42 +858,46 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP7]], 5
 // CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK1-NEXT:    store i32 [[ADD]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[TMP8]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND:%.*]]
 // CHECK1:       for.cond:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP8]], 5
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
-// CHECK1-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[ADD3]], 4
-// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP2]], [[CMP4]]
-// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body:
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD2:%.*]] = add nsw i32 [[TMP10]], 5
+// CHECK1-NEXT:    [[CMP3:%.*]] = icmp slt i32 4, [[ADD2]]
+// CHECK1-NEXT:    br i1 [[CMP3]], label [[COND_TRUE4:%.*]], label [[COND_FALSE5:%.*]]
+// CHECK1:       cond.true4:
+// CHECK1-NEXT:    br label [[COND_END7:%.*]]
+// CHECK1:       cond.false5:
 // CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD5:%.*]] = add nsw i32 [[TMP11]], [[TMP12]]
-// CHECK1-NEXT:    store i32 [[ADD5]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL6:%.*]] = mul nsw i32 [[TMP13]], 3
-// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL6]]
-// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP14]])
+// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP11]], 5
+// CHECK1-NEXT:    br label [[COND_END7]]
+// CHECK1:       cond.end7:
+// CHECK1-NEXT:    [[COND8:%.*]] = phi i32 [ 4, [[COND_TRUE4]] ], [ [[ADD6]], [[COND_FALSE5]] ]
+// CHECK1-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP9]], [[COND8]]
+// CHECK1-NEXT:    br i1 [[CMP9]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL10:%.*]] = mul nsw i32 [[TMP12]], 3
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 7, [[MUL10]]
+// CHECK1-NEXT:    store i32 [[ADD11]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP13]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP15]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP19:![0-9]+]]
 // CHECK1:       for.end:
 // CHECK1-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK1:       omp.body.continue:
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK1:       omp.inner.for.inc:
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP16]], 1
-// CHECK1-NEXT:    store i32 [[ADD8]], ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP15]], 1
+// CHECK1-NEXT:    store i32 [[ADD12]], ptr [[DOTOMP_IV]], align 4
 // CHECK1-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK1:       omp.inner.for.end:
 // CHECK1-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -906,9 +922,8 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK1-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[START_ADDR]], align 4
@@ -932,47 +947,53 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK1-NEXT:    [[ADD5:%.*]] = add i32 [[TMP6]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP5]], [[ADD5]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK1:       for.cond6:
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP7]], 5
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD8:%.*]] = add i32 [[TMP8]], [[TMP9]]
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK1-NEXT:    [[ADD9:%.*]] = add i32 [[TMP10]], 1
-// CHECK1-NEXT:    [[CMP10:%.*]] = icmp ult i32 [[ADD8]], [[ADD9]]
-// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP7]], [[CMP10]]
-// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY11:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body11:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD12:%.*]] = add i32 [[TMP11]], [[TMP12]]
-// CHECK1-NEXT:    store i32 [[ADD12]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD7:%.*]] = add i32 [[TMP9]], 1
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD8:%.*]] = add i32 [[TMP10]], 5
+// CHECK1-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
+// CHECK1-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK1-NEXT:    [[ADD10:%.*]] = add i32 [[TMP11]], 1
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD11:%.*]] = add i32 [[TMP12]], 5
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP8]], [[COND]]
+// CHECK1-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body13:
 // CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
 // CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    [[MUL:%.*]] = mul i32 [[TMP14]], 3
-// CHECK1-NEXT:    [[ADD13:%.*]] = add i32 [[TMP13]], [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD13]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[ADD14:%.*]] = add i32 [[TMP13]], [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
 // CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP15]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    [[INC:%.*]] = add i32 [[TMP16]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP22:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC14:%.*]]
-// CHECK1:       for.inc14:
+// CHECK1-NEXT:    br label [[FOR_INC15:%.*]]
+// CHECK1:       for.inc15:
 // CHECK1-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD15:%.*]] = add i32 [[TMP17]], 5
-// CHECK1-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD16:%.*]] = add i32 [[TMP17]], 5
+// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP23:![0-9]+]]
-// CHECK1:       for.end16:
+// CHECK1:       for.end17:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -981,9 +1002,8 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:  entry:
 // CHECK1-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK1-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK1-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
 // CHECK1-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK1-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
@@ -991,64 +1011,78 @@ extern "C" void foo10(data_t data) {
 // CHECK1:       for.cond:
 // CHECK1-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END15:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK1:       for.cond1:
-// CHECK1-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP2]], 0
+// CHECK1-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], 0
 // CHECK1-NEXT:    br i1 [[CMP2]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK1:       cond.true:
 // CHECK1-NEXT:    br label [[COND_END:%.*]]
 // CHECK1:       cond.false:
-// CHECK1-NEXT:    [[TMP3:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[A_ADDR]], align 4
 // CHECK1-NEXT:    br label [[COND_END]]
 // CHECK1:       cond.end:
-// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP3]], [[COND_FALSE]] ]
-// CHECK1-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[TMP1]], [[COND]]
-// CHECK1-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
-// CHECK1-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[ADD]], 4
-// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP3]], [[CMP4]]
-// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY5:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body5:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP5]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[COND]]
+// CHECK1-NEXT:    [[CMP3:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK1-NEXT:    br i1 [[CMP3]], label [[COND_TRUE4:%.*]], label [[COND_FALSE5:%.*]]
+// CHECK1:       cond.true4:
+// CHECK1-NEXT:    br label [[COND_END12:%.*]]
+// CHECK1:       cond.false5:
 // CHECK1-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP6]], [[TMP7]]
-// CHECK1-NEXT:    store i32 [[ADD6]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP8]], 3
-// CHECK1-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK1-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[I]], align 4
-// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP9]])
+// CHECK1-NEXT:    [[TMP7:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    [[CMP6:%.*]] = icmp sle i32 [[TMP7]], 0
+// CHECK1-NEXT:    br i1 [[CMP6]], label [[COND_TRUE7:%.*]], label [[COND_FALSE8:%.*]]
+// CHECK1:       cond.true7:
+// CHECK1-NEXT:    br label [[COND_END9:%.*]]
+// CHECK1:       cond.false8:
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    br label [[COND_END9]]
+// CHECK1:       cond.end9:
+// CHECK1-NEXT:    [[COND10:%.*]] = phi i32 [ 1, [[COND_TRUE7]] ], [ [[TMP8]], [[COND_FALSE8]] ]
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP6]], [[COND10]]
+// CHECK1-NEXT:    br label [[COND_END12]]
+// CHECK1:       cond.end12:
+// CHECK1-NEXT:    [[COND13:%.*]] = phi i32 [ 4, [[COND_TRUE4]] ], [ [[ADD11]], [[COND_END9]] ]
+// CHECK1-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP2]], [[COND13]]
+// CHECK1-NEXT:    br i1 [[CMP14]], label [[FOR_BODY15:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body15:
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP9]], 3
+// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK1-NEXT:    store i32 [[ADD16]], ptr [[I]], align 4
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
+// CHECK1-NEXT:    call void (...) @body(i32 noundef [[TMP10]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
-// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK1-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
+// CHECK1-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP24:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC8:%.*]]
-// CHECK1:       for.inc8:
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    [[CMP9:%.*]] = icmp sle i32 [[TMP11]], 0
-// CHECK1-NEXT:    br i1 [[CMP9]], label [[COND_TRUE10:%.*]], label [[COND_FALSE11:%.*]]
-// CHECK1:       cond.true10:
-// CHECK1-NEXT:    br label [[COND_END12:%.*]]
-// CHECK1:       cond.false11:
+// CHECK1-NEXT:    br label [[FOR_INC17:%.*]]
+// CHECK1:       for.inc17:
 // CHECK1-NEXT:    [[TMP12:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK1-NEXT:    br label [[COND_END12]]
-// CHECK1:       cond.end12:
-// CHECK1-NEXT:    [[COND13:%.*]] = phi i32 [ 1, [[COND_TRUE10]] ], [ [[TMP12]], [[COND_FALSE11]] ]
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK1-NEXT:    [[ADD14:%.*]] = add nsw i32 [[TMP13]], [[COND13]]
-// CHECK1-NEXT:    store i32 [[ADD14]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[CMP18:%.*]] = icmp sle i32 [[TMP12]], 0
+// CHECK1-NEXT:    br i1 [[CMP18]], label [[COND_TRUE19:%.*]], label [[COND_FALSE20:%.*]]
+// CHECK1:       cond.true19:
+// CHECK1-NEXT:    br label [[COND_END21:%.*]]
+// CHECK1:       cond.false20:
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK1-NEXT:    br label [[COND_END21]]
+// CHECK1:       cond.end21:
+// CHECK1-NEXT:    [[COND22:%.*]] = phi i32 [ 1, [[COND_TRUE19]] ], [ [[TMP13]], [[COND_FALSE20]] ]
+// CHECK1-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK1-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP14]], [[COND22]]
+// CHECK1-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP25:![0-9]+]]
-// CHECK1:       for.end15:
+// CHECK1:       for.end24:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -1061,9 +1095,8 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK1-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
 // CHECK1-NEXT:    store ptr [[ARRAY]], ptr [[__RANGE2]], align 8
@@ -1097,31 +1130,37 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK1:       for.cond7:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK1-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP11]], 1
-// CHECK1-NEXT:    [[CMP11:%.*]] = icmp slt i64 [[ADD9]], [[ADD10]]
-// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP8]], [[CMP11]]
-// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY12:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body12:
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i64 [[TMP12]], [[TMP13]]
-// CHECK1-NEXT:    store i64 [[ADD13]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
+// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
+// CHECK1-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
+// CHECK1-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body14:
 // CHECK1-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
-// CHECK1-NEXT:    [[ADD_PTR14:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
-// CHECK1-NEXT:    store ptr [[ADD_PTR14]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
+// CHECK1-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
 // CHECK1-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
 // CHECK1-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
 // CHECK1-NEXT:    store double [[TMP17]], ptr [[V]], align 8
@@ -1129,18 +1168,18 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    call void (...) @body(double noundef [[TMP18]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP19]], 1
-// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP27:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC15:%.*]]
-// CHECK1:       for.inc15:
+// CHECK1-NEXT:    br label [[FOR_INC16:%.*]]
+// CHECK1:       for.inc16:
 // CHECK1-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i64 [[TMP20]], 5
-// CHECK1-NEXT:    store i64 [[ADD16]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP20]], 5
+// CHECK1-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP28:![0-9]+]]
-// CHECK1:       for.end17:
+// CHECK1:       for.end18:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -1154,9 +1193,8 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK1-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
-// CHECK1-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK1-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK1-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK1-NEXT:    store double 4.200000e+01, ptr [[C]], align 8
 // CHECK1-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
@@ -1191,31 +1229,37 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK1-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK1-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
+// CHECK1-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
 // CHECK1:       for.body:
-// CHECK1-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK1:       for.cond7:
-// CHECK1-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
-// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
-// CHECK1-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK1-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP11]], 1
-// CHECK1-NEXT:    [[CMP11:%.*]] = icmp slt i64 [[ADD9]], [[ADD10]]
-// CHECK1-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP8]], [[CMP11]]
-// CHECK1-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY12:%.*]], label [[FOR_END:%.*]]
-// CHECK1:       for.body12:
-// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD13:%.*]] = add nsw i64 [[TMP12]], [[TMP13]]
-// CHECK1-NEXT:    store i64 [[ADD13]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK1-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
+// CHECK1-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
+// CHECK1-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
+// CHECK1-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK1:       cond.true:
+// CHECK1-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK1-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
+// CHECK1-NEXT:    br label [[COND_END:%.*]]
+// CHECK1:       cond.false:
+// CHECK1-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
+// CHECK1-NEXT:    br label [[COND_END]]
+// CHECK1:       cond.end:
+// CHECK1-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
+// CHECK1-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
+// CHECK1-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
+// CHECK1:       for.body14:
 // CHECK1-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK1-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
-// CHECK1-NEXT:    [[ADD_PTR14:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
-// CHECK1-NEXT:    store ptr [[ADD_PTR14]], ptr [[__BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
+// CHECK1-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
 // CHECK1-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
 // CHECK1-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
 // CHECK1-NEXT:    store double [[TMP17]], ptr [[V]], align 8
@@ -1224,18 +1268,18 @@ extern "C" void foo10(data_t data) {
 // CHECK1-NEXT:    call void (...) @body(double noundef [[TMP18]], double noundef [[TMP19]])
 // CHECK1-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK1:       for.inc:
-// CHECK1-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP20]], 1
-// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP29:![0-9]+]]
 // CHECK1:       for.end:
-// CHECK1-NEXT:    br label [[FOR_INC15:%.*]]
-// CHECK1:       for.inc15:
+// CHECK1-NEXT:    br label [[FOR_INC16:%.*]]
+// CHECK1:       for.inc16:
 // CHECK1-NEXT:    [[TMP21:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK1-NEXT:    [[ADD16:%.*]] = add nsw i64 [[TMP21]], 5
-// CHECK1-NEXT:    store i64 [[ADD16]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK1-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP21]], 5
+// CHECK1-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
 // CHECK1-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP30:![0-9]+]]
-// CHECK1:       for.end17:
+// CHECK1:       for.end18:
 // CHECK1-NEXT:    ret void
 //
 //
@@ -1268,9 +1312,8 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:  entry:
 // CHECK2-NEXT:    [[THIS_ADDR:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[I2:%.*]] = alloca ptr, align 8
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store ptr [[THIS]], ptr [[THIS_ADDR]], align 8
 // CHECK2-NEXT:    [[THIS1:%.*]] = load ptr, ptr [[THIS_ADDR]], align 8
 // CHECK2-NEXT:    [[I:%.*]] = getelementptr inbounds nuw [[STRUCT_S:%.*]], ptr [[THIS1]], i32 0, i32 0
@@ -1284,41 +1327,45 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
 // CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END12:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK2:       for.cond4:
-// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP1]], 5
-// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
-// CHECK2-NEXT:    [[CMP6:%.*]] = icmp slt i32 [[ADD]], 4
-// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP5]], [[CMP6]]
-// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY7:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body7:
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 5
+// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK2-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
 // CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
-// CHECK2-NEXT:    store i32 [[ADD8]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
+// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP4]], 5
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
+// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body8:
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
 // CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK2-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
-// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[TMP7]], align 4
-// CHECK2-NEXT:    [[TMP8:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2]], !align [[META3]]
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[TMP8]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP9]])
+// CHECK2-NEXT:    [[TMP6:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2:![0-9]+]], !align [[META3:![0-9]+]]
+// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[TMP6]], align 4
+// CHECK2-NEXT:    [[TMP7:%.*]] = load ptr, ptr [[I2]], align 8, !nonnull [[META2]], !align [[META3]]
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TMP7]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP8]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP9]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP4:![0-9]+]]
 // CHECK2:       for.end:
 // CHECK2-NEXT:    br label [[FOR_INC10:%.*]]
 // CHECK2:       for.inc10:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP11]], 5
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP10]], 5
 // CHECK2-NEXT:    store i32 [[ADD11]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
 // CHECK2:       for.end12:
@@ -1342,9 +1389,8 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -1373,48 +1419,54 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK2-NEXT:    [[ADD5:%.*]] = add i32 [[TMP9]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP8]], [[ADD5]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[TMP10]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK2:       for.cond6:
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP10]], 5
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD8:%.*]] = add i32 [[TMP11]], [[TMP12]]
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[ADD9:%.*]] = add i32 [[TMP13]], 1
-// CHECK2-NEXT:    [[CMP10:%.*]] = icmp ult i32 [[ADD8]], [[ADD9]]
-// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP7]], [[CMP10]]
-// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY11:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body11:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD12:%.*]] = add i32 [[TMP14]], [[TMP15]]
-// CHECK2-NEXT:    store i32 [[ADD12]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD7:%.*]] = add i32 [[TMP12]], 1
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD8:%.*]] = add i32 [[TMP13]], 5
+// CHECK2-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
+// CHECK2-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD10:%.*]] = add i32 [[TMP14]], 1
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD11:%.*]] = add i32 [[TMP15]], 5
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP11]], [[COND]]
+// CHECK2-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body13:
 // CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
 // CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
 // CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP17]], [[TMP18]]
-// CHECK2-NEXT:    [[ADD13:%.*]] = add i32 [[TMP16]], [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD13]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[ADD14:%.*]] = add i32 [[TMP16]], [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
 // CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[I]], align 4
 // CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP19]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP20]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP7:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC14:%.*]]
-// CHECK2:       for.inc14:
+// CHECK2-NEXT:    br label [[FOR_INC15:%.*]]
+// CHECK2:       for.inc15:
 // CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD15:%.*]] = add i32 [[TMP21]], 5
-// CHECK2-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD16:%.*]] = add i32 [[TMP21]], 5
+// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
-// CHECK2:       for.end16:
+// CHECK2:       for.end17:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -1428,9 +1480,8 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK2-NEXT:    store double 4.200000e+01, ptr [[C]], align 8
 // CHECK2-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
@@ -1465,31 +1516,37 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK2:       for.cond7:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK2-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP11]], 1
-// CHECK2-NEXT:    [[CMP11:%.*]] = icmp slt i64 [[ADD9]], [[ADD10]]
-// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP8]], [[CMP11]]
-// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY12:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body12:
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i64 [[TMP12]], [[TMP13]]
-// CHECK2-NEXT:    store i64 [[ADD13]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
+// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
+// CHECK2-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
+// CHECK2-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body14:
 // CHECK2-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
-// CHECK2-NEXT:    [[ADD_PTR14:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
-// CHECK2-NEXT:    store ptr [[ADD_PTR14]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
+// CHECK2-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
 // CHECK2-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
 // CHECK2-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
 // CHECK2-NEXT:    store double [[TMP17]], ptr [[V]], align 8
@@ -1498,18 +1555,18 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    call void (...) @body(double noundef [[TMP18]], double noundef [[TMP19]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP20]], 1
-// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP10:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC15:%.*]]
-// CHECK2:       for.inc15:
+// CHECK2-NEXT:    br label [[FOR_INC16:%.*]]
+// CHECK2:       for.inc16:
 // CHECK2-NEXT:    [[TMP21:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i64 [[TMP21]], 5
-// CHECK2-NEXT:    store i64 [[ADD16]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP21]], 5
+// CHECK2-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP11:![0-9]+]]
-// CHECK2:       for.end17:
+// CHECK2:       for.end18:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -1520,13 +1577,11 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
@@ -1537,90 +1592,94 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END26:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END30:%.*]]
 // CHECK2:       for.body:
 // CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK2:       for.cond1:
 // CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 4
-// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END23:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY3:%.*]], label [[FOR_END27:%.*]]
 // CHECK2:       for.body3:
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[TMP2]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND4:%.*]]
 // CHECK2:       for.cond4:
-// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP2]], 5
-// CHECK2-NEXT:    br i1 [[CMP5]], label [[FOR_BODY6:%.*]], label [[FOR_END20:%.*]]
-// CHECK2:       for.body6:
-// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[TMP4]]
-// CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 3
-// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND8:%.*]]
-// CHECK2:       for.cond8:
-// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP6]], 5
-// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP7]], [[TMP8]]
-// CHECK2-NEXT:    [[CMP11:%.*]] = icmp slt i32 [[ADD10]], 4
-// CHECK2-NEXT:    br i1 [[CMP11]], label [[LAND_RHS:%.*]], label [[LAND_END:%.*]]
-// CHECK2:       land.rhs:
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], 5
+// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK2-NEXT:    br i1 [[CMP5]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP5]], 5
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 4, [[COND_TRUE]] ], [ [[ADD6]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP3]], [[COND]]
+// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY8:%.*]], label [[FOR_END24:%.*]]
+// CHECK2:       for.body8:
+// CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 3
+// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND10:%.*]]
+// CHECK2:       for.cond10:
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
 // CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
-// CHECK2-NEXT:    [[CMP13:%.*]] = icmp slt i32 [[ADD12]], 4
-// CHECK2-NEXT:    br label [[LAND_END]]
-// CHECK2:       land.end:
-// CHECK2-NEXT:    [[TMP11:%.*]] = phi i1 [ false, [[FOR_COND8]] ], [ [[CMP13]], [[LAND_RHS]] ]
-// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP9]], [[TMP11]]
-// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body14:
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP12]], [[TMP13]]
-// CHECK2-NEXT:    store i32 [[ADD15]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[MUL16:%.*]] = mul nsw i32 [[TMP14]], 3
-// CHECK2-NEXT:    [[ADD17:%.*]] = add nsw i32 7, [[MUL16]]
-// CHECK2-NEXT:    store i32 [[ADD17]], ptr [[J]], align 4
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[J]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP15]], i32 noundef [[TMP16]])
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP9]], 5
+// CHECK2-NEXT:    [[CMP12:%.*]] = icmp slt i32 4, [[ADD11]]
+// CHECK2-NEXT:    br i1 [[CMP12]], label [[COND_TRUE13:%.*]], label [[COND_FALSE14:%.*]]
+// CHECK2:       cond.true13:
+// CHECK2-NEXT:    br label [[COND_END16:%.*]]
+// CHECK2:       cond.false14:
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP10]], 5
+// CHECK2-NEXT:    br label [[COND_END16]]
+// CHECK2:       cond.end16:
+// CHECK2-NEXT:    [[COND17:%.*]] = phi i32 [ 4, [[COND_TRUE13]] ], [ [[ADD15]], [[COND_FALSE14]] ]
+// CHECK2-NEXT:    [[CMP18:%.*]] = icmp slt i32 [[TMP8]], [[COND17]]
+// CHECK2-NEXT:    br i1 [[CMP18]], label [[FOR_BODY19:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body19:
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[TMP11]], 3
+// CHECK2-NEXT:    [[ADD21:%.*]] = add nsw i32 7, [[MUL20]]
+// CHECK2-NEXT:    store i32 [[ADD21]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP12]], i32 noundef [[TMP13]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP17]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP12:![0-9]+]]
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND10]], !llvm.loop [[LOOP12:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC18:%.*]]
-// CHECK2:       for.inc18:
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC19:%.*]] = add nsw i32 [[TMP18]], 1
-// CHECK2-NEXT:    store i32 [[INC19]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_INC22:%.*]]
+// CHECK2:       for.inc22:
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC23:%.*]] = add nsw i32 [[TMP15]], 1
+// CHECK2-NEXT:    store i32 [[INC23]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND4]], !llvm.loop [[LOOP13:![0-9]+]]
-// CHECK2:       for.end20:
-// CHECK2-NEXT:    br label [[FOR_INC21:%.*]]
-// CHECK2:       for.inc21:
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD22:%.*]] = add nsw i32 [[TMP19]], 5
-// CHECK2-NEXT:    store i32 [[ADD22]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2:       for.end24:
+// CHECK2-NEXT:    br label [[FOR_INC25:%.*]]
+// CHECK2:       for.inc25:
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD26:%.*]] = add nsw i32 [[TMP16]], 5
+// CHECK2-NEXT:    store i32 [[ADD26]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP14:![0-9]+]]
-// CHECK2:       for.end23:
-// CHECK2-NEXT:    br label [[FOR_INC24:%.*]]
-// CHECK2:       for.inc24:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD25:%.*]] = add nsw i32 [[TMP20]], 5
-// CHECK2-NEXT:    store i32 [[ADD25]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2:       for.end27:
+// CHECK2-NEXT:    br label [[FOR_INC28:%.*]]
+// CHECK2:       for.inc28:
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD29:%.*]] = add nsw i32 [[TMP17]], 5
+// CHECK2-NEXT:    store i32 [[ADD29]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP15:![0-9]+]]
-// CHECK2:       for.end26:
+// CHECK2:       for.end30:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -1630,17 +1689,15 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
 // CHECK2-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK2-NEXT:    store i32 7, ptr [[J]], align 4
@@ -1678,83 +1735,87 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP7]], 4
-// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP2]], label [[FOR_BODY:%.*]], label [[FOR_END32:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[TMP8]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND3:%.*]]
 // CHECK2:       for.cond3:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP8]], 5
-// CHECK2-NEXT:    br i1 [[CMP4]], label [[FOR_BODY5:%.*]], label [[FOR_END21:%.*]]
-// CHECK2:       for.body5:
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
-// CHECK2-NEXT:    store i32 [[ADD6]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL7:%.*]] = mul nsw i32 [[TMP11]], 3
-// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i32 7, [[MUL7]]
-// CHECK2-NEXT:    store i32 [[ADD8]], ptr [[I]], align 4
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND9:%.*]]
-// CHECK2:       for.cond9:
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i32 [[TMP12]], 5
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP13]], [[TMP14]]
-// CHECK2-NEXT:    [[CMP12:%.*]] = icmp slt i32 [[ADD11]], 4
-// CHECK2-NEXT:    br i1 [[CMP12]], label [[LAND_RHS:%.*]], label [[LAND_END:%.*]]
-// CHECK2:       land.rhs:
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD4:%.*]] = add nsw i32 [[TMP10]], 5
+// CHECK2-NEXT:    [[CMP5:%.*]] = icmp slt i32 4, [[ADD4]]
+// CHECK2-NEXT:    br i1 [[CMP5]], label [[COND_TRUE6:%.*]], label [[COND_FALSE7:%.*]]
+// CHECK2:       cond.true6:
+// CHECK2-NEXT:    br label [[COND_END9:%.*]]
+// CHECK2:       cond.false7:
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP11]], 5
+// CHECK2-NEXT:    br label [[COND_END9]]
+// CHECK2:       cond.end9:
+// CHECK2-NEXT:    [[COND10:%.*]] = phi i32 [ 4, [[COND_TRUE6]] ], [ [[ADD8]], [[COND_FALSE7]] ]
+// CHECK2-NEXT:    [[CMP11:%.*]] = icmp slt i32 [[TMP9]], [[COND10]]
+// CHECK2-NEXT:    br i1 [[CMP11]], label [[FOR_BODY12:%.*]], label [[FOR_END29:%.*]]
+// CHECK2:       for.body12:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL13:%.*]] = mul nsw i32 [[TMP12]], 3
+// CHECK2-NEXT:    [[ADD14:%.*]] = add nsw i32 7, [[MUL13]]
+// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    store i32 [[TMP13]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND15:%.*]]
+// CHECK2:       for.cond15:
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP15]], [[TMP16]]
-// CHECK2-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[ADD13]], 4
-// CHECK2-NEXT:    br label [[LAND_END]]
-// CHECK2:       land.end:
-// CHECK2-NEXT:    [[TMP17:%.*]] = phi i1 [ false, [[FOR_COND9]] ], [ [[CMP14]], [[LAND_RHS]] ]
-// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP10]], [[TMP17]]
-// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY15:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body15:
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP18]], [[TMP19]]
-// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[MUL17:%.*]] = mul nsw i32 [[TMP20]], 3
-// CHECK2-NEXT:    [[ADD18:%.*]] = add nsw i32 7, [[MUL17]]
-// CHECK2-NEXT:    store i32 [[ADD18]], ptr [[J]], align 4
-// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[J]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP21]], i32 noundef [[TMP22]])
+// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP15]], 5
+// CHECK2-NEXT:    [[CMP17:%.*]] = icmp slt i32 4, [[ADD16]]
+// CHECK2-NEXT:    br i1 [[CMP17]], label [[COND_TRUE18:%.*]], label [[COND_FALSE19:%.*]]
+// CHECK2:       cond.true18:
+// CHECK2-NEXT:    br label [[COND_END21:%.*]]
+// CHECK2:       cond.false19:
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP16]], 5
+// CHECK2-NEXT:    br label [[COND_END21]]
+// CHECK2:       cond.end21:
+// CHECK2-NEXT:    [[COND22:%.*]] = phi i32 [ 4, [[COND_TRUE18]] ], [ [[ADD20]], [[COND_FALSE19]] ]
+// CHECK2-NEXT:    [[CMP23:%.*]] = icmp slt i32 [[TMP14]], [[COND22]]
+// CHECK2-NEXT:    br i1 [[CMP23]], label [[FOR_BODY24:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body24:
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[MUL25:%.*]] = mul nsw i32 [[TMP17]], 3
+// CHECK2-NEXT:    [[ADD26:%.*]] = add nsw i32 7, [[MUL25]]
+// CHECK2-NEXT:    store i32 [[ADD26]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP18]], i32 noundef [[TMP19]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP23]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND9]], !llvm.loop [[LOOP16:![0-9]+]]
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP20]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND15]], !llvm.loop [[LOOP16:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC19:%.*]]
-// CHECK2:       for.inc19:
-// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC20:%.*]] = add nsw i32 [[TMP24]], 1
-// CHECK2-NEXT:    store i32 [[INC20]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_INC27:%.*]]
+// CHECK2:       for.inc27:
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC28:%.*]] = add nsw i32 [[TMP21]], 1
+// CHECK2-NEXT:    store i32 [[INC28]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND3]], !llvm.loop [[LOOP17:![0-9]+]]
-// CHECK2:       for.end21:
-// CHECK2-NEXT:    br label [[FOR_INC22:%.*]]
-// CHECK2:       for.inc22:
-// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP25]], 5
-// CHECK2-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2:       for.end29:
+// CHECK2-NEXT:    br label [[FOR_INC30:%.*]]
+// CHECK2:       for.inc30:
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD31:%.*]] = add nsw i32 [[TMP22]], 5
+// CHECK2-NEXT:    store i32 [[ADD31]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP18:![0-9]+]]
-// CHECK2:       for.end24:
+// CHECK2:       for.end32:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
-// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[ADD25:%.*]] = add nsw i32 [[TMP26]], 1
-// CHECK2-NEXT:    store i32 [[ADD25]], ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[ADD33:%.*]] = add nsw i32 [[TMP23]], 1
+// CHECK2-NEXT:    store i32 [[ADD33]], ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -1771,9 +1832,7 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[_TMP1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
@@ -1781,8 +1840,8 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[K:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK2-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK2-NEXT:    store i32 7, ptr [[J]], align 4
@@ -1829,83 +1888,87 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP9]], 4
-// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY:%.*]], label [[FOR_END29:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP7]], label [[FOR_BODY:%.*]], label [[FOR_END37:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[TMP10]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND8:%.*]]
 // CHECK2:       for.cond8:
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP10]], 5
-// CHECK2-NEXT:    br i1 [[CMP9]], label [[FOR_BODY10:%.*]], label [[FOR_END26:%.*]]
-// CHECK2:       for.body10:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP11]], [[TMP12]]
-// CHECK2-NEXT:    store i32 [[ADD11]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL12:%.*]] = mul nsw i32 [[TMP13]], 3
-// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i32 7, [[MUL12]]
-// CHECK2-NEXT:    store i32 [[ADD13]], ptr [[I]], align 4
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND14:%.*]]
-// CHECK2:       for.cond14:
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[CMP15:%.*]] = icmp slt i32 [[TMP14]], 5
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i32 [[TMP15]], [[TMP16]]
-// CHECK2-NEXT:    [[CMP17:%.*]] = icmp slt i32 [[ADD16]], 4
-// CHECK2-NEXT:    br i1 [[CMP17]], label [[LAND_RHS:%.*]], label [[LAND_END:%.*]]
-// CHECK2:       land.rhs:
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i32 [[TMP12]], 5
+// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i32 4, [[ADD9]]
+// CHECK2-NEXT:    br i1 [[CMP10]], label [[COND_TRUE11:%.*]], label [[COND_FALSE12:%.*]]
+// CHECK2:       cond.true11:
+// CHECK2-NEXT:    br label [[COND_END14:%.*]]
+// CHECK2:       cond.false12:
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i32 [[TMP13]], 5
+// CHECK2-NEXT:    br label [[COND_END14]]
+// CHECK2:       cond.end14:
+// CHECK2-NEXT:    [[COND15:%.*]] = phi i32 [ 4, [[COND_TRUE11]] ], [ [[ADD13]], [[COND_FALSE12]] ]
+// CHECK2-NEXT:    [[CMP16:%.*]] = icmp slt i32 [[TMP11]], [[COND15]]
+// CHECK2-NEXT:    br i1 [[CMP16]], label [[FOR_BODY17:%.*]], label [[FOR_END34:%.*]]
+// CHECK2:       for.body17:
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL18:%.*]] = mul nsw i32 [[TMP14]], 3
+// CHECK2-NEXT:    [[ADD19:%.*]] = add nsw i32 7, [[MUL18]]
+// CHECK2-NEXT:    store i32 [[ADD19]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    store i32 [[TMP15]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND20:%.*]]
+// CHECK2:       for.cond20:
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
 // CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD18:%.*]] = add nsw i32 [[TMP17]], [[TMP18]]
-// CHECK2-NEXT:    [[CMP19:%.*]] = icmp slt i32 [[ADD18]], 4
-// CHECK2-NEXT:    br label [[LAND_END]]
-// CHECK2:       land.end:
-// CHECK2-NEXT:    [[TMP19:%.*]] = phi i1 [ false, [[FOR_COND14]] ], [ [[CMP19]], [[LAND_RHS]] ]
-// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP15]], [[TMP19]]
-// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY20:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body20:
-// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD21:%.*]] = add nsw i32 [[TMP20]], [[TMP21]]
-// CHECK2-NEXT:    store i32 [[ADD21]], ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// CHECK2-NEXT:    [[MUL22:%.*]] = mul nsw i32 [[TMP22]], 3
-// CHECK2-NEXT:    [[ADD23:%.*]] = add nsw i32 7, [[MUL22]]
-// CHECK2-NEXT:    store i32 [[ADD23]], ptr [[J]], align 4
-// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[J]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP23]], i32 noundef [[TMP24]])
+// CHECK2-NEXT:    [[ADD21:%.*]] = add nsw i32 [[TMP17]], 5
+// CHECK2-NEXT:    [[CMP22:%.*]] = icmp slt i32 4, [[ADD21]]
+// CHECK2-NEXT:    br i1 [[CMP22]], label [[COND_TRUE23:%.*]], label [[COND_FALSE24:%.*]]
+// CHECK2:       cond.true23:
+// CHECK2-NEXT:    br label [[COND_END26:%.*]]
+// CHECK2:       cond.false24:
+// CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD25:%.*]] = add nsw i32 [[TMP18]], 5
+// CHECK2-NEXT:    br label [[COND_END26]]
+// CHECK2:       cond.end26:
+// CHECK2-NEXT:    [[COND27:%.*]] = phi i32 [ 4, [[COND_TRUE23]] ], [ [[ADD25]], [[COND_FALSE24]] ]
+// CHECK2-NEXT:    [[CMP28:%.*]] = icmp slt i32 [[TMP16]], [[COND27]]
+// CHECK2-NEXT:    br i1 [[CMP28]], label [[FOR_BODY29:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body29:
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[MUL30:%.*]] = mul nsw i32 [[TMP19]], 3
+// CHECK2-NEXT:    [[ADD31:%.*]] = add nsw i32 7, [[MUL30]]
+// CHECK2-NEXT:    store i32 [[ADD31]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP20:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP21:%.*]] = load i32, ptr [[J]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP20]], i32 noundef [[TMP21]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP25]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// CHECK2-NEXT:    br label [[FOR_COND14]], !llvm.loop [[LOOP19:![0-9]+]]
+// CHECK2-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP22]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
+// CHECK2-NEXT:    br label [[FOR_COND20]], !llvm.loop [[LOOP19:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC24:%.*]]
-// CHECK2:       for.inc24:
-// CHECK2-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC25:%.*]] = add nsw i32 [[TMP26]], 1
-// CHECK2-NEXT:    store i32 [[INC25]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    br label [[FOR_INC32:%.*]]
+// CHECK2:       for.inc32:
+// CHECK2-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC33:%.*]] = add nsw i32 [[TMP23]], 1
+// CHECK2-NEXT:    store i32 [[INC33]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND8]], !llvm.loop [[LOOP20:![0-9]+]]
-// CHECK2:       for.end26:
-// CHECK2-NEXT:    br label [[FOR_INC27:%.*]]
-// CHECK2:       for.inc27:
-// CHECK2-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// CHECK2-NEXT:    [[ADD28:%.*]] = add nsw i32 [[TMP27]], 5
-// CHECK2-NEXT:    store i32 [[ADD28]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2:       for.end34:
+// CHECK2-NEXT:    br label [[FOR_INC35:%.*]]
+// CHECK2:       for.inc35:
+// CHECK2-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// CHECK2-NEXT:    [[ADD36:%.*]] = add nsw i32 [[TMP24]], 5
+// CHECK2-NEXT:    store i32 [[ADD36]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP21:![0-9]+]]
-// CHECK2:       for.end29:
+// CHECK2:       for.end37:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
-// CHECK2-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[ADD30:%.*]] = add nsw i32 [[TMP28]], 1
-// CHECK2-NEXT:    store i32 [[ADD30]], ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[ADD38:%.*]] = add nsw i32 [[TMP25]], 1
+// CHECK2-NEXT:    store i32 [[ADD38]], ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -1923,91 +1986,90 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[_TMP1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[_TMP2:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[J:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2]])
 // CHECK2-NEXT:    store i32 7, ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[TMP]], align 4
+// CHECK2-NEXT:    store i32 [[TMP1]], ptr [[DOTCAPTURE_EXPR_]], align 4
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TMP]], align 4
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], 5
+// CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTCAPTURE_EXPR_3]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
 // CHECK2-NEXT:    store i32 19, ptr [[DOTOMP_UB]], align 4
 // CHECK2-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
 // CHECK2-NEXT:    call void @__kmpc_for_static_init_4(ptr @[[GLOB1]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
-// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// CHECK2-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP1]], 19
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[TMP3]], 19
 // CHECK2-NEXT:    br i1 [[CMP]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK2:       cond.true:
 // CHECK2-NEXT:    br label [[COND_END:%.*]]
 // CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
 // CHECK2-NEXT:    br label [[COND_END]]
 // CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 19, [[COND_TRUE]] ], [ [[TMP2]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 19, [[COND_TRUE]] ], [ [[TMP4]], [[COND_FALSE]] ]
 // CHECK2-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
-// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
-// CHECK2-NEXT:    store i32 [[TMP3]], ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
+// CHECK2-NEXT:    store i32 [[TMP5]], ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND:%.*]]
 // CHECK2:       omp.inner.for.cond:
-// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// CHECK2-NEXT:    [[CMP3:%.*]] = icmp sle i32 [[TMP4]], [[TMP5]]
-// CHECK2-NEXT:    br i1 [[CMP3]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
-// CHECK2:       omp.inner.for.body:
 // CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP6]], 20
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[DIV]], 5
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// CHECK2-NEXT:    [[CMP4:%.*]] = icmp sle i32 [[TMP6]], [[TMP7]]
+// CHECK2-NEXT:    br i1 [[CMP4]], label [[OMP_INNER_FOR_BODY:%.*]], label [[OMP_INNER_FOR_END:%.*]]
+// CHECK2:       omp.inner.for.body:
 // CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[DIV4:%.*]] = sdiv i32 [[TMP8]], 20
-// CHECK2-NEXT:    [[MUL5:%.*]] = mul nsw i32 [[DIV4]], 20
-// CHECK2-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP7]], [[MUL5]]
-// CHECK2-NEXT:    [[DIV6:%.*]] = sdiv i32 [[SUB]], 4
-// CHECK2-NEXT:    [[MUL7:%.*]] = mul nsw i32 [[DIV6]], 1
-// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i32 0, [[MUL7]]
-// CHECK2-NEXT:    store i32 [[ADD8]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[DIV:%.*]] = sdiv i32 [[TMP8]], 20
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[DIV]], 5
+// CHECK2-NEXT:    [[ADD5:%.*]] = add nsw i32 0, [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD5]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[DIV9:%.*]] = sdiv i32 [[TMP10]], 20
-// CHECK2-NEXT:    [[MUL10:%.*]] = mul nsw i32 [[DIV9]], 20
-// CHECK2-NEXT:    [[SUB11:%.*]] = sub nsw i32 [[TMP9]], [[MUL10]]
 // CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[DIV6:%.*]] = sdiv i32 [[TMP11]], 20
+// CHECK2-NEXT:    [[MUL7:%.*]] = mul nsw i32 [[DIV6]], 20
+// CHECK2-NEXT:    [[SUB:%.*]] = sub nsw i32 [[TMP10]], [[MUL7]]
+// CHECK2-NEXT:    [[DIV8:%.*]] = sdiv i32 [[SUB]], 4
+// CHECK2-NEXT:    [[MUL9:%.*]] = mul nsw i32 [[DIV8]], 1
+// CHECK2-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP9]], [[MUL9]]
+// CHECK2-NEXT:    store i32 [[ADD10]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[DIV12:%.*]] = sdiv i32 [[TMP12]], 20
-// CHECK2-NEXT:    [[MUL13:%.*]] = mul nsw i32 [[DIV12]], 20
-// CHECK2-NEXT:    [[SUB14:%.*]] = sub nsw i32 [[TMP11]], [[MUL13]]
-// CHECK2-NEXT:    [[DIV15:%.*]] = sdiv i32 [[SUB14]], 4
-// CHECK2-NEXT:    [[MUL16:%.*]] = mul nsw i32 [[DIV15]], 4
-// CHECK2-NEXT:    [[SUB17:%.*]] = sub nsw i32 [[SUB11]], [[MUL16]]
-// CHECK2-NEXT:    [[MUL18:%.*]] = mul nsw i32 [[SUB17]], 3
-// CHECK2-NEXT:    [[ADD19:%.*]] = add nsw i32 7, [[MUL18]]
-// CHECK2-NEXT:    store i32 [[ADD19]], ptr [[J]], align 4
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD20:%.*]] = add nsw i32 [[TMP13]], [[TMP14]]
-// CHECK2-NEXT:    store i32 [[ADD20]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL21:%.*]] = mul nsw i32 [[TMP15]], 3
-// CHECK2-NEXT:    [[ADD22:%.*]] = add nsw i32 7, [[MUL21]]
-// CHECK2-NEXT:    store i32 [[ADD22]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP16]], [[TMP17]]
-// CHECK2-NEXT:    [[CMP24:%.*]] = icmp slt i32 [[ADD23]], 4
-// CHECK2-NEXT:    br i1 [[CMP24]], label [[OMP_TILE_PRED_THEN:%.*]], label [[OMP_TILE_PRED_END:%.*]]
-// CHECK2:       omp_tile.pred.then:
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[DIV11:%.*]] = sdiv i32 [[TMP13]], 20
+// CHECK2-NEXT:    [[MUL12:%.*]] = mul nsw i32 [[DIV11]], 20
+// CHECK2-NEXT:    [[SUB13:%.*]] = sub nsw i32 [[TMP12]], [[MUL12]]
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[DIV14:%.*]] = sdiv i32 [[TMP15]], 20
+// CHECK2-NEXT:    [[MUL15:%.*]] = mul nsw i32 [[DIV14]], 20
+// CHECK2-NEXT:    [[SUB16:%.*]] = sub nsw i32 [[TMP14]], [[MUL15]]
+// CHECK2-NEXT:    [[DIV17:%.*]] = sdiv i32 [[SUB16]], 4
+// CHECK2-NEXT:    [[MUL18:%.*]] = mul nsw i32 [[DIV17]], 4
+// CHECK2-NEXT:    [[SUB19:%.*]] = sub nsw i32 [[SUB13]], [[MUL18]]
+// CHECK2-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[SUB19]], 3
+// CHECK2-NEXT:    [[ADD21:%.*]] = add nsw i32 7, [[MUL20]]
+// CHECK2-NEXT:    store i32 [[ADD21]], ptr [[J]], align 4
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP22:%.*]] = icmp slt i32 [[TMP16]], 4
+// CHECK2-NEXT:    br i1 [[CMP22]], label [[OMP_BODY_NEXT:%.*]], label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK2:       omp.body.next:
+// CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL23:%.*]] = mul nsw i32 [[TMP17]], 3
+// CHECK2-NEXT:    [[ADD24:%.*]] = add nsw i32 7, [[MUL23]]
+// CHECK2-NEXT:    store i32 [[ADD24]], ptr [[I]], align 4
 // CHECK2-NEXT:    [[TMP18:%.*]] = load i32, ptr [[I]], align 4
 // CHECK2-NEXT:    [[TMP19:%.*]] = load i32, ptr [[J]], align 4
 // CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP18]], i32 noundef [[TMP19]])
-// CHECK2-NEXT:    br label [[OMP_TILE_PRED_END]]
-// CHECK2:       omp_tile.pred.end:
-// CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
+// CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
@@ -2038,13 +2100,12 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[TMP:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store ptr [[DOTGLOBAL_TID_]], ptr [[DOTGLOBAL_TID__ADDR]], align 8
 // CHECK2-NEXT:    store ptr [[DOTBOUND_TID_]], ptr [[DOTBOUND_TID__ADDR]], align 8
 // CHECK2-NEXT:    store i32 7, ptr [[I]], align 4
@@ -2079,42 +2140,46 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP7]], 5
 // CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 0, [[MUL]]
 // CHECK2-NEXT:    store i32 [[ADD]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[TMP8]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND:%.*]]
 // CHECK2:       for.cond:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP8]], 5
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
-// CHECK2-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[ADD3]], 4
-// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP2]], [[CMP4]]
-// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body:
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD2:%.*]] = add nsw i32 [[TMP10]], 5
+// CHECK2-NEXT:    [[CMP3:%.*]] = icmp slt i32 4, [[ADD2]]
+// CHECK2-NEXT:    br i1 [[CMP3]], label [[COND_TRUE4:%.*]], label [[COND_FALSE5:%.*]]
+// CHECK2:       cond.true4:
+// CHECK2-NEXT:    br label [[COND_END7:%.*]]
+// CHECK2:       cond.false5:
 // CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD5:%.*]] = add nsw i32 [[TMP11]], [[TMP12]]
-// CHECK2-NEXT:    store i32 [[ADD5]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL6:%.*]] = mul nsw i32 [[TMP13]], 3
-// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL6]]
-// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP14]])
+// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP11]], 5
+// CHECK2-NEXT:    br label [[COND_END7]]
+// CHECK2:       cond.end7:
+// CHECK2-NEXT:    [[COND8:%.*]] = phi i32 [ 4, [[COND_TRUE4]] ], [ [[ADD6]], [[COND_FALSE5]] ]
+// CHECK2-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP9]], [[COND8]]
+// CHECK2-NEXT:    br i1 [[CMP9]], label [[FOR_BODY:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL10:%.*]] = mul nsw i32 [[TMP12]], 3
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 7, [[MUL10]]
+// CHECK2-NEXT:    store i32 [[ADD11]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP13]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP15]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP22:![0-9]+]]
 // CHECK2:       for.end:
 // CHECK2-NEXT:    br label [[OMP_BODY_CONTINUE:%.*]]
 // CHECK2:       omp.body.continue:
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_INC:%.*]]
 // CHECK2:       omp.inner.for.inc:
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP16]], 1
-// CHECK2-NEXT:    store i32 [[ADD8]], ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
+// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP15]], 1
+// CHECK2-NEXT:    store i32 [[ADD12]], ptr [[DOTOMP_IV]], align 4
 // CHECK2-NEXT:    br label [[OMP_INNER_FOR_COND]]
 // CHECK2:       omp.inner.for.end:
 // CHECK2-NEXT:    br label [[OMP_LOOP_EXIT:%.*]]
@@ -2128,9 +2193,8 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:  entry:
 // CHECK2-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[A]], ptr [[A_ADDR]], align 4
 // CHECK2-NEXT:    store i32 7, ptr [[I]], align 4
 // CHECK2-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
@@ -2138,64 +2202,78 @@ extern "C" void foo10(data_t data) {
 // CHECK2:       for.cond:
 // CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 4
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END15:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END24:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND1:%.*]]
 // CHECK2:       for.cond1:
-// CHECK2-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP2]], 0
+// CHECK2-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], 0
 // CHECK2-NEXT:    br i1 [[CMP2]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
 // CHECK2:       cond.true:
 // CHECK2-NEXT:    br label [[COND_END:%.*]]
 // CHECK2:       cond.false:
-// CHECK2-NEXT:    [[TMP3:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[A_ADDR]], align 4
 // CHECK2-NEXT:    br label [[COND_END]]
 // CHECK2:       cond.end:
-// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP3]], [[COND_FALSE]] ]
-// CHECK2-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[TMP1]], [[COND]]
-// CHECK2-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
-// CHECK2-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[ADD]], 4
-// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP3]], [[CMP4]]
-// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY5:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body5:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ 1, [[COND_TRUE]] ], [ [[TMP5]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[COND]]
+// CHECK2-NEXT:    [[CMP3:%.*]] = icmp slt i32 4, [[ADD]]
+// CHECK2-NEXT:    br i1 [[CMP3]], label [[COND_TRUE4:%.*]], label [[COND_FALSE5:%.*]]
+// CHECK2:       cond.true4:
+// CHECK2-NEXT:    br label [[COND_END12:%.*]]
+// CHECK2:       cond.false5:
 // CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP6]], [[TMP7]]
-// CHECK2-NEXT:    store i32 [[ADD6]], ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP8]], 3
-// CHECK2-NEXT:    [[ADD7:%.*]] = add nsw i32 7, [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[I]], align 4
-// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP9]])
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    [[CMP6:%.*]] = icmp sle i32 [[TMP7]], 0
+// CHECK2-NEXT:    br i1 [[CMP6]], label [[COND_TRUE7:%.*]], label [[COND_FALSE8:%.*]]
+// CHECK2:       cond.true7:
+// CHECK2-NEXT:    br label [[COND_END9:%.*]]
+// CHECK2:       cond.false8:
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    br label [[COND_END9]]
+// CHECK2:       cond.end9:
+// CHECK2-NEXT:    [[COND10:%.*]] = phi i32 [ 1, [[COND_TRUE7]] ], [ [[TMP8]], [[COND_FALSE8]] ]
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP6]], [[COND10]]
+// CHECK2-NEXT:    br label [[COND_END12]]
+// CHECK2:       cond.end12:
+// CHECK2-NEXT:    [[COND13:%.*]] = phi i32 [ 4, [[COND_TRUE4]] ], [ [[ADD11]], [[COND_END9]] ]
+// CHECK2-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP2]], [[COND13]]
+// CHECK2-NEXT:    br i1 [[CMP14]], label [[FOR_BODY15:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body15:
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP9]], 3
+// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i32 7, [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
+// CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP10]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND1]], !llvm.loop [[LOOP25:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC8:%.*]]
-// CHECK2:       for.inc8:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    [[CMP9:%.*]] = icmp sle i32 [[TMP11]], 0
-// CHECK2-NEXT:    br i1 [[CMP9]], label [[COND_TRUE10:%.*]], label [[COND_FALSE11:%.*]]
-// CHECK2:       cond.true10:
-// CHECK2-NEXT:    br label [[COND_END12:%.*]]
-// CHECK2:       cond.false11:
+// CHECK2-NEXT:    br label [[FOR_INC17:%.*]]
+// CHECK2:       for.inc17:
 // CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[A_ADDR]], align 4
-// CHECK2-NEXT:    br label [[COND_END12]]
-// CHECK2:       cond.end12:
-// CHECK2-NEXT:    [[COND13:%.*]] = phi i32 [ 1, [[COND_TRUE10]] ], [ [[TMP12]], [[COND_FALSE11]] ]
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD14:%.*]] = add nsw i32 [[TMP13]], [[COND13]]
-// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[CMP18:%.*]] = icmp sle i32 [[TMP12]], 0
+// CHECK2-NEXT:    br i1 [[CMP18]], label [[COND_TRUE19:%.*]], label [[COND_FALSE20:%.*]]
+// CHECK2:       cond.true19:
+// CHECK2-NEXT:    br label [[COND_END21:%.*]]
+// CHECK2:       cond.false20:
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[A_ADDR]], align 4
+// CHECK2-NEXT:    br label [[COND_END21]]
+// CHECK2:       cond.end21:
+// CHECK2-NEXT:    [[COND22:%.*]] = phi i32 [ 1, [[COND_TRUE19]] ], [ [[TMP13]], [[COND_FALSE20]] ]
+// CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP14]], [[COND22]]
+// CHECK2-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP26:![0-9]+]]
-// CHECK2:       for.end15:
+// CHECK2:       for.end24:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -2208,9 +2286,8 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_3:%.*]] = alloca ptr, align 8
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_4:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV___BEGIN2:%.*]] = alloca i64, align 8
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV___BEGIN2:%.*]] = alloca i64, align 8
+// CHECK2-NEXT:    [[DOTTILE_0_IV___BEGIN2:%.*]] = alloca i64, align 8
 // CHECK2-NEXT:    [[V:%.*]] = alloca double, align 8
 // CHECK2-NEXT:    [[ARRAY:%.*]] = getelementptr inbounds nuw [[STRUCT_DATA_T]], ptr [[DATA]], i32 0, i32 0
 // CHECK2-NEXT:    store ptr [[ARRAY]], ptr [[__RANGE2]], align 8
@@ -2244,31 +2321,37 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP7:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
 // CHECK2-NEXT:    [[ADD6:%.*]] = add nsw i64 [[TMP7]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp slt i64 [[TMP6]], [[ADD6]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END18:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    store i64 0, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    store i64 [[TMP8]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND7:%.*]]
 // CHECK2:       for.cond7:
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[CMP8:%.*]] = icmp slt i64 [[TMP8]], 5
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP9]], [[TMP10]]
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
-// CHECK2-NEXT:    [[ADD10:%.*]] = add nsw i64 [[TMP11]], 1
-// CHECK2-NEXT:    [[CMP11:%.*]] = icmp slt i64 [[ADD9]], [[ADD10]]
-// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP8]], [[CMP11]]
-// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY12:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body12:
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD13:%.*]] = add nsw i64 [[TMP12]], [[TMP13]]
-// CHECK2-NEXT:    store i64 [[ADD13]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK2-NEXT:    [[ADD8:%.*]] = add nsw i64 [[TMP10]], 1
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD9:%.*]] = add nsw i64 [[TMP11]], 5
+// CHECK2-NEXT:    [[CMP10:%.*]] = icmp slt i64 [[ADD8]], [[ADD9]]
+// CHECK2-NEXT:    br i1 [[CMP10]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i64, ptr [[DOTCAPTURE_EXPR_4]], align 8
+// CHECK2-NEXT:    [[ADD11:%.*]] = add nsw i64 [[TMP12]], 1
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP13:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD12:%.*]] = add nsw i64 [[TMP13]], 5
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i64 [ [[ADD11]], [[COND_TRUE]] ], [ [[ADD12]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[CMP13:%.*]] = icmp slt i64 [[TMP9]], [[COND]]
+// CHECK2-NEXT:    br i1 [[CMP13]], label [[FOR_BODY14:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body14:
 // CHECK2-NEXT:    [[TMP14:%.*]] = load ptr, ptr [[DOTCAPTURE_EXPR_]], align 8
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    [[MUL:%.*]] = mul nsw i64 [[TMP15]], 1
-// CHECK2-NEXT:    [[ADD_PTR14:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
-// CHECK2-NEXT:    store ptr [[ADD_PTR14]], ptr [[__BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD_PTR15:%.*]] = getelementptr inbounds double, ptr [[TMP14]], i64 [[MUL]]
+// CHECK2-NEXT:    store ptr [[ADD_PTR15]], ptr [[__BEGIN2]], align 8
 // CHECK2-NEXT:    [[TMP16:%.*]] = load ptr, ptr [[__BEGIN2]], align 8
 // CHECK2-NEXT:    [[TMP17:%.*]] = load double, ptr [[TMP16]], align 8
 // CHECK2-NEXT:    store double [[TMP17]], ptr [[V]], align 8
@@ -2276,18 +2359,18 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    call void (...) @body(double noundef [[TMP18]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[TMP19:%.*]] = load i64, ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    [[INC:%.*]] = add nsw i64 [[TMP19]], 1
-// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_CNT_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    store i64 [[INC]], ptr [[DOTTILE_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND7]], !llvm.loop [[LOOP27:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC15:%.*]]
-// CHECK2:       for.inc15:
+// CHECK2-NEXT:    br label [[FOR_INC16:%.*]]
+// CHECK2:       for.inc16:
 // CHECK2-NEXT:    [[TMP20:%.*]] = load i64, ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
-// CHECK2-NEXT:    [[ADD16:%.*]] = add nsw i64 [[TMP20]], 5
-// CHECK2-NEXT:    store i64 [[ADD16]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
+// CHECK2-NEXT:    [[ADD17:%.*]] = add nsw i64 [[TMP20]], 5
+// CHECK2-NEXT:    store i64 [[ADD17]], ptr [[DOTFLOOR_0_IV___BEGIN2]], align 8
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP28:![0-9]+]]
-// CHECK2:       for.end17:
+// CHECK2:       for.end18:
 // CHECK2-NEXT:    ret void
 //
 //
@@ -2307,9 +2390,8 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// CHECK2-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// CHECK2-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // CHECK2-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // CHECK2-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
 // CHECK2-NEXT:    [[TMP0:%.*]] = load i32, ptr [[START_ADDR]], align 4
@@ -2333,47 +2415,53 @@ extern "C" void foo10(data_t data) {
 // CHECK2-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
 // CHECK2-NEXT:    [[ADD5:%.*]] = add i32 [[TMP6]], 1
 // CHECK2-NEXT:    [[CMP:%.*]] = icmp ult i32 [[TMP5]], [[ADD5]]
-// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END16:%.*]]
+// CHECK2-NEXT:    br i1 [[CMP]], label [[FOR_BODY:%.*]], label [[FOR_END17:%.*]]
 // CHECK2:       for.body:
-// CHECK2-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND6:%.*]]
 // CHECK2:       for.cond6:
-// CHECK2-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[CMP7:%.*]] = icmp ult i32 [[TMP7]], 5
-// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD8:%.*]] = add i32 [[TMP8]], [[TMP9]]
-// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// CHECK2-NEXT:    [[ADD9:%.*]] = add i32 [[TMP10]], 1
-// CHECK2-NEXT:    [[CMP10:%.*]] = icmp ult i32 [[ADD8]], [[ADD9]]
-// CHECK2-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP7]], [[CMP10]]
-// CHECK2-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label [[FOR_BODY11:%.*]], label [[FOR_END:%.*]]
-// CHECK2:       for.body11:
-// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD12:%.*]] = add i32 [[TMP11]], [[TMP12]]
-// CHECK2-NEXT:    store i32 [[ADD12]], ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD7:%.*]] = add i32 [[TMP9]], 1
+// CHECK2-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD8:%.*]] = add i32 [[TMP10]], 5
+// CHECK2-NEXT:    [[CMP9:%.*]] = icmp ult i32 [[ADD7]], [[ADD8]]
+// CHECK2-NEXT:    br i1 [[CMP9]], label [[COND_TRUE:%.*]], label [[COND_FALSE:%.*]]
+// CHECK2:       cond.true:
+// CHECK2-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// CHECK2-NEXT:    [[ADD10:%.*]] = add i32 [[TMP11]], 1
+// CHECK2-NEXT:    br label [[COND_END:%.*]]
+// CHECK2:       cond.false:
+// CHECK2-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD11:%.*]] = add i32 [[TMP12]], 5
+// CHECK2-NEXT:    br label [[COND_END]]
+// CHECK2:       cond.end:
+// CHECK2-NEXT:    [[COND:%.*]] = phi i32 [ [[ADD10]], [[COND_TRUE]] ], [ [[ADD11]], [[COND_FALSE]] ]
+// CHECK2-NEXT:    [[CMP12:%.*]] = icmp ult i32 [[TMP8]], [[COND]]
+// CHECK2-NEXT:    br i1 [[CMP12]], label [[FOR_BODY13:%.*]], label [[FOR_END:%.*]]
+// CHECK2:       for.body13:
 // CHECK2-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
 // CHECK2-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    [[MUL:%.*]] = mul i32 [[TMP14]], 3
-// CHECK2-NEXT:    [[ADD13:%.*]] = add i32 [[TMP13]], [[MUL]]
-// CHECK2-NEXT:    store i32 [[ADD13]], ptr [[I]], align 4
+// CHECK2-NEXT:    [[ADD14:%.*]] = add i32 [[TMP13]], [[MUL]]
+// CHECK2-NEXT:    store i32 [[ADD14]], ptr [[I]], align 4
 // CHECK2-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
 // CHECK2-NEXT:    call void (...) @body(i32 noundef [[TMP15]])
 // CHECK2-NEXT:    br label [[FOR_INC:%.*]]
 // CHECK2:       for.inc:
-// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    [[INC:%.*]] = add i32 [[TMP16]], 1
-// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// CHECK2-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND6]], !llvm.loop [[LOOP29:![0-9]+]]
 // CHECK2:       for.end:
-// CHECK2-NEXT:    br label [[FOR_INC14:%.*]]
-// CHECK2:       for.inc14:
+// CHECK2-NEXT:    br label [[FOR_INC15:%.*]]
+// CHECK2:       for.inc15:
 // CHECK2-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// CHECK2-NEXT:    [[ADD15:%.*]] = add i32 [[TMP17]], 5
-// CHECK2-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// CHECK2-NEXT:    [[ADD16:%.*]] = add i32 [[TMP17]], 5
+// CHECK2-NEXT:    store i32 [[ADD16]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // CHECK2-NEXT:    br label [[FOR_COND]], !llvm.loop [[LOOP30:![0-9]+]]
-// CHECK2:       for.end16:
+// CHECK2:       for.end17:
 // CHECK2-NEXT:    ret void
 //
 //
diff --git a/clang/test/OpenMP/tile_codegen_for_dependent.cpp b/clang/test/OpenMP/tile_codegen_for_dependent.cpp
index 808f0e71dc3b0..7401ab1a2ded3 100644
--- a/clang/test/OpenMP/tile_codegen_for_dependent.cpp
+++ b/clang/test/OpenMP/tile_codegen_for_dependent.cpp
@@ -31,7 +31,6 @@ extern "C" void body(...) {}
 // IR-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTCAPTURE_EXPR_5:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
@@ -40,7 +39,7 @@ extern "C" void body(...) {}
 // IR-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_0_IV_I11:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
 // IR-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // IR-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
@@ -109,30 +108,36 @@ extern "C" void body(...) {}
 // IR-NEXT:    [[MUL:%.*]] = mul i32 [[TMP20]], 4
 // IR-NEXT:    [[ADD15:%.*]] = add i32 0, [[MUL]]
 // IR-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I11]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I11]], align 4
+// IR-NEXT:    store i32 [[TMP21]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND:.*]]
 // IR:       [[FOR_COND]]:
-// IR-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[CMP16:%.*]] = icmp ult i32 [[TMP21]], 4
-// IR-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I11]], align 4
-// IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD17:%.*]] = add i32 [[TMP22]], [[TMP23]]
-// IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[ADD18:%.*]] = add i32 [[TMP24]], 1
-// IR-NEXT:    [[CMP19:%.*]] = icmp ult i32 [[ADD17]], [[ADD18]]
-// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP16]], [[CMP19]]
-// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
+// IR-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    [[ADD16:%.*]] = add i32 [[TMP23]], 1
+// IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I11]], align 4
+// IR-NEXT:    [[ADD17:%.*]] = add i32 [[TMP24]], 4
+// IR-NEXT:    [[CMP18:%.*]] = icmp ult i32 [[ADD16]], [[ADD17]]
+// IR-NEXT:    br i1 [[CMP18]], label %[[COND_TRUE19:.*]], label %[[COND_FALSE21:.*]]
+// IR:       [[COND_TRUE19]]:
+// IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    [[ADD20:%.*]] = add i32 [[TMP25]], 1
+// IR-NEXT:    br label %[[COND_END23:.*]]
+// IR:       [[COND_FALSE21]]:
+// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I11]], align 4
+// IR-NEXT:    [[ADD22:%.*]] = add i32 [[TMP26]], 4
+// IR-NEXT:    br label %[[COND_END23]]
+// IR:       [[COND_END23]]:
+// IR-NEXT:    [[COND24:%.*]] = phi i32 [ [[ADD20]], %[[COND_TRUE19]] ], [ [[ADD22]], %[[COND_FALSE21]] ]
+// IR-NEXT:    [[CMP25:%.*]] = icmp ult i32 [[TMP22]], [[COND24]]
+// IR-NEXT:    br i1 [[CMP25]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
 // IR:       [[FOR_BODY]]:
-// IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I11]], align 4
-// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD20:%.*]] = add i32 [[TMP25]], [[TMP26]]
-// IR-NEXT:    store i32 [[ADD20]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
 // IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[MUL21:%.*]] = mul i32 [[TMP28]], [[TMP29]]
-// IR-NEXT:    [[ADD22:%.*]] = add i32 [[TMP27]], [[MUL21]]
-// IR-NEXT:    store i32 [[ADD22]], ptr [[I]], align 4
+// IR-NEXT:    [[MUL26:%.*]] = mul i32 [[TMP28]], [[TMP29]]
+// IR-NEXT:    [[ADD27:%.*]] = add i32 [[TMP27]], [[MUL26]]
+// IR-NEXT:    store i32 [[ADD27]], ptr [[I]], align 4
 // IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[START_ADDR]], align 4
 // IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[END_ADDR]], align 4
 // IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
@@ -140,9 +145,9 @@ extern "C" void body(...) {}
 // IR-NEXT:    call void (...) @body(i32 noundef [[TMP30]], i32 noundef [[TMP31]], i32 noundef [[TMP32]], i32 noundef [[TMP33]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    [[INC:%.*]] = add i32 [[TMP34]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP2:![0-9]+]]
 // IR:       [[FOR_END]]:
 // IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.*]]
@@ -150,8 +155,8 @@ extern "C" void body(...) {}
 // IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.*]]
 // IR:       [[OMP_INNER_FOR_INC]]:
 // IR-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[ADD23:%.*]] = add i32 [[TMP35]], 1
-// IR-NEXT:    store i32 [[ADD23]], ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[ADD28:%.*]] = add i32 [[TMP35]], 1
+// IR-NEXT:    store i32 [[ADD28]], ptr [[DOTOMP_IV]], align 4
 // IR-NEXT:    br label %[[OMP_INNER_FOR_COND]]
 // IR:       [[OMP_INNER_FOR_END]]:
 // IR-NEXT:    br label %[[OMP_LOOP_EXIT:.*]]
diff --git a/clang/test/OpenMP/tile_codegen_tile_for.cpp b/clang/test/OpenMP/tile_codegen_tile_for.cpp
index 9ef96834fcfc0..3d0092042cdfc 100644
--- a/clang/test/OpenMP/tile_codegen_tile_for.cpp
+++ b/clang/test/OpenMP/tile_codegen_tile_for.cpp
@@ -32,11 +32,9 @@ extern "C" void body(...) {}
 // IR-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTCAPTURE_EXPR_5:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV__FLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTCAPTURE_EXPR_13:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_0_IV__FLOOR_0_IV_I:%.*]] = alloca i32, align 4
@@ -45,8 +43,8 @@ extern "C" void body(...) {}
 // IR-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_0_IV__FLOOR_0_IV_I17:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV__FLOOR_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
 // IR-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
 // IR-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
@@ -124,52 +122,64 @@ extern "C" void body(...) {}
 // IR-NEXT:    [[MUL:%.*]] = mul i32 [[TMP22]], 3
 // IR-NEXT:    [[ADD21:%.*]] = add i32 0, [[MUL]]
 // IR-NEXT:    store i32 [[ADD21]], ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
+// IR-NEXT:    store i32 [[TMP23]], ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND:.*]]
 // IR:       [[FOR_COND]]:
-// IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[CMP22:%.*]] = icmp ult i32 [[TMP23]], 3
-// IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
-// IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD23:%.*]] = add i32 [[TMP24]], [[TMP25]]
-// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    [[ADD24:%.*]] = add i32 [[TMP26]], 1
-// IR-NEXT:    [[CMP25:%.*]] = icmp ult i32 [[ADD23]], [[ADD24]]
-// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP22]], [[CMP25]]
-// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label %[[FOR_BODY:.*]], label %[[FOR_END41:.*]]
+// IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    [[ADD22:%.*]] = add i32 [[TMP25]], 1
+// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
+// IR-NEXT:    [[ADD23:%.*]] = add i32 [[TMP26]], 3
+// IR-NEXT:    [[CMP24:%.*]] = icmp ult i32 [[ADD22]], [[ADD23]]
+// IR-NEXT:    br i1 [[CMP24]], label %[[COND_TRUE25:.*]], label %[[COND_FALSE27:.*]]
+// IR:       [[COND_TRUE25]]:
+// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    [[ADD26:%.*]] = add i32 [[TMP27]], 1
+// IR-NEXT:    br label %[[COND_END29:.*]]
+// IR:       [[COND_FALSE27]]:
+// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
+// IR-NEXT:    [[ADD28:%.*]] = add i32 [[TMP28]], 3
+// IR-NEXT:    br label %[[COND_END29]]
+// IR:       [[COND_END29]]:
+// IR-NEXT:    [[COND30:%.*]] = phi i32 [ [[ADD26]], %[[COND_TRUE25]] ], [ [[ADD28]], %[[COND_FALSE27]] ]
+// IR-NEXT:    [[CMP31:%.*]] = icmp ult i32 [[TMP24]], [[COND30]]
+// IR-NEXT:    br i1 [[CMP31]], label %[[FOR_BODY:.*]], label %[[FOR_END50:.*]]
 // IR:       [[FOR_BODY]]:
-// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
-// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD26:%.*]] = add i32 [[TMP27]], [[TMP28]]
-// IR-NEXT:    store i32 [[ADD26]], ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
 // IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[MUL27:%.*]] = mul i32 [[TMP29]], 4
-// IR-NEXT:    [[ADD28:%.*]] = add i32 0, [[MUL27]]
-// IR-NEXT:    store i32 [[ADD28]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND29:.*]]
-// IR:       [[FOR_COND29]]:
-// IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[CMP30:%.*]] = icmp ult i32 [[TMP30]], 4
-// IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD31:%.*]] = add i32 [[TMP31]], [[TMP32]]
-// IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[ADD32:%.*]] = add i32 [[TMP33]], 1
-// IR-NEXT:    [[CMP33:%.*]] = icmp ult i32 [[ADD31]], [[ADD32]]
-// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND34:%.*]] = and i1 [[CMP30]], [[CMP33]]
-// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND34]], label %[[FOR_BODY35:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY35]]:
-// IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD36:%.*]] = add i32 [[TMP34]], [[TMP35]]
-// IR-NEXT:    store i32 [[ADD36]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL32:%.*]] = mul i32 [[TMP29]], 4
+// IR-NEXT:    [[ADD33:%.*]] = add i32 0, [[MUL32]]
+// IR-NEXT:    store i32 [[ADD33]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP30]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND34:.*]]
+// IR:       [[FOR_COND34]]:
+// IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    [[ADD35:%.*]] = add i32 [[TMP32]], 1
+// IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD36:%.*]] = add i32 [[TMP33]], 4
+// IR-NEXT:    [[CMP37:%.*]] = icmp ult i32 [[ADD35]], [[ADD36]]
+// IR-NEXT:    br i1 [[CMP37]], label %[[COND_TRUE38:.*]], label %[[COND_FALSE40:.*]]
+// IR:       [[COND_TRUE38]]:
+// IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    [[ADD39:%.*]] = add i32 [[TMP34]], 1
+// IR-NEXT:    br label %[[COND_END42:.*]]
+// IR:       [[COND_FALSE40]]:
+// IR-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD41:%.*]] = add i32 [[TMP35]], 4
+// IR-NEXT:    br label %[[COND_END42]]
+// IR:       [[COND_END42]]:
+// IR-NEXT:    [[COND43:%.*]] = phi i32 [ [[ADD39]], %[[COND_TRUE38]] ], [ [[ADD41]], %[[COND_FALSE40]] ]
+// IR-NEXT:    [[CMP44:%.*]] = icmp ult i32 [[TMP31]], [[COND43]]
+// IR-NEXT:    br i1 [[CMP44]], label %[[FOR_BODY45:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY45]]:
 // IR-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
 // IR-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[MUL37:%.*]] = mul i32 [[TMP37]], [[TMP38]]
-// IR-NEXT:    [[ADD38:%.*]] = add i32 [[TMP36]], [[MUL37]]
-// IR-NEXT:    store i32 [[ADD38]], ptr [[I]], align 4
+// IR-NEXT:    [[MUL46:%.*]] = mul i32 [[TMP37]], [[TMP38]]
+// IR-NEXT:    [[ADD47:%.*]] = add i32 [[TMP36]], [[MUL46]]
+// IR-NEXT:    store i32 [[ADD47]], ptr [[I]], align 4
 // IR-NEXT:    [[TMP39:%.*]] = load i32, ptr [[START_ADDR]], align 4
 // IR-NEXT:    [[TMP40:%.*]] = load i32, ptr [[END_ADDR]], align 4
 // IR-NEXT:    [[TMP41:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
@@ -177,25 +187,25 @@ extern "C" void body(...) {}
 // IR-NEXT:    call void (...) @body(i32 noundef [[TMP39]], i32 noundef [[TMP40]], i32 noundef [[TMP41]], i32 noundef [[TMP42]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    [[INC:%.*]] = add i32 [[TMP43]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND29]], !llvm.loop [[LOOP2:![0-9]+]]
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND34]], !llvm.loop [[LOOP2:![0-9]+]]
 // IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC39:.*]]
-// IR:       [[FOR_INC39]]:
-// IR-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[INC40:%.*]] = add i32 [[TMP44]], 1
-// IR-NEXT:    store i32 [[INC40]], ptr [[DOTTILE_CNT_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_INC48:.*]]
+// IR:       [[FOR_INC48]]:
+// IR-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[INC49:%.*]] = add i32 [[TMP44]], 1
+// IR-NEXT:    store i32 [[INC49]], ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP4:![0-9]+]]
-// IR:       [[FOR_END41]]:
+// IR:       [[FOR_END50]]:
 // IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.*]]
 // IR:       [[OMP_BODY_CONTINUE]]:
 // IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.*]]
 // IR:       [[OMP_INNER_FOR_INC]]:
 // IR-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[ADD42:%.*]] = add i32 [[TMP45]], 1
-// IR-NEXT:    store i32 [[ADD42]], ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[ADD51:%.*]] = add i32 [[TMP45]], 1
+// IR-NEXT:    store i32 [[ADD51]], ptr [[DOTOMP_IV]], align 4
 // IR-NEXT:    br label %[[OMP_INNER_FOR_COND]]
 // IR:       [[OMP_INNER_FOR_END]]:
 // IR-NEXT:    br label %[[OMP_LOOP_EXIT:.*]]
diff --git a/clang/test/OpenMP/tile_collapse_reinterpret_codegen.cpp b/clang/test/OpenMP/tile_collapse_reinterpret_codegen.cpp
new file mode 100644
index 0000000000000..f88db73778f2b
--- /dev/null
+++ b/clang/test/OpenMP/tile_collapse_reinterpret_codegen.cpp
@@ -0,0 +1,108 @@
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fclang-abi-compat=latest -std=c++20 -fopenmp -ast-print %s | FileCheck %s --check-prefix=PRINT
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fclang-abi-compat=latest -std=c++20 -fopenmp -emit-llvm %s -o - | FileCheck %s --check-prefix=IR
+// expected-no-diagnostics
+
+// Codegen tests for how a loop-associated directive (e.g. 'omp for' with a
+// 'collapse' clause) consumes a '#pragma omp tile' intra-tile loop.
+//
+// These tests pin the key IR invariants of the below design:
+// The intra-tile loop is stored in its min-bounded form
+//   for (.tile.iv = .floor.iv; .tile.iv < min(.floor.iv + tilesize, N); ++.tile.iv)
+// and carries an internal OMPInvariantPredicateBoundAttr hint. When a directive
+// consumes the loop, it is reinterpreted as a rectangular loop with a constant
+// per-tile trip count (the tile size), and the overshoot of a partial last tile
+// is handled by a predicate emitted as a body guard.
+
+extern "C" void body(...) {}
+
+// PRINT-LABEL: void collapse2_tile1d(
+// PRINT:       #pragma omp parallel for collapse(2)
+// PRINT-NEXT:  #pragma omp tile sizes(2)
+// IR-LABEL: define internal void @collapse2_tile1d.omp_outlined
+// IR:       %.floor_0.iv{{[0-9a-z.]*}} = alloca i32
+// IR:       %.tile_0.iv{{[0-9a-z.]*}} = alloca i32
+// IR-NOT:   .tile.cnt
+// IR:       omp.inner.for.body:
+// IR:       store i32 %{{.*}}, ptr %.tile_0.iv
+// IR:       %[[PRED:.*]] = icmp slt i32 %{{.*}}, %{{.*}}
+// IR-NEXT:  br i1 %[[PRED]], label %omp.body.next, label %omp.body.continue
+// IR:       omp.body.next:
+// IR:       call void (...) @body
+// IR-NEXT:  br label %omp.body.continue
+// IR:       omp.body.continue:
+extern "C" void collapse2_tile1d(int n) {
+#pragma omp parallel for collapse(2)
+#pragma omp tile sizes(2)
+  for (int i = 0; i < n; ++i)
+    body(i);
+}
+
+// PRINT-LABEL: void collapse3_tile1d_nested(
+// PRINT:       #pragma omp parallel for collapse(3)
+// PRINT-NEXT:  #pragma omp tile sizes(5)
+// IR-LABEL: define internal void @collapse3_tile1d_nested.omp_outlined
+// IR-NOT:   .tile.cnt
+// IR:       omp.inner.for.body:
+// IR:       store i32 %{{.*}}, ptr %.tile_0.iv
+// IR:       %[[PRED:.*]] = icmp slt i32 %{{.*}}, %{{.*}}
+// IR-NEXT:  br i1 %[[PRED]], label %omp.body.next, label %omp.body.continue
+// IR:       omp.body.next:
+// IR:       call void (...) @body
+// IR-NEXT:  br label %omp.body.continue
+extern "C" void collapse3_tile1d_nested(int n) {
+#pragma omp parallel for collapse(3)
+#pragma omp tile sizes(5)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+}
+
+// PRINT-LABEL: void tile2d_inside_tile1d(
+// PRINT:       #pragma omp tile sizes(3)
+// PRINT-NEXT:  #pragma omp tile sizes(2)
+// IR-LABEL: define {{.*}} @tile2d_inside_tile1d(
+// IR:       %.tile_0.iv{{[0-9a-z.]*}} = alloca i32
+// IR-NOT:   .tile.cnt
+// IR:       cond.true:
+// IR:       cond.false:
+// IR:       cond.end:
+// IR:       %{{.*}} = phi i32
+// IR-NOT:   omp.body.next
+extern "C" void tile2d_inside_tile1d(int n) {
+#pragma omp tile sizes(3)
+#pragma omp tile sizes(2)
+  for (int i = 0; i < n; ++i)
+    body(i);
+}
+
+// PRINT-LABEL: void collapse3_tile2d(
+// PRINT:       #pragma omp parallel for collapse(3)
+// PRINT-NEXT:  #pragma omp tile sizes(2, 3)
+// IR-LABEL: define internal void @collapse3_tile2d.omp_outlined
+// IR-NOT:   .tile.cnt
+// IR:       omp.inner.for.body:
+// IR:       store i32 %{{.*}}, ptr %.tile_0.iv
+// IR:       br i1 %{{.*}}, label %omp.body.next, label %omp.body.continue
+// IR:       omp.body.next:
+// IR:       call void (...) @body
+extern "C" void collapse3_tile2d(int n) {
+#pragma omp parallel for collapse(3)
+#pragma omp tile sizes(2, 3)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+}
+
+// PRINT-LABEL: void tile1d_constant_div_no_predicate(
+// PRINT:       #pragma omp tile sizes(5)
+// IR-LABEL: define {{.*}} @tile1d_constant_div_no_predicate(
+// IR:       %.floor_0.iv{{[0-9a-z.]*}} = alloca i32
+// IR:       %.tile_0.iv{{[0-9a-z.]*}} = alloca i32
+// IR-NOT:   .tile.cnt
+// IR-NOT:   omp.body.next
+// IR:       call void (...) @body
+extern "C" void tile1d_constant_div_no_predicate() {
+#pragma omp tile sizes(5)
+  for (int i = 0; i < 10; ++i)
+    body(i);
+}
diff --git a/clang/test/OpenMP/tile_rect_codegen.cpp b/clang/test/OpenMP/tile_rect_codegen.cpp
index 589914eb9d63e..ab63de9343fd2 100644
--- a/clang/test/OpenMP/tile_rect_codegen.cpp
+++ b/clang/test/OpenMP/tile_rect_codegen.cpp
@@ -1,14 +1,15 @@
 // RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fclang-abi-compat=latest -std=c++20 -fopenmp -ast-print %s | FileCheck %s
 // expected-no-diagnostics
 
+// -ast-print round-trip tests: a `#pragma omp tile` directive and its associated
+// loop must print back as written. This checks parsing/printing only; for the
+// generated IR (min-bounded lowering) see tile_rect_codegen_ir.cpp.
+
 extern "C" void body(...) {}
 
 // CHECK-LABEL: void rect_tile_1d(
 void rect_tile_1d() {
-  // Tile size 5, trip count 4: 4 % 5 != 0, so predicate is needed.
-  // The tile loop should have rectangular bound (the tile size) and
-  // the body should be guarded by a validity predicate.
-  //
+  // Partial last tile (trip count 4, tile 5); -ast-print prints the original loop.
   // CHECK: #pragma omp tile sizes(5)
   // CHECK-NEXT: for (int i = 7; i < 17; i += 3)
   #pragma omp tile sizes(5)
@@ -29,7 +30,7 @@ void rect_tile_2d() {
 
 // CHECK-LABEL: void rect_tile_exact_div(
 void rect_tile_exact_div() {
-  // Tile size 5, trip count 10: 10 % 5 == 0, so predicate is NOT needed.
+  // Exact-dividing trip count (10, tile 5); -ast-print prints the original loop.
   // CHECK: #pragma omp tile sizes(5)
   // CHECK-NEXT: for (int i = 0; i < 10; i += 1)
   #pragma omp tile sizes(5)
@@ -39,8 +40,8 @@ void rect_tile_exact_div() {
 
 // CHECK-LABEL: void rect_tile_nested_body_loop(
 void rect_tile_nested_body_loop(int n) {
-  // After tiling i, the j loop (from the associated body) should be part
-  // of the same canonical nest as the floor and tile loops.
+  // The inner `j` loop lives in the tiled construct's body; -ast-print shows the
+  // directive and the outer `i` loop printed back.
   // CHECK: #pragma omp tile sizes(4)
   // CHECK-NEXT: for (int i = 0; i < 6; i += 1)
   #pragma omp tile sizes(4)
diff --git a/clang/test/OpenMP/tile_rect_codegen_ir.cpp b/clang/test/OpenMP/tile_rect_codegen_ir.cpp
index f2b64ced7c46b..027cd8385044c 100644
--- a/clang/test/OpenMP/tile_rect_codegen_ir.cpp
+++ b/clang/test/OpenMP/tile_rect_codegen_ir.cpp
@@ -1,22 +1,26 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
-// Regression tests for rectangular #pragma omp tile lowering:
-// - Inner loop counter .tile.cnt.* with bound == tile size (constant).
-// - Validity guard (icmp + branch) when remainder iterations exist or trip count
-//   is not proven divisible at compile time.
+// Regression tests for standalone `#pragma omp tile` lowering (no consuming
+// directive). The intra-tile loop is emitted in its min-bounded form
+//   for (.tile.iv = .floor.iv; .tile.iv < min(.floor.iv + tilesize, N); ++.tile.iv)
+// so the upper bound is materialized as a `min()` (cond.true/cond.false/phi over
+// the two `.tile_*.iv`/`.floor_*.iv` counters). There is no `.tile.cnt`
+// rectangular counter and no body-guard predicate here; those only appear when a
+// directive (e.g. `collapse`) consumes the tile -- see
+// tile_collapse_reinterpret_codegen.cpp.
 //
 // RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fopenmp -emit-llvm -o - %s | FileCheck %s --check-prefix=IR
 // expected-no-diagnostics
 
 extern "C" void body(int);
 
-// Trip count 6, tile 4 → remainder; body must be guarded.
+// Trip count 6, tile 4 → the last tile is partial; the min() upper bound stops
+// the tile loop at the trip count (no separate body guard).
 // IR-LABEL: define dso_local void @_Z18remainder_6_tile_4v(
 // IR-SAME: ) #[[ATTR0:[0-9]+]] {
 // IR-NEXT:  [[ENTRY:.*:]]
 // IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    store i32 0, ptr [[I]], align 4
 // IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND:.*]]
@@ -25,39 +29,43 @@ extern "C" void body(int);
 // IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 6
 // IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END9:.*]]
 // IR:       [[FOR_BODY]]:
-// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1:.*]]
 // IR:       [[FOR_COND1]]:
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 4
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
-// IR-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[ADD]], 6
-// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP2]], [[CMP3]]
-// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label %[[FOR_BODY4:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY4]]:
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 6, [[ADD]]
+// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
 // IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD5:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
-// IR-NEXT:    store i32 [[ADD5]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 1
+// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 4
+// IR-NEXT:    br label %[[COND_END]]
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ 6, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
+// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
+// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY5]]:
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
 // IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
 // IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void @body(i32 noundef [[TMP7]])
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void @body(i32 noundef [[TMP6]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP8]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP7]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP2:![0-9]+]]
 // IR:       [[FOR_END]]:
 // IR-NEXT:    br label %[[FOR_INC7:.*]]
 // IR:       [[FOR_INC7]]:
-// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP9]], 4
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP8]], 4
 // IR-NEXT:    store i32 [[ADD8]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP4:![0-9]+]]
 // IR:       [[FOR_END9]]:
@@ -75,48 +83,56 @@ void remainder_6_tile_4(void) {
 // IR-SAME: ) #[[ATTR0]] {
 // IR-NEXT:  [[ENTRY:.*:]]
 // IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    store i32 0, ptr [[I]], align 4
 // IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND:.*]]
 // IR:       [[FOR_COND]]:
 // IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 10
-// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END7:.*]]
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END9:.*]]
 // IR:       [[FOR_BODY]]:
-// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1:.*]]
 // IR:       [[FOR_COND1]]:
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 5
-// IR-NEXT:    br i1 [[CMP2]], label %[[FOR_BODY3:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY3]]:
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
-// IR-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP4]], 1
-// IR-NEXT:    [[ADD4:%.*]] = add nsw i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD4]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void @body(i32 noundef [[TMP5]])
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 5
+// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 10, [[ADD]]
+// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 5
+// IR-NEXT:    br label %[[COND_END]]
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ 10, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
+// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
+// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY5]]:
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
+// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void @body(i32 noundef [[TMP6]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP6]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP7]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP5:![0-9]+]]
 // IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC5:.*]]
-// IR:       [[FOR_INC5]]:
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP7]], 5
-// IR-NEXT:    store i32 [[ADD6]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_INC7:.*]]
+// IR:       [[FOR_INC7]]:
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP8]], 5
+// IR-NEXT:    store i32 [[ADD8]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
-// IR:       [[FOR_END7]]:
+// IR:       [[FOR_END9]]:
 // IR-NEXT:    ret void
 //
 void full_tiles_10_5(void) {
@@ -131,9 +147,8 @@ void full_tiles_10_5(void) {
 // IR-NEXT:  [[ENTRY:.*:]]
 // IR-NEXT:    [[TS_ADDR:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    store i32 [[TS]], ptr [[TS_ADDR]], align 4
 // IR-NEXT:    store i32 0, ptr [[I]], align 4
 // IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
@@ -141,64 +156,78 @@ void full_tiles_10_5(void) {
 // IR:       [[FOR_COND]]:
 // IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 12
-// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END15:.*]]
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END24:.*]]
 // IR:       [[FOR_BODY]]:
-// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1:.*]]
 // IR:       [[FOR_COND1]]:
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[TS_ADDR]], align 4
-// IR-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP2]], 0
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TS_ADDR]], align 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], 0
 // IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
 // IR:       [[COND_TRUE]]:
 // IR-NEXT:    br label %[[COND_END:.*]]
 // IR:       [[COND_FALSE]]:
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[TS_ADDR]], align 4
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TS_ADDR]], align 4
 // IR-NEXT:    br label %[[COND_END]]
 // IR:       [[COND_END]]:
-// IR-NEXT:    [[COND:%.*]] = phi i32 [ 1, %[[COND_TRUE]] ], [ [[TMP3]], %[[COND_FALSE]] ]
-// IR-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[TMP1]], [[COND]]
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
-// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[ADD]], 12
-// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP3]], [[CMP4]]
-// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label %[[FOR_BODY5:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY5]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ 1, %[[COND_TRUE]] ], [ [[TMP5]], %[[COND_FALSE]] ]
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[COND]]
+// IR-NEXT:    [[CMP3:%.*]] = icmp slt i32 12, [[ADD]]
+// IR-NEXT:    br i1 [[CMP3]], label %[[COND_TRUE4:.*]], label %[[COND_FALSE5:.*]]
+// IR:       [[COND_TRUE4]]:
+// IR-NEXT:    br label %[[COND_END12:.*]]
+// IR:       [[COND_FALSE5]]:
 // IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP6]], [[TMP7]]
-// IR-NEXT:    store i32 [[ADD6]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP8]], 1
-// IR-NEXT:    [[ADD7:%.*]] = add nsw i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void @body(i32 noundef [[TMP9]])
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TS_ADDR]], align 4
+// IR-NEXT:    [[CMP6:%.*]] = icmp sle i32 [[TMP7]], 0
+// IR-NEXT:    br i1 [[CMP6]], label %[[COND_TRUE7:.*]], label %[[COND_FALSE8:.*]]
+// IR:       [[COND_TRUE7]]:
+// IR-NEXT:    br label %[[COND_END9:.*]]
+// IR:       [[COND_FALSE8]]:
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TS_ADDR]], align 4
+// IR-NEXT:    br label %[[COND_END9]]
+// IR:       [[COND_END9]]:
+// IR-NEXT:    [[COND10:%.*]] = phi i32 [ 1, %[[COND_TRUE7]] ], [ [[TMP8]], %[[COND_FALSE8]] ]
+// IR-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP6]], [[COND10]]
+// IR-NEXT:    br label %[[COND_END12]]
+// IR:       [[COND_END12]]:
+// IR-NEXT:    [[COND13:%.*]] = phi i32 [ 12, %[[COND_TRUE4]] ], [ [[ADD11]], %[[COND_END9]] ]
+// IR-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP2]], [[COND13]]
+// IR-NEXT:    br i1 [[CMP14]], label %[[FOR_BODY15:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY15]]:
+// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP9]], 1
+// IR-NEXT:    [[ADD16:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD16]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void @body(i32 noundef [[TMP10]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP7:![0-9]+]]
 // IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC8:.*]]
-// IR:       [[FOR_INC8]]:
-// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[TS_ADDR]], align 4
-// IR-NEXT:    [[CMP9:%.*]] = icmp sle i32 [[TMP11]], 0
-// IR-NEXT:    br i1 [[CMP9]], label %[[COND_TRUE10:.*]], label %[[COND_FALSE11:.*]]
-// IR:       [[COND_TRUE10]]:
-// IR-NEXT:    br label %[[COND_END12:.*]]
-// IR:       [[COND_FALSE11]]:
+// IR-NEXT:    br label %[[FOR_INC17:.*]]
+// IR:       [[FOR_INC17]]:
 // IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TS_ADDR]], align 4
-// IR-NEXT:    br label %[[COND_END12]]
-// IR:       [[COND_END12]]:
-// IR-NEXT:    [[COND13:%.*]] = phi i32 [ 1, %[[COND_TRUE10]] ], [ [[TMP12]], %[[COND_FALSE11]] ]
-// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD14:%.*]] = add nsw i32 [[TMP13]], [[COND13]]
-// IR-NEXT:    store i32 [[ADD14]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[CMP18:%.*]] = icmp sle i32 [[TMP12]], 0
+// IR-NEXT:    br i1 [[CMP18]], label %[[COND_TRUE19:.*]], label %[[COND_FALSE20:.*]]
+// IR:       [[COND_TRUE19]]:
+// IR-NEXT:    br label %[[COND_END21:.*]]
+// IR:       [[COND_FALSE20]]:
+// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TS_ADDR]], align 4
+// IR-NEXT:    br label %[[COND_END21]]
+// IR:       [[COND_END21]]:
+// IR-NEXT:    [[COND22:%.*]] = phi i32 [ 1, %[[COND_TRUE19]] ], [ [[TMP13]], %[[COND_FALSE20]] ]
+// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP14]], [[COND22]]
+// IR-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
-// IR:       [[FOR_END15]]:
+// IR:       [[FOR_END24]]:
 // IR-NEXT:    ret void
 //
 void var_tile(int ts) {
@@ -212,13 +241,11 @@ void var_tile(int ts) {
 // IR-SAME: ) #[[ATTR0]] {
 // IR-NEXT:  [[ENTRY:.*:]]
 // IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[J:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_CNT_1_IV_J:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
 // IR-NEXT:    store i32 0, ptr [[I]], align 4
 // IR-NEXT:    store i32 0, ptr [[J]], align 4
 // IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
@@ -226,91 +253,95 @@ void var_tile(int ts) {
 // IR:       [[FOR_COND]]:
 // IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 3
-// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END27:.*]]
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END31:.*]]
 // IR:       [[FOR_BODY]]:
 // IR-NEXT:    store i32 0, ptr [[DOTFLOOR_1_IV_J]], align 4
 // IR-NEXT:    br label %[[FOR_COND1:.*]]
 // IR:       [[FOR_COND1]]:
 // IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
 // IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 3
-// IR-NEXT:    br i1 [[CMP2]], label %[[FOR_BODY3:.*]], label %[[FOR_END24:.*]]
+// IR-NEXT:    br i1 [[CMP2]], label %[[FOR_BODY3:.*]], label %[[FOR_END28:.*]]
 // IR:       [[FOR_BODY3]]:
-// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP2]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND4:.*]]
 // IR:       [[FOR_COND4]]:
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[CMP5:%.*]] = icmp slt i32 [[TMP2]], 2
-// IR-NEXT:    br i1 [[CMP5]], label %[[FOR_BODY6:.*]], label %[[FOR_END21:.*]]
-// IR:       [[FOR_BODY6]]:
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[TMP4]]
-// IR-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
-// IR-NEXT:    [[ADD7:%.*]] = add nsw i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD7]], ptr [[I]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// IR-NEXT:    br label %[[FOR_COND8:.*]]
-// IR:       [[FOR_COND8]]:
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// IR-NEXT:    [[CMP9:%.*]] = icmp slt i32 [[TMP6]], 2
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP7]], [[TMP8]]
-// IR-NEXT:    [[CMP11:%.*]] = icmp slt i32 [[ADD10]], 3
-// IR-NEXT:    br i1 [[CMP11]], label %[[LAND_RHS:.*]], label %[[LAND_END:.*]]
-// IR:       [[LAND_RHS]]:
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], 2
+// IR-NEXT:    [[CMP5:%.*]] = icmp slt i32 3, [[ADD]]
+// IR-NEXT:    br i1 [[CMP5]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP5]], 2
+// IR-NEXT:    br label %[[COND_END]]
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ 3, %[[COND_TRUE]] ], [ [[ADD6]], %[[COND_FALSE]] ]
+// IR-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP3]], [[COND]]
+// IR-NEXT:    br i1 [[CMP7]], label %[[FOR_BODY8:.*]], label %[[FOR_END25:.*]]
+// IR:       [[FOR_BODY8]]:
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 1
+// IR-NEXT:    [[ADD9:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// IR-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_1_IV_J]], align 4
+// IR-NEXT:    br label %[[FOR_COND10:.*]]
+// IR:       [[FOR_COND10]]:
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
 // IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// IR-NEXT:    [[ADD12:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
-// IR-NEXT:    [[CMP13:%.*]] = icmp slt i32 [[ADD12]], 3
-// IR-NEXT:    br label %[[LAND_END]]
-// IR:       [[LAND_END]]:
-// IR-NEXT:    [[TMP11:%.*]] = phi i1 [ false, %[[FOR_COND8]] ], [ [[CMP13]], %[[LAND_RHS]] ]
-// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP9]], [[TMP11]]
-// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label %[[FOR_BODY14:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY14]]:
-// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// IR-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP12]], [[TMP13]]
-// IR-NEXT:    store i32 [[ADD15]], ptr [[DOTTILE_1_IV_J]], align 4
-// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// IR-NEXT:    [[MUL16:%.*]] = mul nsw i32 [[TMP14]], 1
-// IR-NEXT:    [[ADD17:%.*]] = add nsw i32 0, [[MUL16]]
-// IR-NEXT:    store i32 [[ADD17]], ptr [[J]], align 4
-// IR-NEXT:    [[TMP15:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    [[TMP16:%.*]] = load i32, ptr [[J]], align 4
-// IR-NEXT:    [[ADD18:%.*]] = add nsw i32 [[TMP15]], [[TMP16]]
-// IR-NEXT:    call void @body(i32 noundef [[ADD18]])
+// IR-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP9]], 2
+// IR-NEXT:    [[CMP12:%.*]] = icmp slt i32 3, [[ADD11]]
+// IR-NEXT:    br i1 [[CMP12]], label %[[COND_TRUE13:.*]], label %[[COND_FALSE14:.*]]
+// IR:       [[COND_TRUE13]]:
+// IR-NEXT:    br label %[[COND_END16:.*]]
+// IR:       [[COND_FALSE14]]:
+// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// IR-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP10]], 2
+// IR-NEXT:    br label %[[COND_END16]]
+// IR:       [[COND_END16]]:
+// IR-NEXT:    [[COND17:%.*]] = phi i32 [ 3, %[[COND_TRUE13]] ], [ [[ADD15]], %[[COND_FALSE14]] ]
+// IR-NEXT:    [[CMP18:%.*]] = icmp slt i32 [[TMP8]], [[COND17]]
+// IR-NEXT:    br i1 [[CMP18]], label %[[FOR_BODY19:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY19]]:
+// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// IR-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[TMP11]], 1
+// IR-NEXT:    [[ADD21:%.*]] = add nsw i32 0, [[MUL20]]
+// IR-NEXT:    store i32 [[ADD21]], ptr [[J]], align 4
+// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
+// IR-NEXT:    [[ADD22:%.*]] = add nsw i32 [[TMP12]], [[TMP13]]
+// IR-NEXT:    call void @body(i32 noundef [[ADD22]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP17]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_1_IV_J]], align 4
-// IR-NEXT:    br label %[[FOR_COND8]], !llvm.loop [[LOOP9:![0-9]+]]
+// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
+// IR-NEXT:    br label %[[FOR_COND10]], !llvm.loop [[LOOP9:![0-9]+]]
 // IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC19:.*]]
-// IR:       [[FOR_INC19]]:
-// IR-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[INC20:%.*]] = add nsw i32 [[TMP18]], 1
-// IR-NEXT:    store i32 [[INC20]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_INC23:.*]]
+// IR:       [[FOR_INC23]]:
+// IR-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[INC24:%.*]] = add nsw i32 [[TMP15]], 1
+// IR-NEXT:    store i32 [[INC24]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND4]], !llvm.loop [[LOOP10:![0-9]+]]
-// IR:       [[FOR_END21]]:
-// IR-NEXT:    br label %[[FOR_INC22:.*]]
-// IR:       [[FOR_INC22]]:
-// IR-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// IR-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP19]], 2
-// IR-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_1_IV_J]], align 4
+// IR:       [[FOR_END25]]:
+// IR-NEXT:    br label %[[FOR_INC26:.*]]
+// IR:       [[FOR_INC26]]:
+// IR-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
+// IR-NEXT:    [[ADD27:%.*]] = add nsw i32 [[TMP16]], 2
+// IR-NEXT:    store i32 [[ADD27]], ptr [[DOTFLOOR_1_IV_J]], align 4
 // IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP11:![0-9]+]]
-// IR:       [[FOR_END24]]:
-// IR-NEXT:    br label %[[FOR_INC25:.*]]
-// IR:       [[FOR_INC25]]:
-// IR-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD26:%.*]] = add nsw i32 [[TMP20]], 2
-// IR-NEXT:    store i32 [[ADD26]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR:       [[FOR_END28]]:
+// IR-NEXT:    br label %[[FOR_INC29:.*]]
+// IR:       [[FOR_INC29]]:
+// IR-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD30:%.*]] = add nsw i32 [[TMP17]], 2
+// IR-NEXT:    store i32 [[ADD30]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP12:![0-9]+]]
-// IR:       [[FOR_END27]]:
+// IR:       [[FOR_END31]]:
 // IR-NEXT:    ret void
 //
 void two_dim(void) {
@@ -320,15 +351,15 @@ void two_dim(void) {
       body(i + j);
 }
 
-// Nested loop in tile body: outer nest still uses .tile.cnt for tiled dim.
+// Nested loop in the tile body: the tiled `i` dim is the floor/tile pair in
+// min-bounded form; the inner `j` loop stays a plain nested loop.
 // IR-LABEL: define dso_local void @_Z14nested_inner_ji(
 // IR-SAME: i32 noundef [[N:%.*]]) #[[ATTR0]] {
 // IR-NEXT:  [[ENTRY:.*:]]
 // IR-NEXT:    [[N_ADDR:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[J:%.*]] = alloca i32, align 4
 // IR-NEXT:    store i32 [[N]], ptr [[N_ADDR]], align 4
 // IR-NEXT:    store i32 0, ptr [[I]], align 4
@@ -339,56 +370,60 @@ void two_dim(void) {
 // IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 6
 // IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END16:.*]]
 // IR:       [[FOR_BODY]]:
-// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1:.*]]
 // IR:       [[FOR_COND1]]:
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 4
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
-// IR-NEXT:    [[CMP3:%.*]] = icmp slt i32 [[ADD]], 6
-// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP2]], [[CMP3]]
-// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label %[[FOR_BODY4:.*]], label %[[FOR_END13:.*]]
-// IR:       [[FOR_BODY4]]:
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 6, [[ADD]]
+// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
 // IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD5:%.*]] = add nsw i32 [[TMP4]], [[TMP5]]
-// IR-NEXT:    store i32 [[ADD5]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 1
+// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 4
+// IR-NEXT:    br label %[[COND_END]]
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ 6, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
+// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
+// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END13:.*]]
+// IR:       [[FOR_BODY5]]:
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
 // IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
 // IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
 // IR-NEXT:    store i32 0, ptr [[J]], align 4
 // IR-NEXT:    br label %[[FOR_COND7:.*]]
 // IR:       [[FOR_COND7]]:
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[J]], align 4
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[N_ADDR]], align 4
-// IR-NEXT:    [[CMP8:%.*]] = icmp slt i32 [[TMP7]], [[TMP8]]
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[J]], align 4
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[N_ADDR]], align 4
+// IR-NEXT:    [[CMP8:%.*]] = icmp slt i32 [[TMP6]], [[TMP7]]
 // IR-NEXT:    br i1 [[CMP8]], label %[[FOR_BODY9:.*]], label %[[FOR_END:.*]]
 // IR:       [[FOR_BODY9]]:
-// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[J]], align 4
-// IR-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP9]], [[TMP10]]
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[J]], align 4
+// IR-NEXT:    [[ADD10:%.*]] = add nsw i32 [[TMP8]], [[TMP9]]
 // IR-NEXT:    call void @body(i32 noundef [[ADD10]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[J]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
+// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[J]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
 // IR-NEXT:    store i32 [[INC]], ptr [[J]], align 4
 // IR-NEXT:    br label %[[FOR_COND7]], !llvm.loop [[LOOP13:![0-9]+]]
 // IR:       [[FOR_END]]:
 // IR-NEXT:    br label %[[FOR_INC11:.*]]
 // IR:       [[FOR_INC11]]:
-// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[INC12:%.*]] = add nsw i32 [[TMP12]], 1
-// IR-NEXT:    store i32 [[INC12]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[INC12:%.*]] = add nsw i32 [[TMP11]], 1
+// IR-NEXT:    store i32 [[INC12]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP14:![0-9]+]]
 // IR:       [[FOR_END13]]:
 // IR-NEXT:    br label %[[FOR_INC14:.*]]
 // IR:       [[FOR_INC14]]:
-// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP13]], 4
+// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP12]], 4
 // IR-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP15:![0-9]+]]
 // IR:       [[FOR_END16]]:
@@ -401,54 +436,63 @@ void nested_inner_j(int n) {
       body(i + j);
 }
 
-// Predicate elision: when trip count is evenly divisible by constant tile size,
-// no if-guard is needed (no icmp against trip count in tile body).
+// Exact division (trip count 12, tile 4): a standalone tile still emits the
+// min() cond-phi upper bound (it is always present, regardless of divisibility)
+// and never a body-guard predicate.
 // IR-LABEL: define dso_local void @_Z16elided_predicatev(
 // IR-SAME: ) #[[ATTR0]] {
 // IR-NEXT:  [[ENTRY:.*:]]
 // IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_CNT_0_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    store i32 0, ptr [[I]], align 4
 // IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND:.*]]
 // IR:       [[FOR_COND]]:
 // IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 12
-// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END7:.*]]
+// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END9:.*]]
 // IR:       [[FOR_BODY]]:
-// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1:.*]]
 // IR:       [[FOR_COND1]]:
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 4
-// IR-NEXT:    br i1 [[CMP2]], label %[[FOR_BODY3:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY3]]:
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP2]], [[TMP3]]
-// IR-NEXT:    store i32 [[ADD]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP4]], 1
-// IR-NEXT:    [[ADD4:%.*]] = add nsw i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD4]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void @body(i32 noundef [[TMP5]])
+// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 4
+// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 12, [[ADD]]
+// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
+// IR:       [[COND_TRUE]]:
+// IR-NEXT:    br label %[[COND_END:.*]]
+// IR:       [[COND_FALSE]]:
+// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 4
+// IR-NEXT:    br label %[[COND_END]]
+// IR:       [[COND_END]]:
+// IR-NEXT:    [[COND:%.*]] = phi i32 [ 12, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
+// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
+// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY5]]:
+// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
+// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
+// IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
+// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[I]], align 4
+// IR-NEXT:    call void @body(i32 noundef [[TMP6]])
 // IR-NEXT:    br label %[[FOR_INC:.*]]
 // IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP6]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_CNT_0_IV_I]], align 4
+// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP7]], 1
+// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP16:![0-9]+]]
 // IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC5:.*]]
-// IR:       [[FOR_INC5]]:
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP7]], 4
-// IR-NEXT:    store i32 [[ADD6]], ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_INC7:.*]]
+// IR:       [[FOR_INC7]]:
+// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP8]], 4
+// IR-NEXT:    store i32 [[ADD8]], ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP17:![0-9]+]]
-// IR:       [[FOR_END7]]:
+// IR:       [[FOR_END9]]:
 // IR-NEXT:    ret void
 //
 void elided_predicate(void) {
diff --git a/clang/test/OpenMP/tile_vectorize_codegen.cpp b/clang/test/OpenMP/tile_vectorize_codegen.cpp
new file mode 100644
index 0000000000000..46fa51b8da57c
--- /dev/null
+++ b/clang/test/OpenMP/tile_vectorize_codegen.cpp
@@ -0,0 +1,14 @@
+// RUN: %clang_cc1 -verify -triple x86_64-unknown-linux-gnu -fclang-abi-compat=latest -std=c++17 -fopenmp -O2 -vectorize-loops -vectorize-slp -emit-llvm %s -o - | FileCheck %s
+// REQUIRES: x86-registered-target
+// expected-no-diagnostics
+
+// Standalone tile stays min-bounded (no body predicate) and vectorizes.
+
+// CHECK-LABEL: define {{.*}} @vec_tile(
+// CHECK: fadd <4 x float>
+// CHECK-NOT: omp.body.next
+extern "C" void vec_tile(float *a, float *b, int n) {
+#pragma omp tile sizes(64)
+  for (int i = 0; i < n; ++i)
+    a[i] = a[i] + b[i];
+}
diff --git a/clang/test/OpenMP/unroll_codegen_tile_for.cpp b/clang/test/OpenMP/unroll_codegen_tile_for.cpp
index cd28ff73a11ec..2e80731c4465d 100644
--- a/clang/test/OpenMP/unroll_codegen_tile_for.cpp
+++ b/clang/test/OpenMP/unroll_codegen_tile_for.cpp
@@ -30,7 +30,6 @@ extern "C" void body(...) {}
 // IR-NEXT:    [[DOTUNROLLED_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTCAPTURE_EXPR_5:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV__UNROLLED_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTCAPTURE_EXPR_13:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_0_IV__UNROLLED_IV_I:%.*]] = alloca i32, align 4
@@ -39,7 +38,7 @@ extern "C" void body(...) {}
 // IR-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTFLOOR_0_IV__UNROLLED_IV_I17:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_CNT_0_IV__UNROLLED_IV_I:%.*]] = alloca i32, align 4
+// IR-NEXT:    [[DOTTILE_0_IV__UNROLLED_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[DOTUNROLL_INNER_IV_I:%.*]] = alloca i32, align 4
 // IR-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
 // IR-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
@@ -118,53 +117,59 @@ extern "C" void body(...) {}
 // IR-NEXT:    [[MUL:%.*]] = mul i32 [[TMP22]], 4
 // IR-NEXT:    [[ADD21:%.*]] = add i32 0, [[MUL]]
 // IR-NEXT:    store i32 [[ADD21]], ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I17]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I17]], align 4
+// IR-NEXT:    store i32 [[TMP23]], ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND:.*]]
 // IR:       [[FOR_COND]]:
-// IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    [[CMP22:%.*]] = icmp ult i32 [[TMP23]], 4
-// IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I17]], align 4
-// IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    [[ADD23:%.*]] = add i32 [[TMP24]], [[TMP25]]
-// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    [[ADD24:%.*]] = add i32 [[TMP26]], 1
-// IR-NEXT:    [[CMP25:%.*]] = icmp ult i32 [[ADD23]], [[ADD24]]
-// IR-NEXT:    [[OMP_TILE_INVARIANT_PRED_BOUND:%.*]] = and i1 [[CMP22]], [[CMP25]]
-// IR-NEXT:    br i1 [[OMP_TILE_INVARIANT_PRED_BOUND]], label %[[FOR_BODY:.*]], label %[[FOR_END39:.*]]
+// IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    [[ADD22:%.*]] = add i32 [[TMP25]], 1
+// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I17]], align 4
+// IR-NEXT:    [[ADD23:%.*]] = add i32 [[TMP26]], 4
+// IR-NEXT:    [[CMP24:%.*]] = icmp ult i32 [[ADD22]], [[ADD23]]
+// IR-NEXT:    br i1 [[CMP24]], label %[[COND_TRUE25:.*]], label %[[COND_FALSE27:.*]]
+// IR:       [[COND_TRUE25]]:
+// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    [[ADD26:%.*]] = add i32 [[TMP27]], 1
+// IR-NEXT:    br label %[[COND_END29:.*]]
+// IR:       [[COND_FALSE27]]:
+// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I17]], align 4
+// IR-NEXT:    [[ADD28:%.*]] = add i32 [[TMP28]], 4
+// IR-NEXT:    br label %[[COND_END29]]
+// IR:       [[COND_END29]]:
+// IR-NEXT:    [[COND30:%.*]] = phi i32 [ [[ADD26]], %[[COND_TRUE25]] ], [ [[ADD28]], %[[COND_FALSE27]] ]
+// IR-NEXT:    [[CMP31:%.*]] = icmp ult i32 [[TMP24]], [[COND30]]
+// IR-NEXT:    br i1 [[CMP31]], label %[[FOR_BODY:.*]], label %[[FOR_END44:.*]]
 // IR:       [[FOR_BODY]]:
-// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I17]], align 4
-// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    [[ADD26:%.*]] = add i32 [[TMP27]], [[TMP28]]
-// IR-NEXT:    store i32 [[ADD26]], ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
 // IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    [[MUL27:%.*]] = mul i32 [[TMP29]], 2
-// IR-NEXT:    [[ADD28:%.*]] = add i32 0, [[MUL27]]
-// IR-NEXT:    store i32 [[ADD28]], ptr [[DOTUNROLLED_IV_I]], align 4
+// IR-NEXT:    [[MUL32:%.*]] = mul i32 [[TMP29]], 2
+// IR-NEXT:    [[ADD33:%.*]] = add i32 0, [[MUL32]]
+// IR-NEXT:    store i32 [[ADD33]], ptr [[DOTUNROLLED_IV_I]], align 4
 // IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTUNROLLED_IV_I]], align 4
 // IR-NEXT:    store i32 [[TMP30]], ptr [[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND29:.*]]
-// IR:       [[FOR_COND29]]:
+// IR-NEXT:    br label %[[FOR_COND34:.*]]
+// IR:       [[FOR_COND34]]:
 // IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
 // IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTUNROLLED_IV_I]], align 4
-// IR-NEXT:    [[ADD30:%.*]] = add i32 [[TMP32]], 2
-// IR-NEXT:    [[CMP31:%.*]] = icmp ult i32 [[TMP31]], [[ADD30]]
-// IR-NEXT:    br i1 [[CMP31]], label %[[LAND_RHS:.*]], label %[[LAND_END:.*]]
+// IR-NEXT:    [[ADD35:%.*]] = add i32 [[TMP32]], 2
+// IR-NEXT:    [[CMP36:%.*]] = icmp ult i32 [[TMP31]], [[ADD35]]
+// IR-NEXT:    br i1 [[CMP36]], label %[[LAND_RHS:.*]], label %[[LAND_END:.*]]
 // IR:       [[LAND_RHS]]:
 // IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
 // IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[ADD32:%.*]] = add i32 [[TMP34]], 1
-// IR-NEXT:    [[CMP33:%.*]] = icmp ult i32 [[TMP33]], [[ADD32]]
+// IR-NEXT:    [[ADD37:%.*]] = add i32 [[TMP34]], 1
+// IR-NEXT:    [[CMP38:%.*]] = icmp ult i32 [[TMP33]], [[ADD37]]
 // IR-NEXT:    br label %[[LAND_END]]
 // IR:       [[LAND_END]]:
-// IR-NEXT:    [[TMP35:%.*]] = phi i1 [ false, %[[FOR_COND29]] ], [ [[CMP33]], %[[LAND_RHS]] ]
-// IR-NEXT:    br i1 [[TMP35]], label %[[FOR_BODY34:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY34]]:
+// IR-NEXT:    [[TMP35:%.*]] = phi i1 [ false, %[[FOR_COND34]] ], [ [[CMP38]], %[[LAND_RHS]] ]
+// IR-NEXT:    br i1 [[TMP35]], label %[[FOR_BODY39:.*]], label %[[FOR_END:.*]]
+// IR:       [[FOR_BODY39]]:
 // IR-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
 // IR-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
 // IR-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[MUL35:%.*]] = mul i32 [[TMP37]], [[TMP38]]
-// IR-NEXT:    [[ADD36:%.*]] = add i32 [[TMP36]], [[MUL35]]
-// IR-NEXT:    store i32 [[ADD36]], ptr [[I]], align 4
+// IR-NEXT:    [[MUL40:%.*]] = mul i32 [[TMP37]], [[TMP38]]
+// IR-NEXT:    [[ADD41:%.*]] = add i32 [[TMP36]], [[MUL40]]
+// IR-NEXT:    store i32 [[ADD41]], ptr [[I]], align 4
 // IR-NEXT:    [[TMP39:%.*]] = load i32, ptr [[START_ADDR]], align 4
 // IR-NEXT:    [[TMP40:%.*]] = load i32, ptr [[END_ADDR]], align 4
 // IR-NEXT:    [[TMP41:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
@@ -175,22 +180,22 @@ extern "C" void body(...) {}
 // IR-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
 // IR-NEXT:    [[INC:%.*]] = add i32 [[TMP43]], 1
 // IR-NEXT:    store i32 [[INC]], ptr [[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND29]], !llvm.loop [[LOOP2:![0-9]+]]
+// IR-NEXT:    br label %[[FOR_COND34]], !llvm.loop [[LOOP2:![0-9]+]]
 // IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC37:.*]]
-// IR:       [[FOR_INC37]]:
-// IR-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    [[INC38:%.*]] = add i32 [[TMP44]], 1
-// IR-NEXT:    store i32 [[INC38]], ptr [[DOTTILE_CNT_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_INC42:.*]]
+// IR:       [[FOR_INC42]]:
+// IR-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    [[INC43:%.*]] = add i32 [[TMP44]], 1
+// IR-NEXT:    store i32 [[INC43]], ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
 // IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP5:![0-9]+]]
-// IR:       [[FOR_END39]]:
+// IR:       [[FOR_END44]]:
 // IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.*]]
 // IR:       [[OMP_BODY_CONTINUE]]:
 // IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.*]]
 // IR:       [[OMP_INNER_FOR_INC]]:
 // IR-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[ADD40:%.*]] = add i32 [[TMP45]], 1
-// IR-NEXT:    store i32 [[ADD40]], ptr [[DOTOMP_IV]], align 4
+// IR-NEXT:    [[ADD45:%.*]] = add i32 [[TMP45]], 1
+// IR-NEXT:    store i32 [[ADD45]], ptr [[DOTOMP_IV]], align 4
 // IR-NEXT:    br label %[[OMP_INNER_FOR_COND]]
 // IR:       [[OMP_INNER_FOR_END]]:
 // IR-NEXT:    br label %[[OMP_LOOP_EXIT:.*]]
diff --git a/openmp/runtime/test/transform/tile/foreach.cpp b/openmp/runtime/test/transform/tile/foreach.cpp
index 608a1c896fcb4..4fb3595760974 100644
--- a/openmp/runtime/test/transform/tile/foreach.cpp
+++ b/openmp/runtime/test/transform/tile/foreach.cpp
@@ -206,10 +206,6 @@ int main() {
 // CHECK-NEXT: v=2 w=1
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
-// CHECK-NEXT: [A] iterator advance: 0 += 3
-// CHECK-NEXT: [A] iterator move assign
-// CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 2
@@ -219,10 +215,6 @@ int main() {
 // CHECK-NEXT: v=2 w=2
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
-// CHECK-NEXT: [A] iterator advance: 0 += 3
-// CHECK-NEXT: [A] iterator move assign
-// CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
diff --git a/openmp/runtime/test/transform/tile/intfor.c b/openmp/runtime/test/transform/tile/intfor.c
index c5ceddd83da70..05ef5e85a0dad 100644
--- a/openmp/runtime/test/transform/tile/intfor.c
+++ b/openmp/runtime/test/transform/tile/intfor.c
@@ -30,49 +30,83 @@ int main() {
 // CHECK:      do
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: i=7 j=7
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: i=7 j=10
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: i=7 j=13
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: i=10 j=7
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: i=10 j=10
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: i=10 j=13
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: i=13 j=7
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: i=13 j=10
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: i=13 j=13
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: i=7 j=16
@@ -83,6 +117,8 @@ int main() {
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: i=10 j=16
@@ -93,6 +129,8 @@ int main() {
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: i=13 j=16
@@ -103,6 +141,8 @@ int main() {
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
@@ -111,21 +151,21 @@ int main() {
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: i=16 j=7
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
+// CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: i=16 j=10
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: i=16 j=13
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
+// CHECK-NEXT: i=16 j=13
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
@@ -148,12 +188,4 @@ int main() {
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(2)
-// CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/tile/iterfor.cpp b/openmp/runtime/test/transform/tile/iterfor.cpp
index 4b9bf6f41ada6..05b5677bf9c4f 100644
--- a/openmp/runtime/test/transform/tile/iterfor.cpp
+++ b/openmp/runtime/test/transform/tile/iterfor.cpp
@@ -215,9 +215,6 @@ int main() {
 // CHECK-NEXT: i=2 j=1
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
-// CHECK-NEXT: [A] iterator advance: 0 += 3
-// CHECK-NEXT: [A] iterator move assign
-// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [B] iterator advance: 0 += 2
@@ -227,9 +224,6 @@ int main() {
 // CHECK-NEXT: i=2 j=2
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
-// CHECK-NEXT: [A] iterator advance: 0 += 3
-// CHECK-NEXT: [A] iterator move assign
-// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
diff --git a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-foreach.cpp b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-foreach.cpp
index a007a369453d4..b1f4d98a52ddc 100644
--- a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-foreach.cpp
+++ b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-foreach.cpp
@@ -206,10 +206,6 @@ int main() {
 // CHECK-NEXT: i=0 v=2 w=1
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
-// CHECK-NEXT: [A] iterator advance: 0 += 3
-// CHECK-NEXT: [A] iterator move assign
-// CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 2
@@ -219,10 +215,6 @@ int main() {
 // CHECK-NEXT: i=0 v=2 w=2
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
-// CHECK-NEXT: [A] iterator advance: 0 += 3
-// CHECK-NEXT: [A] iterator move assign
-// CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 0
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 0
@@ -283,10 +275,6 @@ int main() {
 // CHECK-NEXT: i=1 v=2 w=1
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
-// CHECK-NEXT: [A] iterator advance: 0 += 3
-// CHECK-NEXT: [A] iterator move assign
-// CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 2
@@ -296,10 +284,6 @@ int main() {
 // CHECK-NEXT: i=1 v=2 w=2
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
-// CHECK-NEXT: [A] iterator advance: 0 += 3
-// CHECK-NEXT: [A] iterator move assign
-// CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 0
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 0
@@ -360,10 +344,6 @@ int main() {
 // CHECK-NEXT: i=2 v=2 w=1
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
-// CHECK-NEXT: [A] iterator advance: 0 += 3
-// CHECK-NEXT: [A] iterator move assign
-// CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [A] iterator advance: 0 += 2
 // CHECK-NEXT: [A] iterator move assign
 // CHECK-NEXT: [A] iterator deref: 2
@@ -373,10 +353,6 @@ int main() {
 // CHECK-NEXT: i=2 v=2 w=2
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [A] iterator dtor
-// CHECK-NEXT: [A] iterator advance: 0 += 3
-// CHECK-NEXT: [A] iterator move assign
-// CHECK-NEXT: [A] iterator deref: 3
-// CHECK-NEXT: [A] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
 // CHECK-NEXT: [B] iterator dtor
diff --git a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-stacked-tile-deep.cpp b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-stacked-tile-deep.cpp
new file mode 100644
index 0000000000000..1e835db20fb81
--- /dev/null
+++ b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-stacked-tile-deep.cpp
@@ -0,0 +1,32 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+
+// XFAIL: *
+// collapse(3) through stacked tiles is not supported.
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdio>
+
+int main() {
+  printf("do\n");
+#pragma omp parallel for collapse(3) num_threads(1)
+#pragma omp tile sizes(2)
+#pragma omp tile sizes(4)
+  for (int i = 0; i < 6; ++i)
+    printf("i=%d\n", i);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: i=0
+// CHECK-NEXT: i=1
+// CHECK-NEXT: i=2
+// CHECK-NEXT: i=3
+// CHECK-NEXT: i=4
+// CHECK-NEXT: i=5
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-stacked-tile.cpp b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-stacked-tile.cpp
new file mode 100644
index 0000000000000..feb780e8d3aec
--- /dev/null
+++ b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-stacked-tile.cpp
@@ -0,0 +1,31 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+
+// collapse(2) on stacked tiles: only outer floor+tile consumed (works).
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdio>
+
+int main() {
+  printf("do\n");
+#pragma omp parallel for collapse(2) num_threads(1)
+#pragma omp tile sizes(2)
+#pragma omp tile sizes(4)
+  for (int i = 0; i < 6; ++i)
+    printf("i=%d\n", i);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: i=0
+// CHECK-NEXT: i=1
+// CHECK-NEXT: i=2
+// CHECK-NEXT: i=3
+// CHECK-NEXT: i=4
+// CHECK-NEXT: i=5
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile-2d.cpp b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile-2d.cpp
new file mode 100644
index 0000000000000..64c31fd9470a5
--- /dev/null
+++ b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile-2d.cpp
@@ -0,0 +1,56 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+// RUN: %libomp-cxx-compile -O2 && %libomp-run | FileCheck %s --match-full-lines
+
+// 2-D tile consumed by collapse(4) (full) and collapse(3) (partial).
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdio>
+
+int main() {
+  printf("full4\n");
+#pragma omp parallel for collapse(4) num_threads(1)
+#pragma omp tile sizes(2, 2)
+  for (int i = 0; i < 3; ++i)
+    for (int j = 0; j < 3; ++j)
+      printf("i=%d j=%d\n", i, j);
+
+  printf("part3\n");
+#pragma omp parallel for collapse(3) num_threads(1)
+#pragma omp tile sizes(2, 2)
+  for (int i = 0; i < 3; ++i)
+    for (int j = 0; j < 4; ++j)
+      printf("i=%d j=%d\n", i, j);
+
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      full4
+// CHECK-NEXT: i=0 j=0
+// CHECK-NEXT: i=0 j=1
+// CHECK-NEXT: i=1 j=0
+// CHECK-NEXT: i=1 j=1
+// CHECK-NEXT: i=0 j=2
+// CHECK-NEXT: i=1 j=2
+// CHECK-NEXT: i=2 j=0
+// CHECK-NEXT: i=2 j=1
+// CHECK-NEXT: i=2 j=2
+// CHECK-NEXT: part3
+// CHECK-NEXT: i=0 j=0
+// CHECK-NEXT: i=0 j=1
+// CHECK-NEXT: i=1 j=0
+// CHECK-NEXT: i=1 j=1
+// CHECK-NEXT: i=0 j=2
+// CHECK-NEXT: i=0 j=3
+// CHECK-NEXT: i=1 j=2
+// CHECK-NEXT: i=1 j=3
+// CHECK-NEXT: i=2 j=0
+// CHECK-NEXT: i=2 j=1
+// CHECK-NEXT: i=2 j=2
+// CHECK-NEXT: i=2 j=3
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile-compose.cpp b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile-compose.cpp
new file mode 100644
index 0000000000000..15fe9e72564fc
--- /dev/null
+++ b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile-compose.cpp
@@ -0,0 +1,51 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+// RUN: %libomp-cxx-compile -O2 && %libomp-run | FileCheck %s --match-full-lines
+
+// collapse consuming a tile composed with reverse.
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdio>
+
+int main() {
+  printf("rev2\n");
+#pragma omp parallel for collapse(2) num_threads(1)
+#pragma omp tile sizes(2)
+#pragma omp reverse
+  for (int i = 0; i < 5; ++i)
+    printf("i=%d\n", i);
+
+  printf("rev3\n");
+#pragma omp parallel for collapse(3) num_threads(1)
+#pragma omp tile sizes(2)
+#pragma omp reverse
+  for (int i = 0; i < 5; ++i)
+    for (int j = 0; j < 2; ++j)
+      printf("i=%d j=%d\n", i, j);
+
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      rev2
+// CHECK-NEXT: i=4
+// CHECK-NEXT: i=3
+// CHECK-NEXT: i=2
+// CHECK-NEXT: i=1
+// CHECK-NEXT: i=0
+// CHECK-NEXT: rev3
+// CHECK-NEXT: i=4 j=0
+// CHECK-NEXT: i=4 j=1
+// CHECK-NEXT: i=3 j=0
+// CHECK-NEXT: i=3 j=1
+// CHECK-NEXT: i=2 j=0
+// CHECK-NEXT: i=2 j=1
+// CHECK-NEXT: i=1 j=0
+// CHECK-NEXT: i=1 j=1
+// CHECK-NEXT: i=0 j=0
+// CHECK-NEXT: i=0 j=1
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile-iterator.cpp b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile-iterator.cpp
new file mode 100644
index 0000000000000..94f3399aef9e9
--- /dev/null
+++ b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile-iterator.cpp
@@ -0,0 +1,74 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+// RUN: %libomp-cxx-compile -O2 && %libomp-run | FileCheck %s --match-full-lines
+
+// collapse consuming a tiled range-based-for loop.
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdio>
+
+struct Range {
+  int lo, hi;
+  struct It {
+    int v;
+    int operator*() const { return v; }
+    It &operator++() {
+      ++v;
+      return *this;
+    }
+    It operator++(int) {
+      It t = *this;
+      ++v;
+      return t;
+    }
+    bool operator==(const It &o) const { return v == o.v; }
+    bool operator!=(const It &o) const { return v != o.v; }
+    int operator-(const It &o) const { return v - o.v; }
+    It operator+(int n) const { return It{v + n}; }
+  };
+  It begin() const { return It{lo}; }
+  It end() const { return It{hi}; }
+};
+
+int main() {
+  Range r{0, 5};
+
+  printf("iter2\n");
+#pragma omp parallel for collapse(2) num_threads(1)
+#pragma omp tile sizes(2)
+  for (int v : r)
+    printf("v=%d\n", v);
+
+  printf("iter3\n");
+#pragma omp parallel for collapse(3) num_threads(1)
+#pragma omp tile sizes(2)
+  for (int v : r)
+    for (int j = 0; j < 2; ++j)
+      printf("v=%d j=%d\n", v, j);
+
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      iter2
+// CHECK-NEXT: v=0
+// CHECK-NEXT: v=1
+// CHECK-NEXT: v=2
+// CHECK-NEXT: v=3
+// CHECK-NEXT: v=4
+// CHECK-NEXT: iter3
+// CHECK-NEXT: v=0 j=0
+// CHECK-NEXT: v=0 j=1
+// CHECK-NEXT: v=1 j=0
+// CHECK-NEXT: v=1 j=1
+// CHECK-NEXT: v=2 j=0
+// CHECK-NEXT: v=2 j=1
+// CHECK-NEXT: v=3 j=0
+// CHECK-NEXT: v=3 j=1
+// CHECK-NEXT: v=4 j=0
+// CHECK-NEXT: v=4 j=1
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile.cpp b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile.cpp
index c7f7e143f7983..2cdd3ea33703f 100644
--- a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile.cpp
+++ b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-through-tile.cpp
@@ -1,9 +1,7 @@
 // RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+// RUN: %libomp-cxx-compile -O2 && %libomp-run | FileCheck %s --match-full-lines
 
-// After tiling `i`, the logical loops are `floor_i`, `tile_i`, and `j`, so
-// `collapse(3)` must reach through the tile guard to include `j`. CodeGen must
-// then keep that guard around the collapsed body once, instead of emitting `j`
-// again as a normal nested loop.
+// collapse(3) reaches through a tiled loop to include the inner `j` loop.
 
 #ifndef HEADER
 #define HEADER
@@ -25,8 +23,6 @@ int main() {
 #endif /* HEADER */
 
 // CHECK:      do
-
-// Full tile (.floor.i=0): i=0..3
 // CHECK-NEXT: i=0 j=0
 // CHECK-NEXT: i=0 j=1
 // CHECK-NEXT: i=1 j=0
@@ -35,8 +31,6 @@ int main() {
 // CHECK-NEXT: i=2 j=1
 // CHECK-NEXT: i=3 j=0
 // CHECK-NEXT: i=3 j=1
-
-// Partial tile (.floor.i=4): i=4,5 in bounds; i=6,7 skipped by the predicate
 // CHECK-NEXT: i=4 j=0
 // CHECK-NEXT: i=4 j=1
 // CHECK-NEXT: i=5 j=0

>From 8444507cc1cf84cc91ad87a6a920a40d7e9a8257 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Tue, 11 Aug 2026 15:08:11 -0400
Subject: [PATCH 17/19] revised

---
 clang/include/clang/AST/StmtOpenMP.h          |  36 ++++-
 clang/include/clang/Basic/Attr.td             |   4 +-
 .../clang/Basic/DiagnosticSemaKinds.td        |   3 +
 clang/lib/AST/StmtOpenMP.cpp                  |  47 +++---
 clang/lib/CodeGen/CGStmtOpenMP.cpp            |  17 +--
 clang/lib/Sema/SemaOpenMP.cpp                 | 143 ++++++++++++------
 6 files changed, 163 insertions(+), 87 deletions(-)

diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index dbc76e7df8ecd..33aada14bb642 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -25,6 +25,8 @@
 
 namespace clang {
 
+class OMPInvariantPredicateBoundAttr;
+
 //===----------------------------------------------------------------------===//
 // AST classes for directives.
 //===----------------------------------------------------------------------===//
@@ -892,6 +894,31 @@ class OMPLoopBasedDirective : public OMPExecutableDirective {
                                   TryImperfectlyNestedLoops);
   }
 
+  /// Returns the intra-tile reinterpretation hint attached to \p S, or nullptr
+  /// if \p S does not carry one. See OMPInvariantPredicateBoundAttr.
+  static const OMPInvariantPredicateBoundAttr *getIntraTileHint(const Stmt *S);
+
+  /// If \p S is an intra-tile reinterpretation wrapper, returns the loop it
+  /// annotates; otherwise returns \p S unchanged.
+  static Stmt *ignoreIntraTileHint(Stmt *S);
+  static const Stmt *ignoreIntraTileHint(const Stmt *S) {
+    return ignoreIntraTileHint(const_cast<Stmt *>(S));
+  }
+
+  /// Calls the specified callback function for all the loops in \p CurStmt,
+  /// from the outermost to the innermost. The callback is always passed a
+  /// ForStmt or CXXForRangeStmt; when that loop carries the intra-tile
+  /// reinterpretation hint, it is delivered as a separate argument instead of
+  /// wrapping the loop, so callers that only expect a loop cannot trip over it.
+  static bool
+  doForAllLoops(Stmt *CurStmt, bool TryImperfectlyNestedLoops,
+                unsigned NumLoops,
+                llvm::function_ref<bool(unsigned, Stmt *,
+                                        const OMPInvariantPredicateBoundAttr *)>
+                    Callback,
+                llvm::function_ref<void(OMPLoopTransformationDirective *)>
+                    OnTransformationCallback);
+
   /// Calls the specified callback function for all the loops in \p CurStmt,
   /// from the outermost to the innermost.
   static bool
@@ -899,7 +926,14 @@ class OMPLoopBasedDirective : public OMPExecutableDirective {
                 unsigned NumLoops,
                 llvm::function_ref<bool(unsigned, Stmt *)> Callback,
                 llvm::function_ref<void(OMPLoopTransformationDirective *)>
-                    OnTransformationCallback);
+                    OnTransformationCallback) {
+    auto &&NewCallback = [Callback](unsigned Cnt, Stmt *Loop,
+                                    const OMPInvariantPredicateBoundAttr *) {
+      return Callback(Cnt, Loop);
+    };
+    return doForAllLoops(CurStmt, TryImperfectlyNestedLoops, NumLoops,
+                         NewCallback, OnTransformationCallback);
+  }
   static bool
   doForAllLoops(const Stmt *CurStmt, bool TryImperfectlyNestedLoops,
                 unsigned NumLoops,
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 7d90c40c5ecf9..443a4874fd3e0 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -4881,8 +4881,8 @@ def OMPInvariantPredicateBound : StmtAttr {
   let Spellings = [];
   let SemaHandler = 0;
   let Subjects = SubjectList<[Stmt], ErrorDiag>;
-  let Args = [ExprArgument<"RectCond">, ExprArgument<"Predicate">,
-              ExprArgument<"TileSize">];
+  let Args = [ExprArgument<"RectCond">, ExprArgument<"TileSize">,
+              ExprArgument<"Predicate", 1>];
   let Documentation = [InternalOnly];
 }
 
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 01527e87c903f..b27e4e9e03e04 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12303,6 +12303,9 @@ def err_omp_loop_bad_collapse_var : Error<
   "cannot use variable %1 in collapsed imperfectly-nested loop %select{init|condition|increment}0 statement">;
 def err_omp_loop_var_reused_in_collapsed_loop : Error<
   "loop iteration variable %0 cannot be reused in a nested loop of a collapsed loop nest">;
+def err_omp_collapse_stacked_tile : Error<
+  "cannot collapse the intra-tile loop of a '#pragma omp tile' that is nested "
+  "inside another loop-transforming directive">;
 def err_omp_simd_region_cannot_use_stmt : Error<
   "'%0' statement cannot be used in OpenMP simd region">;
 def warn_omp_loop_64_bit_var : Warning<
diff --git a/clang/lib/AST/StmtOpenMP.cpp b/clang/lib/AST/StmtOpenMP.cpp
index 26000198518a8..827f8143cf744 100644
--- a/clang/lib/AST/StmtOpenMP.cpp
+++ b/clang/lib/AST/StmtOpenMP.cpp
@@ -18,22 +18,19 @@
 using namespace clang;
 using namespace llvm::omp;
 
-/// Returns true if the attributed statement carries the internal `omp tile`
-/// intra-tile reinterpretation hint (see OMPInvariantPredicateBoundAttr).
-static bool hasOMPInvariantPredicateBound(const AttributedStmt *AS) {
-  for (const Attr *A : AS->getAttrs())
-    if (isa<OMPInvariantPredicateBoundAttr>(A))
-      return true;
-  return false;
-}
-
-/// Peeks through an intra-tile hint wrapper to return the underlying loop;
-/// otherwise returns \p S unchanged. Used by the loop walker once the hint has
-/// already been delivered to the analysis callback and we need to look through
-/// the wrapper to reach the loop body.
-static Stmt *ignoreOMPIntraTileHint(Stmt *S) {
+/// Returns the intra-tile hint attribute for the given statement.
+const OMPInvariantPredicateBoundAttr *
+OMPLoopBasedDirective::getIntraTileHint(const Stmt *S) {
+  if (const auto *AS = dyn_cast_or_null<AttributedStmt>(S))
+    return getSpecificAttr<OMPInvariantPredicateBoundAttr>(AS->getAttrs());
+  return nullptr;
+}
+
+/// Peek through an intra-tile hint wrapper to return the underlying loop.
+/// The hint is delivered as the third argument to the loop callback.
+Stmt *OMPLoopBasedDirective::ignoreIntraTileHint(Stmt *S) {
   if (auto *AS = dyn_cast_or_null<AttributedStmt>(S))
-    if (hasOMPInvariantPredicateBound(AS))
+    if (hasSpecificAttr<OMPInvariantPredicateBoundAttr>(AS->getAttrs()))
       return AS->getSubStmt();
   return S;
 }
@@ -45,7 +42,7 @@ static Stmt *ignoreOMPIntraTileHint(Stmt *S) {
 static Stmt *ignoreContainersKeepingIntraTileHint(Stmt *S) {
   while (true) {
     if (auto *AS = dyn_cast_or_null<AttributedStmt>(S)) {
-      if (hasOMPInvariantPredicateBound(AS))
+      if (hasSpecificAttr<OMPInvariantPredicateBoundAttr>(AS->getAttrs()))
         break;
       S = AS->getSubStmt();
       continue;
@@ -144,7 +141,7 @@ OMPLoopBasedDirective::tryToFindNextInnerLoop(Stmt *CurStmt,
           Stmt *Inner = S;
           if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(Inner))
             Inner = CanonLoop->getLoopStmt();
-          Inner = ignoreOMPIntraTileHint(Inner);
+          Inner = OMPLoopBasedDirective::ignoreIntraTileHint(Inner);
           if (isa<ForStmt>(Inner) || isa<CXXForRangeStmt>(Inner) ||
               (isa<OMPLoopBasedDirective>(Inner) &&
                !isa<OMPLoopDirective>(Inner))) {
@@ -176,7 +173,9 @@ OMPLoopBasedDirective::tryToFindNextInnerLoop(Stmt *CurStmt,
 
 bool OMPLoopBasedDirective::doForAllLoops(
     Stmt *CurStmt, bool TryImperfectlyNestedLoops, unsigned NumLoops,
-    llvm::function_ref<bool(unsigned, Stmt *)> Callback,
+    llvm::function_ref<bool(unsigned, Stmt *,
+                            const OMPInvariantPredicateBoundAttr *)>
+        Callback,
     llvm::function_ref<void(OMPLoopTransformationDirective *)>
         OnTransformationCallback) {
   CurStmt = ignoreContainersKeepingIntraTileHint(CurStmt);
@@ -209,12 +208,12 @@ bool OMPLoopBasedDirective::doForAllLoops(
     }
     if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(CurStmt))
       CurStmt = CanonLoop->getLoopStmt();
-    // Deliver the (possibly hint-wrapped) loop node to the callback so it can
-    // read the intra-tile reinterpretation hint, then look through the wrapper
-    // to reach the loop body.
-    if (Callback(Cnt, CurStmt))
+    // The walker keeps the intra-tile hint wrapper around so that it can be
+    // handed to the analysis, but the callback only ever sees the loop itself.
+    const OMPInvariantPredicateBoundAttr *Hint = getIntraTileHint(CurStmt);
+    Stmt *LoopStmt = ignoreIntraTileHint(CurStmt);
+    if (Callback(Cnt, LoopStmt, Hint))
       return false;
-    Stmt *LoopStmt = ignoreOMPIntraTileHint(CurStmt);
     // Move on to the next nested for loop, or to the loop body.
     // OpenMP [2.8.1, simd construct, Restrictions]
     // All loops associated with the construct must be perfectly nested; that
@@ -239,8 +238,6 @@ void OMPLoopBasedDirective::doForAllLoopsBodies(
   bool Res = OMPLoopBasedDirective::doForAllLoops(
       CurStmt, TryImperfectlyNestedLoops, NumLoops,
       [Callback](unsigned Cnt, Stmt *Loop) {
-        // Look through an intra-tile hint wrapper to reach the loop itself.
-        Loop = ignoreOMPIntraTileHint(Loop);
         Stmt *Body = nullptr;
         if (auto *For = dyn_cast<ForStmt>(Loop)) {
           Body = For->getBody();
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 509d4f3d35815..3e87db54d9fd1 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -2231,17 +2231,6 @@ class OMPTransformDirectiveScopeRAII {
 };
 } // namespace
 
-/// Helper function to look through the droppable intra-tile hint wrapper
-/// (OMPInvariantPredicateBoundAttr) to the loop statement it annotates.
-static const Stmt *lookThroughOMPIntraTileHint(const Stmt *S) {
-  if (const auto *AS = dyn_cast_or_null<AttributedStmt>(S))
-    if (llvm::any_of(AS->getAttrs(), [](const Attr *A) {
-          return isa<OMPInvariantPredicateBoundAttr>(A);
-        }))
-      return AS->getSubStmt();
-  return S;
-}
-
 static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop,
                      int MaxLevel, int Level = 0) {
   assert(Level < MaxLevel && "Too deep lookup during loop body codegen.");
@@ -2258,10 +2247,10 @@ static void emitBody(CodeGenFunction &CGF, const Stmt *S, const Stmt *NextLoop,
     return;
   }
 
-  // The loop walker may hand back the intra-tile hint wrapper as `NextLoop`;
-  // match against the loop it annotates. The tile overshoot guard is emitted
+  // `tryToFindNextInnerLoop` keeps the intra-tile hint wrapper around, so match
+  // against the loop it annotates. The tile overshoot guard is emitted
   // separately via EmitOMPLoopBody's finals-conditions handling.
-  if (SimplifiedS == lookThroughOMPIntraTileHint(NextLoop)) {
+  if (SimplifiedS == OMPLoopBasedDirective::ignoreIntraTileHint(NextLoop)) {
     if (auto *Dir = dyn_cast<OMPLoopTransformationDirective>(SimplifiedS))
       SimplifiedS = Dir->getTransformedStmt();
     if (const auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(SimplifiedS))
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 701ba2d9f127d..e1097e75c66d4 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -17,6 +17,7 @@
 #include "TreeTransform.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/ASTMutationListener.h"
+#include "clang/AST/Attr.h"
 #include "clang/AST/CXXInheritance.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclCXX.h"
@@ -8021,6 +8022,8 @@ struct LoopIterationSpace final {
   /// True if this is a reinterpreted intra-tile loop whose lower bound is a
   /// live floor dependence.
   bool ReinterpretTileLB = false;
+  /// Location of the reinterpreted intra-tile loop, for diagnostics.
+  SourceLocation ReinterpretTileLoc;
 };
 
 /// Scan an AST subtree, checking that no decls in the CollapsedLoopVarDecls
@@ -9578,24 +9581,24 @@ static bool checkOpenMPIterationSpace(
     llvm::MutableArrayRef<LoopIterationSpace> ResultIterSpaces,
     llvm::MapVector<const Expr *, DeclRefExpr *> &Captures,
     const llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopVarDecls,
-    llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopInductionVars) {
+    llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopInductionVars,
+    const OMPInvariantPredicateBoundAttr *IntraTileHint) {
   bool SupportsNonRectangular = !isOpenMPLoopTransformationDirective(DKind);
   // See the tile reinterpretation design note in ActOnOpenMPTileDirective.
   // If the loop carries the hint, analyze its rectangular form instead.
+  //
+  // Only directives that emit the per-iteration body guard (i.e. those going
+  // through EmitOMPLoopBody's finals-conditions handling) may reinterpret the
+  // loop; a loop transformation such as an enclosing 'tile' has nowhere to put
+  // the overshoot guard, so for those the stored min-bounded form is analyzed
+  // exactly as it is on a build without this hint.
   Expr *TileRectCond = nullptr;
   Expr *TileBodyPredicate = nullptr;
   Expr *TileTripCount = nullptr;
-  if (auto *AS = dyn_cast_or_null<AttributedStmt>(S)) {
-    for (const Attr *A : AS->getAttrs()) {
-      if (const auto *H = dyn_cast<OMPInvariantPredicateBoundAttr>(A)) {
-        TileRectCond = H->getRectCond();
-        TileBodyPredicate = H->getPredicate();
-        TileTripCount = H->getTileSize();
-        break;
-      }
-    }
-    if (TileRectCond)
-      S = AS->getSubStmt();
+  if (IntraTileHint && !isOpenMPLoopTransformationDirective(DKind)) {
+    TileRectCond = IntraTileHint->getRectCond();
+    TileBodyPredicate = IntraTileHint->getPredicate();
+    TileTripCount = IntraTileHint->getTileSize();
   }
   // OpenMP [2.9.1, Canonical Loop Form]
   //   for (init-expr; test-expr; incr-expr) structured-block
@@ -9693,11 +9696,13 @@ static bool checkOpenMPIterationSpace(
     return HasErrors;
 
   // Build the loop's iteration space representation.
-  ResultIterSpaces[CurrentNestedLoopCount].PreCond =
-      ISC.buildPreCond(DSA.getCurScope(), EffectiveCond, Captures);
-  // Reinterpreted intra-tile: precondition is trivially true (.floor.iv
-  // unreadable here); overshoot handled by Predicate.
-  if (TileRectCond)
+  //
+  // A reinterpreted intra-tile loop takes all three of these from the hint
+  // instead of deriving them: the precondition is trivially true (the floor
+  // loop's own precondition already covers an empty iteration space), the trip
+  // count is the constant tile size, and any overshoot on the remainder tile
+  // is handled by the body guard.
+  if (TileRectCond) {
     ResultIterSpaces[CurrentNestedLoopCount].PreCond =
         SemaRef
             .PerformImplicitConversion(
@@ -9705,17 +9710,19 @@ static bool checkOpenMPIterationSpace(
                 SemaRef.Context.BoolTy, AssignmentAction::Casting,
                 /*AllowExplicit=*/true)
             .get();
-  ResultIterSpaces[CurrentNestedLoopCount].NumIterations =
-      ISC.buildNumIterations(DSA.getCurScope(), ResultIterSpaces,
-                             (isOpenMPWorksharingDirective(DKind) ||
-                              isOpenMPGenericLoopDirective(DKind) ||
-                              isOpenMPTaskLoopDirective(DKind) ||
-                              isOpenMPDistributeDirective(DKind) ||
-                              isOpenMPLoopTransformationDirective(DKind)),
-                             Captures);
-  // Reinterpreted intra-tile: trip count is the constant TileSize.
-  if (TileTripCount)
     ResultIterSpaces[CurrentNestedLoopCount].NumIterations = TileTripCount;
+  } else {
+    ResultIterSpaces[CurrentNestedLoopCount].PreCond =
+        ISC.buildPreCond(DSA.getCurScope(), EffectiveCond, Captures);
+    ResultIterSpaces[CurrentNestedLoopCount].NumIterations =
+        ISC.buildNumIterations(DSA.getCurScope(), ResultIterSpaces,
+                               (isOpenMPWorksharingDirective(DKind) ||
+                                isOpenMPGenericLoopDirective(DKind) ||
+                                isOpenMPTaskLoopDirective(DKind) ||
+                                isOpenMPDistributeDirective(DKind) ||
+                                isOpenMPLoopTransformationDirective(DKind)),
+                               Captures);
+  }
   ResultIterSpaces[CurrentNestedLoopCount].CounterVar =
       ISC.buildCounterVar(Captures, DSA);
   ResultIterSpaces[CurrentNestedLoopCount].PrivateCounterVar =
@@ -9733,15 +9740,19 @@ static bool checkOpenMPIterationSpace(
   std::tie(ResultIterSpaces[CurrentNestedLoopCount].MinValue,
            ResultIterSpaces[CurrentNestedLoopCount].MaxValue) =
       ISC.buildMinMaxValues(DSA.getCurScope(), Captures);
-  ResultIterSpaces[CurrentNestedLoopCount].FinalCondition =
-      ISC.buildFinalCondition(DSA.getCurScope());
-  // Reinterpreted intra-tile: body guard is the overshoot Predicate.
-  if (TileBodyPredicate)
+  if (TileRectCond) {
+    // The body guard is the overshoot predicate, which is absent when the tile
+    // can never be partial. Lower bound is the live .floor.iv; wire as a floor
+    // dependence for collapse
+    // (see design note in ActOnOpenMPTileDirective).
     ResultIterSpaces[CurrentNestedLoopCount].FinalCondition = TileBodyPredicate;
-  // Lower bound is the live .floor.iv; wire as a floor dependence for collapse
-  // (see design note in ActOnOpenMPTileDirective).
-  if (TileRectCond)
     ResultIterSpaces[CurrentNestedLoopCount].ReinterpretTileLB = true;
+    ResultIterSpaces[CurrentNestedLoopCount].ReinterpretTileLoc =
+        ISC.getInitSrcRange().getBegin();
+  } else {
+    ResultIterSpaces[CurrentNestedLoopCount].FinalCondition =
+        ISC.buildFinalCondition(DSA.getCurScope());
+  }
   ResultIterSpaces[CurrentNestedLoopCount].IsNonRectangularLB =
       ISC.doesInitDependOnLC();
   ResultIterSpaces[CurrentNestedLoopCount].IsNonRectangularUB =
@@ -10139,12 +10150,15 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
           [DKind, &SemaRef, &DSA, NumLoops, NestedLoopCount,
            CollapseLoopCountExpr, OrderedLoopCountExpr, &VarsWithImplicitDSA,
            &IterSpaces, &Captures, &CollapsedLoopVarDecls,
-           &CollapsedLoopInductionVars](unsigned Cnt, Stmt *CurStmt) {
+           &CollapsedLoopInductionVars](
+              unsigned Cnt, Stmt *CurStmt,
+              const OMPInvariantPredicateBoundAttr *IntraTileHint) {
             if (checkOpenMPIterationSpace(
                     DKind, CurStmt, SemaRef, DSA, Cnt, NestedLoopCount,
                     NumLoops, CollapseLoopCountExpr, OrderedLoopCountExpr,
                     VarsWithImplicitDSA, IterSpaces, Captures,
-                    CollapsedLoopVarDecls, CollapsedLoopInductionVars))
+                    CollapsedLoopVarDecls, CollapsedLoopInductionVars,
+                    IntraTileHint))
               return true;
             if (Cnt > 0 && Cnt >= NestedLoopCount &&
                 IterSpaces[Cnt].CounterVar) {
@@ -10669,6 +10683,7 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
       // Wire the reinterpreted lower bound to its floor dependence (see design
       // note in ActOnOpenMPTileDirective).
       if (IS.ReinterpretTileLB && IS.CounterInit) {
+        bool FoundFloor = false;
         if (const auto *LBRef =
                 dyn_cast<DeclRefExpr>(IS.CounterInit->IgnoreParenImpCasts())) {
           const Decl *FloorDecl = LBRef->getDecl()->getCanonicalDecl();
@@ -10677,10 +10692,22 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
             if (CV && CV->getDecl()->getCanonicalDecl() == FloorDecl) {
               IS.IsNonRectangularLB = true;
               IS.LoopDependentIdx = K + 1;
+              FoundFloor = true;
               break;
             }
           }
         }
+        if (!FoundFloor) {
+          // The floor this intra-tile loop starts from is not one of the
+          // collapsed counters: it is assigned in the body of an enclosing
+          // transformed loop, so the collapsed nest would read a stale value
+          // instead of recomputing it per iteration. Diagnose to avoid silently
+          // producing the wrong iteration space.
+          SemaRef.Diag(IS.ReinterpretTileLoc,
+                       diag::err_omp_collapse_stacked_tile);
+          HasErrors = true;
+          break;
+        }
       }
 
       // Compute prod
@@ -15242,9 +15269,11 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
   // below carrying an equivalent rectangular reinterpretation:
   //   RectCond  : .tile.iv < .floor.iv + T   -- rectangular bound, analyzed by
   //               checkOpenMPIterationSpace in place of the stored condition
-  //   Predicate : .tile.iv < N               -- remainder-tile overshoot,
-  //               applied as a body guard instead of shortening the trip count
   //   TileSize  : T                          -- constant per-tile trip count
+  //   Predicate : .tile.iv < N               -- remainder-tile overshoot,
+  //               applied as a body guard instead of shortening the trip count;
+  //               omitted when N is known to be a multiple of T, as there is
+  //               then no partial tile to guard against
   // Because the reinterpreted lower bound is still `.floor.iv`, checkOpenMPLoop
   // wires it as a live dependence on the matching floor counter, so a
   // collapsed `.tile.iv` re-reads the floor's current value instead of a
@@ -15314,8 +15343,8 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     if (!IncrStmt.isUsable())
       return StmtError();
 
-    // Build OMPInvariantPredicateBoundAttr hint (RectCond, Predicate,
-    // TileSize).
+    // Build OMPInvariantPredicateBoundAttr hint (RectCond, TileSize,
+    // Predicate).
     Expr *RectDimTileSize = MakeDimTileSize(I);
     if (!RectDimTileSize)
       return StmtError();
@@ -15328,18 +15357,41 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
     ExprResult RectCond =
         SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
                            MakeTileIVRef(), RectEndOfTile.get());
-    ExprResult Predicate =
-        SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(), BO_LT,
-                           MakeTileIVRef(), NumIterations);
     Expr *TileSizeExpr = MakeDimTileSize(I);
     if (!TileSizeExpr)
       return StmtError();
     ExprResult TileSize = SemaRef.PerformImplicitConversion(
         TileSizeExpr, IVTy, AssignmentAction::Converting,
         /*AllowExplicit=*/true);
-    if (!RectCond.isUsable() || !Predicate.isUsable() || !TileSize.isUsable())
+    if (!RectCond.isUsable() || !TileSize.isUsable())
       return StmtError();
 
+    // The overshoot guard is only needed if the last tile can be partial. When
+    // both the trip count and the tile size are known at compile time and the
+    // former is a multiple of the latter, every tile is full, so leave the
+    // predicate out. A dependent or run-time value makes
+    // the evaluation fail, which keeps the guard.
+    bool NoPartialTile = false;
+    Expr::EvalResult TileSizeVal, NumIterationsVal;
+    if (TileSizeExpr->EvaluateAsInt(TileSizeVal, Context) &&
+        NumIterations->EvaluateAsInt(NumIterationsVal, Context)) {
+      llvm::APSInt TS = TileSizeVal.Val.getInt();
+      llvm::APSInt N = NumIterationsVal.Val.getInt();
+      unsigned Width = std::max(TS.getBitWidth(), N.getBitWidth());
+      TS = TS.extend(Width);
+      N = N.extend(Width);
+      NoPartialTile =
+          TS.isStrictlyPositive() && N.isStrictlyPositive() && N.urem(TS) == 0;
+    }
+
+    ExprResult Predicate;
+    if (!NoPartialTile) {
+      Predicate = SemaRef.BuildBinOp(CurScope, LoopHelper.Cond->getExprLoc(),
+                                     BO_LT, MakeTileIVRef(), NumIterations);
+      if (!Predicate.isUsable())
+        return StmtError();
+    }
+
     // Statements to set the original iteration variable's value from the
     // logical iteration number.
     // Generated for loop is:
@@ -15370,7 +15422,8 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
 
     // Attach the droppable reinterpretation attribute to the intra-tile loop.
     auto *Hint = OMPInvariantPredicateBoundAttr::CreateImplicit(
-        Context, RectCond.get(), Predicate.get(), TileSize.get(),
+        Context, RectCond.get(), TileSize.get(),
+        Predicate.isUsable() ? Predicate.get() : nullptr,
         Inner->getSourceRange());
     Inner =
         AttributedStmt::Create(Context, Inner->getBeginLoc(), {Hint}, Inner);

>From 0cff8d5501e2430d82a04594da6190f7fddc20e7 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Tue, 11 Aug 2026 16:22:13 -0400
Subject: [PATCH 18/19] tests-updated

---
 .../OpenMP/tile_codegen_for_dependent.cpp     | 310 ++++++-------
 clang/test/OpenMP/tile_codegen_tile_for.cpp   | 421 +++++++++---------
 clang/test/OpenMP/tile_collapse_messages.cpp  |  63 +++
 .../tile_collapse_reinterpret_codegen.cpp     | 116 +++--
 clang/test/OpenMP/tile_messages.cpp           |  30 ++
 clang/test/OpenMP/tile_rect_codegen.cpp       |  51 ---
 clang/test/OpenMP/tile_rect_codegen_ir.cpp    | 375 +---------------
 clang/test/OpenMP/tile_vectorize_codegen.cpp  |  18 +-
 clang/test/OpenMP/unroll_codegen_tile_for.cpp | 408 +++++++++--------
 clang/unittests/AST/AttrTest.cpp              |  66 +++
 openmp/runtime/test/transform/tile/intfor.c   |   2 +-
 ...llel-wsloop-collapse-stacked-tile-deep.cpp |  32 --
 12 files changed, 875 insertions(+), 1017 deletions(-)
 create mode 100644 clang/test/OpenMP/tile_collapse_messages.cpp
 delete mode 100644 clang/test/OpenMP/tile_rect_codegen.cpp
 delete mode 100644 openmp/runtime/test/transform/tile/parallel-wsloop-collapse-stacked-tile-deep.cpp

diff --git a/clang/test/OpenMP/tile_codegen_for_dependent.cpp b/clang/test/OpenMP/tile_codegen_for_dependent.cpp
index 7401ab1a2ded3..820d33d15287b 100644
--- a/clang/test/OpenMP/tile_codegen_for_dependent.cpp
+++ b/clang/test/OpenMP/tile_codegen_for_dependent.cpp
@@ -1,4 +1,3 @@
-// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
 // Check code generation
 // RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fopenmp -emit-llvm %s -o - | FileCheck %s --check-prefix=IR
 
@@ -18,155 +17,172 @@
 extern "C" void body(...) {}
 
 
-// IR-LABEL: define dso_local void @func(
-// IR-SAME: i32 noundef [[START:%.*]], i32 noundef [[END:%.*]], i32 noundef [[STEP:%.*]]) #[[ATTR0:[0-9]+]] {
-// IR-NEXT:  [[ENTRY:.*:]]
-// IR-NEXT:    [[START_ADDR:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[TMP:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_5:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV_I11:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
-// IR-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
-// IR-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
-// IR-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// IR-NEXT:    store i32 [[TMP1]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// IR-NEXT:    store i32 [[TMP2]], ptr [[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// IR-NEXT:    store i32 [[TMP3]], ptr [[DOTCAPTURE_EXPR_1]], align 4
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// IR-NEXT:    store i32 [[TMP4]], ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    [[SUB:%.*]] = sub i32 [[TMP5]], [[TMP6]]
-// IR-NEXT:    [[SUB3:%.*]] = sub i32 [[SUB]], 1
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add i32 [[SUB3]], [[TMP7]]
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP8]]
-// IR-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
-// IR-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[ADD6:%.*]] = add i32 [[TMP9]], 1
-// IR-NEXT:    store i32 [[ADD6]], ptr [[DOTCAPTURE_EXPR_5]], align 4
-// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// IR-NEXT:    [[SUB8:%.*]] = sub i32 [[TMP10]], -3
-// IR-NEXT:    [[DIV9:%.*]] = udiv i32 [[SUB8]], 4
-// IR-NEXT:    [[SUB10:%.*]] = sub i32 [[DIV9]], 1
-// IR-NEXT:    store i32 [[SUB10]], ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// IR-NEXT:    [[CMP:%.*]] = icmp ult i32 0, [[TMP11]]
-// IR-NEXT:    br i1 [[CMP]], label %[[OMP_PRECOND_THEN:.*]], label %[[OMP_PRECOND_END:.*]]
-// IR:       [[OMP_PRECOND_THEN]]:
-// IR-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
-// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    store i32 [[TMP12]], ptr [[DOTOMP_UB]], align 4
-// IR-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
-// IR-NEXT:    call void @__kmpc_for_static_init_4u(ptr @[[GLOB1:[0-9]+]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
-// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    [[CMP12:%.*]] = icmp ugt i32 [[TMP13]], [[TMP14]]
-// IR-NEXT:    br i1 [[CMP12]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
-// IR:       [[COND_TRUE]]:
-// IR-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    br label %[[COND_END:.*]]
-// IR:       [[COND_FALSE]]:
-// IR-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// IR-LABEL: define {{.*}}@func(
+// IR-NEXT:  [[ENTRY:.*]]:
+// IR-NEXT:    %[[START_ADDR:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[END_ADDR:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[STEP_ADDR:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_IV:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[TMP:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[I:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_1:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTNEW_STEP:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_2:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_5:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_7:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTFLOOR_0_IV_I:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_LB:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_UB:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_STRIDE:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_IS_LAST:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTFLOOR_0_IV_I11:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTTILE_0_IV_I:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[TMP0:.+]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:.+]])
+// IR-NEXT:    store i32 %[[START:.+]], ptr %[[START_ADDR]], align 4
+// IR-NEXT:    store i32 %[[END:.+]], ptr %[[END_ADDR]], align 4
+// IR-NEXT:    store i32 %[[STEP:.+]], ptr %[[STEP_ADDR]], align 4
+// IR-NEXT:    %[[TMP1:.+]] = load i32, ptr %[[START_ADDR]], align 4
+// IR-NEXT:    store i32 %[[TMP1]], ptr %[[I]], align 4
+// IR-NEXT:    %[[TMP2:.+]] = load i32, ptr %[[START_ADDR]], align 4
+// IR-NEXT:    store i32 %[[TMP2]], ptr %[[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    %[[TMP3:.+]] = load i32, ptr %[[END_ADDR]], align 4
+// IR-NEXT:    store i32 %[[TMP3]], ptr %[[DOTCAPTURE_EXPR_1]], align 4
+// IR-NEXT:    %[[TMP4:.+]] = load i32, ptr %[[STEP_ADDR]], align 4
+// IR-NEXT:    store i32 %[[TMP4]], ptr %[[DOTNEW_STEP]], align 4
+// IR-NEXT:    %[[TMP5:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_1]], align 4
+// IR-NEXT:    %[[TMP6:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    %[[SUB:.+]] = sub i32 %[[TMP5]], %[[TMP6]]
+// IR-NEXT:    %[[SUB3:.+]] = sub i32 %[[SUB]], 1
+// IR-NEXT:    %[[TMP7:.+]] = load i32, ptr %[[DOTNEW_STEP]], align 4
+// IR-NEXT:    %[[ADD:.+]] = add i32 %[[SUB3]], %[[TMP7]]
+// IR-NEXT:    %[[TMP8:.+]] = load i32, ptr %[[DOTNEW_STEP]], align 4
+// IR-NEXT:    %[[DIV:.+]] = udiv i32 %[[ADD]], %[[TMP8]]
+// IR-NEXT:    %[[SUB4:.+]] = sub i32 %[[DIV]], 1
+// IR-NEXT:    store i32 %[[SUB4]], ptr %[[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    %[[TMP9:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    %[[ADD6:.+]] = add i32 %[[TMP9]], 1
+// IR-NEXT:    store i32 %[[ADD6]], ptr %[[DOTCAPTURE_EXPR_5]], align 4
+// IR-NEXT:    %[[TMP10:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_5]], align 4
+// IR-NEXT:    %[[SUB8:.+]] = sub i32 %[[TMP10]], -3
+// IR-NEXT:    %[[DIV9:.+]] = udiv i32 %[[SUB8]], 4
+// IR-NEXT:    %[[SUB10:.+]] = sub i32 %[[DIV9]], 1
+// IR-NEXT:    store i32 %[[SUB10]], ptr %[[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    store i32 0, ptr %[[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    %[[TMP11:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_5]], align 4
+// IR-NEXT:    %[[CMP:.+]] = icmp ult i32 0, %[[TMP11]]
+// IR-NEXT:    br i1 %[[CMP]], label %[[OMP_PRECOND_THEN:.+]], label %[[OMP_PRECOND_END:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_PRECOND_THEN]]:
+// IR-NEXT:    store i32 0, ptr %[[DOTOMP_LB]], align 4
+// IR-NEXT:    %[[TMP12:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    store i32 %[[TMP12]], ptr %[[DOTOMP_UB]], align 4
+// IR-NEXT:    store i32 1, ptr %[[DOTOMP_STRIDE]], align 4
+// IR-NEXT:    store i32 0, ptr %[[DOTOMP_IS_LAST]], align 4
+// IR-NEXT:    call void @__kmpc_for_static_init_4u(ptr @[[GLOB1:.+]], i32 %[[TMP0]], i32 34, ptr %[[DOTOMP_IS_LAST]], ptr %[[DOTOMP_LB]], ptr %[[DOTOMP_UB]], ptr %[[DOTOMP_STRIDE]], i32 1, i32 1)
+// IR-NEXT:    %[[TMP13:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
+// IR-NEXT:    %[[TMP14:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    %[[CMP12:.+]] = icmp ugt i32 %[[TMP13]], %[[TMP14]]
+// IR-NEXT:    br i1 %[[CMP12]], label %[[COND_TRUE:.+]], label %[[COND_FALSE:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_TRUE]]:
+// IR-NEXT:    %[[TMP15:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    br label %[[COND_END:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_FALSE]]:
+// IR-NEXT:    %[[TMP16:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
 // IR-NEXT:    br label %[[COND_END]]
-// IR:       [[COND_END]]:
-// IR-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP15]], %[[COND_TRUE]] ], [ [[TMP16]], %[[COND_FALSE]] ]
-// IR-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
-// IR-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
-// IR-NEXT:    store i32 [[TMP17]], ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    br label %[[OMP_INNER_FOR_COND:.*]]
-// IR:       [[OMP_INNER_FOR_COND]]:
-// IR-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// IR-NEXT:    [[ADD13:%.*]] = add i32 [[TMP19]], 1
-// IR-NEXT:    [[CMP14:%.*]] = icmp ult i32 [[TMP18]], [[ADD13]]
-// IR-NEXT:    br i1 [[CMP14]], label %[[OMP_INNER_FOR_BODY:.*]], label %[[OMP_INNER_FOR_END:.*]]
-// IR:       [[OMP_INNER_FOR_BODY]]:
-// IR-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul i32 [[TMP20]], 4
-// IR-NEXT:    [[ADD15:%.*]] = add i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I11]], align 4
-// IR-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I11]], align 4
-// IR-NEXT:    store i32 [[TMP21]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND:.*]]
-// IR:       [[FOR_COND]]:
-// IR-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[ADD16:%.*]] = add i32 [[TMP23]], 1
-// IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I11]], align 4
-// IR-NEXT:    [[ADD17:%.*]] = add i32 [[TMP24]], 4
-// IR-NEXT:    [[CMP18:%.*]] = icmp ult i32 [[ADD16]], [[ADD17]]
-// IR-NEXT:    br i1 [[CMP18]], label %[[COND_TRUE19:.*]], label %[[COND_FALSE21:.*]]
-// IR:       [[COND_TRUE19]]:
-// IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[ADD20:%.*]] = add i32 [[TMP25]], 1
-// IR-NEXT:    br label %[[COND_END23:.*]]
-// IR:       [[COND_FALSE21]]:
-// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I11]], align 4
-// IR-NEXT:    [[ADD22:%.*]] = add i32 [[TMP26]], 4
+// IR-EMPTY:
+// IR-NEXT:  [[COND_END]]:
+// IR-NEXT:    %[[COND:.+]] = phi i32 [ %[[TMP15]], %[[COND_TRUE]] ], [ %[[TMP16]], %[[COND_FALSE]] ]
+// IR-NEXT:    store i32 %[[COND]], ptr %[[DOTOMP_UB]], align 4
+// IR-NEXT:    %[[TMP17:.+]] = load i32, ptr %[[DOTOMP_LB]], align 4
+// IR-NEXT:    store i32 %[[TMP17]], ptr %[[DOTOMP_IV]], align 4
+// IR-NEXT:    br label %[[OMP_INNER_FOR_COND:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_INNER_FOR_COND]]:
+// IR-NEXT:    %[[TMP18:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
+// IR-NEXT:    %[[TMP19:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
+// IR-NEXT:    %[[ADD13:.+]] = add i32 %[[TMP19]], 1
+// IR-NEXT:    %[[CMP14:.+]] = icmp ult i32 %[[TMP18]], %[[ADD13]]
+// IR-NEXT:    br i1 %[[CMP14]], label %[[OMP_INNER_FOR_BODY:.+]], label %[[OMP_INNER_FOR_END:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_INNER_FOR_BODY]]:
+// IR-NEXT:    %[[TMP20:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
+// IR-NEXT:    %[[MUL:.+]] = mul i32 %[[TMP20]], 4
+// IR-NEXT:    %[[ADD15:.+]] = add i32 0, %[[MUL]]
+// IR-NEXT:    store i32 %[[ADD15]], ptr %[[DOTFLOOR_0_IV_I11]], align 4
+// IR-NEXT:    %[[TMP21:.+]] = load i32, ptr %[[DOTFLOOR_0_IV_I11]], align 4
+// IR-NEXT:    store i32 %[[TMP21]], ptr %[[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_COND]]:
+// IR-NEXT:    %[[TMP22:.+]] = load i32, ptr %[[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    %[[TMP23:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    %[[ADD16:.+]] = add i32 %[[TMP23]], 1
+// IR-NEXT:    %[[TMP24:.+]] = load i32, ptr %[[DOTFLOOR_0_IV_I11]], align 4
+// IR-NEXT:    %[[ADD17:.+]] = add i32 %[[TMP24]], 4
+// IR-NEXT:    %[[CMP18:.+]] = icmp ult i32 %[[ADD16]], %[[ADD17]]
+// IR-NEXT:    br i1 %[[CMP18]], label %[[COND_TRUE19:.+]], label %[[COND_FALSE21:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_TRUE19]]:
+// IR-NEXT:    %[[TMP25:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    %[[ADD20:.+]] = add i32 %[[TMP25]], 1
+// IR-NEXT:    br label %[[COND_END23:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_FALSE21]]:
+// IR-NEXT:    %[[TMP26:.+]] = load i32, ptr %[[DOTFLOOR_0_IV_I11]], align 4
+// IR-NEXT:    %[[ADD22:.+]] = add i32 %[[TMP26]], 4
 // IR-NEXT:    br label %[[COND_END23]]
-// IR:       [[COND_END23]]:
-// IR-NEXT:    [[COND24:%.*]] = phi i32 [ [[ADD20]], %[[COND_TRUE19]] ], [ [[ADD22]], %[[COND_FALSE21]] ]
-// IR-NEXT:    [[CMP25:%.*]] = icmp ult i32 [[TMP22]], [[COND24]]
-// IR-NEXT:    br i1 [[CMP25]], label %[[FOR_BODY:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY]]:
-// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[MUL26:%.*]] = mul i32 [[TMP28]], [[TMP29]]
-// IR-NEXT:    [[ADD27:%.*]] = add i32 [[TMP27]], [[MUL26]]
-// IR-NEXT:    store i32 [[ADD27]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void (...) @body(i32 noundef [[TMP30]], i32 noundef [[TMP31]], i32 noundef [[TMP32]], i32 noundef [[TMP33]])
-// IR-NEXT:    br label %[[FOR_INC:.*]]
-// IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add i32 [[TMP34]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP2:![0-9]+]]
-// IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.*]]
-// IR:       [[OMP_BODY_CONTINUE]]:
-// IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.*]]
-// IR:       [[OMP_INNER_FOR_INC]]:
-// IR-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[ADD28:%.*]] = add i32 [[TMP35]], 1
-// IR-NEXT:    store i32 [[ADD28]], ptr [[DOTOMP_IV]], align 4
+// IR-EMPTY:
+// IR-NEXT:  [[COND_END23]]:
+// IR-NEXT:    %[[COND24:.+]] = phi i32 [ %[[ADD20]], %[[COND_TRUE19]] ], [ %[[ADD22]], %[[COND_FALSE21]] ]
+// IR-NEXT:    %[[CMP25:.+]] = icmp ult i32 %[[TMP22]], %[[COND24]]
+// IR-NEXT:    br i1 %[[CMP25]], label %[[FOR_BODY:.+]], label %[[FOR_END:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_BODY]]:
+// IR-NEXT:    %[[TMP27:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    %[[TMP28:.+]] = load i32, ptr %[[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    %[[TMP29:.+]] = load i32, ptr %[[DOTNEW_STEP]], align 4
+// IR-NEXT:    %[[MUL26:.+]] = mul i32 %[[TMP28]], %[[TMP29]]
+// IR-NEXT:    %[[ADD27:.+]] = add i32 %[[TMP27]], %[[MUL26]]
+// IR-NEXT:    store i32 %[[ADD27]], ptr %[[I]], align 4
+// IR-NEXT:    %[[TMP30:.+]] = load i32, ptr %[[START_ADDR]], align 4
+// IR-NEXT:    %[[TMP31:.+]] = load i32, ptr %[[END_ADDR]], align 4
+// IR-NEXT:    %[[TMP32:.+]] = load i32, ptr %[[STEP_ADDR]], align 4
+// IR-NEXT:    %[[TMP33:.+]] = load i32, ptr %[[I]], align 4
+// IR-NEXT:    call void (...) @body(i32 noundef %[[TMP30]], i32 noundef %[[TMP31]], i32 noundef %[[TMP32]], i32 noundef %[[TMP33]])
+// IR-NEXT:    br label %[[FOR_INC:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_INC]]:
+// IR-NEXT:    %[[TMP34:.+]] = load i32, ptr %[[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    %[[INC:.+]] = add i32 %[[TMP34]], 1
+// IR-NEXT:    store i32 %[[INC]], ptr %[[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop ![[LOOP3:[0-9]+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_END]]:
+// IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_BODY_CONTINUE]]:
+// IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_INNER_FOR_INC]]:
+// IR-NEXT:    %[[TMP35:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
+// IR-NEXT:    %[[ADD28:.+]] = add i32 %[[TMP35]], 1
+// IR-NEXT:    store i32 %[[ADD28]], ptr %[[DOTOMP_IV]], align 4
 // IR-NEXT:    br label %[[OMP_INNER_FOR_COND]]
-// IR:       [[OMP_INNER_FOR_END]]:
-// IR-NEXT:    br label %[[OMP_LOOP_EXIT:.*]]
-// IR:       [[OMP_LOOP_EXIT]]:
-// IR-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP0]])
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_INNER_FOR_END]]:
+// IR-NEXT:    br label %[[OMP_LOOP_EXIT:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_LOOP_EXIT]]:
+// IR-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 %[[TMP0]])
 // IR-NEXT:    br label %[[OMP_PRECOND_END]]
-// IR:       [[OMP_PRECOND_END]]:
-// IR-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3:[0-9]+]], i32 [[TMP0]])
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_PRECOND_END]]:
+// IR-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3:.+]], i32 %[[TMP0]])
 // IR-NEXT:    ret void
-//
+// IR-NEXT:  }
 extern "C" void func(int start, int end, int step) {
 #pragma omp for
 #pragma omp tile sizes(4)
@@ -175,7 +191,3 @@ extern "C" void func(int start, int end, int step) {
 }
 
 #endif /* HEADER */
-//.
-// IR: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]]}
-// IR: [[META3]] = !{!"llvm.loop.mustprogress"}
-//.
diff --git a/clang/test/OpenMP/tile_codegen_tile_for.cpp b/clang/test/OpenMP/tile_codegen_tile_for.cpp
index 3d0092042cdfc..8a844c78c54a6 100644
--- a/clang/test/OpenMP/tile_codegen_tile_for.cpp
+++ b/clang/test/OpenMP/tile_codegen_tile_for.cpp
@@ -1,4 +1,3 @@
-// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
 // Check code generation
 // RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fopenmp -emit-llvm %s -o - | FileCheck %s --check-prefix=IR
 
@@ -17,205 +16,229 @@
 extern "C" void body(...) {}
 
 
-
-
-// IR-LABEL: define dso_local void @func(
-// IR-SAME: i32 noundef [[START:%.*]], i32 noundef [[END:%.*]], i32 noundef [[STEP:%.*]]) #[[ATTR0:[0-9]+]] {
-// IR-NEXT:  [[ENTRY:.*:]]
-// IR-NEXT:    [[START_ADDR:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[TMP:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_5:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_13:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV__FLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV__FLOOR_0_IV_I17:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV__FLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
-// IR-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
-// IR-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
-// IR-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// IR-NEXT:    store i32 [[TMP1]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// IR-NEXT:    store i32 [[TMP2]], ptr [[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// IR-NEXT:    store i32 [[TMP3]], ptr [[DOTCAPTURE_EXPR_1]], align 4
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// IR-NEXT:    store i32 [[TMP4]], ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    [[SUB:%.*]] = sub i32 [[TMP5]], [[TMP6]]
-// IR-NEXT:    [[SUB3:%.*]] = sub i32 [[SUB]], 1
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add i32 [[SUB3]], [[TMP7]]
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP8]]
-// IR-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
-// IR-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[ADD6:%.*]] = add i32 [[TMP9]], 1
-// IR-NEXT:    store i32 [[ADD6]], ptr [[DOTCAPTURE_EXPR_5]], align 4
-// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// IR-NEXT:    [[SUB8:%.*]] = sub i32 [[TMP10]], -3
-// IR-NEXT:    [[DIV9:%.*]] = udiv i32 [[SUB8]], 4
-// IR-NEXT:    [[SUB10:%.*]] = sub i32 [[DIV9]], 1
-// IR-NEXT:    store i32 [[SUB10]], ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    [[ADD12:%.*]] = add i32 [[TMP11]], 1
-// IR-NEXT:    store i32 [[ADD12]], ptr [[DOTCAPTURE_EXPR_11]], align 4
-// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// IR-NEXT:    [[SUB14:%.*]] = sub i32 [[TMP12]], -2
-// IR-NEXT:    [[DIV15:%.*]] = udiv i32 [[SUB14]], 3
-// IR-NEXT:    [[SUB16:%.*]] = sub i32 [[DIV15]], 1
-// IR-NEXT:    store i32 [[SUB16]], ptr [[DOTCAPTURE_EXPR_13]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// IR-NEXT:    [[CMP:%.*]] = icmp ult i32 0, [[TMP13]]
-// IR-NEXT:    br i1 [[CMP]], label %[[OMP_PRECOND_THEN:.*]], label %[[OMP_PRECOND_END:.*]]
-// IR:       [[OMP_PRECOND_THEN]]:
-// IR-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
-// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_13]], align 4
-// IR-NEXT:    store i32 [[TMP14]], ptr [[DOTOMP_UB]], align 4
-// IR-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
-// IR-NEXT:    call void @__kmpc_for_static_init_4u(ptr @[[GLOB1:[0-9]+]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
-// IR-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// IR-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_13]], align 4
-// IR-NEXT:    [[CMP18:%.*]] = icmp ugt i32 [[TMP15]], [[TMP16]]
-// IR-NEXT:    br i1 [[CMP18]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
-// IR:       [[COND_TRUE]]:
-// IR-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_13]], align 4
-// IR-NEXT:    br label %[[COND_END:.*]]
-// IR:       [[COND_FALSE]]:
-// IR-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// IR-LABEL: define {{.*}}@func(
+// IR-NEXT:  [[ENTRY:.*]]:
+// IR-NEXT:    %[[START_ADDR:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[END_ADDR:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[STEP_ADDR:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_IV:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[TMP:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[I:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_1:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTNEW_STEP:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_2:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTFLOOR_0_IV_I:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_5:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_7:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_11:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_13:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTFLOOR_0_IV__FLOOR_0_IV_I:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_LB:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_UB:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_STRIDE:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_IS_LAST:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTFLOOR_0_IV__FLOOR_0_IV_I17:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTTILE_0_IV__FLOOR_0_IV_I:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTTILE_0_IV_I:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[TMP0:.+]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:.+]])
+// IR-NEXT:    store i32 %[[START:.+]], ptr %[[START_ADDR]], align 4
+// IR-NEXT:    store i32 %[[END:.+]], ptr %[[END_ADDR]], align 4
+// IR-NEXT:    store i32 %[[STEP:.+]], ptr %[[STEP_ADDR]], align 4
+// IR-NEXT:    %[[TMP1:.+]] = load i32, ptr %[[START_ADDR]], align 4
+// IR-NEXT:    store i32 %[[TMP1]], ptr %[[I]], align 4
+// IR-NEXT:    %[[TMP2:.+]] = load i32, ptr %[[START_ADDR]], align 4
+// IR-NEXT:    store i32 %[[TMP2]], ptr %[[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    %[[TMP3:.+]] = load i32, ptr %[[END_ADDR]], align 4
+// IR-NEXT:    store i32 %[[TMP3]], ptr %[[DOTCAPTURE_EXPR_1]], align 4
+// IR-NEXT:    %[[TMP4:.+]] = load i32, ptr %[[STEP_ADDR]], align 4
+// IR-NEXT:    store i32 %[[TMP4]], ptr %[[DOTNEW_STEP]], align 4
+// IR-NEXT:    %[[TMP5:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_1]], align 4
+// IR-NEXT:    %[[TMP6:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    %[[SUB:.+]] = sub i32 %[[TMP5]], %[[TMP6]]
+// IR-NEXT:    %[[SUB3:.+]] = sub i32 %[[SUB]], 1
+// IR-NEXT:    %[[TMP7:.+]] = load i32, ptr %[[DOTNEW_STEP]], align 4
+// IR-NEXT:    %[[ADD:.+]] = add i32 %[[SUB3]], %[[TMP7]]
+// IR-NEXT:    %[[TMP8:.+]] = load i32, ptr %[[DOTNEW_STEP]], align 4
+// IR-NEXT:    %[[DIV:.+]] = udiv i32 %[[ADD]], %[[TMP8]]
+// IR-NEXT:    %[[SUB4:.+]] = sub i32 %[[DIV]], 1
+// IR-NEXT:    store i32 %[[SUB4]], ptr %[[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    store i32 0, ptr %[[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    %[[TMP9:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    %[[ADD6:.+]] = add i32 %[[TMP9]], 1
+// IR-NEXT:    store i32 %[[ADD6]], ptr %[[DOTCAPTURE_EXPR_5]], align 4
+// IR-NEXT:    %[[TMP10:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_5]], align 4
+// IR-NEXT:    %[[SUB8:.+]] = sub i32 %[[TMP10]], -3
+// IR-NEXT:    %[[DIV9:.+]] = udiv i32 %[[SUB8]], 4
+// IR-NEXT:    %[[SUB10:.+]] = sub i32 %[[DIV9]], 1
+// IR-NEXT:    store i32 %[[SUB10]], ptr %[[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    %[[TMP11:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    %[[ADD12:.+]] = add i32 %[[TMP11]], 1
+// IR-NEXT:    store i32 %[[ADD12]], ptr %[[DOTCAPTURE_EXPR_11]], align 4
+// IR-NEXT:    %[[TMP12:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_11]], align 4
+// IR-NEXT:    %[[SUB14:.+]] = sub i32 %[[TMP12]], -2
+// IR-NEXT:    %[[DIV15:.+]] = udiv i32 %[[SUB14]], 3
+// IR-NEXT:    %[[SUB16:.+]] = sub i32 %[[DIV15]], 1
+// IR-NEXT:    store i32 %[[SUB16]], ptr %[[DOTCAPTURE_EXPR_13]], align 4
+// IR-NEXT:    store i32 0, ptr %[[DOTFLOOR_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    %[[TMP13:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_11]], align 4
+// IR-NEXT:    %[[CMP:.+]] = icmp ult i32 0, %[[TMP13]]
+// IR-NEXT:    br i1 %[[CMP]], label %[[OMP_PRECOND_THEN:.+]], label %[[OMP_PRECOND_END:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_PRECOND_THEN]]:
+// IR-NEXT:    store i32 0, ptr %[[DOTOMP_LB]], align 4
+// IR-NEXT:    %[[TMP14:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_13]], align 4
+// IR-NEXT:    store i32 %[[TMP14]], ptr %[[DOTOMP_UB]], align 4
+// IR-NEXT:    store i32 1, ptr %[[DOTOMP_STRIDE]], align 4
+// IR-NEXT:    store i32 0, ptr %[[DOTOMP_IS_LAST]], align 4
+// IR-NEXT:    call void @__kmpc_for_static_init_4u(ptr @[[GLOB1:.+]], i32 %[[TMP0]], i32 34, ptr %[[DOTOMP_IS_LAST]], ptr %[[DOTOMP_LB]], ptr %[[DOTOMP_UB]], ptr %[[DOTOMP_STRIDE]], i32 1, i32 1)
+// IR-NEXT:    %[[TMP15:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
+// IR-NEXT:    %[[TMP16:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_13]], align 4
+// IR-NEXT:    %[[CMP18:.+]] = icmp ugt i32 %[[TMP15]], %[[TMP16]]
+// IR-NEXT:    br i1 %[[CMP18]], label %[[COND_TRUE:.+]], label %[[COND_FALSE:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_TRUE]]:
+// IR-NEXT:    %[[TMP17:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_13]], align 4
+// IR-NEXT:    br label %[[COND_END:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_FALSE]]:
+// IR-NEXT:    %[[TMP18:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
 // IR-NEXT:    br label %[[COND_END]]
-// IR:       [[COND_END]]:
-// IR-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP17]], %[[COND_TRUE]] ], [ [[TMP18]], %[[COND_FALSE]] ]
-// IR-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
-// IR-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
-// IR-NEXT:    store i32 [[TMP19]], ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    br label %[[OMP_INNER_FOR_COND:.*]]
-// IR:       [[OMP_INNER_FOR_COND]]:
-// IR-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// IR-NEXT:    [[ADD19:%.*]] = add i32 [[TMP21]], 1
-// IR-NEXT:    [[CMP20:%.*]] = icmp ult i32 [[TMP20]], [[ADD19]]
-// IR-NEXT:    br i1 [[CMP20]], label %[[OMP_INNER_FOR_BODY:.*]], label %[[OMP_INNER_FOR_END:.*]]
-// IR:       [[OMP_INNER_FOR_BODY]]:
-// IR-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul i32 [[TMP22]], 3
-// IR-NEXT:    [[ADD21:%.*]] = add i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD21]], ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
-// IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
-// IR-NEXT:    store i32 [[TMP23]], ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND:.*]]
-// IR:       [[FOR_COND]]:
-// IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    [[ADD22:%.*]] = add i32 [[TMP25]], 1
-// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
-// IR-NEXT:    [[ADD23:%.*]] = add i32 [[TMP26]], 3
-// IR-NEXT:    [[CMP24:%.*]] = icmp ult i32 [[ADD22]], [[ADD23]]
-// IR-NEXT:    br i1 [[CMP24]], label %[[COND_TRUE25:.*]], label %[[COND_FALSE27:.*]]
-// IR:       [[COND_TRUE25]]:
-// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    [[ADD26:%.*]] = add i32 [[TMP27]], 1
-// IR-NEXT:    br label %[[COND_END29:.*]]
-// IR:       [[COND_FALSE27]]:
-// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
-// IR-NEXT:    [[ADD28:%.*]] = add i32 [[TMP28]], 3
+// IR-EMPTY:
+// IR-NEXT:  [[COND_END]]:
+// IR-NEXT:    %[[COND:.+]] = phi i32 [ %[[TMP17]], %[[COND_TRUE]] ], [ %[[TMP18]], %[[COND_FALSE]] ]
+// IR-NEXT:    store i32 %[[COND]], ptr %[[DOTOMP_UB]], align 4
+// IR-NEXT:    %[[TMP19:.+]] = load i32, ptr %[[DOTOMP_LB]], align 4
+// IR-NEXT:    store i32 %[[TMP19]], ptr %[[DOTOMP_IV]], align 4
+// IR-NEXT:    br label %[[OMP_INNER_FOR_COND:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_INNER_FOR_COND]]:
+// IR-NEXT:    %[[TMP20:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
+// IR-NEXT:    %[[TMP21:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
+// IR-NEXT:    %[[ADD19:.+]] = add i32 %[[TMP21]], 1
+// IR-NEXT:    %[[CMP20:.+]] = icmp ult i32 %[[TMP20]], %[[ADD19]]
+// IR-NEXT:    br i1 %[[CMP20]], label %[[OMP_INNER_FOR_BODY:.+]], label %[[OMP_INNER_FOR_END:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_INNER_FOR_BODY]]:
+// IR-NEXT:    %[[TMP22:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
+// IR-NEXT:    %[[MUL:.+]] = mul i32 %[[TMP22]], 3
+// IR-NEXT:    %[[ADD21:.+]] = add i32 0, %[[MUL]]
+// IR-NEXT:    store i32 %[[ADD21]], ptr %[[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
+// IR-NEXT:    %[[TMP23:.+]] = load i32, ptr %[[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
+// IR-NEXT:    store i32 %[[TMP23]], ptr %[[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_COND]]:
+// IR-NEXT:    %[[TMP24:.+]] = load i32, ptr %[[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    %[[TMP25:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    %[[ADD22:.+]] = add i32 %[[TMP25]], 1
+// IR-NEXT:    %[[TMP26:.+]] = load i32, ptr %[[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
+// IR-NEXT:    %[[ADD23:.+]] = add i32 %[[TMP26]], 3
+// IR-NEXT:    %[[CMP24:.+]] = icmp ult i32 %[[ADD22]], %[[ADD23]]
+// IR-NEXT:    br i1 %[[CMP24]], label %[[COND_TRUE25:.+]], label %[[COND_FALSE27:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_TRUE25]]:
+// IR-NEXT:    %[[TMP27:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_7]], align 4
+// IR-NEXT:    %[[ADD26:.+]] = add i32 %[[TMP27]], 1
+// IR-NEXT:    br label %[[COND_END29:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_FALSE27]]:
+// IR-NEXT:    %[[TMP28:.+]] = load i32, ptr %[[DOTFLOOR_0_IV__FLOOR_0_IV_I17]], align 4
+// IR-NEXT:    %[[ADD28:.+]] = add i32 %[[TMP28]], 3
 // IR-NEXT:    br label %[[COND_END29]]
-// IR:       [[COND_END29]]:
-// IR-NEXT:    [[COND30:%.*]] = phi i32 [ [[ADD26]], %[[COND_TRUE25]] ], [ [[ADD28]], %[[COND_FALSE27]] ]
-// IR-NEXT:    [[CMP31:%.*]] = icmp ult i32 [[TMP24]], [[COND30]]
-// IR-NEXT:    br i1 [[CMP31]], label %[[FOR_BODY:.*]], label %[[FOR_END50:.*]]
-// IR:       [[FOR_BODY]]:
-// IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[MUL32:%.*]] = mul i32 [[TMP29]], 4
-// IR-NEXT:    [[ADD33:%.*]] = add i32 0, [[MUL32]]
-// IR-NEXT:    store i32 [[ADD33]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    store i32 [[TMP30]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND34:.*]]
-// IR:       [[FOR_COND34]]:
-// IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[ADD35:%.*]] = add i32 [[TMP32]], 1
-// IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD36:%.*]] = add i32 [[TMP33]], 4
-// IR-NEXT:    [[CMP37:%.*]] = icmp ult i32 [[ADD35]], [[ADD36]]
-// IR-NEXT:    br i1 [[CMP37]], label %[[COND_TRUE38:.*]], label %[[COND_FALSE40:.*]]
-// IR:       [[COND_TRUE38]]:
-// IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[ADD39:%.*]] = add i32 [[TMP34]], 1
-// IR-NEXT:    br label %[[COND_END42:.*]]
-// IR:       [[COND_FALSE40]]:
-// IR-NEXT:    [[TMP35:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD41:%.*]] = add i32 [[TMP35]], 4
+// IR-EMPTY:
+// IR-NEXT:  [[COND_END29]]:
+// IR-NEXT:    %[[COND30:.+]] = phi i32 [ %[[ADD26]], %[[COND_TRUE25]] ], [ %[[ADD28]], %[[COND_FALSE27]] ]
+// IR-NEXT:    %[[CMP31:.+]] = icmp ult i32 %[[TMP24]], %[[COND30]]
+// IR-NEXT:    br i1 %[[CMP31]], label %[[FOR_BODY:.+]], label %[[FOR_END50:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_BODY]]:
+// IR-NEXT:    %[[TMP29:.+]] = load i32, ptr %[[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    %[[MUL32:.+]] = mul i32 %[[TMP29]], 4
+// IR-NEXT:    %[[ADD33:.+]] = add i32 0, %[[MUL32]]
+// IR-NEXT:    store i32 %[[ADD33]], ptr %[[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    %[[TMP30:.+]] = load i32, ptr %[[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    store i32 %[[TMP30]], ptr %[[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND34:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_COND34]]:
+// IR-NEXT:    %[[TMP31:.+]] = load i32, ptr %[[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    %[[TMP32:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    %[[ADD35:.+]] = add i32 %[[TMP32]], 1
+// IR-NEXT:    %[[TMP33:.+]] = load i32, ptr %[[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    %[[ADD36:.+]] = add i32 %[[TMP33]], 4
+// IR-NEXT:    %[[CMP37:.+]] = icmp ult i32 %[[ADD35]], %[[ADD36]]
+// IR-NEXT:    br i1 %[[CMP37]], label %[[COND_TRUE38:.+]], label %[[COND_FALSE40:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_TRUE38]]:
+// IR-NEXT:    %[[TMP34:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    %[[ADD39:.+]] = add i32 %[[TMP34]], 1
+// IR-NEXT:    br label %[[COND_END42:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_FALSE40]]:
+// IR-NEXT:    %[[TMP35:.+]] = load i32, ptr %[[DOTFLOOR_0_IV_I]], align 4
+// IR-NEXT:    %[[ADD41:.+]] = add i32 %[[TMP35]], 4
 // IR-NEXT:    br label %[[COND_END42]]
-// IR:       [[COND_END42]]:
-// IR-NEXT:    [[COND43:%.*]] = phi i32 [ [[ADD39]], %[[COND_TRUE38]] ], [ [[ADD41]], %[[COND_FALSE40]] ]
-// IR-NEXT:    [[CMP44:%.*]] = icmp ult i32 [[TMP31]], [[COND43]]
-// IR-NEXT:    br i1 [[CMP44]], label %[[FOR_BODY45:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY45]]:
-// IR-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[MUL46:%.*]] = mul i32 [[TMP37]], [[TMP38]]
-// IR-NEXT:    [[ADD47:%.*]] = add i32 [[TMP36]], [[MUL46]]
-// IR-NEXT:    store i32 [[ADD47]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP39:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// IR-NEXT:    [[TMP40:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// IR-NEXT:    [[TMP41:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// IR-NEXT:    [[TMP42:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void (...) @body(i32 noundef [[TMP39]], i32 noundef [[TMP40]], i32 noundef [[TMP41]], i32 noundef [[TMP42]])
-// IR-NEXT:    br label %[[FOR_INC:.*]]
-// IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add i32 [[TMP43]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND34]], !llvm.loop [[LOOP2:![0-9]+]]
-// IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC48:.*]]
-// IR:       [[FOR_INC48]]:
-// IR-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[INC49:%.*]] = add i32 [[TMP44]], 1
-// IR-NEXT:    store i32 [[INC49]], ptr [[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP4:![0-9]+]]
-// IR:       [[FOR_END50]]:
-// IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.*]]
-// IR:       [[OMP_BODY_CONTINUE]]:
-// IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.*]]
-// IR:       [[OMP_INNER_FOR_INC]]:
-// IR-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[ADD51:%.*]] = add i32 [[TMP45]], 1
-// IR-NEXT:    store i32 [[ADD51]], ptr [[DOTOMP_IV]], align 4
+// IR-EMPTY:
+// IR-NEXT:  [[COND_END42]]:
+// IR-NEXT:    %[[COND43:.+]] = phi i32 [ %[[ADD39]], %[[COND_TRUE38]] ], [ %[[ADD41]], %[[COND_FALSE40]] ]
+// IR-NEXT:    %[[CMP44:.+]] = icmp ult i32 %[[TMP31]], %[[COND43]]
+// IR-NEXT:    br i1 %[[CMP44]], label %[[FOR_BODY45:.+]], label %[[FOR_END:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_BODY45]]:
+// IR-NEXT:    %[[TMP36:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    %[[TMP37:.+]] = load i32, ptr %[[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    %[[TMP38:.+]] = load i32, ptr %[[DOTNEW_STEP]], align 4
+// IR-NEXT:    %[[MUL46:.+]] = mul i32 %[[TMP37]], %[[TMP38]]
+// IR-NEXT:    %[[ADD47:.+]] = add i32 %[[TMP36]], %[[MUL46]]
+// IR-NEXT:    store i32 %[[ADD47]], ptr %[[I]], align 4
+// IR-NEXT:    %[[TMP39:.+]] = load i32, ptr %[[START_ADDR]], align 4
+// IR-NEXT:    %[[TMP40:.+]] = load i32, ptr %[[END_ADDR]], align 4
+// IR-NEXT:    %[[TMP41:.+]] = load i32, ptr %[[STEP_ADDR]], align 4
+// IR-NEXT:    %[[TMP42:.+]] = load i32, ptr %[[I]], align 4
+// IR-NEXT:    call void (...) @body(i32 noundef %[[TMP39]], i32 noundef %[[TMP40]], i32 noundef %[[TMP41]], i32 noundef %[[TMP42]])
+// IR-NEXT:    br label %[[FOR_INC:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_INC]]:
+// IR-NEXT:    %[[TMP43:.+]] = load i32, ptr %[[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    %[[INC:.+]] = add i32 %[[TMP43]], 1
+// IR-NEXT:    store i32 %[[INC]], ptr %[[DOTTILE_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND34]], !llvm.loop ![[LOOP2:[0-9]+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_END]]:
+// IR-NEXT:    br label %[[FOR_INC48:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_INC48]]:
+// IR-NEXT:    %[[TMP44:.+]] = load i32, ptr %[[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    %[[INC49:.+]] = add i32 %[[TMP44]], 1
+// IR-NEXT:    store i32 %[[INC49]], ptr %[[DOTTILE_0_IV__FLOOR_0_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop ![[LOOP4:[0-9]+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_END50]]:
+// IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_BODY_CONTINUE]]:
+// IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_INNER_FOR_INC]]:
+// IR-NEXT:    %[[TMP45:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
+// IR-NEXT:    %[[ADD51:.+]] = add i32 %[[TMP45]], 1
+// IR-NEXT:    store i32 %[[ADD51]], ptr %[[DOTOMP_IV]], align 4
 // IR-NEXT:    br label %[[OMP_INNER_FOR_COND]]
-// IR:       [[OMP_INNER_FOR_END]]:
-// IR-NEXT:    br label %[[OMP_LOOP_EXIT:.*]]
-// IR:       [[OMP_LOOP_EXIT]]:
-// IR-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP0]])
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_INNER_FOR_END]]:
+// IR-NEXT:    br label %[[OMP_LOOP_EXIT:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_LOOP_EXIT]]:
+// IR-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 %[[TMP0]])
 // IR-NEXT:    br label %[[OMP_PRECOND_END]]
-// IR:       [[OMP_PRECOND_END]]:
-// IR-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3:[0-9]+]], i32 [[TMP0]])
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_PRECOND_END]]:
+// IR-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3:.+]], i32 %[[TMP0]])
 // IR-NEXT:    ret void
-//
+// IR-NEXT:  }
+
+
 extern "C" void func(int start, int end, int step) {
 #pragma omp for
 #pragma omp tile sizes(3)
@@ -226,8 +249,8 @@ extern "C" void func(int start, int end, int step) {
 
 #endif /* HEADER */
 
-//.
-// IR: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]]}
-// IR: [[META3]] = !{!"llvm.loop.mustprogress"}
-// IR: [[LOOP4]] = distinct !{[[LOOP4]], [[META3]]}
-//.
+// IR: ![[META0:[0-9]+]] = !{i32 7, !"openmp", i32 51}
+// IR: ![[META1:[0-9]+]] =
+// IR: ![[LOOP2]] = distinct !{![[LOOP2]], ![[LOOPPROP3:[0-9]+]]}
+// IR: ![[LOOPPROP3]] = !{!"llvm.loop.mustprogress"}
+// IR: ![[LOOP4]] = distinct !{![[LOOP4]], ![[LOOPPROP3]]}
diff --git a/clang/test/OpenMP/tile_collapse_messages.cpp b/clang/test/OpenMP/tile_collapse_messages.cpp
new file mode 100644
index 0000000000000..6c5cb24b30289
--- /dev/null
+++ b/clang/test/OpenMP/tile_collapse_messages.cpp
@@ -0,0 +1,63 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++17 -fopenmp -fsyntax-only -Wuninitialized -verify %s
+
+// A collapsed intra-tile loop keeps a live dependence on its floor counter, so
+// that floor must itself be one of the collapsed loops. Inside another loop
+// transformation it is assigned in an enclosing loop's body and the nest would
+// read a stale value, which is diagnosed rather than silently miscomputed.
+
+extern void body(int);
+
+void collapse_through_single_tile() {
+#pragma omp for collapse(2)
+#pragma omp tile sizes(4)
+  for (int i = 0; i < 6; ++i)
+    body(i);
+}
+
+void collapse_through_tile_2d() {
+#pragma omp for collapse(4)
+#pragma omp tile sizes(2, 3)
+  for (int i = 0; i < 6; ++i)
+    for (int j = 0; j < 7; ++j)
+      body(i + j);
+}
+
+void collapse_outer_of_stacked_tiles() {
+  // Only the outer tile's floor and intra-tile loops are collapsed, and that
+  // floor is a collapsed counter, so this is fine.
+#pragma omp for collapse(2)
+#pragma omp tile sizes(2)
+#pragma omp tile sizes(4)
+  for (int i = 0; i < 6; ++i)
+    body(i);
+}
+
+void collapse_into_stacked_tiles() {
+  // The third loop is the inner tile's intra-tile loop, whose floor is computed
+  // inside the outer intra-tile loop's body.
+#pragma omp for collapse(3)
+#pragma omp tile sizes(2)
+#pragma omp tile sizes(4)
+  // expected-error at +1 {{cannot collapse the intra-tile loop of a '#pragma omp tile' that is nested inside another loop-transforming directive}}
+  for (int i = 0; i < 6; ++i)
+    body(i);
+}
+
+void collapse_simd_over_tile() {
+  // Rejected by the pre-existing restriction on nesting inside a 'simd' region.
+#pragma omp simd collapse(2)
+  // expected-error at +1 {{OpenMP constructs may not be nested inside a simd region except for ordered simd, simd, scan, or atomic directive}}
+#pragma omp tile sizes(4)
+  for (int i = 0; i < 6; ++i)
+    body(i);
+}
+
+void collapse_outer_of_stacked_tiles_2d() {
+  // The outer tile consumes both loops of the inner tile, so both of its
+  // intra-tile loops find their floor among the collapsed counters.
+#pragma omp for collapse(4)
+#pragma omp tile sizes(2, 3)
+#pragma omp tile sizes(4)
+  for (int i = 0; i < 6; ++i)
+    body(i);
+}
diff --git a/clang/test/OpenMP/tile_collapse_reinterpret_codegen.cpp b/clang/test/OpenMP/tile_collapse_reinterpret_codegen.cpp
index f88db73778f2b..477c86b4ead44 100644
--- a/clang/test/OpenMP/tile_collapse_reinterpret_codegen.cpp
+++ b/clang/test/OpenMP/tile_collapse_reinterpret_codegen.cpp
@@ -1,17 +1,23 @@
 // RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fclang-abi-compat=latest -std=c++20 -fopenmp -ast-print %s | FileCheck %s --check-prefix=PRINT
 // RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fclang-abi-compat=latest -std=c++20 -fopenmp -emit-llvm %s -o - | FileCheck %s --check-prefix=IR
+
+// Check same results after serialization round-trip, which exercises reading
+// and writing OMPInvariantPredicateBoundAttr including its optional predicate.
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fclang-abi-compat=latest -std=c++20 -fopenmp -emit-pch -o %t %s
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fclang-abi-compat=latest -std=c++20 -fopenmp -include-pch %t -ast-print %s | FileCheck %s --check-prefix=PRINT
+
 // expected-no-diagnostics
 
-// Codegen tests for how a loop-associated directive (e.g. 'omp for' with a
-// 'collapse' clause) consumes a '#pragma omp tile' intra-tile loop.
-//
-// These tests pin the key IR invariants of the below design:
-// The intra-tile loop is stored in its min-bounded form
-//   for (.tile.iv = .floor.iv; .tile.iv < min(.floor.iv + tilesize, N); ++.tile.iv)
-// and carries an internal OMPInvariantPredicateBoundAttr hint. When a directive
-// consumes the loop, it is reinterpreted as a rectangular loop with a constant
-// per-tile trip count (the tile size), and the overshoot of a partial last tile
-// is handled by a predicate emitted as a body guard.
+// How a loop-associated directive consumes a '#pragma omp tile': the stored
+// min-bounded intra-tile loop is reinterpreted as a rectangular one with a
+// constant trip count (the tile size), and a partial last tile is handled by a
+// body guard. Worksharing, 'distribute' and 'ordered(n)' consumers each read
+// the iteration space differently, so all three are covered. The hint is
+// internal and never dumped, so it is observed through the guard it produces
+// ('omp.body.next') and through the PCH round-trip above.
+
+#ifndef HEADER
+#define HEADER
 
 extern "C" void body(...) {}
 
@@ -21,7 +27,6 @@ extern "C" void body(...) {}
 // IR-LABEL: define internal void @collapse2_tile1d.omp_outlined
 // IR:       %.floor_0.iv{{[0-9a-z.]*}} = alloca i32
 // IR:       %.tile_0.iv{{[0-9a-z.]*}} = alloca i32
-// IR-NOT:   .tile.cnt
 // IR:       omp.inner.for.body:
 // IR:       store i32 %{{.*}}, ptr %.tile_0.iv
 // IR:       %[[PRED:.*]] = icmp slt i32 %{{.*}}, %{{.*}}
@@ -37,18 +42,18 @@ extern "C" void collapse2_tile1d(int n) {
     body(i);
 }
 
+// A non-tiled inner loop joins the collapsed nest as a third counter, and the
+// guard still applies to the tile counter alone.
 // PRINT-LABEL: void collapse3_tile1d_nested(
 // PRINT:       #pragma omp parallel for collapse(3)
 // PRINT-NEXT:  #pragma omp tile sizes(5)
 // IR-LABEL: define internal void @collapse3_tile1d_nested.omp_outlined
-// IR-NOT:   .tile.cnt
 // IR:       omp.inner.for.body:
 // IR:       store i32 %{{.*}}, ptr %.tile_0.iv
-// IR:       %[[PRED:.*]] = icmp slt i32 %{{.*}}, %{{.*}}
+// IR:       store i32 %{{.*}}, ptr %j
+// IR:       %[[TILEIV:.*]] = load i32, ptr %.tile_0.iv
+// IR:       %[[PRED:.*]] = icmp slt i32 %[[TILEIV]], %{{.*}}
 // IR-NEXT:  br i1 %[[PRED]], label %omp.body.next, label %omp.body.continue
-// IR:       omp.body.next:
-// IR:       call void (...) @body
-// IR-NEXT:  br label %omp.body.continue
 extern "C" void collapse3_tile1d_nested(int n) {
 #pragma omp parallel for collapse(3)
 #pragma omp tile sizes(5)
@@ -57,12 +62,13 @@ extern "C" void collapse3_tile1d_nested(int n) {
       body(i, j);
 }
 
+// An enclosing 'tile' is another loop transformation rather than a consumer, so
+// the inner intra-tile loop stays min-bounded and gets no guard.
 // PRINT-LABEL: void tile2d_inside_tile1d(
 // PRINT:       #pragma omp tile sizes(3)
 // PRINT-NEXT:  #pragma omp tile sizes(2)
 // IR-LABEL: define {{.*}} @tile2d_inside_tile1d(
 // IR:       %.tile_0.iv{{[0-9a-z.]*}} = alloca i32
-// IR-NOT:   .tile.cnt
 // IR:       cond.true:
 // IR:       cond.false:
 // IR:       cond.end:
@@ -79,7 +85,6 @@ extern "C" void tile2d_inside_tile1d(int n) {
 // PRINT:       #pragma omp parallel for collapse(3)
 // PRINT-NEXT:  #pragma omp tile sizes(2, 3)
 // IR-LABEL: define internal void @collapse3_tile2d.omp_outlined
-// IR-NOT:   .tile.cnt
 // IR:       omp.inner.for.body:
 // IR:       store i32 %{{.*}}, ptr %.tile_0.iv
 // IR:       br i1 %{{.*}}, label %omp.body.next, label %omp.body.continue
@@ -93,16 +98,77 @@ extern "C" void collapse3_tile2d(int n) {
       body(i, j);
 }
 
-// PRINT-LABEL: void tile1d_constant_div_no_predicate(
-// PRINT:       #pragma omp tile sizes(5)
-// IR-LABEL: define {{.*}} @tile1d_constant_div_no_predicate(
-// IR:       %.floor_0.iv{{[0-9a-z.]*}} = alloca i32
-// IR:       %.tile_0.iv{{[0-9a-z.]*}} = alloca i32
-// IR-NOT:   .tile.cnt
+// Trip count 10 is a multiple of the tile size 5, so no tile is partial, the
+// hint carries no predicate, and the body is emitted with no guard.
+// PRINT-LABEL: void collapse2_tile1d_no_partial_tile(
+// PRINT:       #pragma omp parallel for collapse(2)
+// PRINT-NEXT:  #pragma omp tile sizes(5)
+// IR-LABEL: define internal void @collapse2_tile1d_no_partial_tile.omp_outlined
+// IR:       omp.inner.for.body:
 // IR-NOT:   omp.body.next
 // IR:       call void (...) @body
-extern "C" void tile1d_constant_div_no_predicate() {
+// IR:       omp.body.continue:
+extern "C" void collapse2_tile1d_no_partial_tile() {
+#pragma omp parallel for collapse(2)
 #pragma omp tile sizes(5)
   for (int i = 0; i < 10; ++i)
     body(i);
 }
+
+// Trip count 11 is not a multiple of the tile size 5, so the predicate is kept
+// even though both values are compile-time constants.
+// PRINT-LABEL: void collapse2_tile1d_constant_partial_tile(
+// PRINT:       #pragma omp parallel for collapse(2)
+// PRINT-NEXT:  #pragma omp tile sizes(5)
+// IR-LABEL: define internal void @collapse2_tile1d_constant_partial_tile.omp_outlined
+// IR:       omp.inner.for.body:
+// IR:       br i1 %{{.*}}, label %omp.body.next, label %omp.body.continue
+// IR:       omp.body.next:
+// IR:       call void (...) @body
+extern "C" void collapse2_tile1d_constant_partial_tile() {
+#pragma omp parallel for collapse(2)
+#pragma omp tile sizes(5)
+  for (int i = 0; i < 11; ++i)
+    body(i);
+}
+
+// 'distribute' computes bounds through its own LB/UB/EUB/DistCond fields, so
+// check that the reinterpretation and its guard survive that path too.
+// PRINT-LABEL: void dist_collapse2_tile1d(
+// PRINT:       #pragma omp teams distribute collapse(2)
+// PRINT-NEXT:  #pragma omp tile sizes(5)
+// IR-LABEL: define internal void @dist_collapse2_tile1d.omp_outlined
+// IR:       omp.inner.for.body:
+// IR:       store i32 %{{.*}}, ptr %.tile_0.iv
+// IR:       br i1 %{{.*}}, label %omp.body.next, label %omp.body.continue
+// IR:       omp.body.next:
+// IR:       call void (...) @body
+extern "C" void dist_collapse2_tile1d(int n) {
+#pragma omp teams distribute collapse(2)
+#pragma omp tile sizes(5)
+  for (int i = 0; i < n; ++i)
+    body(i);
+}
+
+// 'ordered(n)' feeds each collapsed loop's trip count to __kmpc_doacross_init,
+// so the intra-tile dimension must report the constant tile size, matching the
+// space 'collapse' linearizes.
+// PRINT-LABEL: void ordered_collapse2_tile1d(
+// PRINT:       #pragma omp for collapse(2) ordered(2)
+// PRINT-NEXT:  #pragma omp tile sizes(5)
+// IR-LABEL: define {{.*}} @ordered_collapse2_tile1d(
+// IR:       %[[DIM1:.*]] = getelementptr inbounds [2 x %struct.kmp_dim], ptr %dims, i64 0, i64 1
+// IR:       %[[UP1:.*]] = getelementptr inbounds nuw %struct.kmp_dim, ptr %[[DIM1]], i32 0, i32 1
+// IR-NEXT:  store i64 5, ptr %[[UP1]]
+// IR:       call void @__kmpc_doacross_init(ptr @{{[0-9]+}}, i32 %{{.*}}, i32 2, ptr %{{.*}})
+// IR:       omp.inner.for.body:
+// IR:       br i1 %{{.*}}, label %omp.body.next, label %omp.body.continue
+// IR:       call void @__kmpc_doacross_fini(
+extern "C" void ordered_collapse2_tile1d(int n) {
+#pragma omp for collapse(2) ordered(2)
+#pragma omp tile sizes(5)
+  for (int i = 0; i < n; ++i)
+    body(i);
+}
+
+#endif /* HEADER */
diff --git a/clang/test/OpenMP/tile_messages.cpp b/clang/test/OpenMP/tile_messages.cpp
index e1f5155c924e5..6ac905d937159 100644
--- a/clang/test/OpenMP/tile_messages.cpp
+++ b/clang/test/OpenMP/tile_messages.cpp
@@ -170,3 +170,33 @@ void f(void) {
     ;
 }
 } // namespace GH139073
+
+// A tile's intra-tile loop is wrapped in an internal reinterpretation hint. A
+// loop transformation consuming that loop must see through the wrapper, so
+// check that these still parse and analyze without diagnostics.
+namespace consumed_by_transformation {
+// Codegen: tile_collapse_reinterpret_codegen.cpp and tile_codegen_tile_for.cpp;
+// run time: libomp transform/tile/parallel-wsloop-collapse-stacked-tile.cpp.
+void tile_of_tile() {
+#pragma omp tile sizes(3, 5)
+#pragma omp tile sizes(2)
+  for (int i = 0; i < 100; ++i)
+    ;
+}
+
+// Same, for a range-based for loop.
+void tile_of_tile_range_for(int (&A)[100]) {
+#pragma omp tile sizes(3, 5)
+#pragma omp tile sizes(2)
+  for (int &v : A)
+    (void)v;
+}
+
+// 'unroll' is a loop transformation too, so it consumes the tile the same way.
+void unroll_of_tile() {
+#pragma omp unroll partial(4)
+#pragma omp tile sizes(2)
+  for (int i = 0; i < 100; ++i)
+    ;
+}
+} // namespace consumed_by_transformation
diff --git a/clang/test/OpenMP/tile_rect_codegen.cpp b/clang/test/OpenMP/tile_rect_codegen.cpp
deleted file mode 100644
index ab63de9343fd2..0000000000000
--- a/clang/test/OpenMP/tile_rect_codegen.cpp
+++ /dev/null
@@ -1,51 +0,0 @@
-// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fclang-abi-compat=latest -std=c++20 -fopenmp -ast-print %s | FileCheck %s
-// expected-no-diagnostics
-
-// -ast-print round-trip tests: a `#pragma omp tile` directive and its associated
-// loop must print back as written. This checks parsing/printing only; for the
-// generated IR (min-bounded lowering) see tile_rect_codegen_ir.cpp.
-
-extern "C" void body(...) {}
-
-// CHECK-LABEL: void rect_tile_1d(
-void rect_tile_1d() {
-  // Partial last tile (trip count 4, tile 5); -ast-print prints the original loop.
-  // CHECK: #pragma omp tile sizes(5)
-  // CHECK-NEXT: for (int i = 7; i < 17; i += 3)
-  #pragma omp tile sizes(5)
-  for (int i = 7; i < 17; i += 3)
-    body(i);
-}
-
-// CHECK-LABEL: void rect_tile_2d(
-void rect_tile_2d() {
-  // CHECK: #pragma omp tile sizes(5, 5)
-  // CHECK-NEXT: for (int i = 7; i < 17; i += 3)
-  // CHECK-NEXT:   for (int j = 7; j < 17; j += 3)
-  #pragma omp tile sizes(5, 5)
-  for (int i = 7; i < 17; i += 3)
-    for (int j = 7; j < 17; j += 3)
-      body(i, j);
-}
-
-// CHECK-LABEL: void rect_tile_exact_div(
-void rect_tile_exact_div() {
-  // Exact-dividing trip count (10, tile 5); -ast-print prints the original loop.
-  // CHECK: #pragma omp tile sizes(5)
-  // CHECK-NEXT: for (int i = 0; i < 10; i += 1)
-  #pragma omp tile sizes(5)
-  for (int i = 0; i < 10; i += 1)
-    body(i);
-}
-
-// CHECK-LABEL: void rect_tile_nested_body_loop(
-void rect_tile_nested_body_loop(int n) {
-  // The inner `j` loop lives in the tiled construct's body; -ast-print shows the
-  // directive and the outer `i` loop printed back.
-  // CHECK: #pragma omp tile sizes(4)
-  // CHECK-NEXT: for (int i = 0; i < 6; i += 1)
-  #pragma omp tile sizes(4)
-  for (int i = 0; i < 6; i += 1)
-    for (int j = 0; j < n; ++j)
-      body(i, j);
-}
diff --git a/clang/test/OpenMP/tile_rect_codegen_ir.cpp b/clang/test/OpenMP/tile_rect_codegen_ir.cpp
index 027cd8385044c..33b3415d36b6d 100644
--- a/clang/test/OpenMP/tile_rect_codegen_ir.cpp
+++ b/clang/test/OpenMP/tile_rect_codegen_ir.cpp
@@ -1,11 +1,8 @@
 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
-// Regression tests for standalone `#pragma omp tile` lowering (no consuming
-// directive). The intra-tile loop is emitted in its min-bounded form
-//   for (.tile.iv = .floor.iv; .tile.iv < min(.floor.iv + tilesize, N); ++.tile.iv)
-// so the upper bound is materialized as a `min()` (cond.true/cond.false/phi over
-// the two `.tile_*.iv`/`.floor_*.iv` counters). There is no `.tile.cnt`
-// rectangular counter and no body-guard predicate here; those only appear when a
-// directive (e.g. `collapse`) consumes the tile -- see
+// A standalone `#pragma omp tile` keeps the intra-tile loop min-bounded: the
+// upper bound `min(.floor.iv + tilesize, N)` is materialized as a
+// cond.true/cond.false/phi and there is no body guard. The guarded rectangular
+// form appears only once a directive consumes the tile, see
 // tile_collapse_reinterpret_codegen.cpp.
 //
 // RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fopenmp -emit-llvm -o - %s | FileCheck %s --check-prefix=IR
@@ -13,8 +10,8 @@
 
 extern "C" void body(int);
 
-// Trip count 6, tile 4 → the last tile is partial; the min() upper bound stops
-// the tile loop at the trip count (no separate body guard).
+// Trip count 6, tile 4: the last tile is partial, and the min() bound alone
+// stops it at the trip count.
 // IR-LABEL: define dso_local void @_Z18remainder_6_tile_4v(
 // IR-SAME: ) #[[ATTR0:[0-9]+]] {
 // IR-NEXT:  [[ENTRY:.*:]]
@@ -72,287 +69,13 @@ extern "C" void body(int);
 // IR-NEXT:    ret void
 //
 void remainder_6_tile_4(void) {
-  // Rectangular tile counter and constant tile-size bound.
 #pragma omp tile sizes(4)
   for (int i = 0; i < 6; ++i)
     body(i);
 }
 
-// Simple stride-1 loop; tile size matches full tiles only (10, tile 5).
-// IR-LABEL: define dso_local void @_Z15full_tiles_10_5v(
-// IR-SAME: ) #[[ATTR0]] {
-// IR-NEXT:  [[ENTRY:.*:]]
-// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    store i32 0, ptr [[I]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND:.*]]
-// IR:       [[FOR_COND]]:
-// IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 10
-// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END9:.*]]
-// IR:       [[FOR_BODY]]:
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND1:.*]]
-// IR:       [[FOR_COND1]]:
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 5
-// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 10, [[ADD]]
-// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
-// IR:       [[COND_TRUE]]:
-// IR-NEXT:    br label %[[COND_END:.*]]
-// IR:       [[COND_FALSE]]:
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 5
-// IR-NEXT:    br label %[[COND_END]]
-// IR:       [[COND_END]]:
-// IR-NEXT:    [[COND:%.*]] = phi i32 [ 10, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
-// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
-// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY5]]:
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
-// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void @body(i32 noundef [[TMP6]])
-// IR-NEXT:    br label %[[FOR_INC:.*]]
-// IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP7]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP5:![0-9]+]]
-// IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC7:.*]]
-// IR:       [[FOR_INC7]]:
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP8]], 5
-// IR-NEXT:    store i32 [[ADD8]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP6:![0-9]+]]
-// IR:       [[FOR_END9]]:
-// IR-NEXT:    ret void
-//
-void full_tiles_10_5(void) {
-#pragma omp tile sizes(5)
-  for (int i = 0; i < 10; ++i)
-    body(i);
-}
-
-// Variable tile size: clamp (TS<=0 ? 1 : TS) is inlined; bound compares to %cond.
-// IR-LABEL: define dso_local void @_Z8var_tilei(
-// IR-SAME: i32 noundef [[TS:%.*]]) #[[ATTR0]] {
-// IR-NEXT:  [[ENTRY:.*:]]
-// IR-NEXT:    [[TS_ADDR:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    store i32 [[TS]], ptr [[TS_ADDR]], align 4
-// IR-NEXT:    store i32 0, ptr [[I]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND:.*]]
-// IR:       [[FOR_COND]]:
-// IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 12
-// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END24:.*]]
-// IR:       [[FOR_BODY]]:
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND1:.*]]
-// IR:       [[FOR_COND1]]:
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[TS_ADDR]], align 4
-// IR-NEXT:    [[CMP2:%.*]] = icmp sle i32 [[TMP4]], 0
-// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
-// IR:       [[COND_TRUE]]:
-// IR-NEXT:    br label %[[COND_END:.*]]
-// IR:       [[COND_FALSE]]:
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[TS_ADDR]], align 4
-// IR-NEXT:    br label %[[COND_END]]
-// IR:       [[COND_END]]:
-// IR-NEXT:    [[COND:%.*]] = phi i32 [ 1, %[[COND_TRUE]] ], [ [[TMP5]], %[[COND_FALSE]] ]
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], [[COND]]
-// IR-NEXT:    [[CMP3:%.*]] = icmp slt i32 12, [[ADD]]
-// IR-NEXT:    br i1 [[CMP3]], label %[[COND_TRUE4:.*]], label %[[COND_FALSE5:.*]]
-// IR:       [[COND_TRUE4]]:
-// IR-NEXT:    br label %[[COND_END12:.*]]
-// IR:       [[COND_FALSE5]]:
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[TS_ADDR]], align 4
-// IR-NEXT:    [[CMP6:%.*]] = icmp sle i32 [[TMP7]], 0
-// IR-NEXT:    br i1 [[CMP6]], label %[[COND_TRUE7:.*]], label %[[COND_FALSE8:.*]]
-// IR:       [[COND_TRUE7]]:
-// IR-NEXT:    br label %[[COND_END9:.*]]
-// IR:       [[COND_FALSE8]]:
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[TS_ADDR]], align 4
-// IR-NEXT:    br label %[[COND_END9]]
-// IR:       [[COND_END9]]:
-// IR-NEXT:    [[COND10:%.*]] = phi i32 [ 1, %[[COND_TRUE7]] ], [ [[TMP8]], %[[COND_FALSE8]] ]
-// IR-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP6]], [[COND10]]
-// IR-NEXT:    br label %[[COND_END12]]
-// IR:       [[COND_END12]]:
-// IR-NEXT:    [[COND13:%.*]] = phi i32 [ 12, %[[COND_TRUE4]] ], [ [[ADD11]], %[[COND_END9]] ]
-// IR-NEXT:    [[CMP14:%.*]] = icmp slt i32 [[TMP2]], [[COND13]]
-// IR-NEXT:    br i1 [[CMP14]], label %[[FOR_BODY15:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY15]]:
-// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP9]], 1
-// IR-NEXT:    [[ADD16:%.*]] = add nsw i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD16]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void @body(i32 noundef [[TMP10]])
-// IR-NEXT:    br label %[[FOR_INC:.*]]
-// IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP11]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP7:![0-9]+]]
-// IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC17:.*]]
-// IR:       [[FOR_INC17]]:
-// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[TS_ADDR]], align 4
-// IR-NEXT:    [[CMP18:%.*]] = icmp sle i32 [[TMP12]], 0
-// IR-NEXT:    br i1 [[CMP18]], label %[[COND_TRUE19:.*]], label %[[COND_FALSE20:.*]]
-// IR:       [[COND_TRUE19]]:
-// IR-NEXT:    br label %[[COND_END21:.*]]
-// IR:       [[COND_FALSE20]]:
-// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[TS_ADDR]], align 4
-// IR-NEXT:    br label %[[COND_END21]]
-// IR:       [[COND_END21]]:
-// IR-NEXT:    [[COND22:%.*]] = phi i32 [ 1, %[[COND_TRUE19]] ], [ [[TMP13]], %[[COND_FALSE20]] ]
-// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD23:%.*]] = add nsw i32 [[TMP14]], [[COND22]]
-// IR-NEXT:    store i32 [[ADD23]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP8:![0-9]+]]
-// IR:       [[FOR_END24]]:
-// IR-NEXT:    ret void
-//
-void var_tile(int ts) {
-#pragma omp tile sizes(ts)
-  for (int i = 0; i < 12; ++i)
-    body(i);
-}
-
-// Two tiled dimensions → two tile counters.
-// IR-LABEL: define dso_local void @_Z7two_dimv(
-// IR-SAME: ) #[[ATTR0]] {
-// IR-NEXT:  [[ENTRY:.*:]]
-// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[J:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_1_IV_J:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_1_IV_J:%.*]] = alloca i32, align 4
-// IR-NEXT:    store i32 0, ptr [[I]], align 4
-// IR-NEXT:    store i32 0, ptr [[J]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND:.*]]
-// IR:       [[FOR_COND]]:
-// IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 3
-// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END31:.*]]
-// IR:       [[FOR_BODY]]:
-// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_1_IV_J]], align 4
-// IR-NEXT:    br label %[[FOR_COND1:.*]]
-// IR:       [[FOR_COND1]]:
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 [[TMP1]], 3
-// IR-NEXT:    br i1 [[CMP2]], label %[[FOR_BODY3:.*]], label %[[FOR_END28:.*]]
-// IR:       [[FOR_BODY3]]:
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    store i32 [[TMP2]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND4:.*]]
-// IR:       [[FOR_COND4]]:
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP4]], 2
-// IR-NEXT:    [[CMP5:%.*]] = icmp slt i32 3, [[ADD]]
-// IR-NEXT:    br i1 [[CMP5]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
-// IR:       [[COND_TRUE]]:
-// IR-NEXT:    br label %[[COND_END:.*]]
-// IR:       [[COND_FALSE]]:
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 [[TMP5]], 2
-// IR-NEXT:    br label %[[COND_END]]
-// IR:       [[COND_END]]:
-// IR-NEXT:    [[COND:%.*]] = phi i32 [ 3, %[[COND_TRUE]] ], [ [[ADD6]], %[[COND_FALSE]] ]
-// IR-NEXT:    [[CMP7:%.*]] = icmp slt i32 [[TMP3]], [[COND]]
-// IR-NEXT:    br i1 [[CMP7]], label %[[FOR_BODY8:.*]], label %[[FOR_END25:.*]]
-// IR:       [[FOR_BODY8]]:
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP6]], 1
-// IR-NEXT:    [[ADD9:%.*]] = add nsw i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD9]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// IR-NEXT:    store i32 [[TMP7]], ptr [[DOTTILE_1_IV_J]], align 4
-// IR-NEXT:    br label %[[FOR_COND10:.*]]
-// IR:       [[FOR_COND10]]:
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// IR-NEXT:    [[ADD11:%.*]] = add nsw i32 [[TMP9]], 2
-// IR-NEXT:    [[CMP12:%.*]] = icmp slt i32 3, [[ADD11]]
-// IR-NEXT:    br i1 [[CMP12]], label %[[COND_TRUE13:.*]], label %[[COND_FALSE14:.*]]
-// IR:       [[COND_TRUE13]]:
-// IR-NEXT:    br label %[[COND_END16:.*]]
-// IR:       [[COND_FALSE14]]:
-// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// IR-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP10]], 2
-// IR-NEXT:    br label %[[COND_END16]]
-// IR:       [[COND_END16]]:
-// IR-NEXT:    [[COND17:%.*]] = phi i32 [ 3, %[[COND_TRUE13]] ], [ [[ADD15]], %[[COND_FALSE14]] ]
-// IR-NEXT:    [[CMP18:%.*]] = icmp slt i32 [[TMP8]], [[COND17]]
-// IR-NEXT:    br i1 [[CMP18]], label %[[FOR_BODY19:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY19]]:
-// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// IR-NEXT:    [[MUL20:%.*]] = mul nsw i32 [[TMP11]], 1
-// IR-NEXT:    [[ADD21:%.*]] = add nsw i32 0, [[MUL20]]
-// IR-NEXT:    store i32 [[ADD21]], ptr [[J]], align 4
-// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[J]], align 4
-// IR-NEXT:    [[ADD22:%.*]] = add nsw i32 [[TMP12]], [[TMP13]]
-// IR-NEXT:    call void @body(i32 noundef [[ADD22]])
-// IR-NEXT:    br label %[[FOR_INC:.*]]
-// IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTTILE_1_IV_J]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP14]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_1_IV_J]], align 4
-// IR-NEXT:    br label %[[FOR_COND10]], !llvm.loop [[LOOP9:![0-9]+]]
-// IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC23:.*]]
-// IR:       [[FOR_INC23]]:
-// IR-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[INC24:%.*]] = add nsw i32 [[TMP15]], 1
-// IR-NEXT:    store i32 [[INC24]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND4]], !llvm.loop [[LOOP10:![0-9]+]]
-// IR:       [[FOR_END25]]:
-// IR-NEXT:    br label %[[FOR_INC26:.*]]
-// IR:       [[FOR_INC26]]:
-// IR-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTFLOOR_1_IV_J]], align 4
-// IR-NEXT:    [[ADD27:%.*]] = add nsw i32 [[TMP16]], 2
-// IR-NEXT:    store i32 [[ADD27]], ptr [[DOTFLOOR_1_IV_J]], align 4
-// IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP11:![0-9]+]]
-// IR:       [[FOR_END28]]:
-// IR-NEXT:    br label %[[FOR_INC29:.*]]
-// IR:       [[FOR_INC29]]:
-// IR-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD30:%.*]] = add nsw i32 [[TMP17]], 2
-// IR-NEXT:    store i32 [[ADD30]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP12:![0-9]+]]
-// IR:       [[FOR_END31]]:
-// IR-NEXT:    ret void
-//
-void two_dim(void) {
-#pragma omp tile sizes(2, 2)
-  for (int i = 0; i < 3; ++i)
-    for (int j = 0; j < 3; ++j)
-      body(i + j);
-}
-
-// Nested loop in the tile body: the tiled `i` dim is the floor/tile pair in
-// min-bounded form; the inner `j` loop stays a plain nested loop.
+// The inner `j` loop is not one of the tiled loops, so it is emitted untouched
+// inside the intra-tile loop.
 // IR-LABEL: define dso_local void @_Z14nested_inner_ji(
 // IR-SAME: i32 noundef [[N:%.*]]) #[[ATTR0]] {
 // IR-NEXT:  [[ENTRY:.*:]]
@@ -411,21 +134,21 @@ void two_dim(void) {
 // IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[J]], align 4
 // IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP10]], 1
 // IR-NEXT:    store i32 [[INC]], ptr [[J]], align 4
-// IR-NEXT:    br label %[[FOR_COND7]], !llvm.loop [[LOOP13:![0-9]+]]
+// IR-NEXT:    br label %[[FOR_COND7]], !llvm.loop [[LOOP5:![0-9]+]]
 // IR:       [[FOR_END]]:
 // IR-NEXT:    br label %[[FOR_INC11:.*]]
 // IR:       [[FOR_INC11]]:
 // IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
 // IR-NEXT:    [[INC12:%.*]] = add nsw i32 [[TMP11]], 1
 // IR-NEXT:    store i32 [[INC12]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP14:![0-9]+]]
+// IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP6:![0-9]+]]
 // IR:       [[FOR_END13]]:
 // IR-NEXT:    br label %[[FOR_INC14:.*]]
 // IR:       [[FOR_INC14]]:
 // IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
 // IR-NEXT:    [[ADD15:%.*]] = add nsw i32 [[TMP12]], 4
 // IR-NEXT:    store i32 [[ADD15]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP15:![0-9]+]]
+// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP7:![0-9]+]]
 // IR:       [[FOR_END16]]:
 // IR-NEXT:    ret void
 //
@@ -435,72 +158,6 @@ void nested_inner_j(int n) {
     for (int j = 0; j < n; ++j)
       body(i + j);
 }
-
-// Exact division (trip count 12, tile 4): a standalone tile still emits the
-// min() cond-phi upper bound (it is always present, regardless of divisibility)
-// and never a body-guard predicate.
-// IR-LABEL: define dso_local void @_Z16elided_predicatev(
-// IR-SAME: ) #[[ATTR0]] {
-// IR-NEXT:  [[ENTRY:.*:]]
-// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    store i32 0, ptr [[I]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND:.*]]
-// IR:       [[FOR_COND]]:
-// IR-NEXT:    [[TMP0:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[CMP:%.*]] = icmp slt i32 [[TMP0]], 12
-// IR-NEXT:    br i1 [[CMP]], label %[[FOR_BODY:.*]], label %[[FOR_END9:.*]]
-// IR:       [[FOR_BODY]]:
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    store i32 [[TMP1]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND1:.*]]
-// IR:       [[FOR_COND1]]:
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add nsw i32 [[TMP3]], 4
-// IR-NEXT:    [[CMP2:%.*]] = icmp slt i32 12, [[ADD]]
-// IR-NEXT:    br i1 [[CMP2]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
-// IR:       [[COND_TRUE]]:
-// IR-NEXT:    br label %[[COND_END:.*]]
-// IR:       [[COND_FALSE]]:
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD3:%.*]] = add nsw i32 [[TMP4]], 4
-// IR-NEXT:    br label %[[COND_END]]
-// IR:       [[COND_END]]:
-// IR-NEXT:    [[COND:%.*]] = phi i32 [ 12, %[[COND_TRUE]] ], [ [[ADD3]], %[[COND_FALSE]] ]
-// IR-NEXT:    [[CMP4:%.*]] = icmp slt i32 [[TMP2]], [[COND]]
-// IR-NEXT:    br i1 [[CMP4]], label %[[FOR_BODY5:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY5]]:
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul nsw i32 [[TMP5]], 1
-// IR-NEXT:    [[ADD6:%.*]] = add nsw i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD6]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void @body(i32 noundef [[TMP6]])
-// IR-NEXT:    br label %[[FOR_INC:.*]]
-// IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add nsw i32 [[TMP7]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTTILE_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND1]], !llvm.loop [[LOOP16:![0-9]+]]
-// IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC7:.*]]
-// IR:       [[FOR_INC7]]:
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    [[ADD8:%.*]] = add nsw i32 [[TMP8]], 4
-// IR-NEXT:    store i32 [[ADD8]], ptr [[DOTFLOOR_0_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP17:![0-9]+]]
-// IR:       [[FOR_END9]]:
-// IR-NEXT:    ret void
-//
-void elided_predicate(void) {
-  // Trip count 12, tile size 4 → 12 % 4 == 0, no predicate.
-#pragma omp tile sizes(4)
-  for (int i = 0; i < 12; ++i)
-    body(i);
-}
 //.
 // IR: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]]}
 // IR: [[META3]] = !{!"llvm.loop.mustprogress"}
@@ -508,14 +165,4 @@ void elided_predicate(void) {
 // IR: [[LOOP5]] = distinct !{[[LOOP5]], [[META3]]}
 // IR: [[LOOP6]] = distinct !{[[LOOP6]], [[META3]]}
 // IR: [[LOOP7]] = distinct !{[[LOOP7]], [[META3]]}
-// IR: [[LOOP8]] = distinct !{[[LOOP8]], [[META3]]}
-// IR: [[LOOP9]] = distinct !{[[LOOP9]], [[META3]]}
-// IR: [[LOOP10]] = distinct !{[[LOOP10]], [[META3]]}
-// IR: [[LOOP11]] = distinct !{[[LOOP11]], [[META3]]}
-// IR: [[LOOP12]] = distinct !{[[LOOP12]], [[META3]]}
-// IR: [[LOOP13]] = distinct !{[[LOOP13]], [[META3]]}
-// IR: [[LOOP14]] = distinct !{[[LOOP14]], [[META3]]}
-// IR: [[LOOP15]] = distinct !{[[LOOP15]], [[META3]]}
-// IR: [[LOOP16]] = distinct !{[[LOOP16]], [[META3]]}
-// IR: [[LOOP17]] = distinct !{[[LOOP17]], [[META3]]}
 //.
diff --git a/clang/test/OpenMP/tile_vectorize_codegen.cpp b/clang/test/OpenMP/tile_vectorize_codegen.cpp
index 46fa51b8da57c..f2f110c0372c4 100644
--- a/clang/test/OpenMP/tile_vectorize_codegen.cpp
+++ b/clang/test/OpenMP/tile_vectorize_codegen.cpp
@@ -2,13 +2,27 @@
 // REQUIRES: x86-registered-target
 // expected-no-diagnostics
 
-// Standalone tile stays min-bounded (no body predicate) and vectorizes.
+// Why the intra-tile loop is stored min-bounded: that form carries its trip
+// count in the loop bound and vectorizes, while the rectangular form a
+// 'collapse' consumer needs adds a body guard that blocks the vectorizer. Both
+// sides are checked here so the tradeoff is visible in one place.
 
 // CHECK-LABEL: define {{.*}} @vec_tile(
 // CHECK: fadd <4 x float>
-// CHECK-NOT: omp.body.next
 extern "C" void vec_tile(float *a, float *b, int n) {
 #pragma omp tile sizes(64)
   for (int i = 0; i < n; ++i)
     a[i] = a[i] + b[i];
 }
+
+// CHECK-LABEL: define internal void @vec_tile_collapsed.omp_outlined(
+// CHECK-NOT: fadd <4 x float>
+// CHECK: br i1 %{{.*}}, label %omp.body.next, label %omp.inner.for.inc
+// CHECK-NOT: fadd <4 x float>
+// CHECK: ret void
+extern "C" void vec_tile_collapsed(float *a, float *b, int n) {
+#pragma omp parallel for collapse(2)
+#pragma omp tile sizes(64)
+  for (int i = 0; i < n; ++i)
+    a[i] = a[i] + b[i];
+}
diff --git a/clang/test/OpenMP/unroll_codegen_tile_for.cpp b/clang/test/OpenMP/unroll_codegen_tile_for.cpp
index 2e80731c4465d..ccfc9f38f5a24 100644
--- a/clang/test/OpenMP/unroll_codegen_tile_for.cpp
+++ b/clang/test/OpenMP/unroll_codegen_tile_for.cpp
@@ -1,4 +1,3 @@
-// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
 // Check code generation
 // RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -fopenmp -emit-llvm %s -o - | FileCheck %s --check-prefix=IR
 
@@ -14,198 +13,221 @@
 extern "C" void body(...) {}
 
 
-// IR-LABEL: define dso_local void @func(
-// IR-SAME: i32 noundef [[START:%.*]], i32 noundef [[END:%.*]], i32 noundef [[STEP:%.*]]) #[[ATTR0:[0-9]+]] {
-// IR-NEXT:  [[ENTRY:.*:]]
-// IR-NEXT:    [[START_ADDR:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[END_ADDR:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[STEP_ADDR:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_IV:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[TMP:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_1:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTNEW_STEP:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_2:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTUNROLLED_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_5:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_7:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_11:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTCAPTURE_EXPR_13:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV__UNROLLED_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_LB:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_UB:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_STRIDE:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTOMP_IS_LAST:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTFLOOR_0_IV__UNROLLED_IV_I17:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTTILE_0_IV__UNROLLED_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[DOTUNROLL_INNER_IV_I:%.*]] = alloca i32, align 4
-// IR-NEXT:    [[TMP0:%.*]] = call i32 @__kmpc_global_thread_num(ptr @[[GLOB2:[0-9]+]])
-// IR-NEXT:    store i32 [[START]], ptr [[START_ADDR]], align 4
-// IR-NEXT:    store i32 [[END]], ptr [[END_ADDR]], align 4
-// IR-NEXT:    store i32 [[STEP]], ptr [[STEP_ADDR]], align 4
-// IR-NEXT:    [[TMP1:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// IR-NEXT:    store i32 [[TMP1]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP2:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// IR-NEXT:    store i32 [[TMP2]], ptr [[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    [[TMP3:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// IR-NEXT:    store i32 [[TMP3]], ptr [[DOTCAPTURE_EXPR_1]], align 4
-// IR-NEXT:    [[TMP4:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// IR-NEXT:    store i32 [[TMP4]], ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[TMP5:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_1]], align 4
-// IR-NEXT:    [[TMP6:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    [[SUB:%.*]] = sub i32 [[TMP5]], [[TMP6]]
-// IR-NEXT:    [[SUB3:%.*]] = sub i32 [[SUB]], 1
-// IR-NEXT:    [[TMP7:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[ADD:%.*]] = add i32 [[SUB3]], [[TMP7]]
-// IR-NEXT:    [[TMP8:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[DIV:%.*]] = udiv i32 [[ADD]], [[TMP8]]
-// IR-NEXT:    [[SUB4:%.*]] = sub i32 [[DIV]], 1
-// IR-NEXT:    store i32 [[SUB4]], ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTUNROLLED_IV_I]], align 4
-// IR-NEXT:    [[TMP9:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[ADD6:%.*]] = add i32 [[TMP9]], 1
-// IR-NEXT:    store i32 [[ADD6]], ptr [[DOTCAPTURE_EXPR_5]], align 4
-// IR-NEXT:    [[TMP10:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_5]], align 4
-// IR-NEXT:    [[SUB8:%.*]] = sub i32 [[TMP10]], -1
-// IR-NEXT:    [[DIV9:%.*]] = udiv i32 [[SUB8]], 2
-// IR-NEXT:    [[SUB10:%.*]] = sub i32 [[DIV9]], 1
-// IR-NEXT:    store i32 [[SUB10]], ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    [[TMP11:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    [[ADD12:%.*]] = add i32 [[TMP11]], 1
-// IR-NEXT:    store i32 [[ADD12]], ptr [[DOTCAPTURE_EXPR_11]], align 4
-// IR-NEXT:    [[TMP12:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// IR-NEXT:    [[SUB14:%.*]] = sub i32 [[TMP12]], -3
-// IR-NEXT:    [[DIV15:%.*]] = udiv i32 [[SUB14]], 4
-// IR-NEXT:    [[SUB16:%.*]] = sub i32 [[DIV15]], 1
-// IR-NEXT:    store i32 [[SUB16]], ptr [[DOTCAPTURE_EXPR_13]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    [[TMP13:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_11]], align 4
-// IR-NEXT:    [[CMP:%.*]] = icmp ult i32 0, [[TMP13]]
-// IR-NEXT:    br i1 [[CMP]], label %[[OMP_PRECOND_THEN:.*]], label %[[OMP_PRECOND_END:.*]]
-// IR:       [[OMP_PRECOND_THEN]]:
-// IR-NEXT:    store i32 0, ptr [[DOTOMP_LB]], align 4
-// IR-NEXT:    [[TMP14:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_13]], align 4
-// IR-NEXT:    store i32 [[TMP14]], ptr [[DOTOMP_UB]], align 4
-// IR-NEXT:    store i32 1, ptr [[DOTOMP_STRIDE]], align 4
-// IR-NEXT:    store i32 0, ptr [[DOTOMP_IS_LAST]], align 4
-// IR-NEXT:    call void @__kmpc_for_static_init_4u(ptr @[[GLOB1:[0-9]+]], i32 [[TMP0]], i32 34, ptr [[DOTOMP_IS_LAST]], ptr [[DOTOMP_LB]], ptr [[DOTOMP_UB]], ptr [[DOTOMP_STRIDE]], i32 1, i32 1)
-// IR-NEXT:    [[TMP15:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// IR-NEXT:    [[TMP16:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_13]], align 4
-// IR-NEXT:    [[CMP18:%.*]] = icmp ugt i32 [[TMP15]], [[TMP16]]
-// IR-NEXT:    br i1 [[CMP18]], label %[[COND_TRUE:.*]], label %[[COND_FALSE:.*]]
-// IR:       [[COND_TRUE]]:
-// IR-NEXT:    [[TMP17:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_13]], align 4
-// IR-NEXT:    br label %[[COND_END:.*]]
-// IR:       [[COND_FALSE]]:
-// IR-NEXT:    [[TMP18:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
+// IR-LABEL: @func(
+// IR-NEXT:  [[ENTRY:.*]]:
+// IR-NEXT:    %[[START_ADDR:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[END_ADDR:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[STEP_ADDR:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_IV:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[TMP:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[I:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_1:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_2:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_3:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTUNROLLED_IV_I:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_6:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_8:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_12:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTCAPTURE_EXPR_14:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTFLOOR_0_IV__UNROLLED_IV_I:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_LB:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_UB:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_STRIDE:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTOMP_IS_LAST:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTFLOOR_0_IV__UNROLLED_IV_I18:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTTILE_0_IV__UNROLLED_IV_I:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[DOTUNROLL_INNER_IV_I:.+]] = alloca i32, align 4
+// IR-NEXT:    %[[TMP0:.+]] = call i32 @__kmpc_global_thread_num(ptr @2)
+// IR-NEXT:    store i32 %[[START:.+]], ptr %[[START_ADDR]], align 4
+// IR-NEXT:    store i32 %[[END:.+]], ptr %[[END_ADDR]], align 4
+// IR-NEXT:    store i32 %[[STEP:.+]], ptr %[[STEP_ADDR]], align 4
+// IR-NEXT:    %[[TMP1:.+]] = load i32, ptr %[[START_ADDR]], align 4
+// IR-NEXT:    store i32 %[[TMP1]], ptr %[[I]], align 4
+// IR-NEXT:    %[[TMP2:.+]] = load i32, ptr %[[START_ADDR]], align 4
+// IR-NEXT:    store i32 %[[TMP2]], ptr %[[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    %[[TMP3:.+]] = load i32, ptr %[[END_ADDR]], align 4
+// IR-NEXT:    store i32 %[[TMP3]], ptr %[[DOTCAPTURE_EXPR_1]], align 4
+// IR-NEXT:    %[[TMP4:.+]] = load i32, ptr %[[STEP_ADDR]], align 4
+// IR-NEXT:    store i32 %[[TMP4]], ptr %[[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    %[[TMP5:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_1]], align 4
+// IR-NEXT:    %[[TMP6:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    %[[SUB:.+]] = sub i32 %[[TMP5]], %[[TMP6]]
+// IR-NEXT:    %[[SUB4:.+]] = sub i32 %[[SUB]], 1
+// IR-NEXT:    %[[TMP7:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    %[[ADD:.+]] = add i32 %[[SUB4]], %[[TMP7]]
+// IR-NEXT:    %[[TMP8:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    %[[DIV:.+]] = udiv i32 %[[ADD]], %[[TMP8]]
+// IR-NEXT:    %[[SUB5:.+]] = sub i32 %[[DIV]], 1
+// IR-NEXT:    store i32 %[[SUB5]], ptr %[[DOTCAPTURE_EXPR_3]], align 4
+// IR-NEXT:    store i32 0, ptr %[[DOTUNROLLED_IV_I]], align 4
+// IR-NEXT:    %[[TMP9:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_3]], align 4
+// IR-NEXT:    %[[ADD7:.+]] = add i32 %[[TMP9]], 1
+// IR-NEXT:    store i32 %[[ADD7]], ptr %[[DOTCAPTURE_EXPR_6]], align 4
+// IR-NEXT:    %[[TMP10:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_6]], align 4
+// IR-NEXT:    %[[SUB9:.+]] = sub i32 %[[TMP10]], -1
+// IR-NEXT:    %[[DIV10:.+]] = udiv i32 %[[SUB9]], 2
+// IR-NEXT:    %[[SUB11:.+]] = sub i32 %[[DIV10]], 1
+// IR-NEXT:    store i32 %[[SUB11]], ptr %[[DOTCAPTURE_EXPR_8]], align 4
+// IR-NEXT:    %[[TMP11:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_8]], align 4
+// IR-NEXT:    %[[ADD13:.+]] = add i32 %[[TMP11]], 1
+// IR-NEXT:    store i32 %[[ADD13]], ptr %[[DOTCAPTURE_EXPR_12]], align 4
+// IR-NEXT:    %[[TMP12:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_12]], align 4
+// IR-NEXT:    %[[SUB15:.+]] = sub i32 %[[TMP12]], -3
+// IR-NEXT:    %[[DIV16:.+]] = udiv i32 %[[SUB15]], 4
+// IR-NEXT:    %[[SUB17:.+]] = sub i32 %[[DIV16]], 1
+// IR-NEXT:    store i32 %[[SUB17]], ptr %[[DOTCAPTURE_EXPR_14]], align 4
+// IR-NEXT:    store i32 0, ptr %[[DOTFLOOR_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    %[[TMP13:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_12]], align 4
+// IR-NEXT:    %[[CMP:.+]] = icmp ult i32 0, %[[TMP13]]
+// IR-NEXT:    br i1 %[[CMP]], label %[[OMP_PRECOND_THEN:.+]], label %[[OMP_PRECOND_END:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_PRECOND_THEN]]:
+// IR-NEXT:    store i32 0, ptr %[[DOTOMP_LB]], align 4
+// IR-NEXT:    %[[TMP14:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_14]], align 4
+// IR-NEXT:    store i32 %[[TMP14]], ptr %[[DOTOMP_UB]], align 4
+// IR-NEXT:    store i32 1, ptr %[[DOTOMP_STRIDE]], align 4
+// IR-NEXT:    store i32 0, ptr %[[DOTOMP_IS_LAST]], align 4
+// IR-NEXT:    call void @__kmpc_for_static_init_4u(ptr @1, i32 %[[TMP0]], i32 34, ptr %[[DOTOMP_IS_LAST]], ptr %[[DOTOMP_LB]], ptr %[[DOTOMP_UB]], ptr %[[DOTOMP_STRIDE]], i32 1, i32 1)
+// IR-NEXT:    %[[TMP15:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
+// IR-NEXT:    %[[TMP16:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_14]], align 4
+// IR-NEXT:    %[[CMP19:.+]] = icmp ugt i32 %[[TMP15]], %[[TMP16]]
+// IR-NEXT:    br i1 %[[CMP19]], label %[[COND_TRUE:.+]], label %[[COND_FALSE:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_TRUE]]:
+// IR-NEXT:    %[[TMP17:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_14]], align 4
+// IR-NEXT:    br label %[[COND_END:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_FALSE]]:
+// IR-NEXT:    %[[TMP18:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
 // IR-NEXT:    br label %[[COND_END]]
-// IR:       [[COND_END]]:
-// IR-NEXT:    [[COND:%.*]] = phi i32 [ [[TMP17]], %[[COND_TRUE]] ], [ [[TMP18]], %[[COND_FALSE]] ]
-// IR-NEXT:    store i32 [[COND]], ptr [[DOTOMP_UB]], align 4
-// IR-NEXT:    [[TMP19:%.*]] = load i32, ptr [[DOTOMP_LB]], align 4
-// IR-NEXT:    store i32 [[TMP19]], ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    br label %[[OMP_INNER_FOR_COND:.*]]
-// IR:       [[OMP_INNER_FOR_COND]]:
-// IR-NEXT:    [[TMP20:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[TMP21:%.*]] = load i32, ptr [[DOTOMP_UB]], align 4
-// IR-NEXT:    [[ADD19:%.*]] = add i32 [[TMP21]], 1
-// IR-NEXT:    [[CMP20:%.*]] = icmp ult i32 [[TMP20]], [[ADD19]]
-// IR-NEXT:    br i1 [[CMP20]], label %[[OMP_INNER_FOR_BODY:.*]], label %[[OMP_INNER_FOR_END:.*]]
-// IR:       [[OMP_INNER_FOR_BODY]]:
-// IR-NEXT:    [[TMP22:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[MUL:%.*]] = mul i32 [[TMP22]], 4
-// IR-NEXT:    [[ADD21:%.*]] = add i32 0, [[MUL]]
-// IR-NEXT:    store i32 [[ADD21]], ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I17]], align 4
-// IR-NEXT:    [[TMP23:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I17]], align 4
-// IR-NEXT:    store i32 [[TMP23]], ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND:.*]]
-// IR:       [[FOR_COND]]:
-// IR-NEXT:    [[TMP24:%.*]] = load i32, ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    [[TMP25:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    [[ADD22:%.*]] = add i32 [[TMP25]], 1
-// IR-NEXT:    [[TMP26:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I17]], align 4
-// IR-NEXT:    [[ADD23:%.*]] = add i32 [[TMP26]], 4
-// IR-NEXT:    [[CMP24:%.*]] = icmp ult i32 [[ADD22]], [[ADD23]]
-// IR-NEXT:    br i1 [[CMP24]], label %[[COND_TRUE25:.*]], label %[[COND_FALSE27:.*]]
-// IR:       [[COND_TRUE25]]:
-// IR-NEXT:    [[TMP27:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_7]], align 4
-// IR-NEXT:    [[ADD26:%.*]] = add i32 [[TMP27]], 1
-// IR-NEXT:    br label %[[COND_END29:.*]]
-// IR:       [[COND_FALSE27]]:
-// IR-NEXT:    [[TMP28:%.*]] = load i32, ptr [[DOTFLOOR_0_IV__UNROLLED_IV_I17]], align 4
-// IR-NEXT:    [[ADD28:%.*]] = add i32 [[TMP28]], 4
-// IR-NEXT:    br label %[[COND_END29]]
-// IR:       [[COND_END29]]:
-// IR-NEXT:    [[COND30:%.*]] = phi i32 [ [[ADD26]], %[[COND_TRUE25]] ], [ [[ADD28]], %[[COND_FALSE27]] ]
-// IR-NEXT:    [[CMP31:%.*]] = icmp ult i32 [[TMP24]], [[COND30]]
-// IR-NEXT:    br i1 [[CMP31]], label %[[FOR_BODY:.*]], label %[[FOR_END44:.*]]
-// IR:       [[FOR_BODY]]:
-// IR-NEXT:    [[TMP29:%.*]] = load i32, ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    [[MUL32:%.*]] = mul i32 [[TMP29]], 2
-// IR-NEXT:    [[ADD33:%.*]] = add i32 0, [[MUL32]]
-// IR-NEXT:    store i32 [[ADD33]], ptr [[DOTUNROLLED_IV_I]], align 4
-// IR-NEXT:    [[TMP30:%.*]] = load i32, ptr [[DOTUNROLLED_IV_I]], align 4
-// IR-NEXT:    store i32 [[TMP30]], ptr [[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND34:.*]]
-// IR:       [[FOR_COND34]]:
-// IR-NEXT:    [[TMP31:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    [[TMP32:%.*]] = load i32, ptr [[DOTUNROLLED_IV_I]], align 4
-// IR-NEXT:    [[ADD35:%.*]] = add i32 [[TMP32]], 2
-// IR-NEXT:    [[CMP36:%.*]] = icmp ult i32 [[TMP31]], [[ADD35]]
-// IR-NEXT:    br i1 [[CMP36]], label %[[LAND_RHS:.*]], label %[[LAND_END:.*]]
-// IR:       [[LAND_RHS]]:
-// IR-NEXT:    [[TMP33:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    [[TMP34:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_2]], align 4
-// IR-NEXT:    [[ADD37:%.*]] = add i32 [[TMP34]], 1
-// IR-NEXT:    [[CMP38:%.*]] = icmp ult i32 [[TMP33]], [[ADD37]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_END]]:
+// IR-NEXT:    %[[COND:.+]] = phi i32 [ %[[TMP17]], %[[COND_TRUE]] ], [ %[[TMP18]], %[[COND_FALSE]] ]
+// IR-NEXT:    store i32 %[[COND]], ptr %[[DOTOMP_UB]], align 4
+// IR-NEXT:    %[[TMP19:.+]] = load i32, ptr %[[DOTOMP_LB]], align 4
+// IR-NEXT:    store i32 %[[TMP19]], ptr %[[DOTOMP_IV]], align 4
+// IR-NEXT:    br label %[[OMP_INNER_FOR_COND:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_INNER_FOR_COND]]:
+// IR-NEXT:    %[[TMP20:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
+// IR-NEXT:    %[[TMP21:.+]] = load i32, ptr %[[DOTOMP_UB]], align 4
+// IR-NEXT:    %[[ADD20:.+]] = add i32 %[[TMP21]], 1
+// IR-NEXT:    %[[CMP21:.+]] = icmp ult i32 %[[TMP20]], %[[ADD20]]
+// IR-NEXT:    br i1 %[[CMP21]], label %[[OMP_INNER_FOR_BODY:.+]], label %[[OMP_INNER_FOR_END:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_INNER_FOR_BODY]]:
+// IR-NEXT:    %[[TMP22:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
+// IR-NEXT:    %[[MUL:.+]] = mul i32 %[[TMP22]], 4
+// IR-NEXT:    %[[ADD22:.+]] = add i32 0, %[[MUL]]
+// IR-NEXT:    store i32 %[[ADD22]], ptr %[[DOTFLOOR_0_IV__UNROLLED_IV_I18]], align 4
+// IR-NEXT:    %[[TMP23:.+]] = load i32, ptr %[[DOTFLOOR_0_IV__UNROLLED_IV_I18]], align 4
+// IR-NEXT:    store i32 %[[TMP23]], ptr %[[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_COND]]:
+// IR-NEXT:    %[[TMP24:.+]] = load i32, ptr %[[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    %[[TMP25:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_8]], align 4
+// IR-NEXT:    %[[ADD23:.+]] = add i32 %[[TMP25]], 1
+// IR-NEXT:    %[[TMP26:.+]] = load i32, ptr %[[DOTFLOOR_0_IV__UNROLLED_IV_I18]], align 4
+// IR-NEXT:    %[[ADD24:.+]] = add i32 %[[TMP26]], 4
+// IR-NEXT:    %[[CMP25:.+]] = icmp ult i32 %[[ADD23]], %[[ADD24]]
+// IR-NEXT:    br i1 %[[CMP25]], label %[[COND_TRUE26:.+]], label %[[COND_FALSE28:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_TRUE26]]:
+// IR-NEXT:    %[[TMP27:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_8]], align 4
+// IR-NEXT:    %[[ADD27:.+]] = add i32 %[[TMP27]], 1
+// IR-NEXT:    br label %[[COND_END30:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_FALSE28]]:
+// IR-NEXT:    %[[TMP28:.+]] = load i32, ptr %[[DOTFLOOR_0_IV__UNROLLED_IV_I18]], align 4
+// IR-NEXT:    %[[ADD29:.+]] = add i32 %[[TMP28]], 4
+// IR-NEXT:    br label %[[COND_END30]]
+// IR-EMPTY:
+// IR-NEXT:  [[COND_END30]]:
+// IR-NEXT:    %[[COND31:.+]] = phi i32 [ %[[ADD27]], %[[COND_TRUE26]] ], [ %[[ADD29]], %[[COND_FALSE28]] ]
+// IR-NEXT:    %[[CMP32:.+]] = icmp ult i32 %[[TMP24]], %[[COND31]]
+// IR-NEXT:    br i1 %[[CMP32]], label %[[FOR_BODY:.+]], label %[[FOR_END45:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_BODY]]:
+// IR-NEXT:    %[[TMP29:.+]] = load i32, ptr %[[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    %[[MUL33:.+]] = mul i32 %[[TMP29]], 2
+// IR-NEXT:    %[[ADD34:.+]] = add i32 0, %[[MUL33]]
+// IR-NEXT:    store i32 %[[ADD34]], ptr %[[DOTUNROLLED_IV_I]], align 4
+// IR-NEXT:    %[[TMP30:.+]] = load i32, ptr %[[DOTUNROLLED_IV_I]], align 4
+// IR-NEXT:    store i32 %[[TMP30]], ptr %[[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND35:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_COND35]]:
+// IR-NEXT:    %[[TMP31:.+]] = load i32, ptr %[[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    %[[TMP32:.+]] = load i32, ptr %[[DOTUNROLLED_IV_I]], align 4
+// IR-NEXT:    %[[ADD36:.+]] = add i32 %[[TMP32]], 2
+// IR-NEXT:    %[[CMP37:.+]] = icmp ult i32 %[[TMP31]], %[[ADD36]]
+// IR-NEXT:    br i1 %[[CMP37]], label %[[LAND_RHS:.+]], label %[[LAND_END:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[LAND_RHS]]:
+// IR-NEXT:    %[[TMP33:.+]] = load i32, ptr %[[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    %[[TMP34:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_3]], align 4
+// IR-NEXT:    %[[ADD38:.+]] = add i32 %[[TMP34]], 1
+// IR-NEXT:    %[[CMP39:.+]] = icmp ult i32 %[[TMP33]], %[[ADD38]]
 // IR-NEXT:    br label %[[LAND_END]]
-// IR:       [[LAND_END]]:
-// IR-NEXT:    [[TMP35:%.*]] = phi i1 [ false, %[[FOR_COND34]] ], [ [[CMP38]], %[[LAND_RHS]] ]
-// IR-NEXT:    br i1 [[TMP35]], label %[[FOR_BODY39:.*]], label %[[FOR_END:.*]]
-// IR:       [[FOR_BODY39]]:
-// IR-NEXT:    [[TMP36:%.*]] = load i32, ptr [[DOTCAPTURE_EXPR_]], align 4
-// IR-NEXT:    [[TMP37:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    [[TMP38:%.*]] = load i32, ptr [[DOTNEW_STEP]], align 4
-// IR-NEXT:    [[MUL40:%.*]] = mul i32 [[TMP37]], [[TMP38]]
-// IR-NEXT:    [[ADD41:%.*]] = add i32 [[TMP36]], [[MUL40]]
-// IR-NEXT:    store i32 [[ADD41]], ptr [[I]], align 4
-// IR-NEXT:    [[TMP39:%.*]] = load i32, ptr [[START_ADDR]], align 4
-// IR-NEXT:    [[TMP40:%.*]] = load i32, ptr [[END_ADDR]], align 4
-// IR-NEXT:    [[TMP41:%.*]] = load i32, ptr [[STEP_ADDR]], align 4
-// IR-NEXT:    [[TMP42:%.*]] = load i32, ptr [[I]], align 4
-// IR-NEXT:    call void (...) @body(i32 noundef [[TMP39]], i32 noundef [[TMP40]], i32 noundef [[TMP41]], i32 noundef [[TMP42]])
-// IR-NEXT:    br label %[[FOR_INC:.*]]
-// IR:       [[FOR_INC]]:
-// IR-NEXT:    [[TMP43:%.*]] = load i32, ptr [[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    [[INC:%.*]] = add i32 [[TMP43]], 1
-// IR-NEXT:    store i32 [[INC]], ptr [[DOTUNROLL_INNER_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND34]], !llvm.loop [[LOOP2:![0-9]+]]
-// IR:       [[FOR_END]]:
-// IR-NEXT:    br label %[[FOR_INC42:.*]]
-// IR:       [[FOR_INC42]]:
-// IR-NEXT:    [[TMP44:%.*]] = load i32, ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    [[INC43:%.*]] = add i32 [[TMP44]], 1
-// IR-NEXT:    store i32 [[INC43]], ptr [[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
-// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop [[LOOP5:![0-9]+]]
-// IR:       [[FOR_END44]]:
-// IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.*]]
-// IR:       [[OMP_BODY_CONTINUE]]:
-// IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.*]]
-// IR:       [[OMP_INNER_FOR_INC]]:
-// IR-NEXT:    [[TMP45:%.*]] = load i32, ptr [[DOTOMP_IV]], align 4
-// IR-NEXT:    [[ADD45:%.*]] = add i32 [[TMP45]], 1
-// IR-NEXT:    store i32 [[ADD45]], ptr [[DOTOMP_IV]], align 4
+// IR-EMPTY:
+// IR-NEXT:  [[LAND_END]]:
+// IR-NEXT:    %[[TMP35:.+]] = phi i1 [ false, %[[FOR_COND35]] ], [ %[[CMP39]], %[[LAND_RHS]] ]
+// IR-NEXT:    br i1 %[[TMP35]], label %[[FOR_BODY40:.+]], label %[[FOR_END:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_BODY40]]:
+// IR-NEXT:    %[[TMP36:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_]], align 4
+// IR-NEXT:    %[[TMP37:.+]] = load i32, ptr %[[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    %[[TMP38:.+]] = load i32, ptr %[[DOTCAPTURE_EXPR_2]], align 4
+// IR-NEXT:    %[[MUL41:.+]] = mul i32 %[[TMP37]], %[[TMP38]]
+// IR-NEXT:    %[[ADD42:.+]] = add i32 %[[TMP36]], %[[MUL41]]
+// IR-NEXT:    store i32 %[[ADD42]], ptr %[[I]], align 4
+// IR-NEXT:    %[[TMP39:.+]] = load i32, ptr %[[START_ADDR]], align 4
+// IR-NEXT:    %[[TMP40:.+]] = load i32, ptr %[[END_ADDR]], align 4
+// IR-NEXT:    %[[TMP41:.+]] = load i32, ptr %[[STEP_ADDR]], align 4
+// IR-NEXT:    %[[TMP42:.+]] = load i32, ptr %[[I]], align 4
+// IR-NEXT:    call void (...) @body(i32 noundef %[[TMP39]], i32 noundef %[[TMP40]], i32 noundef %[[TMP41]], i32 noundef %[[TMP42]])
+// IR-NEXT:    br label %[[FOR_INC:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_INC]]:
+// IR-NEXT:    %[[TMP43:.+]] = load i32, ptr %[[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    %[[INC:.+]] = add i32 %[[TMP43]], 1
+// IR-NEXT:    store i32 %[[INC]], ptr %[[DOTUNROLL_INNER_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND35]], !llvm.loop ![[LOOP2:[0-9]+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_END]]:
+// IR-NEXT:    br label %[[FOR_INC43:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_INC43]]:
+// IR-NEXT:    %[[TMP44:.+]] = load i32, ptr %[[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    %[[INC44:.+]] = add i32 %[[TMP44]], 1
+// IR-NEXT:    store i32 %[[INC44]], ptr %[[DOTTILE_0_IV__UNROLLED_IV_I]], align 4
+// IR-NEXT:    br label %[[FOR_COND]], !llvm.loop ![[LOOP5:[0-9]+]]
+// IR-EMPTY:
+// IR-NEXT:  [[FOR_END45]]:
+// IR-NEXT:    br label %[[OMP_BODY_CONTINUE:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_BODY_CONTINUE]]:
+// IR-NEXT:    br label %[[OMP_INNER_FOR_INC:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_INNER_FOR_INC]]:
+// IR-NEXT:    %[[TMP45:.+]] = load i32, ptr %[[DOTOMP_IV]], align 4
+// IR-NEXT:    %[[ADD46:.+]] = add i32 %[[TMP45]], 1
+// IR-NEXT:    store i32 %[[ADD46]], ptr %[[DOTOMP_IV]], align 4
 // IR-NEXT:    br label %[[OMP_INNER_FOR_COND]]
-// IR:       [[OMP_INNER_FOR_END]]:
-// IR-NEXT:    br label %[[OMP_LOOP_EXIT:.*]]
-// IR:       [[OMP_LOOP_EXIT]]:
-// IR-NEXT:    call void @__kmpc_for_static_fini(ptr @[[GLOB1]], i32 [[TMP0]])
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_INNER_FOR_END]]:
+// IR-NEXT:    br label %[[OMP_LOOP_EXIT:.+]]
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_LOOP_EXIT]]:
+// IR-NEXT:    call void @__kmpc_for_static_fini(ptr @1, i32 %[[TMP0]])
 // IR-NEXT:    br label %[[OMP_PRECOND_END]]
-// IR:       [[OMP_PRECOND_END]]:
-// IR-NEXT:    call void @__kmpc_barrier(ptr @[[GLOB3:[0-9]+]], i32 [[TMP0]])
+// IR-EMPTY:
+// IR-NEXT:  [[OMP_PRECOND_END]]:
+// IR-NEXT:    call void @__kmpc_barrier(ptr @3, i32 %[[TMP0]])
 // IR-NEXT:    ret void
-//
+// IR-NEXT:  }
 extern "C" void func(int start, int end, int step) {
   #pragma omp for
   #pragma omp tile sizes(4)
@@ -217,9 +239,7 @@ extern "C" void func(int start, int end, int step) {
 #endif /* HEADER */
 
 
-//.
-// IR: [[LOOP2]] = distinct !{[[LOOP2]], [[META3:![0-9]+]], [[META4:![0-9]+]]}
-// IR: [[META3]] = !{!"llvm.loop.mustprogress"}
-// IR: [[META4]] = !{!"llvm.loop.unroll.count", i32 2}
-// IR: [[LOOP5]] = distinct !{[[LOOP5]], [[META3]]}
-//.
+// IR: ![[LOOP2]] = distinct !{![[LOOP2]], ![[LOOPPROP3:[0-9]+]], ![[LOOPPROP4:[0-9]+]]}
+// IR: ![[LOOPPROP3]] = !{!"llvm.loop.mustprogress"}
+// IR: ![[LOOPPROP4]] = !{!"llvm.loop.unroll.count", i32 2}
+// IR: ![[LOOP5]] = distinct !{![[LOOP5]], ![[LOOPPROP3]]}
diff --git a/clang/unittests/AST/AttrTest.cpp b/clang/unittests/AST/AttrTest.cpp
index cbdc3a1cc59a9..56096054dd267 100644
--- a/clang/unittests/AST/AttrTest.cpp
+++ b/clang/unittests/AST/AttrTest.cpp
@@ -8,6 +8,7 @@
 
 #include "clang/AST/Attr.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/StmtOpenMP.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
 #include "clang/ASTMatchers/ASTMatchers.h"
 #include "clang/Basic/AttrKinds.h"
@@ -272,4 +273,69 @@ TEST(Attr, UnknownScopedAttributeSpellingIndex) {
   EXPECT_EQ(CI.getAttributeSpellingListIndex(), 0u);
 }
 
+template <typename NodeT> const NodeT *findFirst(const Stmt *S) {
+  if (!S)
+    return nullptr;
+  if (const auto *N = dyn_cast<NodeT>(S))
+    return N;
+  for (const Stmt *C : S->children())
+    if (const NodeT *N = findFirst<NodeT>(C))
+      return N;
+  return nullptr;
+}
+
+const OMPInvariantPredicateBoundAttr *findIntraTileHint(const Stmt *S) {
+  if (const auto *AS = findFirst<AttributedStmt>(S))
+    for (const Attr *A : AS->getAttrs())
+      if (const auto *Hint = dyn_cast<OMPInvariantPredicateBoundAttr>(A))
+        return Hint;
+  return nullptr;
+}
+
+// The intra-tile loop generated by '#pragma omp tile' carries an internal
+// OMPInvariantPredicateBoundAttr; that marker is what lets a consuming
+// directive such as 'collapse' reinterpret the min-bounded loop as a
+// rectangular one with a constant trip count.
+//
+// Assert on the attribute directly here,
+// including the fact that the overshoot predicate is dropped when the trip
+// count is a compile-time multiple of the tile size.
+TEST(Attr, OMPInvariantPredicateBoundOnIntraTileLoop) {
+  auto AST = buildASTFromCodeWithArgs(
+      R"cpp(
+        void body(int);
+        void partial_last_tile() {
+          #pragma omp tile sizes(5)
+          for (int i = 0; i < 11; ++i)
+            body(i);
+        }
+        void full_tiles_only() {
+          #pragma omp tile sizes(5)
+          for (int i = 0; i < 10; ++i)
+            body(i);
+        }
+      )cpp",
+      {"-fopenmp"});
+  ASSERT_TRUE(AST);
+
+  auto GetHint = [&AST](const char *FuncName) {
+    const FunctionDecl *Func = getFunctionNode(AST.get(), FuncName);
+    const auto *Tile = findFirst<OMPTileDirective>(Func->getBody());
+    EXPECT_TRUE(Tile);
+    return Tile ? findIntraTileHint(Tile->getTransformedStmt()) : nullptr;
+  };
+
+  const OMPInvariantPredicateBoundAttr *Partial = GetHint("partial_last_tile");
+  ASSERT_TRUE(Partial);
+  EXPECT_TRUE(Partial->getRectCond());
+  EXPECT_TRUE(Partial->getTileSize());
+  EXPECT_TRUE(Partial->getPredicate());
+
+  const OMPInvariantPredicateBoundAttr *Full = GetHint("full_tiles_only");
+  ASSERT_TRUE(Full);
+  EXPECT_TRUE(Full->getRectCond());
+  EXPECT_TRUE(Full->getTileSize());
+  EXPECT_FALSE(Full->getPredicate());
+}
+
 } // namespace
diff --git a/openmp/runtime/test/transform/tile/intfor.c b/openmp/runtime/test/transform/tile/intfor.c
index 05ef5e85a0dad..4a930eab6730a 100644
--- a/openmp/runtime/test/transform/tile/intfor.c
+++ b/openmp/runtime/test/transform/tile/intfor.c
@@ -188,4 +188,4 @@ int main() {
 // CHECK-NEXT: tilesize(2)
 // CHECK-NEXT: tilesize(1)
 // CHECK-NEXT: tilesize(1)
-// CHECK-NEXT: done
+// CHECK-NEXT: done
\ No newline at end of file
diff --git a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-stacked-tile-deep.cpp b/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-stacked-tile-deep.cpp
deleted file mode 100644
index 1e835db20fb81..0000000000000
--- a/openmp/runtime/test/transform/tile/parallel-wsloop-collapse-stacked-tile-deep.cpp
+++ /dev/null
@@ -1,32 +0,0 @@
-// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
-
-// XFAIL: *
-// collapse(3) through stacked tiles is not supported.
-
-#ifndef HEADER
-#define HEADER
-
-#include <cstdlib>
-#include <cstdio>
-
-int main() {
-  printf("do\n");
-#pragma omp parallel for collapse(3) num_threads(1)
-#pragma omp tile sizes(2)
-#pragma omp tile sizes(4)
-  for (int i = 0; i < 6; ++i)
-    printf("i=%d\n", i);
-  printf("done\n");
-  return EXIT_SUCCESS;
-}
-
-#endif /* HEADER */
-
-// CHECK:      do
-// CHECK-NEXT: i=0
-// CHECK-NEXT: i=1
-// CHECK-NEXT: i=2
-// CHECK-NEXT: i=3
-// CHECK-NEXT: i=4
-// CHECK-NEXT: i=5
-// CHECK-NEXT: done

>From 835eb70821ca46add331458809c69eaa1f56d75a Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Mon, 17 Aug 2026 05:50:57 -0400
Subject: [PATCH 19/19] revised

---
 clang/include/clang/AST/StmtOpenMP.h          | 29 +++++--------------
 .../clang/Basic/DiagnosticSemaKinds.td        |  3 +-
 clang/lib/AST/StmtOpenMP.cpp                  | 15 ++++------
 clang/lib/Sema/SemaOpenMP.cpp                 | 28 +++++++++---------
 4 files changed, 30 insertions(+), 45 deletions(-)

diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index 33aada14bb642..124e984360ab7 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -905,35 +905,20 @@ class OMPLoopBasedDirective : public OMPExecutableDirective {
     return ignoreIntraTileHint(const_cast<Stmt *>(S));
   }
 
-  /// Calls the specified callback function for all the loops in \p CurStmt,
-  /// from the outermost to the innermost. The callback is always passed a
-  /// ForStmt or CXXForRangeStmt; when that loop carries the intra-tile
-  /// reinterpretation hint, it is delivered as a separate argument instead of
-  /// wrapping the loop, so callers that only expect a loop cannot trip over it.
-  static bool
-  doForAllLoops(Stmt *CurStmt, bool TryImperfectlyNestedLoops,
-                unsigned NumLoops,
-                llvm::function_ref<bool(unsigned, Stmt *,
-                                        const OMPInvariantPredicateBoundAttr *)>
-                    Callback,
-                llvm::function_ref<void(OMPLoopTransformationDirective *)>
-                    OnTransformationCallback);
-
   /// Calls the specified callback function for all the loops in \p CurStmt,
   /// from the outermost to the innermost.
+  ///
+  /// By default the callback receives a ForStmt or CXXForRangeStmt. Pass
+  /// \p UnwrapIntraTileHint as false to leave an intra-tile
+  /// OMPInvariantPredicateBoundAttr wrapper in place so the caller can read
+  /// the hint itself (see checkOpenMPIterationSpace).
   static bool
   doForAllLoops(Stmt *CurStmt, bool TryImperfectlyNestedLoops,
                 unsigned NumLoops,
                 llvm::function_ref<bool(unsigned, Stmt *)> Callback,
                 llvm::function_ref<void(OMPLoopTransformationDirective *)>
-                    OnTransformationCallback) {
-    auto &&NewCallback = [Callback](unsigned Cnt, Stmt *Loop,
-                                    const OMPInvariantPredicateBoundAttr *) {
-      return Callback(Cnt, Loop);
-    };
-    return doForAllLoops(CurStmt, TryImperfectlyNestedLoops, NumLoops,
-                         NewCallback, OnTransformationCallback);
-  }
+                    OnTransformationCallback,
+                bool UnwrapIntraTileHint = true);
   static bool
   doForAllLoops(const Stmt *CurStmt, bool TryImperfectlyNestedLoops,
                 unsigned NumLoops,
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index b27e4e9e03e04..6c046cf824f7b 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12305,7 +12305,8 @@ def err_omp_loop_var_reused_in_collapsed_loop : Error<
   "loop iteration variable %0 cannot be reused in a nested loop of a collapsed loop nest">;
 def err_omp_collapse_stacked_tile : Error<
   "cannot collapse the intra-tile loop of a '#pragma omp tile' that is nested "
-  "inside another loop-transforming directive">;
+  "inside another loop-transforming directive; OpenMP permits this construct, "
+  "but it is yet not supported">;
 def err_omp_simd_region_cannot_use_stmt : Error<
   "'%0' statement cannot be used in OpenMP simd region">;
 def warn_omp_loop_64_bit_var : Warning<
diff --git a/clang/lib/AST/StmtOpenMP.cpp b/clang/lib/AST/StmtOpenMP.cpp
index 827f8143cf744..f19745bf8e5e4 100644
--- a/clang/lib/AST/StmtOpenMP.cpp
+++ b/clang/lib/AST/StmtOpenMP.cpp
@@ -27,7 +27,6 @@ OMPLoopBasedDirective::getIntraTileHint(const Stmt *S) {
 }
 
 /// Peek through an intra-tile hint wrapper to return the underlying loop.
-/// The hint is delivered as the third argument to the loop callback.
 Stmt *OMPLoopBasedDirective::ignoreIntraTileHint(Stmt *S) {
   if (auto *AS = dyn_cast_or_null<AttributedStmt>(S))
     if (hasSpecificAttr<OMPInvariantPredicateBoundAttr>(AS->getAttrs()))
@@ -173,11 +172,10 @@ OMPLoopBasedDirective::tryToFindNextInnerLoop(Stmt *CurStmt,
 
 bool OMPLoopBasedDirective::doForAllLoops(
     Stmt *CurStmt, bool TryImperfectlyNestedLoops, unsigned NumLoops,
-    llvm::function_ref<bool(unsigned, Stmt *,
-                            const OMPInvariantPredicateBoundAttr *)>
-        Callback,
+    llvm::function_ref<bool(unsigned, Stmt *)> Callback,
     llvm::function_ref<void(OMPLoopTransformationDirective *)>
-        OnTransformationCallback) {
+        OnTransformationCallback,
+    bool UnwrapIntraTileHint) {
   CurStmt = ignoreContainersKeepingIntraTileHint(CurStmt);
   for (unsigned Cnt = 0; Cnt < NumLoops; ++Cnt) {
     while (true) {
@@ -208,11 +206,10 @@ bool OMPLoopBasedDirective::doForAllLoops(
     }
     if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(CurStmt))
       CurStmt = CanonLoop->getLoopStmt();
-    // The walker keeps the intra-tile hint wrapper around so that it can be
-    // handed to the analysis, but the callback only ever sees the loop itself.
-    const OMPInvariantPredicateBoundAttr *Hint = getIntraTileHint(CurStmt);
+    // Keep the wrapper while walking so a caller that asked not to unwrap can
+    // read the hint; look through it to reach the loop body.
     Stmt *LoopStmt = ignoreIntraTileHint(CurStmt);
-    if (Callback(Cnt, LoopStmt, Hint))
+    if (Callback(Cnt, UnwrapIntraTileHint ? LoopStmt : CurStmt))
       return false;
     // Move on to the next nested for loop, or to the loop body.
     // OpenMP [2.8.1, simd construct, Restrictions]
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index e1097e75c66d4..e12c858141622 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -9581,9 +9581,9 @@ static bool checkOpenMPIterationSpace(
     llvm::MutableArrayRef<LoopIterationSpace> ResultIterSpaces,
     llvm::MapVector<const Expr *, DeclRefExpr *> &Captures,
     const llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopVarDecls,
-    llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopInductionVars,
-    const OMPInvariantPredicateBoundAttr *IntraTileHint) {
+    llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopInductionVars) {
   bool SupportsNonRectangular = !isOpenMPLoopTransformationDirective(DKind);
+
   // See the tile reinterpretation design note in ActOnOpenMPTileDirective.
   // If the loop carries the hint, analyze its rectangular form instead.
   //
@@ -9592,6 +9592,15 @@ static bool checkOpenMPIterationSpace(
   // loop; a loop transformation such as an enclosing 'tile' has nowhere to put
   // the overshoot guard, so for those the stored min-bounded form is analyzed
   // exactly as it is on a build without this hint.
+  // OpenMP [2.9.1, Canonical Loop Form]
+  //   for (init-expr; test-expr; incr-expr) structured-block
+  //   for (range-decl: range-expr) structured-block
+  if (auto *CanonLoop = dyn_cast_or_null<OMPCanonicalLoop>(S))
+    S = CanonLoop->getLoopStmt();
+  const OMPInvariantPredicateBoundAttr *IntraTileHint =
+      OMPLoopBasedDirective::getIntraTileHint(S);
+  if (IntraTileHint)
+    S = OMPLoopBasedDirective::ignoreIntraTileHint(S);
   Expr *TileRectCond = nullptr;
   Expr *TileBodyPredicate = nullptr;
   Expr *TileTripCount = nullptr;
@@ -9600,11 +9609,6 @@ static bool checkOpenMPIterationSpace(
     TileBodyPredicate = IntraTileHint->getPredicate();
     TileTripCount = IntraTileHint->getTileSize();
   }
-  // OpenMP [2.9.1, Canonical Loop Form]
-  //   for (init-expr; test-expr; incr-expr) structured-block
-  //   for (range-decl: range-expr) structured-block
-  if (auto *CanonLoop = dyn_cast_or_null<OMPCanonicalLoop>(S))
-    S = CanonLoop->getLoopStmt();
   auto *For = dyn_cast_or_null<ForStmt>(S);
   auto *CXXFor = dyn_cast_or_null<CXXForRangeStmt>(S);
   // Ranged for is supported only in OpenMP 5.0.
@@ -10150,15 +10154,12 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
           [DKind, &SemaRef, &DSA, NumLoops, NestedLoopCount,
            CollapseLoopCountExpr, OrderedLoopCountExpr, &VarsWithImplicitDSA,
            &IterSpaces, &Captures, &CollapsedLoopVarDecls,
-           &CollapsedLoopInductionVars](
-              unsigned Cnt, Stmt *CurStmt,
-              const OMPInvariantPredicateBoundAttr *IntraTileHint) {
+           &CollapsedLoopInductionVars](unsigned Cnt, Stmt *CurStmt) {
             if (checkOpenMPIterationSpace(
                     DKind, CurStmt, SemaRef, DSA, Cnt, NestedLoopCount,
                     NumLoops, CollapseLoopCountExpr, OrderedLoopCountExpr,
                     VarsWithImplicitDSA, IterSpaces, Captures,
-                    CollapsedLoopVarDecls, CollapsedLoopInductionVars,
-                    IntraTileHint))
+                    CollapsedLoopVarDecls, CollapsedLoopInductionVars))
               return true;
             if (Cnt > 0 && Cnt >= NestedLoopCount &&
                 IterSpaces[Cnt].CounterVar) {
@@ -10191,7 +10192,8 @@ checkOpenMPLoop(OpenMPDirectiveKind DKind, Expr *CollapseLoopCountExpr,
                 }
               }
             }
-          }))
+          },
+          /*UnwrapIntraTileHint=*/false))
     return 0;
 
   Built.clear(/*size=*/NestedLoopCount);


From openmp-commits at lists.llvm.org  Mon Aug 17 02:51:50 2026
From: openmp-commits at lists.llvm.org (Amit Tiwari via Openmp-commits)
Date: Mon, 17 Aug 2026 02:51:50 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [mlir] [openmp] [Clang][OpenMP]
 Canonicalize Intra-tiles in Loop Tiling (PR #191114)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/191114@github.com>
Message-ID: <6a82d9b6.4987c784.36d8d6.79cc@mx.google.com>


================
@@ -9575,8 +9581,25 @@ static bool checkOpenMPIterationSpace(
     llvm::MutableArrayRef<LoopIterationSpace> ResultIterSpaces,
     llvm::MapVector<const Expr *, DeclRefExpr *> &Captures,
     const llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopVarDecls,
-    llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopInductionVars) {
+    llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopInductionVars,
+    const OMPInvariantPredicateBoundAttr *IntraTileHint) {
   bool SupportsNonRectangular = !isOpenMPLoopTransformationDirective(DKind);
+  // See the tile reinterpretation design note in ActOnOpenMPTileDirective.
----------------
loopacino wrote:

Done

https://github.com/llvm/llvm-project/pull/191114

From openmp-commits at lists.llvm.org  Mon Aug 17 02:52:04 2026
From: openmp-commits at lists.llvm.org (Amit Tiwari via Openmp-commits)
Date: Mon, 17 Aug 2026 02:52:04 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [mlir] [openmp] [Clang][OpenMP]
 Canonicalize Intra-tiles in Loop Tiling (PR #191114)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/191114@github.com>
Message-ID: <6a82d9c4.a1accbb4.ee747.7934@mx.google.com>


================
@@ -12303,6 +12303,9 @@ def err_omp_loop_bad_collapse_var : Error<
   "cannot use variable %1 in collapsed imperfectly-nested loop %select{init|condition|increment}0 statement">;
 def err_omp_loop_var_reused_in_collapsed_loop : Error<
   "loop iteration variable %0 cannot be reused in a nested loop of a collapsed loop nest">;
+def err_omp_collapse_stacked_tile : Error<
+  "cannot collapse the intra-tile loop of a '#pragma omp tile' that is nested "
+  "inside another loop-transforming directive">;
----------------
loopacino wrote:

Added.

https://github.com/llvm/llvm-project/pull/191114

From openmp-commits at lists.llvm.org  Mon Aug 17 02:54:02 2026
From: openmp-commits at lists.llvm.org (Amit Tiwari via Openmp-commits)
Date: Mon, 17 Aug 2026 02:54:02 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [mlir] [openmp] [Clang][OpenMP]
 Canonicalize Intra-tiles in Loop Tiling (PR #191114)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/191114@github.com>
Message-ID: <6a82da3a.642c3f8d.18da66.34f1@mx.google.com>


================
@@ -9575,8 +9581,25 @@ static bool checkOpenMPIterationSpace(
     llvm::MutableArrayRef<LoopIterationSpace> ResultIterSpaces,
     llvm::MapVector<const Expr *, DeclRefExpr *> &Captures,
     const llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopVarDecls,
-    llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopInductionVars) {
+    llvm::SmallPtrSetImpl<const Decl *> &CollapsedLoopInductionVars,
+    const OMPInvariantPredicateBoundAttr *IntraTileHint) {
----------------
loopacino wrote:

Dropped the 3-arg form in `doForAllLoop`. By default, the walker unwraps before calling and hands only the loop, so `dyn_cast<ForStmt>` (the earlier crash) don't go, while keeps `checkOpenMPLoop` as the only site that asks not to unwrap, so that the `Stmt *S` can be `AttributeStmt`. Only `checkOpenMPIterationSpace` takes the hint off the wrapper (same pattern as the canonical-loop unwrap that was already there). 


https://github.com/llvm/llvm-project/pull/191114

From openmp-commits at lists.llvm.org  Mon Aug 17 02:57:01 2026
From: openmp-commits at lists.llvm.org (Amit Tiwari via Openmp-commits)
Date: Mon, 17 Aug 2026 02:57:01 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [mlir] [openmp] [Clang][OpenMP]
 Canonicalize Intra-tiles in Loop Tiling (PR #191114)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/191114@github.com>
Message-ID: <6a82daed.6c8668f2.38979c.0a9b@mx.google.com>


================
@@ -15257,6 +15367,13 @@ StmtResult SemaOpenMP::ActOnOpenMPTileDirective(ArrayRef<OMPClause *> Clauses,
         ForStmt(Context, InitStmt.get(), CondExpr.get(), nullptr,
                 IncrStmt.get(), Inner, LoopHelper.Init->getBeginLoc(),
                 LoopHelper.Init->getBeginLoc(), LoopHelper.Inc->getEndLoc());
+
+    // Attach the droppable reinterpretation attribute to the intra-tile loop.
+    auto *Hint = OMPInvariantPredicateBoundAttr::CreateImplicit(
----------------
loopacino wrote:

No, it does because without the attribute the outer-loop variable (`.floor.iv`) is not registered in the DSA stack–making it loop invariant. This results in no recomputation of the `.floor.iv` value, and the stale value keeps re-assigning to inner `.tile.iv`.  Here, `ReinterpretTileLB/FoundFloor` trigger this recompuatation for all the cases including the `NoPartialTile` case. By the way, other convenience is that we can directly use `f+T` from the `OMPInvariantPredicateBoundAttr`, instead of extracting it from default `min(f+T,N)`.
	  
NB: The same floor stale value issue arises with `stripe` with an outer `collapse`

https://github.com/llvm/llvm-project/pull/191114

From openmp-commits at lists.llvm.org  Mon Aug 17 05:12:49 2026
From: openmp-commits at lists.llvm.org (Robert Imschweiler via Openmp-commits)
Date: Mon, 17 Aug 2026 05:12:49 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [libomp] Atomically claim library
	unregistration (PR #216713)
Message-ID: <llvm.org/llvm/llvm-project/pull/216713@github.com>


https://github.com/ro-i created https://github.com/llvm/llvm-project/pull/216713

Claude assisted with this patch.

Follow-up to https://github.com/llvm/llvm-project/pull/215988.

>From 4887cfe8db6f8172b0583b83df90e0d6e8b99518 Mon Sep 17 00:00:00 2001
From: Robert Imschweiler <robert.imschweiler at amd.com>
Date: Mon, 17 Aug 2026 02:27:58 -0500
Subject: [PATCH] [libomp] Atomically claim library unregistration

Claude assisted with this patch.
---
 openmp/runtime/src/kmp_runtime.cpp | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/openmp/runtime/src/kmp_runtime.cpp b/openmp/runtime/src/kmp_runtime.cpp
index f16bcf5e2cbbb..22ec2605718a3 100644
--- a/openmp/runtime/src/kmp_runtime.cpp
+++ b/openmp/runtime/src/kmp_runtime.cpp
@@ -6927,12 +6927,17 @@ void __kmp_register_library_startup(void) {
 
 void __kmp_unregister_library(void) {
 
-  // The library can be torn down before it ever registered itself, e.g. when
-  // __kmp_abort_process() runs for a fatal error raised during environment
-  // parsing. There is nothing to unregister then, and __kmp_registration_str
-  // is still NULL, so the strcmp() below would dereference it.
-  if (__kmp_registration_flag == 0)
+  // Claim the unregistration. Teardown can be entered concurrently from
+  // library shutdown, __kmp_abort_process() and the signal handler, none of
+  // which share a lock, and the library may never have registered itself at
+  // all (e.g. a fatal error raised before registration). Atomically take
+  // ownership of __kmp_registration_str so exactly one caller runs the
+  // teardown below; the others return without touching the freed string.
+  char *reg_str = __kmp_registration_str;
+  if (reg_str == NULL ||
+      !KMP_COMPARE_AND_STORE_PTR(&__kmp_registration_str, reg_str, NULL))
     return;
+  __kmp_registration_flag = 0;
 
   char *name = __kmp_reg_status_name();
   char *value = NULL;
@@ -6968,9 +6973,7 @@ void __kmp_unregister_library(void) {
   value = __kmp_env_get(name);
 #endif
 
-  KMP_DEBUG_ASSERT(__kmp_registration_flag != 0);
-  KMP_DEBUG_ASSERT(__kmp_registration_str != NULL);
-  if (value != NULL && strcmp(value, __kmp_registration_str) == 0) {
+  if (value != NULL && strcmp(value, reg_str) == 0) {
 //  Ok, this is our variable. Delete it.
 #if defined(KMP_USE_SHM)
     if (__kmp_shm_available) {
@@ -6992,13 +6995,10 @@ void __kmp_unregister_library(void) {
     KMP_INTERNAL_FREE(temp_reg_status_file_name);
 #endif
 
-  KMP_INTERNAL_FREE(__kmp_registration_str);
+  KMP_INTERNAL_FREE(reg_str);
   KMP_INTERNAL_FREE(value);
   KMP_INTERNAL_FREE(name);
 
-  __kmp_registration_flag = 0;
-  __kmp_registration_str = NULL;
-
 } // __kmp_unregister_library
 
 // End of Library registration stuff.


From openmp-commits at lists.llvm.org  Tue Aug 18 04:20:45 2026
From: openmp-commits at lists.llvm.org (Amit Tiwari via Openmp-commits)
Date: Tue, 18 Aug 2026 04:20:45 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [Clang][OpenMP] Add
 `#pragma omp flatten` loop directive + `depth` clause (PR #206977)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206977@github.com>
Message-ID: <6a84400d.b1b3e0a1.325f8e.eec5@mx.google.com>


https://github.com/loopacino commented:

Will soon post changes for testcases too.

https://github.com/llvm/llvm-project/pull/206977

From openmp-commits at lists.llvm.org  Tue Aug 18 04:21:13 2026
From: openmp-commits at lists.llvm.org (Amit Tiwari via Openmp-commits)
Date: Tue, 18 Aug 2026 04:21:13 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [Clang][OpenMP] Add
 `#pragma omp flatten` loop directive + `depth` clause (PR #206977)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206977@github.com>
Message-ID: <6a844029.436e7ae1.2a65b4.ab61@mx.google.com>


https://github.com/loopacino updated https://github.com/llvm/llvm-project/pull/206977

>From 0fb5bd42f1e127faea963c359b1e3c69cb969c60 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 1 Jul 2026 09:06:58 -0400
Subject: [PATCH 01/16] add OMPFlattenDirective ast node

---
 clang/include/clang/AST/RecursiveASTVisitor.h |  3 +
 clang/include/clang/AST/StmtOpenMP.h          | 75 ++++++++++++++++++-
 clang/include/clang/Basic/StmtNodes.td        |  1 +
 clang/lib/AST/StmtOpenMP.cpp                  | 20 +++++
 clang/lib/AST/StmtPrinter.cpp                 |  5 ++
 clang/lib/AST/StmtProfile.cpp                 |  4 +
 clang/lib/Basic/OpenMPKinds.cpp               |  2 +-
 7 files changed, 108 insertions(+), 2 deletions(-)

diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index b000a34043696..e425f7a31f59c 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -3227,6 +3227,9 @@ DEF_TRAVERSE_STMT(OMPFuseDirective,
 DEF_TRAVERSE_STMT(OMPInterchangeDirective,
                   { TRY_TO(TraverseOMPExecutableDirective(S)); })
 
+DEF_TRAVERSE_STMT(OMPFlattenDirective,
+                  { TRY_TO(TraverseOMPExecutableDirective(S)); })
+
 DEF_TRAVERSE_STMT(OMPSplitDirective,
                   { TRY_TO(TraverseOMPExecutableDirective(S)); })
 
diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index dbc76e7df8ecd..1420331f37c1f 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -1041,7 +1041,7 @@ class OMPCanonicalLoopNestTransformationDirective
     Stmt::StmtClass C = T->getStmtClass();
     return C == OMPTileDirectiveClass || C == OMPUnrollDirectiveClass ||
            C == OMPReverseDirectiveClass || C == OMPInterchangeDirectiveClass ||
-           C == OMPStripeDirectiveClass;
+           C == OMPStripeDirectiveClass || C == OMPFlattenDirectiveClass;
   }
 };
 
@@ -5959,6 +5959,79 @@ class OMPInterchangeDirective final
   }
 };
 
+/// Represents the '#pragma omp flatten' loop transformation directive.
+///
+/// \code{c}
+///   #pragma omp flatten
+///   for (int i = 0; i < m; ++i)
+///     for (int j = 0; j < n; ++j)
+///       ..
+/// \endcode
+class OMPFlattenDirective final
+    : public OMPCanonicalLoopNestTransformationDirective {
+  friend class ASTStmtReader;
+  friend class OMPExecutableDirective;
+
+  /// Offsets of child members.
+  enum {
+    PreInitsOffset = 0,
+    TransformedStmtOffset,
+  };
+
+  explicit OMPFlattenDirective(SourceLocation StartLoc, SourceLocation EndLoc,
+                               unsigned NumLoops)
+      : OMPCanonicalLoopNestTransformationDirective(
+            OMPFlattenDirectiveClass, llvm::omp::OMPD_flatten, StartLoc, EndLoc,
+            NumLoops) {}
+
+  void setPreInits(Stmt *PreInits) {
+    Data->getChildren()[PreInitsOffset] = PreInits;
+  }
+
+  void setTransformedStmt(Stmt *S) {
+    Data->getChildren()[TransformedStmtOffset] = S;
+  }
+
+public:
+  /// Create a new AST node representation for '#pragma omp flatten'.
+  ///
+  /// \param C         Context of the AST.
+  /// \param StartLoc  Location of the introducer (e.g. the 'omp' token).
+  /// \param EndLoc    Location of the directive's end (e.g. the tok::eod).
+  /// \param Clauses   The directive's clauses.
+  /// \param NumLoops  Number of affected loops (the flatten depth, currently
+  ///                  always 2).
+  /// \param AssociatedStmt  The outermost associated loop.
+  /// \param TransformedStmt The flattened loop, or nullptr in dependent
+  ///                        contexts.
+  /// \param PreInits  Helper preinits statements for the loop nest.
+  static OMPFlattenDirective *
+  Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation EndLoc,
+         ArrayRef<OMPClause *> Clauses, unsigned NumLoops, Stmt *AssociatedStmt,
+         Stmt *TransformedStmt, Stmt *PreInits);
+
+  /// Build an empty '#pragma omp flatten' AST node for deserialization.
+  ///
+  /// \param C          Context of the AST.
+  /// \param NumClauses Number of clauses to allocate.
+  /// \param NumLoops   Number of associated loops to allocate.
+  static OMPFlattenDirective *
+  CreateEmpty(const ASTContext &C, unsigned NumClauses, unsigned NumLoops);
+
+  /// Gets the flattened loop after the transformation. This is the de-sugared
+  /// replacement or nullptr in dependent contexts.
+  Stmt *getTransformedStmt() const {
+    return Data->getChildren()[TransformedStmtOffset];
+  }
+
+  /// Return preinits statement.
+  Stmt *getPreInits() const { return Data->getChildren()[PreInitsOffset]; }
+
+  static bool classof(const Stmt *T) {
+    return T->getStmtClass() == OMPFlattenDirectiveClass;
+  }
+};
+
 /// The base class for all transformation directives of canonical loop
 /// sequences (currently only 'fuse')
 class OMPCanonicalLoopSequenceTransformationDirective
diff --git a/clang/include/clang/Basic/StmtNodes.td b/clang/include/clang/Basic/StmtNodes.td
index e166894ea024b..0d7a0c1490a1e 100644
--- a/clang/include/clang/Basic/StmtNodes.td
+++ b/clang/include/clang/Basic/StmtNodes.td
@@ -247,6 +247,7 @@ def OMPReverseDirective : StmtNode<OMPCanonicalLoopNestTransformationDirective>;
 def OMPSplitDirective : StmtNode<OMPCanonicalLoopNestTransformationDirective>;
 def OMPInterchangeDirective
     : StmtNode<OMPCanonicalLoopNestTransformationDirective>;
+def OMPFlattenDirective : StmtNode<OMPCanonicalLoopNestTransformationDirective>;
 def OMPCanonicalLoopSequenceTransformationDirective
     : StmtNode<OMPExecutableDirective, 1>;
 def OMPFuseDirective
diff --git a/clang/lib/AST/StmtOpenMP.cpp b/clang/lib/AST/StmtOpenMP.cpp
index 9d6b315effb41..477604ef3dafb 100644
--- a/clang/lib/AST/StmtOpenMP.cpp
+++ b/clang/lib/AST/StmtOpenMP.cpp
@@ -552,6 +552,26 @@ OMPInterchangeDirective::CreateEmpty(const ASTContext &C, unsigned NumClauses,
       SourceLocation(), SourceLocation(), NumLoops);
 }
 
+OMPFlattenDirective *OMPFlattenDirective::Create(
+    const ASTContext &C, SourceLocation StartLoc, SourceLocation EndLoc,
+    ArrayRef<OMPClause *> Clauses, unsigned NumLoops, Stmt *AssociatedStmt,
+    Stmt *TransformedStmt, Stmt *PreInits) {
+  OMPFlattenDirective *Dir = createDirective<OMPFlattenDirective>(
+      C, Clauses, AssociatedStmt, TransformedStmtOffset + 1, StartLoc, EndLoc,
+      NumLoops);
+  Dir->setTransformedStmt(TransformedStmt);
+  Dir->setPreInits(PreInits);
+  return Dir;
+}
+
+OMPFlattenDirective *OMPFlattenDirective::CreateEmpty(const ASTContext &C,
+                                                      unsigned NumClauses,
+                                                      unsigned NumLoops) {
+  return createEmptyDirective<OMPFlattenDirective>(
+      C, NumClauses, /*HasAssociatedStmt=*/true, TransformedStmtOffset + 1,
+      SourceLocation(), SourceLocation(), NumLoops);
+}
+
 OMPSplitDirective *
 OMPSplitDirective::Create(const ASTContext &C, SourceLocation StartLoc,
                           SourceLocation EndLoc, ArrayRef<OMPClause *> Clauses,
diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp
index 6c3294573e9d4..0bacb79ed86e3 100644
--- a/clang/lib/AST/StmtPrinter.cpp
+++ b/clang/lib/AST/StmtPrinter.cpp
@@ -800,6 +800,11 @@ void StmtPrinter::VisitOMPInterchangeDirective(OMPInterchangeDirective *Node) {
   PrintOMPExecutableDirective(Node);
 }
 
+void StmtPrinter::VisitOMPFlattenDirective(OMPFlattenDirective *Node) {
+  Indent() << "#pragma omp flatten";
+  PrintOMPExecutableDirective(Node);
+}
+
 void StmtPrinter::VisitOMPSplitDirective(OMPSplitDirective *Node) {
   Indent() << "#pragma omp split";
   PrintOMPExecutableDirective(Node);
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index 90eab530e0c2e..de7ed322d7424 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -1071,6 +1071,10 @@ void StmtProfiler::VisitOMPInterchangeDirective(
   VisitOMPCanonicalLoopNestTransformationDirective(S);
 }
 
+void StmtProfiler::VisitOMPFlattenDirective(const OMPFlattenDirective *S) {
+  VisitOMPCanonicalLoopNestTransformationDirective(S);
+}
+
 void StmtProfiler::VisitOMPSplitDirective(const OMPSplitDirective *S) {
   VisitOMPCanonicalLoopNestTransformationDirective(S);
 }
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index 675d86349c933..57e0347f8ea77 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -818,7 +818,7 @@ bool clang::isOpenMPCanonicalLoopNestTransformationDirective(
     OpenMPDirectiveKind DKind) {
   return DKind == OMPD_tile || DKind == OMPD_unroll || DKind == OMPD_reverse ||
          DKind == OMPD_split || DKind == OMPD_interchange ||
-         DKind == OMPD_stripe;
+         DKind == OMPD_stripe || DKind == OMPD_flatten;
 }
 
 bool clang::isOpenMPCanonicalLoopSequenceTransformationDirective(

>From 933007b6333584fb358adc4f89a61db1ae15d4cb Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:02:27 -0400
Subject: [PATCH 02/16] add OMPDepthClause ast node

---
 clang/include/clang/AST/OpenMPClause.h        | 68 +++++++++++++++++++
 clang/include/clang/AST/RecursiveASTVisitor.h |  6 ++
 clang/lib/AST/OpenMPClause.cpp                | 25 +++++++
 clang/lib/AST/StmtProfile.cpp                 |  5 ++
 clang/lib/Parse/ParseOpenMP.cpp               |  1 +
 llvm/include/llvm/Frontend/OpenMP/OMP.td      |  1 +
 6 files changed, 106 insertions(+)

diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 8ceafc4669297..50d1364a9726d 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -1380,6 +1380,74 @@ class OMPPartialClause final : public OMPClause {
   }
 };
 
+/// This represents the 'depth' clause on the '#pragma omp flatten' (and
+/// '#pragma omp fuse') loop-transformation directives.
+///
+/// \code
+/// #pragma omp flatten depth(3)
+/// \endcode
+/// In this example the 'flatten' directive has a 'depth' clause whose argument
+/// '3' specifies how many perfectly nested loops are combined into one.
+/// The argument must be a positive integer constant expression that evaluates
+/// to at most the loop nest depth of the associated loop nest.
+class OMPDepthClause final : public OMPClause {
+  friend class OMPClauseReader;
+
+  /// Location of '('.
+  SourceLocation LParenLoc;
+
+  /// The depth expression (number of loops to flatten).
+  Stmt *Depth = nullptr;
+
+  /// Build an empty clause.
+  explicit OMPDepthClause() : OMPClause(llvm::omp::OMPC_depth, {}, {}) {}
+
+  /// Set the depth expression.
+  void setDepth(Expr *E) { Depth = E; }
+
+  /// Sets the location of '('.
+  void setLParenLoc(SourceLocation Loc) { LParenLoc = Loc; }
+
+public:
+  /// Build an AST node for a 'depth' clause.
+  ///
+  /// \param C         Context of the AST.
+  /// \param StartLoc  Location of the 'depth' identifier.
+  /// \param LParenLoc Location of '('.
+  /// \param EndLoc    Location of ')'.
+  /// \param Depth     The depth expression.
+  static OMPDepthClause *Create(const ASTContext &C, SourceLocation StartLoc,
+                                SourceLocation LParenLoc, SourceLocation EndLoc,
+                                Expr *Depth);
+
+  /// Build an empty 'depth' AST node for deserialization.
+  ///
+  /// \param C     Context of the AST.
+  static OMPDepthClause *CreateEmpty(const ASTContext &C);
+
+  /// Returns the location of '('.
+  SourceLocation getLParenLoc() const { return LParenLoc; }
+
+  /// Returns the depth expression or nullptr if not set.
+  Expr *getDepth() const { return cast_or_null<Expr>(Depth); }
+
+  child_range children() { return child_range(&Depth, &Depth + 1); }
+  const_child_range children() const {
+    return const_child_range(&Depth, &Depth + 1);
+  }
+
+  child_range used_children() {
+    return child_range(child_iterator(), child_iterator());
+  }
+  const_child_range used_children() const {
+    return const_child_range(const_child_iterator(), const_child_iterator());
+  }
+
+  static bool classof(const OMPClause *T) {
+    return T->getClauseKind() == llvm::omp::OMPC_depth;
+  }
+};
+
 /// This represents 'collapse' clause in the '#pragma omp ...'
 /// directive.
 ///
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index e425f7a31f59c..da55e97fdfcb5 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -3568,6 +3568,12 @@ bool RecursiveASTVisitor<Derived>::VisitOMPPartialClause(OMPPartialClause *C) {
   return true;
 }
 
+template <typename Derived>
+bool RecursiveASTVisitor<Derived>::VisitOMPDepthClause(OMPDepthClause *C) {
+  TRY_TO(TraverseStmt(C->getDepth()));
+  return true;
+}
+
 template <typename Derived>
 bool
 RecursiveASTVisitor<Derived>::VisitOMPCollapseClause(OMPCollapseClause *C) {
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index ed00e80144c25..7b2f67a31cad9 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -1057,6 +1057,22 @@ OMPPartialClause *OMPPartialClause::CreateEmpty(const ASTContext &C) {
   return new (C) OMPPartialClause();
 }
 
+OMPDepthClause *OMPDepthClause::Create(const ASTContext &C,
+                                       SourceLocation StartLoc,
+                                       SourceLocation LParenLoc,
+                                       SourceLocation EndLoc, Expr *Depth) {
+  OMPDepthClause *Clause = CreateEmpty(C);
+  Clause->setLocStart(StartLoc);
+  Clause->setLParenLoc(LParenLoc);
+  Clause->setLocEnd(EndLoc);
+  Clause->setDepth(Depth);
+  return Clause;
+}
+
+OMPDepthClause *OMPDepthClause::CreateEmpty(const ASTContext &C) {
+  return new (C) OMPDepthClause();
+}
+
 OMPLoopRangeClause *
 OMPLoopRangeClause::Create(const ASTContext &C, SourceLocation StartLoc,
                            SourceLocation LParenLoc, SourceLocation FirstLoc,
@@ -2069,6 +2085,15 @@ void OMPClausePrinter::VisitOMPPermutationClause(OMPPermutationClause *Node) {
 
 void OMPClausePrinter::VisitOMPFullClause(OMPFullClause *Node) { OS << "full"; }
 
+void OMPClausePrinter::VisitOMPDepthClause(OMPDepthClause *Node) {
+  OS << "depth";
+  if (Expr *Depth = Node->getDepth()) {
+    OS << '(';
+    Depth->printPretty(OS, nullptr, Policy, 0);
+    OS << ')';
+  }
+}
+
 void OMPClausePrinter::VisitOMPPartialClause(OMPPartialClause *Node) {
   OS << "partial";
 
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index de7ed322d7424..838d6d02f7446 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -521,6 +521,11 @@ void OMPClauseProfiler::VisitOMPPartialClause(const OMPPartialClause *C) {
     Profiler->VisitExpr(Factor);
 }
 
+void OMPClauseProfiler::VisitOMPDepthClause(const OMPDepthClause *C) {
+  if (const Expr *Depth = C->getDepth())
+    Profiler->VisitExpr(Depth);
+}
+
 void OMPClauseProfiler::VisitOMPLoopRangeClause(const OMPLoopRangeClause *C) {
   if (const Expr *First = C->getFirst())
     Profiler->VisitExpr(First);
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index af52295df2d8b..952a517205381 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -3247,6 +3247,7 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
   case OMPC_nocontext:
   case OMPC_filter:
   case OMPC_partial:
+  case OMPC_depth:
   case OMPC_align:
   case OMPC_message:
   case OMPC_ompx_dyn_cgroup_mem:
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index 679a944fc4358..d49c5ee1a0db3 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -161,6 +161,7 @@ def OMPC_Depobj : Clause<[Spelling<"depobj">]> {
   let isImplicit = true;
 }
 def OMPC_Depth : Clause<[Spelling<"depth">]> {
+  let clangClass = "OMPDepthClause";
   let flangClass = "ScalarIntConstantExpr";
 }
 def OMPC_Destroy : Clause<[Spelling<"destroy">]> {

>From b63024e1e342cd168ef128a3f6a24909df2a82db Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:04:44 -0400
Subject: [PATCH 03/16] template-transform support for flatten directive &
 depth clause

---
 clang/lib/Sema/TreeTransform.h | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 3b99ff4bb9e23..865622ad50f1c 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -1804,6 +1804,14 @@ class TreeTransform {
                                                        LParenLoc, EndLoc);
   }
 
+  /// Build a new OpenMP 'depth' clause.
+  OMPClause *RebuildOMPDepthClause(Expr *Depth, SourceLocation StartLoc,
+                                   SourceLocation LParenLoc,
+                                   SourceLocation EndLoc) {
+    return getSema().OpenMP().ActOnOpenMPDepthClause(Depth, StartLoc, LParenLoc,
+                                                     EndLoc);
+  }
+
   OMPClause *
   RebuildOMPLoopRangeClause(Expr *First, Expr *Count, SourceLocation StartLoc,
                             SourceLocation LParenLoc, SourceLocation FirstLoc,
@@ -9791,6 +9799,17 @@ TreeTransform<Derived>::TransformOMPSplitDirective(OMPSplitDirective *D) {
   return Res;
 }
 
+template <typename Derived>
+StmtResult
+TreeTransform<Derived>::TransformOMPFlattenDirective(OMPFlattenDirective *D) {
+  DeclarationNameInfo DirName;
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      D->getDirectiveKind(), DirName, nullptr, D->getBeginLoc());
+  StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
+  return Res;
+}
+
 template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPFuseDirective(OMPFuseDirective *D) {
@@ -10717,6 +10736,20 @@ TreeTransform<Derived>::TransformOMPPartialClause(OMPPartialClause *C) {
                                  C->getEndLoc());
 }
 
+template <typename Derived>
+OMPClause *TreeTransform<Derived>::TransformOMPDepthClause(OMPDepthClause *C) {
+  ExprResult T = getDerived().TransformExpr(C->getDepth());
+  if (T.isInvalid())
+    return nullptr;
+  Expr *Depth = T.get();
+  bool Changed = Depth != C->getDepth();
+
+  if (!Changed && !getDerived().AlwaysRebuild())
+    return C;
+  return RebuildOMPDepthClause(Depth, C->getBeginLoc(), C->getLParenLoc(),
+                               C->getEndLoc());
+}
+
 template <typename Derived>
 OMPClause *
 TreeTransform<Derived>::TransformOMPLoopRangeClause(OMPLoopRangeClause *C) {

>From deafb222a7d251069c68a44fa267e58e750f1064 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:08:12 -0400
Subject: [PATCH 04/16] serialize flatten

---
 clang/include/clang/Serialization/ASTBitCodes.h |  1 +
 clang/lib/Serialization/ASTReaderStmt.cpp       | 11 +++++++++++
 clang/lib/Serialization/ASTWriterStmt.cpp       |  5 +++++
 3 files changed, 17 insertions(+)

diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h
index 279380de2f7fe..93a0e0a454b68 100644
--- a/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/clang/include/clang/Serialization/ASTBitCodes.h
@@ -1970,6 +1970,7 @@ enum StmtCode {
   STMT_OMP_REVERSE_DIRECTIVE,
   STMT_OMP_SPLIT_DIRECTIVE,
   STMT_OMP_INTERCHANGE_DIRECTIVE,
+  STMT_OMP_FLATTEN_DIRECTIVE,
   STMT_OMP_FUSE_DIRECTIVE,
   STMT_OMP_FOR_DIRECTIVE,
   STMT_OMP_FOR_SIMD_DIRECTIVE,
diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp
index 02ccf8d4d41c2..8c2debd6cdf32 100644
--- a/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -2537,6 +2537,10 @@ void ASTStmtReader::VisitOMPInterchangeDirective(OMPInterchangeDirective *D) {
   VisitOMPCanonicalLoopNestTransformationDirective(D);
 }
 
+void ASTStmtReader::VisitOMPFlattenDirective(OMPFlattenDirective *D) {
+  VisitOMPCanonicalLoopNestTransformationDirective(D);
+}
+
 void ASTStmtReader::VisitOMPSplitDirective(OMPSplitDirective *D) {
   VisitOMPCanonicalLoopNestTransformationDirective(D);
 }
@@ -3719,6 +3723,13 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) {
       break;
     }
 
+    case STMT_OMP_FLATTEN_DIRECTIVE: {
+      unsigned NumLoops = Record[ASTStmtReader::NumStmtFields];
+      unsigned NumClauses = Record[ASTStmtReader::NumStmtFields + 1];
+      S = OMPFlattenDirective::CreateEmpty(Context, NumClauses, NumLoops);
+      break;
+    }
+
     case STMT_OMP_FOR_DIRECTIVE: {
       unsigned CollapsedNum = Record[ASTStmtReader::NumStmtFields];
       unsigned NumClauses = Record[ASTStmtReader::NumStmtFields + 1];
diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp
index 4e9eadd730a56..dfe28a9b915e3 100644
--- a/clang/lib/Serialization/ASTWriterStmt.cpp
+++ b/clang/lib/Serialization/ASTWriterStmt.cpp
@@ -2547,6 +2547,11 @@ void ASTStmtWriter::VisitOMPInterchangeDirective(OMPInterchangeDirective *D) {
   Code = serialization::STMT_OMP_INTERCHANGE_DIRECTIVE;
 }
 
+void ASTStmtWriter::VisitOMPFlattenDirective(OMPFlattenDirective *D) {
+  VisitOMPCanonicalLoopNestTransformationDirective(D);
+  Code = serialization::STMT_OMP_FLATTEN_DIRECTIVE;
+}
+
 void ASTStmtWriter::VisitOMPSplitDirective(OMPSplitDirective *D) {
   VisitOMPCanonicalLoopNestTransformationDirective(D);
   Code = serialization::STMT_OMP_SPLIT_DIRECTIVE;

>From 3e2dfeb99230057b7cb9c484182ae33befbe1e41 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:09:03 -0400
Subject: [PATCH 05/16] serialize depth

---
 clang/lib/Serialization/ASTReader.cpp | 8 ++++++++
 clang/lib/Serialization/ASTWriter.cpp | 5 +++++
 2 files changed, 13 insertions(+)

diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index f8a6a38bb9b5c..4f51d79a52105 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -11495,6 +11495,9 @@ OMPClause *OMPClauseReader::readClause() {
   case llvm::omp::OMPC_partial:
     C = OMPPartialClause::CreateEmpty(Context);
     break;
+  case llvm::omp::OMPC_depth:
+    C = OMPDepthClause::CreateEmpty(Context);
+    break;
   case llvm::omp::OMPC_looprange:
     C = OMPLoopRangeClause::CreateEmpty(Context);
     break;
@@ -11924,6 +11927,11 @@ void OMPClauseReader::VisitOMPPartialClause(OMPPartialClause *C) {
   C->setLParenLoc(Record.readSourceLocation());
 }
 
+void OMPClauseReader::VisitOMPDepthClause(OMPDepthClause *C) {
+  C->setDepth(Record.readSubExpr());
+  C->setLParenLoc(Record.readSourceLocation());
+}
+
 void OMPClauseReader::VisitOMPLoopRangeClause(OMPLoopRangeClause *C) {
   C->setFirst(Record.readSubExpr());
   C->setCount(Record.readSubExpr());
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 357a7f7e95fa0..cbfeaa9e6638a 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -8106,6 +8106,11 @@ void OMPClauseWriter::VisitOMPPartialClause(OMPPartialClause *C) {
   Record.AddSourceLocation(C->getLParenLoc());
 }
 
+void OMPClauseWriter::VisitOMPDepthClause(OMPDepthClause *C) {
+  Record.AddStmt(C->getDepth());
+  Record.AddSourceLocation(C->getLParenLoc());
+}
+
 void OMPClauseWriter::VisitOMPLoopRangeClause(OMPLoopRangeClause *C) {
   Record.AddStmt(C->getFirst());
   Record.AddStmt(C->getCount());

>From 4ae746ca1eeaf11db10e859572a4500029d01cce Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:17:29 -0400
Subject: [PATCH 06/16] add analysis traversal support for flatten directive

---
 clang/lib/Sema/SemaExceptionSpec.cpp         | 1 +
 clang/lib/StaticAnalyzer/Core/ExprEngine.cpp | 1 +
 2 files changed, 2 insertions(+)

diff --git a/clang/lib/Sema/SemaExceptionSpec.cpp b/clang/lib/Sema/SemaExceptionSpec.cpp
index 40d530a1f3925..995b5b6010978 100644
--- a/clang/lib/Sema/SemaExceptionSpec.cpp
+++ b/clang/lib/Sema/SemaExceptionSpec.cpp
@@ -1508,6 +1508,7 @@ CanThrowResult Sema::canThrow(const Stmt *S) {
   case Stmt::OMPUnrollDirectiveClass:
   case Stmt::OMPReverseDirectiveClass:
   case Stmt::OMPInterchangeDirectiveClass:
+  case Stmt::OMPFlattenDirectiveClass:
   case Stmt::OMPSplitDirectiveClass:
   case Stmt::OMPFuseDirectiveClass:
   case Stmt::OMPSingleDirectiveClass:
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
index cfb294736ee02..3f666be00cc24 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
@@ -1783,6 +1783,7 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
     case Stmt::OMPStripeDirectiveClass:
     case Stmt::OMPTileDirectiveClass:
     case Stmt::OMPInterchangeDirectiveClass:
+    case Stmt::OMPFlattenDirectiveClass:
     case Stmt::OMPSplitDirectiveClass:
     case Stmt::OMPFuseDirectiveClass:
     case Stmt::OMPInteropDirectiveClass:

>From 415eba88ad0af63c552b4c5c2eabd660bcb29c02 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:21:25 -0400
Subject: [PATCH 07/16] codegen flatten directive

---
 clang/lib/CodeGen/CGStmt.cpp        | 3 +++
 clang/lib/CodeGen/CGStmtOpenMP.cpp  | 8 ++++++++
 clang/lib/CodeGen/CodeGenFunction.h | 1 +
 3 files changed, 12 insertions(+)

diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index 232094777f233..27f48cb63b553 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -236,6 +236,9 @@ void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {
   case Stmt::OMPInterchangeDirectiveClass:
     EmitOMPInterchangeDirective(cast<OMPInterchangeDirective>(*S));
     break;
+  case Stmt::OMPFlattenDirectiveClass:
+    EmitOMPFlattenDirective(cast<OMPFlattenDirective>(*S));
+    break;
   case Stmt::OMPFuseDirectiveClass:
     EmitOMPFuseDirective(cast<OMPFuseDirective>(*S));
     break;
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 95fd6694437fe..25da3430a23fe 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -218,6 +218,8 @@ class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
     } else if (const auto *Interchange =
                    dyn_cast<OMPInterchangeDirective>(&S)) {
       PreInits = Interchange->getPreInits();
+    } else if (const auto *Flatten = dyn_cast<OMPFlattenDirective>(&S)) {
+      PreInits = Flatten->getPreInits();
     } else {
       llvm_unreachable("Unknown loop-based directive kind.");
     }
@@ -3248,6 +3250,12 @@ void CodeGenFunction::EmitOMPInterchangeDirective(
   EmitStmt(S.getTransformedStmt());
 }
 
+void CodeGenFunction::EmitOMPFlattenDirective(const OMPFlattenDirective &S) {
+  // Emit the de-sugared statement.
+  OMPTransformDirectiveScopeRAII FlattenScope(*this, &S);
+  EmitStmt(S.getTransformedStmt());
+}
+
 void CodeGenFunction::EmitOMPFuseDirective(const OMPFuseDirective &S) {
   // Emit the de-sugared statement
   OMPTransformDirectiveScopeRAII FuseScope(*this, &S);
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 6d0718c243812..eea0163422cd3 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -3934,6 +3934,7 @@ class CodeGenFunction : public CodeGenTypeCache {
   void EmitOMPReverseDirective(const OMPReverseDirective &S);
   void EmitOMPSplitDirective(const OMPSplitDirective &S);
   void EmitOMPInterchangeDirective(const OMPInterchangeDirective &S);
+  void EmitOMPFlattenDirective(const OMPFlattenDirective &S);
   void EmitOMPFuseDirective(const OMPFuseDirective &S);
   void EmitOMPForDirective(const OMPForDirective &S);
   void EmitOMPForSimdDirective(const OMPForSimdDirective &S);

>From 814e2e9fe9bf4e28e4557999fbc16beec144cc39 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:24:42 -0400
Subject: [PATCH 08/16] add NYI CIR stub

---
 clang/lib/CIR/CodeGen/CIRGenFunction.h     | 1 +
 clang/lib/CIR/CodeGen/CIRGenStmt.cpp       | 2 ++
 clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp | 5 +++++
 3 files changed, 8 insertions(+)

diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.h b/clang/lib/CIR/CodeGen/CIRGenFunction.h
index 322355fde3957..8374583ffd4d0 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.h
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.h
@@ -2559,6 +2559,7 @@ class CIRGenFunction : public CIRGenTypeCache {
   mlir::LogicalResult emitOMPSplitDirective(const OMPSplitDirective &s);
   mlir::LogicalResult
   emitOMPInterchangeDirective(const OMPInterchangeDirective &s);
+  mlir::LogicalResult emitOMPFlattenDirective(const OMPFlattenDirective &s);
   mlir::LogicalResult emitOMPAssumeDirective(const OMPAssumeDirective &s);
   mlir::LogicalResult emitOMPMaskedDirective(const OMPMaskedDirective &s);
   mlir::LogicalResult emitOMPStripeDirective(const OMPStripeDirective &s);
diff --git a/clang/lib/CIR/CodeGen/CIRGenStmt.cpp b/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
index d3acac5801e74..33d425b22d61a 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
@@ -408,6 +408,8 @@ mlir::LogicalResult CIRGenFunction::emitStmt(const Stmt *s,
     return emitOMPSplitDirective(cast<OMPSplitDirective>(*s));
   case Stmt::OMPInterchangeDirectiveClass:
     return emitOMPInterchangeDirective(cast<OMPInterchangeDirective>(*s));
+  case Stmt::OMPFlattenDirectiveClass:
+    return emitOMPFlattenDirective(cast<OMPFlattenDirective>(*s));
   case Stmt::OMPAssumeDirectiveClass:
     return emitOMPAssumeDirective(cast<OMPAssumeDirective>(*s));
   case Stmt::OMPMaskedDirectiveClass:
diff --git a/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp b/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp
index 17a1fb8090f5c..4fffd17b25a16 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp
@@ -587,6 +587,11 @@ CIRGenFunction::emitOMPInterchangeDirective(const OMPInterchangeDirective &s) {
   return mlir::failure();
 }
 mlir::LogicalResult
+CIRGenFunction::emitOMPFlattenDirective(const OMPFlattenDirective &s) {
+  getCIRGenModule().errorNYI(s.getSourceRange(), "OpenMP OMPFlattenDirective");
+  return mlir::failure();
+}
+mlir::LogicalResult
 CIRGenFunction::emitOMPAssumeDirective(const OMPAssumeDirective &s) {
   getCIRGenModule().errorNYI(s.getSourceRange(), "OpenMP OMPAssumeDirective");
   return mlir::failure();

>From 0027ec6ab982a83766bd65f3e1bc45d9a671fc62 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:26:31 -0400
Subject: [PATCH 09/16] add libclang traversal support for depth clause

---
 clang/tools/libclang/CIndex.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index ac2fad38a1348..f8ab63e3e0dae 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2378,6 +2378,10 @@ void OMPClauseEnqueue::VisitOMPLoopRangeClause(const OMPLoopRangeClause *C) {
   Visitor->AddStmt(C->getCount());
 }
 
+void OMPClauseEnqueue::VisitOMPDepthClause(const OMPDepthClause *C) {
+  Visitor->AddStmt(C->getDepth());
+}
+
 void OMPClauseEnqueue::VisitOMPAllocatorClause(const OMPAllocatorClause *C) {
   Visitor->AddStmt(C->getAllocator());
 }

>From f5ac5f740c414310c8482f652f1bed936ae7cda3 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Fri, 10 Jul 2026 06:54:16 -0400
Subject: [PATCH 10/16] sema

---
 clang/include/clang/Sema/SemaOpenMP.h |  10 +
 clang/lib/Sema/SemaOpenMP.cpp         | 295 ++++++++++++++++++++++++++
 2 files changed, 305 insertions(+)

diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index 3621ce96b8724..7b086388416a0 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -470,6 +470,12 @@ class SemaOpenMP : public SemaBase {
                                              SourceLocation StartLoc,
                                              SourceLocation EndLoc);
 
+  /// Called on well-formed '#pragma omp flatten' after parsing of its
+  /// clauses and the associated statement.
+  StmtResult ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
+                                         Stmt *AStmt, SourceLocation StartLoc,
+                                         SourceLocation EndLoc);
+
   /// Called on well-formed '#pragma omp fuse' after parsing of its
   /// clauses and the associated statement.
   StmtResult ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
@@ -935,6 +941,10 @@ class SemaOpenMP : public SemaBase {
   OMPClause *ActOnOpenMPPartialClause(Expr *FactorExpr, SourceLocation StartLoc,
                                       SourceLocation LParenLoc,
                                       SourceLocation EndLoc);
+  /// Called on well-formed 'depth' clause.
+  OMPClause *ActOnOpenMPDepthClause(Expr *DepthExpr, SourceLocation StartLoc,
+                                    SourceLocation LParenLoc,
+                                    SourceLocation EndLoc);
   /// Called on well-formed 'collapse' clause.
   OMPClause *ActOnOpenMPCollapseClause(Expr *NumForLoops,
                                        SourceLocation StartLoc,
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 247a4a6ad9271..fcc256dec3c88 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -4629,6 +4629,7 @@ void SemaOpenMP::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind,
   case OMPD_reverse:
   case OMPD_split:
   case OMPD_interchange:
+  case OMPD_flatten:
   case OMPD_fuse:
   case OMPD_assume:
     break;
@@ -6475,6 +6476,10 @@ StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
     Res = ActOnOpenMPInterchangeDirective(ClausesWithImplicit, AStmt, StartLoc,
                                           EndLoc);
     break;
+  case OMPD_flatten:
+    Res = ActOnOpenMPFlattenDirective(ClausesWithImplicit, AStmt, StartLoc,
+                                      EndLoc);
+    break;
   case OMPD_fuse:
     Res =
         ActOnOpenMPFuseDirective(ClausesWithImplicit, AStmt, StartLoc, EndLoc);
@@ -16390,6 +16395,277 @@ StmtResult SemaOpenMP::ActOnOpenMPInterchangeDirective(
                                          buildPreInits(Context, PreInits));
 }
 
+/// Counts the perfectly nested canonical loop depth at \p AStmt. Returns
+/// std::nullopt if a loop-generating transformation prevents a static count.
+static std::optional<unsigned> getCanonicalLoopNestDepth(Stmt *AStmt) {
+  unsigned Depth = 0;
+  Stmt *CurStmt = AStmt ? AStmt->IgnoreContainers() : nullptr;
+  while (CurStmt) {
+    if (isa<OMPLoopTransformationDirective>(CurStmt))
+      return std::nullopt;
+    if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(CurStmt))
+      CurStmt = CanonLoop->getLoopStmt();
+    Stmt *Body = nullptr;
+    if (auto *For = dyn_cast<ForStmt>(CurStmt))
+      Body = For->getBody();
+    else if (auto *RangeFor = dyn_cast<CXXForRangeStmt>(CurStmt))
+      Body = RangeFor->getBody();
+    else
+      break;
+    ++Depth;
+    CurStmt = Body ? Body->IgnoreContainers() : nullptr;
+  }
+  return Depth;
+}
+
+StmtResult
+SemaOpenMP::ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
+                                        Stmt *AStmt, SourceLocation StartLoc,
+                                        SourceLocation EndLoc) {
+  ASTContext &Context = getASTContext();
+  DeclContext *CurContext = SemaRef.CurContext;
+  Scope *CurScope = SemaRef.getCurScope();
+
+  // Empty statement should only be possible if there already was an error.
+  if (!AStmt)
+    return StmtError();
+
+  // flatten without 'depth' clause combines two loops; 'depth(k)' selects k.
+  unsigned NumLoops = 2;
+  bool DepthIsValueDependent = false;
+  const auto *DepthClause =
+      OMPExecutableDirective::getSingleClause<OMPDepthClause>(Clauses);
+  if (DepthClause) {
+    Expr *DepthExpr = DepthClause->getDepth();
+    if (DepthExpr && DepthExpr->isValueDependent()) {
+      DepthIsValueDependent = true;
+    } else if (DepthExpr) {
+      Expr::EvalResult EvalResult;
+      if (DepthExpr->EvaluateAsInt(EvalResult, Context))
+        NumLoops = EvalResult.Val.getInt().getZExtValue();
+    }
+  }
+
+  // Report "expected k for loops, but found only n" when depth exceeds the
+  // perfect nest (same form as 'collapse'); skip if depth or nest is unknown.
+  if (DepthClause && !DepthIsValueDependent) {
+    if (std::optional<unsigned> NestDepth = getCanonicalLoopNestDepth(AStmt);
+        NestDepth && NumLoops > *NestDepth) {
+      Diag(AStmt->getBeginLoc(), diag::err_omp_not_for)
+          << /*expected N for loops form=*/1
+          << getOpenMPDirectiveName(OMPD_flatten) << NumLoops
+          << (*NestDepth > 0) << *NestDepth;
+      return StmtError();
+    }
+  }
+
+  // Defer when 'depth' is value-dependent (concrete k unknown until
+  // instantiation).
+  if (DepthIsValueDependent)
+    return OMPFlattenDirective::Create(Context, StartLoc, EndLoc, Clauses,
+                                       NumLoops, AStmt, nullptr, nullptr);
+
+  // Verify and diagnose loop nest.
+  SmallVector<OMPLoopBasedDirective::HelperExprs, 4> LoopHelpers(NumLoops);
+  Stmt *Body = nullptr;
+  SmallVector<SmallVector<Stmt *>, 4> OriginalInits;
+  if (!checkTransformableLoopNest(OMPD_flatten, AStmt, NumLoops, LoopHelpers,
+                                  Body, OriginalInits))
+    return StmtError();
+
+  // Delay flattening to when template is completely instantiated.
+  if (CurContext->isDependentContext())
+    return OMPFlattenDirective::Create(Context, StartLoc, EndLoc, Clauses,
+                                       NumLoops, AStmt, nullptr, nullptr);
+
+  assert(LoopHelpers.size() == NumLoops &&
+         "Expecting loop iteration space dimensionality to match number of "
+         "affected loops");
+  assert(OriginalInits.size() == NumLoops &&
+         "Expecting loop iteration space dimensionality to match number of "
+         "affected loops");
+
+  // Find the affected loops.
+  SmallVector<Stmt *> LoopStmts(NumLoops, nullptr);
+  collectLoopStmts(AStmt, LoopStmts);
+
+  // Collect pre-init statements in outer-to-inner order.
+  SmallVector<Stmt *> PreInits;
+  for (auto I : llvm::seq<int>(NumLoops)) {
+    OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
+    assert(LoopHelper.Counters.size() == 1 &&
+           "Single-dimensional loop iteration space expected");
+    addLoopPreInits(Context, LoopHelper, LoopStmts[I], OriginalInits[I],
+                    PreInits);
+  }
+
+  CaptureVars CopyTransformer(SemaRef);
+  auto MakeNumIterations = [&CopyTransformer,
+                            &LoopHelpers](unsigned I) -> Expr * {
+    return AssertSuccess(
+        CopyTransformer.TransformExpr(LoopHelpers[I].NumIterations));
+  };
+
+  OMPLoopBasedDirective::HelperExprs &OutermostHelper = LoopHelpers[0];
+  auto *OutermostCntVar = cast<DeclRefExpr>(OutermostHelper.Counters.front());
+  SourceLocation OrigVarLoc = OutermostCntVar->getExprLoc();
+  SourceLocation OrigVarLocBegin = OutermostCntVar->getBeginLoc();
+  SourceLocation OrigVarLocEnd = OutermostCntVar->getEndLoc();
+  SourceLocation CondLoc = OutermostHelper.Cond->getExprLoc();
+
+  // Product of trip counts; mirror 'collapse' IV-width selection to avoid
+  // overflow when several counts are multiplied.
+  auto BuildTripCount = [&](unsigned Bits) -> ExprResult {
+    ExprResult Product;
+    for (unsigned I = 0; I < NumLoops; ++I) {
+      ExprResult N = widenIterationCount(Bits, MakeNumIterations(I), SemaRef);
+      if (!N.isUsable())
+        return ExprError();
+      if (I == 0)
+        Product = N;
+      else
+        Product = SemaRef.BuildBinOp(CurScope, CondLoc, BO_Mul, Product.get(),
+                                     N.get());
+      if (!Product.isUsable())
+        return ExprError();
+    }
+    return Product;
+  };
+
+  bool AllCountsLessThan32Bits = true;
+  for (unsigned I = 0; I < NumLoops; ++I)
+    AllCountsLessThan32Bits &=
+        Context.getTypeSize(LoopHelpers[I].NumIterations->getType()) < 32;
+
+  ExprResult TripCount32 = BuildTripCount(/*Bits=*/32);
+  ExprResult TripCount64 = BuildTripCount(/*Bits=*/64);
+  if (!TripCount32.isUsable() || !TripCount64.isUsable())
+    return StmtError();
+
+  ExprResult TripCount = TripCount64;
+  if (Context.getTypeSize(TripCount32.get()->getType()) == 32 &&
+      (AllCountsLessThan32Bits || NumLoops == 1 ||
+       fitsInto(
+           /*Bits=*/32,
+           TripCount32.get()->getType()->hasSignedIntegerRepresentation(),
+           TripCount64.get(), SemaRef)))
+    TripCount = TripCount32;
+
+  QualType IVTy = TripCount.get()->getType();
+  uint64_t IVWidth = Context.getTypeSize(IVTy);
+
+  auto MakeNumIterationsInIVTy = [&](unsigned I) -> Expr * {
+    return AssertSuccess(SemaRef.PerformImplicitConversion(
+        MakeNumIterations(I), IVTy, AssignmentAction::Converting,
+        /*AllowExplicit=*/true));
+  };
+
+  // \code{.cpp}
+  //   for (auto .flatten.iv = 0; .flatten.iv < n0 * n1 * ...; ++.flatten.iv) {
+  //     .flatten.iv.0 = .flatten.iv / (n1 * ...);
+  //     i0 = ...;                                        // Updates[0]
+  //     .flatten.iv.1 = (.flatten.iv / ...) % n1;
+  //     i1 = ...;                                        // Updates[1]
+  //     ...
+  //     body(i0, i1, ...);
+  //   }
+  // \endcode
+  SmallString<64> FlattenedIVName(".flatten.iv");
+  VarDecl *FlattenedIVDecl = buildVarDecl(SemaRef, {}, IVTy, FlattenedIVName,
+                                          nullptr, OutermostCntVar);
+  auto MakeFlattenedRef = [&SemaRef = this->SemaRef, FlattenedIVDecl, IVTy,
+                           OrigVarLoc]() {
+    return buildDeclRefExpr(SemaRef, FlattenedIVDecl, IVTy, OrigVarLoc);
+  };
+
+  // For init-statement:
+  // \code{.cpp}
+  //   auto .flatten.iv = 0;
+  // \endcode
+  auto *Zero = IntegerLiteral::Create(Context, llvm::APInt::getZero(IVWidth),
+                                      IVTy, OrigVarLoc);
+  SemaRef.AddInitializerToDecl(FlattenedIVDecl, Zero, /*DirectInit=*/false);
+  StmtResult Init = new (Context)
+      DeclStmt(DeclGroupRef(FlattenedIVDecl), OrigVarLocBegin, OrigVarLocEnd);
+  if (!Init.isUsable())
+    return StmtError();
+
+  // For cond-expression:
+  // \code{.cpp}
+  //   .flatten.iv < n0 * n1 * ... * n(k-1)
+  // \endcode
+  ExprResult Cond = SemaRef.BuildBinOp(CurScope, CondLoc, BO_LT,
+                                       MakeFlattenedRef(), TripCount.get());
+  if (!Cond.isUsable())
+    return StmtError();
+
+  // For incr-statement:
+  // \code{.cpp}
+  //   ++.flatten.iv
+  // \endcode
+  ExprResult Incr =
+      SemaRef.BuildUnaryOp(CurScope, OutermostHelper.Inc->getExprLoc(),
+                           UO_PreInc, MakeFlattenedRef());
+  if (!Incr.isUsable())
+    return StmtError();
+
+  // Recover each logical iteration counter via mixed-radix div/mod; reuse the
+  // iteration variables from checkOpenMPLoop so Updates compute user counters.
+  SmallVector<Stmt *, 8> BodyStmts;
+  for (unsigned I = 0; I < NumLoops; ++I) {
+    OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
+    auto *IVRef = cast<DeclRefExpr>(LoopHelper.IterationVarRef);
+    auto *IVDecl = cast<VarDecl>(IVRef->getDecl());
+    std::string IVName = (".flatten.iv." + llvm::Twine(I)).str();
+    IVDecl->setDeclName(&SemaRef.PP.getIdentifierTable().get(IVName));
+
+    ExprResult Value = MakeFlattenedRef();
+    if (I + 1 < NumLoops) {
+      ExprResult Divisor = MakeNumIterationsInIVTy(I + 1);
+      for (unsigned J = I + 2; J < NumLoops; ++J) {
+        Divisor = SemaRef.BuildBinOp(CurScope, OrigVarLoc, BO_Mul,
+                                     Divisor.get(), MakeNumIterationsInIVTy(J));
+        if (!Divisor.isUsable())
+          return StmtError();
+      }
+      Value = SemaRef.BuildBinOp(CurScope, OrigVarLoc, BO_Div, Value.get(),
+                                 Divisor.get());
+      if (!Value.isUsable())
+        return StmtError();
+    }
+    if (I > 0) {
+      Value = SemaRef.BuildBinOp(CurScope, OrigVarLoc, BO_Rem, Value.get(),
+                                 MakeNumIterationsInIVTy(I));
+      if (!Value.isUsable())
+        return StmtError();
+    }
+
+    SemaRef.AddInitializerToDecl(IVDecl, Value.get(), /*DirectInit=*/false);
+    StmtResult IVStmt = new (Context)
+        DeclStmt(DeclGroupRef(IVDecl), OrigVarLocBegin, OrigVarLocEnd);
+    if (!IVStmt.isUsable())
+      return StmtError();
+
+    BodyStmts.push_back(IVStmt.get());
+    llvm::append_range(BodyStmts, LoopHelper.Updates);
+    if (auto *CXXFor = dyn_cast<CXXForRangeStmt>(LoopStmts[I]))
+      BodyStmts.push_back(CXXFor->getLoopVarStmt());
+  }
+  BodyStmts.push_back(Body);
+  auto *FlattenedBody =
+      CompoundStmt::Create(Context, BodyStmts, FPOptionsOverride(),
+                           Body->getBeginLoc(), Body->getEndLoc());
+
+  auto *FlattenedFor = new (Context) ForStmt(
+      Context, Init.get(), Cond.get(), nullptr, Incr.get(), FlattenedBody,
+      OutermostHelper.Init->getBeginLoc(), OutermostHelper.Init->getBeginLoc(),
+      OutermostHelper.Inc->getEndLoc());
+
+  return OMPFlattenDirective::Create(Context, StartLoc, EndLoc, Clauses,
+                                     NumLoops, AStmt, FlattenedFor,
+                                     buildPreInits(Context, PreInits));
+}
+
 StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
                                                 Stmt *AStmt,
                                                 SourceLocation StartLoc,
@@ -16920,6 +17196,9 @@ OMPClause *SemaOpenMP::ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind,
   case OMPC_partial:
     Res = ActOnOpenMPPartialClause(Expr, StartLoc, LParenLoc, EndLoc);
     break;
+  case OMPC_depth:
+    Res = ActOnOpenMPDepthClause(Expr, StartLoc, LParenLoc, EndLoc);
+    break;
   case OMPC_message:
     Res = ActOnOpenMPMessageClause(Expr, StartLoc, LParenLoc, EndLoc);
     break;
@@ -18241,6 +18520,22 @@ OMPClause *SemaOpenMP::ActOnOpenMPPartialClause(Expr *FactorExpr,
                                   FactorExpr);
 }
 
+OMPClause *SemaOpenMP::ActOnOpenMPDepthClause(Expr *DepthExpr,
+                                              SourceLocation StartLoc,
+                                              SourceLocation LParenLoc,
+                                              SourceLocation EndLoc) {
+  // The depth-expr must be a positive integer constant expression and
+  // not greater than the number of loops in the associated loop nest.
+  ExprResult DepthResult = VerifyPositiveIntegerConstantInClause(
+      DepthExpr, OMPC_depth, /*StrictlyPositive=*/true);
+  if (DepthResult.isInvalid())
+    return nullptr;
+  DepthExpr = DepthResult.get();
+
+  return OMPDepthClause::Create(getASTContext(), StartLoc, LParenLoc, EndLoc,
+                                DepthExpr);
+}
+
 OMPClause *SemaOpenMP::ActOnOpenMPLoopRangeClause(
     Expr *First, Expr *Count, SourceLocation StartLoc, SourceLocation LParenLoc,
     SourceLocation FirstLoc, SourceLocation CountLoc, SourceLocation EndLoc) {

>From 54be7347979638725cd4b6c421d11ec1cd618c43 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Mon, 27 Jul 2026 07:03:06 -0400
Subject: [PATCH 11/16] add libclang traversal support for flatten directive

---
 clang/tools/libclang/CIndex.cpp   | 7 +++++++
 clang/tools/libclang/CXCursor.cpp | 3 +++
 2 files changed, 10 insertions(+)

diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index f8ab63e3e0dae..9eb0ec8c21158 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2160,6 +2160,7 @@ class EnqueueVisitor : public ConstStmtVisitor<EnqueueVisitor, void>,
   void VisitOMPUnrollDirective(const OMPUnrollDirective *D);
   void VisitOMPReverseDirective(const OMPReverseDirective *D);
   void VisitOMPInterchangeDirective(const OMPInterchangeDirective *D);
+  void VisitOMPFlattenDirective(const OMPFlattenDirective *D);
   void VisitOMPCanonicalLoopSequenceTransformationDirective(
       const OMPCanonicalLoopSequenceTransformationDirective *D);
   void VisitOMPFuseDirective(const OMPFuseDirective *D);
@@ -3358,6 +3359,10 @@ void EnqueueVisitor::VisitOMPInterchangeDirective(
   VisitOMPCanonicalLoopNestTransformationDirective(D);
 }
 
+void EnqueueVisitor::VisitOMPFlattenDirective(const OMPFlattenDirective *D) {
+  VisitOMPCanonicalLoopNestTransformationDirective(D);
+}
+
 void EnqueueVisitor::VisitOMPCanonicalLoopSequenceTransformationDirective(
     const OMPCanonicalLoopSequenceTransformationDirective *D) {
   VisitOMPExecutableDirective(D);
@@ -6329,6 +6334,8 @@ CXString clang_getCursorKindSpelling(enum CXCursorKind Kind) {
     return cxstring::createRef("OMPReverseDirective");
   case CXCursor_OMPInterchangeDirective:
     return cxstring::createRef("OMPInterchangeDirective");
+  case CXCursor_OMPFlattenDirective:
+    return cxstring::createRef("OMPFlattenDirective");
   case CXCursor_OMPFuseDirective:
     return cxstring::createRef("OMPFuseDirective");
   case CXCursor_OMPSplitDirective:
diff --git a/clang/tools/libclang/CXCursor.cpp b/clang/tools/libclang/CXCursor.cpp
index b2feed578fc0f..97bbfd36558a9 100644
--- a/clang/tools/libclang/CXCursor.cpp
+++ b/clang/tools/libclang/CXCursor.cpp
@@ -703,6 +703,9 @@ CXCursor cxcursor::MakeCXCursor(const Stmt *S, const Decl *Parent,
   case Stmt::OMPInterchangeDirectiveClass:
     K = CXCursor_OMPInterchangeDirective;
     break;
+  case Stmt::OMPFlattenDirectiveClass:
+    K = CXCursor_OMPFlattenDirective;
+    break;
   case Stmt::OMPFuseDirectiveClass:
     K = CXCursor_OMPFuseDirective;
     break;

>From cdedb1045baf373937b0da858624a6ace14a3d21 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Mon, 27 Jul 2026 07:06:36 -0400
Subject: [PATCH 12/16] bindings

---
 clang/bindings/python/clang/cindex.py | 3 +++
 clang/include/clang-c/Index.h         | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/clang/bindings/python/clang/cindex.py b/clang/bindings/python/clang/cindex.py
index 24b737139dba8..ccad140e8c861 100644
--- a/clang/bindings/python/clang/cindex.py
+++ b/clang/bindings/python/clang/cindex.py
@@ -1456,6 +1456,9 @@ def is_unexposed(self):
     # OpenMP split directive.
     OMP_SPLIT_DIRECTIVE = 312
 
+    # OpenMP flatten directive.
+    OMP_FLATTEN_DIRECTIVE = 313
+
     # OpenACC Compute Construct.
     OPEN_ACC_COMPUTE_DIRECTIVE = 320
 
diff --git a/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h
index 8427236e0b444..f561fb5724897 100644
--- a/clang/include/clang-c/Index.h
+++ b/clang/include/clang-c/Index.h
@@ -2170,6 +2170,10 @@ enum CXCursorKind {
    */
   CXCursor_OMPSplitDirective = 312,
 
+  /** OpenMP flatten directive.
+   */
+  CXCursor_OMPFlattenDirective = 313,
+
   /** OpenACC Compute Construct.
    */
   CXCursor_OpenACCComputeConstruct = 320,

>From 94dee8fe1351f74760df0fb2d9903a291a5ccf66 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 5 Aug 2026 04:46:14 -0400
Subject: [PATCH 13/16] core tests-flatten and depth

---
 clang/test/OpenMP/flatten_codegen.cpp         |  65 ++++++++
 clang/test/OpenMP/flatten_depth_codegen.cpp   | 154 ++++++++++++++++++
 .../test/transform/flatten/depth-intfor.c     |  40 +++++
 .../runtime/test/transform/flatten/intfor.c   |  33 ++++
 .../test/transform/flatten/lit.local.cfg      |   5 +
 5 files changed, 297 insertions(+)
 create mode 100644 clang/test/OpenMP/flatten_codegen.cpp
 create mode 100644 clang/test/OpenMP/flatten_depth_codegen.cpp
 create mode 100644 openmp/runtime/test/transform/flatten/depth-intfor.c
 create mode 100644 openmp/runtime/test/transform/flatten/intfor.c
 create mode 100644 openmp/runtime/test/transform/flatten/lit.local.cfg

diff --git a/clang/test/OpenMP/flatten_codegen.cpp b/clang/test/OpenMP/flatten_codegen.cpp
new file mode 100644
index 0000000000000..f77b2fb00ce9b
--- /dev/null
+++ b/clang/test/OpenMP/flatten_codegen.cpp
@@ -0,0 +1,65 @@
+// Check code generation
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=60 -emit-llvm %s -o - | FileCheck %s
+
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+extern "C" void body(int, int);
+
+// CHECK-LABEL: define {{.*}}void @foo(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK:   store i64 0, ptr %.flatten.iv,
+// CHECK:   %[[COND_IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[MUL:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   icmp slt i64 %[[COND_IV]], %[[MUL]]
+// CHECK:   %[[DIV_IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[DIV:.+]] = sdiv i64 %[[DIV_IV]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   %[[REM_IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[REM:.+]] = srem i64 %[[REM_IV]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+// CHECK:   %[[I_VAL:.+]] = load i32, ptr %i,
+// CHECK:   %[[J_VAL:.+]] = load i32, ptr %j,
+// CHECK:   call void @body(i32{{.*}} %[[I_VAL]], i32{{.*}} %[[J_VAL]])
+// CHECK: for.inc:
+// CHECK:   %[[INC_IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[NEXT:.+]] = add nsw i64 %[[INC_IV]], 1
+// CHECK:   store i64 %[[NEXT]], ptr %.flatten.iv,
+// CHECK:   br label %for.cond
+extern "C" void foo(int n, int m) {
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+
+// CHECK-LABEL: define {{.*}}void @bar(
+// CHECK:   icmp slt i32 %{{.+}}, 63
+// CHECK:   sdiv i32 %{{.+}}, 9
+// CHECK:   srem i32 %{{.+}}, 9
+// CHECK:   call void @body(
+extern "C" void bar() {
+#pragma omp flatten
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+}
+
+// CHECK-LABEL: define {{.*}}void @baz(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   icmp slt i64 %{{.+}}, 10000000000
+// CHECK:   sdiv i64 %{{.+}}, 100000
+// CHECK:   srem i64 %{{.+}}, 100000
+// CHECK:   call void @body(
+extern "C" void baz() {
+#pragma omp flatten
+  for (int i = 0; i < 100000; ++i)
+    for (int j = 0; j < 100000; ++j)
+      body(i, j);
+}
+
+#endif
diff --git a/clang/test/OpenMP/flatten_depth_codegen.cpp b/clang/test/OpenMP/flatten_depth_codegen.cpp
new file mode 100644
index 0000000000000..77408fbcbdcf1
--- /dev/null
+++ b/clang/test/OpenMP/flatten_depth_codegen.cpp
@@ -0,0 +1,154 @@
+// Check code generation for the 'depth' clause on '#pragma omp flatten'.
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=61 -emit-llvm %s -o - | FileCheck %s
+
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+extern "C" void body(int, int, int);
+
+// CHECK-LABEL: define {{.*}}void @foo3(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK:   %.flatten.iv.2 = alloca i32
+// CHECK:   %[[CIV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[T01:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   %[[TALL:.+]] = mul nsw i64 %[[T01]], %{{.+}}
+// CHECK:   icmp slt i64 %[[CIV]], %[[TALL]]
+// CHECK:   %[[D0IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[D0M:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   %[[D0:.+]] = sdiv i64 %[[D0IV]], %[[D0M]]
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   %[[D1IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[D1:.+]] = sdiv i64 %[[D1IV]], %{{.+}}
+// CHECK:   %[[R1:.+]] = srem i64 %[[D1]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+// CHECK:   %[[R2IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[R2:.+]] = srem i64 %[[R2IV]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.2
+// CHECK:   call void @body(
+extern "C" void foo3(int n, int m, int p) {
+#pragma omp flatten depth(3)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        body(i, j, k);
+}
+
+// CHECK-LABEL: define {{.*}}void @foo2(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK:   %[[CIV2:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[M:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   icmp slt i64 %[[CIV2]], %[[M]]
+// CHECK:   sdiv i64 %{{.+}}, %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   srem i64 %{{.+}}, %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+extern "C" void foo2(int n, int m) {
+#pragma omp flatten depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j, 0);
+}
+
+// CHECK-LABEL: define {{.*}}void @foo1(
+// CHECK:   %.flatten.iv = alloca i32
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK: for.body:
+// CHECK:   %[[FV:.+]] = load i32, ptr %.flatten.iv,
+// CHECK:   store i32 %[[FV]], ptr %.flatten.iv.0,
+// CHECK-NOT: sdiv
+// CHECK-NOT: srem
+// CHECK:   call void @body(
+extern "C" void foo1(int n) {
+#pragma omp flatten depth(1)
+  for (int i = 0; i < n; ++i)
+    body(i, 0, 0);
+}
+
+// CHECK-LABEL: define {{.*}}void @foo2partial(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK-NOT: %.flatten.iv.2 = alloca
+// CHECK:   %[[CIV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[M:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   icmp slt i64 %[[CIV]], %[[M]]
+// CHECK:   sdiv i64 %{{.+}}, %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   srem i64 %{{.+}}, %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+// CHECK:   br label %for.cond{{.*}}
+// CHECK: for.cond{{.*}}:
+// CHECK:   icmp slt i32 %{{.+}}, 5
+extern "C" void foo2partial(int n, int m) {
+#pragma omp flatten depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < 5; ++k)
+        body(i, j, k);
+}
+
+// A value-dependent 'depth' (the argument is a non-type template parameter) is
+// deferred until instantiation, where the concrete depth drives the flattening.
+
+// CHECK-LABEL: define {{.*}}void @_Z11tmpl_depth3ILi3EEviii(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK:   %.flatten.iv.2 = alloca i32
+// CHECK:   %[[CIV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[T01:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   %[[TALL:.+]] = mul nsw i64 %[[T01]], %{{.+}}
+// CHECK:   icmp slt i64 %[[CIV]], %[[TALL]]
+// CHECK:   %[[D0IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[D0M:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   %[[D0:.+]] = sdiv i64 %[[D0IV]], %[[D0M]]
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   %[[D1IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[D1:.+]] = sdiv i64 %[[D1IV]], %{{.+}}
+// CHECK:   %[[R1:.+]] = srem i64 %[[D1]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+// CHECK:   %[[R2IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[R2:.+]] = srem i64 %[[R2IV]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.2
+// CHECK:   call void @body(
+template <int D>
+void tmpl_depth3(int n, int m, int p) {
+#pragma omp flatten depth(D)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        body(i, j, k);
+}
+
+// A value-dependent 'depth' that instantiates to depth(1) on a single loop is
+// well-formed: the shallower-than-default nest is accepted because validation is
+// deferred to instantiation. The single loop needs no div/mod recovery.
+
+// CHECK-LABEL: define {{.*}}void @_Z11tmpl_depth1ILi1EEvi(
+// CHECK:   %.flatten.iv = alloca i32
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK: for.body:
+// CHECK:   %[[FV:.+]] = load i32, ptr %.flatten.iv,
+// CHECK:   store i32 %[[FV]], ptr %.flatten.iv.0,
+// CHECK-NOT: sdiv
+// CHECK-NOT: srem
+// CHECK:   call void @body(
+template <int D>
+void tmpl_depth1(int n) {
+#pragma omp flatten depth(D)
+  for (int i = 0; i < n; ++i)
+    body(i, 0, 0);
+}
+
+void inst() {
+  tmpl_depth3<3>(1, 1, 1);
+  tmpl_depth1<1>(1);
+}
+
+#endif
diff --git a/openmp/runtime/test/transform/flatten/depth-intfor.c b/openmp/runtime/test/transform/flatten/depth-intfor.c
new file mode 100644
index 0000000000000..7211e02e25313
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/depth-intfor.c
@@ -0,0 +1,40 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+// 'depth(3)' fully flattens a three-deep perfectly nested loop into one loop.
+// This verifies that the flattened loop preserves the exact row-major
+// visitation order of the original nest, including non-unit steps and non-zero
+// start values.
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+int main() {
+  printf("do\n");
+#pragma omp flatten depth(3)
+  for (int i = 7; i < 11; i += 2)
+    for (int j = 1; j < 4; j += 1)
+      for (int k = 0; k < 2; ++k)
+        printf("i=%d j=%d k=%d\n", i, j, k);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: i=7 j=1 k=0
+// CHECK-NEXT: i=7 j=1 k=1
+// CHECK-NEXT: i=7 j=2 k=0
+// CHECK-NEXT: i=7 j=2 k=1
+// CHECK-NEXT: i=7 j=3 k=0
+// CHECK-NEXT: i=7 j=3 k=1
+// CHECK-NEXT: i=9 j=1 k=0
+// CHECK-NEXT: i=9 j=1 k=1
+// CHECK-NEXT: i=9 j=2 k=0
+// CHECK-NEXT: i=9 j=2 k=1
+// CHECK-NEXT: i=9 j=3 k=0
+// CHECK-NEXT: i=9 j=3 k=1
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/flatten/intfor.c b/openmp/runtime/test/transform/flatten/intfor.c
new file mode 100644
index 0000000000000..f937659fbebe1
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/intfor.c
@@ -0,0 +1,33 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+int main() {
+  printf("do\n");
+#pragma omp flatten
+  for (int i = 5; i < 11; i += 2)
+    for (int j = 1; j < 7; j += 2)
+      printf("i=%d j=%d\n", i, j);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// The flattened loop visits the original iteration space in the original
+// (row-major) order: for each outer iteration the inner loop runs fully.
+// CHECK:      do
+// CHECK-NEXT: i=5 j=1
+// CHECK-NEXT: i=5 j=3
+// CHECK-NEXT: i=5 j=5
+// CHECK-NEXT: i=7 j=1
+// CHECK-NEXT: i=7 j=3
+// CHECK-NEXT: i=7 j=5
+// CHECK-NEXT: i=9 j=1
+// CHECK-NEXT: i=9 j=3
+// CHECK-NEXT: i=9 j=5
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/flatten/lit.local.cfg b/openmp/runtime/test/transform/flatten/lit.local.cfg
new file mode 100644
index 0000000000000..e05bd627df1e9
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/lit.local.cfg
@@ -0,0 +1,5 @@
+# The flatten directive's depth clause requires OpenMP 6.1.
+for i, (pattern, replacement) in enumerate(config.substitutions):
+    if pattern == "%openmp_flags":
+        config.substitutions[i] = (pattern, replacement + " -fopenmp-version=61")
+        break

>From 667fe837a39dbc711dcc5288b736aa52fbe893b6 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 5 Aug 2026 04:48:45 -0400
Subject: [PATCH 14/16] TBD

---
 clang/docs/OpenMPSupport.rst                  |   2 +-
 clang/include/clang/ASTMatchers/ASTMatchers.h |  14 ++
 clang/lib/ASTMatchers/ASTMatchersInternal.cpp |   2 +
 clang/lib/ASTMatchers/Dynamic/Registry.cpp    |   1 +
 clang/test/Index/openmp-flatten.c             |  12 ++
 clang/test/OpenMP/flatten_ast_print.cpp       | 124 ++++++++++++++++++
 clang/test/OpenMP/flatten_codegen_for.cpp     |  67 ++++++++++
 clang/test/OpenMP/flatten_depth_ast_print.cpp |  74 +++++++++++
 clang/test/OpenMP/flatten_depth_messages.cpp  |  48 +++++++
 clang/test/OpenMP/flatten_messages.cpp        |  89 +++++++++++++
 .../test/OpenMP/flatten_serialize_module.cpp  |  25 ++++
 .../ASTMatchers/ASTMatchersNodeTest.cpp       |  22 ++++
 flatten-verify-tmp/01_default_depth2.cpp      |   8 ++
 flatten-verify-tmp/02_depth3_full.cpp         |   9 ++
 flatten-verify-tmp/03_depth2_partial.cpp      |  10 ++
 flatten-verify-tmp/04_depth1_identity.cpp     |   8 ++
 flatten-verify-tmp/05_rangefor_template.cpp   |  20 +++
 flatten-verify-tmp/README.md                  |  39 ++++++
 flatten-verify-tmp/end_result_check.cpp       |  82 ++++++++++++
 .../test/transform/flatten/depth-partial.c    |  45 +++++++
 openmp/runtime/test/transform/flatten/empty.c |  42 ++++++
 .../test/transform/flatten/foreach.cpp        |  31 +++++
 .../flatten/parallel-wsloop-foreach.cpp       |  32 +++++
 23 files changed, 805 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/Index/openmp-flatten.c
 create mode 100644 clang/test/OpenMP/flatten_ast_print.cpp
 create mode 100644 clang/test/OpenMP/flatten_codegen_for.cpp
 create mode 100644 clang/test/OpenMP/flatten_depth_ast_print.cpp
 create mode 100644 clang/test/OpenMP/flatten_depth_messages.cpp
 create mode 100644 clang/test/OpenMP/flatten_messages.cpp
 create mode 100644 clang/test/OpenMP/flatten_serialize_module.cpp
 create mode 100644 flatten-verify-tmp/01_default_depth2.cpp
 create mode 100644 flatten-verify-tmp/02_depth3_full.cpp
 create mode 100644 flatten-verify-tmp/03_depth2_partial.cpp
 create mode 100644 flatten-verify-tmp/04_depth1_identity.cpp
 create mode 100644 flatten-verify-tmp/05_rangefor_template.cpp
 create mode 100644 flatten-verify-tmp/README.md
 create mode 100644 flatten-verify-tmp/end_result_check.cpp
 create mode 100644 openmp/runtime/test/transform/flatten/depth-partial.c
 create mode 100644 openmp/runtime/test/transform/flatten/empty.c
 create mode 100644 openmp/runtime/test/transform/flatten/foreach.cpp
 create mode 100644 openmp/runtime/test/transform/flatten/parallel-wsloop-foreach.cpp

diff --git a/clang/docs/OpenMPSupport.rst b/clang/docs/OpenMPSupport.rst
index 0c21aa41ff269..c11a46c3bb6a6 100644
--- a/clang/docs/OpenMPSupport.rst
+++ b/clang/docs/OpenMPSupport.rst
@@ -666,7 +666,7 @@ implementation.
 +=============================================================+===========================+===========================+==========================================================================+
 | dyn_groupprivate clause                                     | :part:`partial`           | :part:`In Progress`       | C/C++: Host device support missing                                       |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
-| loop flatten transformation                                 | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
+| loop flatten transformation                                 | :part:`partial`           | :none:`unclaimed`         | Clang: depth clause supported (OpenMP 6.1); apply clause not yet supported|
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | loop grid/tile modifiers for sizes clause                   | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
diff --git a/clang/include/clang/ASTMatchers/ASTMatchers.h b/clang/include/clang/ASTMatchers/ASTMatchers.h
index bc0f35898a2c9..602bab9aa8cb8 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchers.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchers.h
@@ -8842,6 +8842,20 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt,
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, OMPSplitDirective>
     ompSplitDirective;
 
+/// Matches OpenMP ``flatten`` directive.
+///
+/// Given
+///
+/// \code
+///   #pragma omp flatten
+///   for (int i = 0; i < m; ++i)
+///     for (int j = 0; j < n; ++j) {}
+/// \endcode
+///
+/// ``ompFlattenDirective()`` matches the flatten directive.
+extern const internal::VariadicDynCastAllOfMatcher<Stmt, OMPFlattenDirective>
+    ompFlattenDirective;
+
 /// Matches OpenMP ``counts`` clause used by ``#pragma omp split``.
 ///
 /// Given ``#pragma omp split counts(1, 2, omp_fill)``, ``ompCountsClause()``
diff --git a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
index 5cbf134620e34..e2c0c25518b70 100644
--- a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
+++ b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
@@ -1141,6 +1141,8 @@ const internal::VariadicDynCastAllOfMatcher<Stmt, OMPTargetUpdateDirective>
     ompTargetUpdateDirective;
 const internal::VariadicDynCastAllOfMatcher<Stmt, OMPSplitDirective>
     ompSplitDirective;
+const internal::VariadicDynCastAllOfMatcher<Stmt, OMPFlattenDirective>
+    ompFlattenDirective;
 const internal::VariadicDynCastAllOfMatcher<OMPClause, OMPCountsClause>
     ompCountsClause;
 const internal::VariadicDynCastAllOfMatcher<OMPClause, OMPDefaultClause>
diff --git a/clang/lib/ASTMatchers/Dynamic/Registry.cpp b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
index a04070971f0eb..a6252363e08ce 100644
--- a/clang/lib/ASTMatchers/Dynamic/Registry.cpp
+++ b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
@@ -534,6 +534,7 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(ompFromClause);
   REGISTER_MATCHER(ompToClause);
   REGISTER_MATCHER(ompExecutableDirective);
+  REGISTER_MATCHER(ompFlattenDirective);
   REGISTER_MATCHER(ompSplitDirective);
   REGISTER_MATCHER(ompTargetUpdateDirective);
   REGISTER_MATCHER(on);
diff --git a/clang/test/Index/openmp-flatten.c b/clang/test/Index/openmp-flatten.c
new file mode 100644
index 0000000000000..74bc3cda3ef1a
--- /dev/null
+++ b/clang/test/Index/openmp-flatten.c
@@ -0,0 +1,12 @@
+// RUN: c-index-test -test-load-source local %s -fopenmp=libomp -fopenmp-version=60 | FileCheck %s
+
+void test(void) {
+#pragma omp flatten
+  for (int i = 0; i < 20; i += 1)
+    for (int j = 0; j < 30; j += 1)
+      ;
+}
+
+// CHECK: openmp-flatten.c:4:1: OMPFlattenDirective= Extent=[4:1 - 4:20]
+// CHECK: openmp-flatten.c:5:3: ForStmt= Extent=[5:3 - 7:8]
+// CHECK: openmp-flatten.c:6:5: ForStmt= Extent=[6:5 - 7:8]
diff --git a/clang/test/OpenMP/flatten_ast_print.cpp b/clang/test/OpenMP/flatten_ast_print.cpp
new file mode 100644
index 0000000000000..e1115bf112c40
--- /dev/null
+++ b/clang/test/OpenMP/flatten_ast_print.cpp
@@ -0,0 +1,124 @@
+// Check no warnings/errors
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -fsyntax-only -verify %s
+// expected-no-diagnostics
+
+// Check AST and unparsing
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -ast-dump  %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -ast-print %s | FileCheck %s --check-prefix=PRINT
+
+// Check same results after serialization round-trip
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -emit-pch -o %t %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -include-pch %t -ast-dump-all %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -include-pch %t -ast-print    %s | FileCheck %s --check-prefix=PRINT
+
+#ifndef HEADER
+#define HEADER
+
+// placeholder for loop body code.
+extern "C" void body(...);
+
+// PRINT-LABEL: void foo1(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo1
+void foo1() {
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (int i = 0; i < 7; i += 1)
+  // DUMP-NEXT: ForStmt
+  for (int i = 0; i < 7; i += 1)
+    // PRINT: for (int j = 0; j < 9; j += 1)
+    // DUMP:  ForStmt
+    for (int j = 0; j < 9; j += 1)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// PRINT-LABEL: void foo2(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo2
+void foo2(int start1, int end1, int step1, int start2, int end2, int step2) {
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (int i = start1; i < end1; i += step1)
+  // DUMP-NEXT: ForStmt
+  for (int i = start1; i < end1; i += step1)
+    // PRINT: for (int j = start2; j < end2; j += step2)
+    // DUMP:  ForStmt
+    for (int j = start2; j < end2; j += step2)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// PRINT-LABEL: void foo3(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo3
+void foo3() {
+  // Verify that a flattened loop can itself be the operand of another loop
+  // transformation. The outer reverse consumes the loop generated by flatten.
+  // PRINT:     #pragma omp reverse
+  // DUMP:      OMPReverseDirective
+  #pragma omp reverse
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (int i = 0; i < 13; i += 1)
+  // DUMP-NEXT: ForStmt
+  for (int i = 0; i < 13; i += 1)
+    // PRINT: for (int j = 0; j < 17; j += 1)
+    // DUMP:  ForStmt
+    for (int j = 0; j < 17; j += 1)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// Range-based for loops are supported as affected loops.
+// PRINT-LABEL: void foo4(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo4
+struct Range {
+  int *begin();
+  int *end();
+};
+void foo4(Range r1, Range r2) {
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (int i : r1)
+  // DUMP-NEXT: CXXForRangeStmt
+  for (int i : r1)
+    // PRINT: for (int j : r2)
+    // DUMP:  CXXForRangeStmt
+    for (int j : r2)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// The directive is instantiated together with its enclosing template.
+// PRINT-LABEL: template <typename T, int LO, int HI> void foo5()
+// DUMP-LABEL:  FunctionTemplateDecl {{.*}} foo5
+template <typename T, int LO, int HI>
+void foo5() {
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (T i = LO; i < HI; ++i)
+  // DUMP-NEXT: ForStmt
+  for (T i = LO; i < HI; ++i)
+    // PRINT: for (T j = LO; j < HI; ++j)
+    // DUMP:  ForStmt
+    for (T j = LO; j < HI; ++j)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// PRINT-LABEL: template<> void foo5<int, 0, 8>()
+// DUMP-LABEL:  FunctionDecl {{.*}} foo5 'void ()' implicit_instantiation
+// DUMP:        OMPFlattenDirective
+void inst() {
+  foo5<int, 0, 8>();
+}
+
+#endif
diff --git a/clang/test/OpenMP/flatten_codegen_for.cpp b/clang/test/OpenMP/flatten_codegen_for.cpp
new file mode 100644
index 0000000000000..d64128bd25d78
--- /dev/null
+++ b/clang/test/OpenMP/flatten_codegen_for.cpp
@@ -0,0 +1,67 @@
+// Check that the loop generated by 'flatten' can be consumed by a loop
+// associated directive ('omp for' / 'omp parallel for'), and that signed and
+// unsigned induction variables generate signed/unsigned division.
+
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=60 -emit-llvm %s -o - | FileCheck %s
+
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+extern "C" void body(int, int);
+extern "C" void ubody(unsigned, unsigned);
+
+// 'omp for' worksharing over the flattened iteration space: the flattened loop
+// uses a 64-bit induction variable for the variable-bound product trip count,
+// so the worksharing loop is distributed with the 64-bit static-init call and
+// the body recovers the logical counters via signed division/remainder.
+//
+// CHECK-LABEL: define {{.*}}void @wsfor(
+// CHECK:   mul nsw i64
+// CHECK:   call void @__kmpc_for_static_init_8(
+// CHECK:   sdiv i64
+// CHECK:   srem i64
+// CHECK:   call void @body(
+// CHECK:   call void @__kmpc_for_static_fini(
+extern "C" void wsfor(int n, int m) {
+#pragma omp for
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+
+// 'omp parallel for' outlines the flattened loop and worksharing-distributes
+// it.
+//
+// CHECK-LABEL: define {{.*}}void @pfor(
+// CHECK:   call {{.*}}@__kmpc_fork_call(
+// CHECK-LABEL: define internal void @pfor.omp_outlined(
+// CHECK:   call void @__kmpc_for_static_init_8(
+// CHECK:   sdiv i64
+// CHECK:   srem i64
+// CHECK:   call void @body(
+extern "C" void pfor(int n, int m) {
+#pragma omp parallel for
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+
+// Unsigned induction variables produce unsigned division/remainder.
+//
+// CHECK-LABEL: define {{.*}}void @ufor(
+// CHECK:   icmp ult i32 %{{.+}}, 63
+// CHECK:   udiv i32 %{{.+}}, 9
+// CHECK:   urem i32 %{{.+}}, 9
+// CHECK:   call void @ubody(
+extern "C" void ufor() {
+#pragma omp flatten
+  for (unsigned i = 0; i < 7u; ++i)
+    for (unsigned j = 0; j < 9u; ++j)
+      ubody(i, j);
+}
+
+#endif
diff --git a/clang/test/OpenMP/flatten_depth_ast_print.cpp b/clang/test/OpenMP/flatten_depth_ast_print.cpp
new file mode 100644
index 0000000000000..ceaaa68e1f79a
--- /dev/null
+++ b/clang/test/OpenMP/flatten_depth_ast_print.cpp
@@ -0,0 +1,74 @@
+// Check no warnings/errors
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -fsyntax-only -verify %s
+// expected-no-diagnostics
+
+// Check AST and unparsing
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-dump  %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-print %s | FileCheck %s --check-prefix=PRINT
+
+// Check same results after serialization round-trip
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -emit-pch -o %t %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -include-pch %t -ast-dump-all %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -include-pch %t -ast-print    %s | FileCheck %s --check-prefix=PRINT
+
+#ifndef HEADER
+#define HEADER
+
+// placeholder for loop body code.
+extern "C" void body(...);
+
+// The depth clause round-trips through -ast-print and PCH serialization.
+// PRINT-LABEL: void foo_depth3(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo_depth3
+void foo_depth3(int n, int m, int p) {
+  // PRINT:     #pragma omp flatten depth(3)
+  // DUMP:      OMPFlattenDirective
+  // DUMP-NEXT: OMPDepthClause
+  #pragma omp flatten depth(3)
+  // PRINT: for (int i = 0; i < n; ++i)
+  // DUMP:  ForStmt
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        // PRINT: body(i, j, k);
+        // DUMP:  CallExpr
+        body(i, j, k);
+}
+
+// depth(2) is accepted and printed like any other depth argument.
+// PRINT-LABEL: void foo_depth2(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo_depth2
+void foo_depth2(int n, int m) {
+  // PRINT:     #pragma omp flatten depth(2)
+  // DUMP:      OMPFlattenDirective
+  // DUMP-NEXT: OMPDepthClause
+  #pragma omp flatten depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+
+// The depth clause is instantiated together with its enclosing template.
+// PRINT-LABEL: template <typename T> void foo_tmpl()
+// DUMP-LABEL:  FunctionTemplateDecl {{.*}} foo_tmpl
+template <typename T>
+void foo_tmpl() {
+  // PRINT:     #pragma omp flatten depth(3)
+  // DUMP:      OMPFlattenDirective
+  // DUMP-NEXT: OMPDepthClause
+  #pragma omp flatten depth(3)
+  for (T i = 0; i < 8; ++i)
+    for (T j = 0; j < 8; ++j)
+      for (T k = 0; k < 8; ++k)
+        body(i, j, k);
+}
+
+// PRINT-LABEL: template<> void foo_tmpl<int>()
+// DUMP-LABEL:  FunctionDecl {{.*}} foo_tmpl 'void ()' implicit_instantiation
+// DUMP:        OMPFlattenDirective
+// DUMP-NEXT:   OMPDepthClause
+void inst() {
+  foo_tmpl<int>();
+}
+
+#endif
diff --git a/clang/test/OpenMP/flatten_depth_messages.cpp b/clang/test/OpenMP/flatten_depth_messages.cpp
new file mode 100644
index 0000000000000..16f60318d0d06
--- /dev/null
+++ b/clang/test/OpenMP/flatten_depth_messages.cpp
@@ -0,0 +1,48 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=61 -fsyntax-only -Wuninitialized -verify %s
+
+extern "C" void body(...);
+
+// expected-note at +1 {{declared here}}
+void func(int n) {
+
+  // The depth argument must be a positive integer constant expression.
+  // expected-error at +1 {{argument to 'depth' clause must be a strictly positive integer value}}
+  #pragma omp flatten depth(0)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // expected-error at +1 {{argument to 'depth' clause must be a strictly positive integer value}}
+  #pragma omp flatten depth(-1)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // A non-constant argument is rejected.
+  // expected-error at +2 {{expression is not an integral constant expression}}
+  // expected-note at +1 {{function parameter 'n' with unknown value cannot be used in a constant expression}}
+  #pragma omp flatten depth(n)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // At most one 'depth' clause may appear on the directive.
+  // expected-error at +1 {{directive '#pragma omp flatten' cannot contain more than one 'depth' clause}}
+  #pragma omp flatten depth(2) depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // The depth must be at most the loop nest depth: depth(3) requires three
+  // perfectly nested loops, but only two are present here.
+  #pragma omp flatten depth(3)
+  // expected-error at +1 {{expected 3 for loops after '#pragma omp flatten', but found only 2}}
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // depth(1) flattens a single loop and is well-formed (no diagnostic).
+  #pragma omp flatten depth(1)
+  for (int i = 0; i < n; ++i)
+    body(i);
+}
diff --git a/clang/test/OpenMP/flatten_messages.cpp b/clang/test/OpenMP/flatten_messages.cpp
new file mode 100644
index 0000000000000..a022e0e8999d2
--- /dev/null
+++ b/clang/test/OpenMP/flatten_messages.cpp
@@ -0,0 +1,89 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=60 -fsyntax-only -Wuninitialized -verify %s
+
+extern "C" void body(...);
+
+void func(int n) {
+
+  // The associated statement must be a for loop.
+  // expected-error at +2 {{statement after '#pragma omp flatten' must be a for loop}}
+  #pragma omp flatten
+  ;
+
+  // A non-loop statement is rejected as well.
+  // expected-error at +2 {{statement after '#pragma omp flatten' must be a for loop}}
+  #pragma omp flatten
+  int b = 0;
+
+  // A single loop is not enough: flatten combines two perfectly nested loops,
+  // so the body of the outer loop must itself be a for loop.
+  #pragma omp flatten
+  for (int i = 0; i < 7; ++i)
+    // expected-error at +1 {{statement after '#pragma omp flatten' must be a for loop}}
+    ;
+
+  // The associated statement of a directive is not a for loop.
+  // expected-error at +2 {{statement after '#pragma omp flatten' must be a for loop}}
+  #pragma omp flatten
+  #pragma omp for
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 7; ++j)
+      body(i, j);
+
+  {
+    // expected-error at +2 {{expected statement}}
+    #pragma omp flatten
+  }
+
+  // The loops must be perfectly nested: no code is allowed between them.
+  #pragma omp flatten
+  for (int i = 0; i < n; ++i) {
+    int x = 0;
+    // expected-error at -2 {{statement after '#pragma omp flatten' must be a for loop}}
+    for (int j = 0; j < n; ++j)
+      body(i, j, x);
+  }
+
+  // Each affected loop must be in OpenMP canonical form.
+  #pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    // expected-error at +1 {{condition of OpenMP for loop must be a relational comparison ('<', '<=', '>', '>=', or '!=') of loop variable 'j'}}
+    for (int j = 0; j / 3 < n; ++j)
+      body(i, j);
+
+  // The affected loops must be rectangular: an inner bound may not depend on an
+  // outer loop counter.
+  #pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    // expected-error at +1 {{expected loop invariant expression}}
+    for (int j = i; j < n; ++j)
+      body(i, j);
+
+  // The 'sizes' clause is not allowed on 'flatten'.
+  // expected-error at +1 {{unexpected OpenMP clause 'sizes' in directive '#pragma omp flatten'}}
+  #pragma omp flatten sizes(2)
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+
+  // The 'permutation' clause is not allowed on 'flatten'.
+  // expected-error at +1 {{unexpected OpenMP clause 'permutation' in directive '#pragma omp flatten'}}
+  #pragma omp flatten permutation(2, 1)
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+
+  // Tokens after the directive name are ignored with a warning.
+  // expected-warning at +1 {{extra tokens at the end of '#pragma omp flatten' are ignored}}
+  #pragma omp flatten foo
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+
+  // The 'depth' clause is only available from OpenMP 6.1; it is rejected here
+  // under -fopenmp-version=60.
+  // expected-error at +1 {{unexpected OpenMP clause 'depth' in directive '#pragma omp flatten'}}
+  #pragma omp flatten depth(2)
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+}
diff --git a/clang/test/OpenMP/flatten_serialize_module.cpp b/clang/test/OpenMP/flatten_serialize_module.cpp
new file mode 100644
index 0000000000000..5d3420500b282
--- /dev/null
+++ b/clang/test/OpenMP/flatten_serialize_module.cpp
@@ -0,0 +1,25 @@
+// C++20 module interface with `#pragma omp flatten` — emit BMI + import; AST retains directive.
+//
+// RUN: rm -rf %t && split-file %s %t && cd %t
+// RUN: %clang_cc1 -std=c++20 -fopenmp -fopenmp-version=60 -triple x86_64-unknown-linux-gnu %t/FlattenMod.cppm -emit-module-interface -o %t/FlattenMod.pcm
+// RUN: %clang_cc1 -std=c++20 -fopenmp -fopenmp-version=60 -triple x86_64-unknown-linux-gnu %t/UseFlattenMod.cpp -fmodule-file=FlattenMod=%t/FlattenMod.pcm -ast-dump-all | FileCheck %t/FlattenMod.cppm
+
+// expected-no-diagnostics
+
+//--- FlattenMod.cppm
+module;
+export module FlattenMod;
+
+export void flattenfoo(int n, int m) {
+// CHECK: OMPFlattenDirective
+// CHECK: ForStmt
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j) {
+    }
+}
+
+//--- UseFlattenMod.cpp
+import FlattenMod;
+
+void g(void) { flattenfoo(10, 20); }
diff --git a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
index 4190d4703e37d..34c0fd27821d1 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
@@ -3126,6 +3126,28 @@ void f() {
   EXPECT_TRUE(notMatchesWithOpenMP60(ParallelOnly, Matcher));
 }
 
+// OpenMP 6 flatten directive
+TEST(ASTMatchersTestOpenMP, OMPFlattenDirective) {
+  auto Matcher = stmt(ompFlattenDirective());
+
+  StringRef FlattenOk = R"(
+void f() {
+#pragma omp flatten
+  for (int i = 0; i < 10; ++i)
+    for (int j = 0; j < 10; ++j) {}
+}
+)";
+  EXPECT_TRUE(matchesWithOpenMP60(FlattenOk, Matcher));
+
+  StringRef ParallelOnly = R"(
+void f() {
+#pragma omp parallel
+  ;
+}
+)";
+  EXPECT_TRUE(notMatchesWithOpenMP60(ParallelOnly, Matcher));
+}
+
 TEST(ASTMatchersTestOpenMP, OMPSplitDirective_HasCountsClause) {
   auto Matcher = stmt(ompSplitDirective(hasAnyClause(ompCountsClause())));
 
diff --git a/flatten-verify-tmp/01_default_depth2.cpp b/flatten-verify-tmp/01_default_depth2.cpp
new file mode 100644
index 0000000000000..900d0263622cd
--- /dev/null
+++ b/flatten-verify-tmp/01_default_depth2.cpp
@@ -0,0 +1,8 @@
+// Default flatten: combine 2 perfectly nested loops (depth defaults to 2).
+extern "C" void body(int, int);
+void t(int n, int m) {
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
diff --git a/flatten-verify-tmp/02_depth3_full.cpp b/flatten-verify-tmp/02_depth3_full.cpp
new file mode 100644
index 0000000000000..ba9f0b7ed8808
--- /dev/null
+++ b/flatten-verify-tmp/02_depth3_full.cpp
@@ -0,0 +1,9 @@
+// depth(3): fully flatten a three-deep perfect nest (OpenMP 6.1).
+extern "C" void body(int, int, int);
+void t(int n, int m, int p) {
+#pragma omp flatten depth(3)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        body(i, j, k);
+}
diff --git a/flatten-verify-tmp/03_depth2_partial.cpp b/flatten-verify-tmp/03_depth2_partial.cpp
new file mode 100644
index 0000000000000..35b9dc83834fd
--- /dev/null
+++ b/flatten-verify-tmp/03_depth2_partial.cpp
@@ -0,0 +1,10 @@
+// depth(2) on a three-deep nest: only the outer two loops are combined; the
+// innermost loop must remain as ordinary body of the flattened loop.
+extern "C" void body(int, int, int);
+void t(int n, int m, int p) {
+#pragma omp flatten depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        body(i, j, k);
+}
diff --git a/flatten-verify-tmp/04_depth1_identity.cpp b/flatten-verify-tmp/04_depth1_identity.cpp
new file mode 100644
index 0000000000000..ae62a11312fe8
--- /dev/null
+++ b/flatten-verify-tmp/04_depth1_identity.cpp
@@ -0,0 +1,8 @@
+// depth(1): identity transform. A single loop is "combined"; no mixed-radix
+// decomposition (no srem), the flattened counter maps straight to the loop var.
+extern "C" void body(int);
+void t(int n) {
+#pragma omp flatten depth(1)
+  for (int i = 0; i < n; ++i)
+    body(i);
+}
diff --git a/flatten-verify-tmp/05_rangefor_template.cpp b/flatten-verify-tmp/05_rangefor_template.cpp
new file mode 100644
index 0000000000000..89928c34a8a15
--- /dev/null
+++ b/flatten-verify-tmp/05_rangefor_template.cpp
@@ -0,0 +1,20 @@
+// Range-for (CXXForRangeStmt) nest + a function template instantiated with a
+// dependent depth(K). Exercises the __begin/__end pre-inits and the deferred
+// (dependent-context) desugaring path.
+#include <stdlibc++.h>
+extern "C" void body(int, int);
+
+void ranges(std::vector<int> &a, std::vector<int> &b) {
+#pragma omp flatten
+  for (int i : a)
+    for (int j : b)
+      body(i, j);
+}
+
+template <int K> void tmpl(int n, int m) {
+#pragma omp flatten depth(K)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+template void tmpl<2>(int, int);
diff --git a/flatten-verify-tmp/README.md b/flatten-verify-tmp/README.md
new file mode 100644
index 0000000000000..f48f4d06cfec0
--- /dev/null
+++ b/flatten-verify-tmp/README.md
@@ -0,0 +1,39 @@
+# Temporary flatten + depth verification cases
+
+Throwaway sandbox for eyeballing the flatten desugaring (AST / IR / runtime).
+Not part of any PR — delete freely. The committed regression tests live under
+`clang/test/OpenMP/flatten*` and `openmp/runtime/test/transform/flatten/*`.
+
+`CLANG=build/bin/clang`, `CLANGXX=build/bin/clang++`, run from the repo root.
+
+| File | Case | What to look for |
+|------|------|------------------|
+| `01_default_depth2.cpp` | default (no clause) = depth 2 | i64 `.flatten.iv`, `mul nsw`, `sdiv`+`srem` |
+| `02_depth3_full.cpp` | `depth(3)` full flatten | 3 `.flatten.iv.N`, two-level mixed radix |
+| `03_depth2_partial.cpp` | `depth(2)` on 3-deep nest | 2 flatten IVs, **inner `%k` loop stays** |
+| `04_depth1_identity.cpp` | `depth(1)` identity | i32 IV, **no `srem`**, direct store |
+| `05_rangefor_template.cpp` | range-for + template `depth(K)` | ast-print round-trips; instantiation |
+| `end_result_check.cpp` | runtime semantic check | prints `OK`, exit 0 (order+set preserved) |
+
+## Commands
+
+AST (round-trip original loops) and the desugared node:
+```
+$CLANG -cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-print FILE
+$CLANG -cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-dump  FILE
+```
+
+IR (the div/mod arithmetic that implements the transform):
+```
+$CLANG -cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -emit-llvm -o - FILE
+```
+
+Range-for / template need the driver (libstdc++ + template instantiation):
+```
+$CLANGXX -fopenmp -fopenmp-version=61 -Xclang -ast-print -fsyntax-only 05_rangefor_template.cpp
+```
+
+Runtime end-result (must print OK, exit 0):
+```
+$CLANGXX -fopenmp -fopenmp-version=61 -O0 end_result_check.cpp -o /tmp/erc && /tmp/erc
+```
diff --git a/flatten-verify-tmp/end_result_check.cpp b/flatten-verify-tmp/end_result_check.cpp
new file mode 100644
index 0000000000000..3aa8850829058
--- /dev/null
+++ b/flatten-verify-tmp/end_result_check.cpp
@@ -0,0 +1,82 @@
+// End-result (semantic) verification: flatten must preserve BOTH the set of
+// iterations and their row-major visitation ORDER. We record the sequence of
+// visited tuples from a flattened nest and compare it against the sequence from
+// an identical non-flattened reference nest. Exit code 0 == identical.
+//
+// flatten is a pure frontend AST desugaring to an ordinary loop, so this runs
+// as a normal program (build with: clang++ -fopenmp -fopenmp-version=61).
+#include <cassert>
+#include <cstdio>
+#include <vector>
+
+using Seq = std::vector<long>;
+
+static void rec(Seq &s, int i, int j, int k = -1) {
+  s.push_back(i);
+  s.push_back(j);
+  if (k >= 0)
+    s.push_back(k);
+}
+
+int main() {
+  const int n = 3, m = 4, p = 2;
+
+  // --- depth(2) default over 2 loops ---
+  {
+    Seq flat, ref;
+#pragma omp flatten
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        rec(flat, i, j);
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        rec(ref, i, j);
+    assert(flat == ref && "depth(2) order/set mismatch");
+  }
+
+  // --- depth(3) full flatten over 3 loops ---
+  {
+    Seq flat, ref;
+#pragma omp flatten depth(3)
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        for (int k = 0; k < p; ++k)
+          rec(flat, i, j, k);
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        for (int k = 0; k < p; ++k)
+          rec(ref, i, j, k);
+    assert(flat == ref && "depth(3) order/set mismatch");
+  }
+
+  // --- depth(2) partial: inner k-loop stays intact ---
+  {
+    Seq flat, ref;
+#pragma omp flatten depth(2)
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        for (int k = 0; k < p; ++k)
+          rec(flat, i, j, k);
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        for (int k = 0; k < p; ++k)
+          rec(ref, i, j, k);
+    assert(flat == ref && "depth(2)-partial order/set mismatch");
+  }
+
+  // --- non-unit step / non-zero start must survive ---
+  {
+    Seq flat, ref;
+#pragma omp flatten
+    for (int i = 5; i < 12; i += 2)
+      for (int j = 1; j < 7; j += 3)
+        rec(flat, i, j);
+    for (int i = 5; i < 12; i += 2)
+      for (int j = 1; j < 7; j += 3)
+        rec(ref, i, j);
+    assert(flat == ref && "stride/offset mismatch");
+  }
+
+  std::puts("flatten end-result preserved: OK");
+  return 0;
+}
diff --git a/openmp/runtime/test/transform/flatten/depth-partial.c b/openmp/runtime/test/transform/flatten/depth-partial.c
new file mode 100644
index 0000000000000..2fda3a04cbd84
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/depth-partial.c
@@ -0,0 +1,45 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+// A depth(2) clause on a three-deep loop nest flattens only the outermost two
+// loops; the innermost loop is left untouched and executes normally inside the
+// flattened loop body.
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+int main() {
+  printf("do\n");
+#pragma omp flatten depth(2)
+  for (int i = 5; i < 11; i += 2)
+    for (int j = 1; j < 7; j += 2)
+      for (int k = 0; k < 2; ++k)
+        printf("i=%d j=%d k=%d\n", i, j, k);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: i=5 j=1 k=0
+// CHECK-NEXT: i=5 j=1 k=1
+// CHECK-NEXT: i=5 j=3 k=0
+// CHECK-NEXT: i=5 j=3 k=1
+// CHECK-NEXT: i=5 j=5 k=0
+// CHECK-NEXT: i=5 j=5 k=1
+// CHECK-NEXT: i=7 j=1 k=0
+// CHECK-NEXT: i=7 j=1 k=1
+// CHECK-NEXT: i=7 j=3 k=0
+// CHECK-NEXT: i=7 j=3 k=1
+// CHECK-NEXT: i=7 j=5 k=0
+// CHECK-NEXT: i=7 j=5 k=1
+// CHECK-NEXT: i=9 j=1 k=0
+// CHECK-NEXT: i=9 j=1 k=1
+// CHECK-NEXT: i=9 j=3 k=0
+// CHECK-NEXT: i=9 j=3 k=1
+// CHECK-NEXT: i=9 j=5 k=0
+// CHECK-NEXT: i=9 j=5 k=1
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/flatten/empty.c b/openmp/runtime/test/transform/flatten/empty.c
new file mode 100644
index 0000000000000..ecf1e8ef622f2
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/empty.c
@@ -0,0 +1,42 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+// The bounds are runtime values so that an empty iteration space does not
+// trigger a compile-time division-by-zero diagnostic for the (never executed)
+// induction-variable recovery.
+static void flatten(int n, int m) {
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      printf("i=%d j=%d\n", i, j);
+}
+
+int main() {
+  printf("empty-inner-begin\n");
+  flatten(3, 0);
+  printf("empty-inner-end\n");
+
+  printf("empty-outer-begin\n");
+  flatten(0, 3);
+  printf("empty-outer-end\n");
+
+  printf("single-begin\n");
+  flatten(1, 1);
+  printf("single-end\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      empty-inner-begin
+// CHECK-NEXT: empty-inner-end
+// CHECK-NEXT: empty-outer-begin
+// CHECK-NEXT: empty-outer-end
+// CHECK-NEXT: single-begin
+// CHECK-NEXT: i=0 j=0
+// CHECK-NEXT: single-end
diff --git a/openmp/runtime/test/transform/flatten/foreach.cpp b/openmp/runtime/test/transform/flatten/foreach.cpp
new file mode 100644
index 0000000000000..14997f7c4051d
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/foreach.cpp
@@ -0,0 +1,31 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdio>
+#include <vector>
+
+int main() {
+  printf("do\n");
+  std::vector<int> outer{10, 20};
+  std::vector<int> inner{1, 2, 3};
+#pragma omp flatten
+  for (int a : outer)
+    for (int b : inner)
+      printf("a=%d b=%d\n", a, b);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: a=10 b=1
+// CHECK-NEXT: a=10 b=2
+// CHECK-NEXT: a=10 b=3
+// CHECK-NEXT: a=20 b=1
+// CHECK-NEXT: a=20 b=2
+// CHECK-NEXT: a=20 b=3
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/flatten/parallel-wsloop-foreach.cpp b/openmp/runtime/test/transform/flatten/parallel-wsloop-foreach.cpp
new file mode 100644
index 0000000000000..7ddcde2fd622c
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/parallel-wsloop-foreach.cpp
@@ -0,0 +1,32 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdio>
+
+int main() {
+  printf("do\n");
+  // The loop generated by 'flatten' is the canonical loop associated with the
+  // worksharing 'omp for'. With a single thread the iteration order is the
+  // original row-major order.
+#pragma omp parallel for num_threads(1)
+#pragma omp flatten
+  for (int i = 0; i < 2; ++i)
+    for (int j = 0; j < 3; ++j)
+      printf("i=%d j=%d\n", i, j);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: i=0 j=0
+// CHECK-NEXT: i=0 j=1
+// CHECK-NEXT: i=0 j=2
+// CHECK-NEXT: i=1 j=0
+// CHECK-NEXT: i=1 j=1
+// CHECK-NEXT: i=1 j=2
+// CHECK-NEXT: done

>From 267ca5079c57bc299a056eb5a7d414c74c6c04f4 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Tue, 18 Aug 2026 07:15:20 -0400
Subject: [PATCH 15/16] conflict-resolve

---
 clang/bindings/python/clang/cindex.py | 5 ++++-
 clang/include/clang-c/Index.h         | 6 +++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/clang/bindings/python/clang/cindex.py b/clang/bindings/python/clang/cindex.py
index ccad140e8c861..56e20fdfad781 100644
--- a/clang/bindings/python/clang/cindex.py
+++ b/clang/bindings/python/clang/cindex.py
@@ -1456,8 +1456,11 @@ def is_unexposed(self):
     # OpenMP split directive.
     OMP_SPLIT_DIRECTIVE = 312
 
+    # OpenMP ordered-blockassoc directive.
+    OMP_ORDERED_BLOCK_ASSOC_DIRECTIVE = 313
+
     # OpenMP flatten directive.
-    OMP_FLATTEN_DIRECTIVE = 313
+    OMP_FLATTEN_DIRECTIVE = 314
 
     # OpenACC Compute Construct.
     OPEN_ACC_COMPUTE_DIRECTIVE = 320
diff --git a/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h
index f561fb5724897..4c4cf392f172e 100644
--- a/clang/include/clang-c/Index.h
+++ b/clang/include/clang-c/Index.h
@@ -2170,9 +2170,13 @@ enum CXCursorKind {
    */
   CXCursor_OMPSplitDirective = 312,
 
+  /** OpenMP ordered-blockassoc directive.
+   */
+  CXCursor_OMPOrderedBlockAssocDirective = 313,
+
   /** OpenMP flatten directive.
    */
-  CXCursor_OMPFlattenDirective = 313,
+  CXCursor_OMPFlattenDirective = 314,
 
   /** OpenACC Compute Construct.
    */

>From 4788e3e9a56751f93a2268d6de146bc9f2bcd34d Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Tue, 18 Aug 2026 07:17:36 -0400
Subject: [PATCH 16/16] revised

---
 clang/include/clang/AST/OpenMPClause.h        |   8 +-
 clang/include/clang/AST/StmtOpenMP.h          |   4 +-
 .../clang/Basic/DiagnosticSemaKinds.td        |   6 +
 clang/lib/AST/OpenMPClause.cpp                |  12 --
 clang/lib/Sema/SemaOpenMP.cpp                 | 112 ++++++++++--------
 5 files changed, 74 insertions(+), 68 deletions(-)

diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 50d1364a9726d..05ccf4db5752b 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -1411,14 +1411,14 @@ class OMPDepthClause final : public OMPClause {
 public:
   /// Build an AST node for a 'depth' clause.
   ///
-  /// \param C         Context of the AST.
   /// \param StartLoc  Location of the 'depth' identifier.
   /// \param LParenLoc Location of '('.
   /// \param EndLoc    Location of ')'.
   /// \param Depth     The depth expression.
-  static OMPDepthClause *Create(const ASTContext &C, SourceLocation StartLoc,
-                                SourceLocation LParenLoc, SourceLocation EndLoc,
-                                Expr *Depth);
+  OMPDepthClause(SourceLocation StartLoc, SourceLocation LParenLoc,
+                 SourceLocation EndLoc, Expr *Depth)
+      : OMPClause(llvm::omp::OMPC_depth, StartLoc, EndLoc),
+        LParenLoc(LParenLoc), Depth(Depth) {}
 
   /// Build an empty 'depth' AST node for deserialization.
   ///
diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index 1420331f37c1f..4ae6f364dced3 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -5999,8 +5999,8 @@ class OMPFlattenDirective final
   /// \param StartLoc  Location of the introducer (e.g. the 'omp' token).
   /// \param EndLoc    Location of the directive's end (e.g. the tok::eod).
   /// \param Clauses   The directive's clauses.
-  /// \param NumLoops  Number of affected loops (the flatten depth, currently
-  ///                  always 2).
+  /// \param NumLoops  Number of affected loops (the flatten depth: the
+  ///                  argument of the 'depth' clause, or 2 if omitted).
   /// \param AssociatedStmt  The outermost associated loop.
   /// \param TransformedStmt The flattened loop, or nullptr in dependent
   ///                        contexts.
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 7e20630708312..fc44b8d15239e 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12172,6 +12172,12 @@ def err_omp_invalid_looprange
 def warn_omp_redundant_fusion : Warning<"looprange clause selects a single "
                                         "loop, resulting in redundant fusion">,
                                 InGroup<OpenMPClauses>;
+def warn_omp_flatten_omitted_depth : Warning<
+  "'flatten' without a 'depth' clause only combines 2 loops, but 3 or more "
+  "loops are perfectly nested">,
+  InGroup<OpenMPClauses>;
+def err_omp_clause_not_supported_yet : Error<
+  "'%0' clause is not yet supported on '#pragma omp %1' directive">;
 def err_omp_not_for : Error<
   "%select{statement after '#pragma omp %1' must be a for loop|"
   "expected %2 for loops after '#pragma omp %1'%select{|, but found only %4}3}0">;
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index 7b2f67a31cad9..7311e404b66c3 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -1057,18 +1057,6 @@ OMPPartialClause *OMPPartialClause::CreateEmpty(const ASTContext &C) {
   return new (C) OMPPartialClause();
 }
 
-OMPDepthClause *OMPDepthClause::Create(const ASTContext &C,
-                                       SourceLocation StartLoc,
-                                       SourceLocation LParenLoc,
-                                       SourceLocation EndLoc, Expr *Depth) {
-  OMPDepthClause *Clause = CreateEmpty(C);
-  Clause->setLocStart(StartLoc);
-  Clause->setLParenLoc(LParenLoc);
-  Clause->setLocEnd(EndLoc);
-  Clause->setDepth(Depth);
-  return Clause;
-}
-
 OMPDepthClause *OMPDepthClause::CreateEmpty(const ASTContext &C) {
   return new (C) OMPDepthClause();
 }
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index fcc256dec3c88..caaba5a3be9fa 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -6839,6 +6839,7 @@ StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
       case OMPC_safelen:
       case OMPC_simdlen:
       case OMPC_sizes:
+      case OMPC_depth:
       case OMPC_default:
       case OMPC_proc_bind:
       case OMPC_private:
@@ -16395,29 +16396,6 @@ StmtResult SemaOpenMP::ActOnOpenMPInterchangeDirective(
                                          buildPreInits(Context, PreInits));
 }
 
-/// Counts the perfectly nested canonical loop depth at \p AStmt. Returns
-/// std::nullopt if a loop-generating transformation prevents a static count.
-static std::optional<unsigned> getCanonicalLoopNestDepth(Stmt *AStmt) {
-  unsigned Depth = 0;
-  Stmt *CurStmt = AStmt ? AStmt->IgnoreContainers() : nullptr;
-  while (CurStmt) {
-    if (isa<OMPLoopTransformationDirective>(CurStmt))
-      return std::nullopt;
-    if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(CurStmt))
-      CurStmt = CanonLoop->getLoopStmt();
-    Stmt *Body = nullptr;
-    if (auto *For = dyn_cast<ForStmt>(CurStmt))
-      Body = For->getBody();
-    else if (auto *RangeFor = dyn_cast<CXXForRangeStmt>(CurStmt))
-      Body = RangeFor->getBody();
-    else
-      break;
-    ++Depth;
-    CurStmt = Body ? Body->IgnoreContainers() : nullptr;
-  }
-  return Depth;
-}
-
 StmtResult
 SemaOpenMP::ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
                                         Stmt *AStmt, SourceLocation StartLoc,
@@ -16432,36 +16410,49 @@ SemaOpenMP::ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
 
   // flatten without 'depth' clause combines two loops; 'depth(k)' selects k.
   unsigned NumLoops = 2;
-  bool DepthIsValueDependent = false;
+  bool DepthIsDependent = false;
   const auto *DepthClause =
       OMPExecutableDirective::getSingleClause<OMPDepthClause>(Clauses);
   if (DepthClause) {
     Expr *DepthExpr = DepthClause->getDepth();
-    if (DepthExpr && DepthExpr->isValueDependent()) {
-      DepthIsValueDependent = true;
+    if (DepthExpr && DepthExpr->isInstantiationDependent()) {
+      DepthIsDependent = true;
     } else if (DepthExpr) {
       Expr::EvalResult EvalResult;
       if (DepthExpr->EvaluateAsInt(EvalResult, Context))
-        NumLoops = EvalResult.Val.getInt().getZExtValue();
+        NumLoops = EvalResult.Val.getInt().getLimitedValue();
     }
   }
 
-  // Report "expected k for loops, but found only n" when depth exceeds the
-  // perfect nest (same form as 'collapse'); skip if depth or nest is unknown.
-  if (DepthClause && !DepthIsValueDependent) {
-    if (std::optional<unsigned> NestDepth = getCanonicalLoopNestDepth(AStmt);
-        NestDepth && NumLoops > *NestDepth) {
+  // Count perfectly nested loops with doForAllLoops. When 'depth' is present,
+  // walk NumLoops iterations to diagnose an insufficient nest. When it is
+  // omitted, walk one extra loop (3 total) so we can warn that default
+  // flatten only combines 2 of a deeper nest.
+  if (!DepthIsDependent) {
+    unsigned WalkLimit = DepthClause ? NumLoops : 3;
+    unsigned Found = 0;
+    bool Enough = OMPLoopBasedDirective::doForAllLoops(
+        AStmt->IgnoreContainers(), /*TryImperfectlyNestedLoops=*/false,
+        WalkLimit, [&](unsigned Cnt, Stmt *S) {
+          if (!isa<ForStmt>(S) && !isa<CXXForRangeStmt>(S))
+            return true;
+          Found = Cnt + 1;
+          return false;
+        });
+    if (DepthClause && !Enough) {
       Diag(AStmt->getBeginLoc(), diag::err_omp_not_for)
           << /*expected N for loops form=*/1
-          << getOpenMPDirectiveName(OMPD_flatten) << NumLoops
-          << (*NestDepth > 0) << *NestDepth;
+          << getOpenMPDirectiveName(OMPD_flatten) << NumLoops << (Found > 0)
+          << Found;
       return StmtError();
     }
+    if (!DepthClause && Found >= 3)
+      Diag(StartLoc, diag::warn_omp_flatten_omitted_depth);
   }
 
-  // Defer when 'depth' is value-dependent (concrete k unknown until
+  // Defer when 'depth' is instantiation-dependent (concrete k unknown until
   // instantiation).
-  if (DepthIsValueDependent)
+  if (DepthIsDependent)
     return OMPFlattenDirective::Create(Context, StartLoc, EndLoc, Clauses,
                                        NumLoops, AStmt, nullptr, nullptr);
 
@@ -16537,20 +16528,30 @@ SemaOpenMP::ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
     AllCountsLessThan32Bits &=
         Context.getTypeSize(LoopHelpers[I].NumIterations->getType()) < 32;
 
-  ExprResult TripCount32 = BuildTripCount(/*Bits=*/32);
-  ExprResult TripCount64 = BuildTripCount(/*Bits=*/64);
-  if (!TripCount32.isUsable() || !TripCount64.isUsable())
+  ExprResult TripCount;
+  if (AllCountsLessThan32Bits || NumLoops == 1) {
+    TripCount = BuildTripCount(/*Bits=*/32);
+  } else {
+    ExprResult TripCount64 = BuildTripCount(/*Bits=*/64);
+    if (!TripCount64.isUsable())
+      return StmtError();
+    if (fitsInto(
+            /*Bits=*/32,
+            TripCount64.get()->getType()->hasSignedIntegerRepresentation(),
+            TripCount64.get(), SemaRef)) {
+      ExprResult TripCount32 = BuildTripCount(/*Bits=*/32);
+      if (TripCount32.isUsable() &&
+          Context.getTypeSize(TripCount32.get()->getType()) == 32)
+        TripCount = TripCount32;
+      else
+        TripCount = TripCount64;
+    } else {
+      TripCount = TripCount64;
+    }
+  }
+  if (!TripCount.isUsable())
     return StmtError();
 
-  ExprResult TripCount = TripCount64;
-  if (Context.getTypeSize(TripCount32.get()->getType()) == 32 &&
-      (AllCountsLessThan32Bits || NumLoops == 1 ||
-       fitsInto(
-           /*Bits=*/32,
-           TripCount32.get()->getType()->hasSignedIntegerRepresentation(),
-           TripCount64.get(), SemaRef)))
-    TripCount = TripCount32;
-
   QualType IVTy = TripCount.get()->getType();
   uint64_t IVWidth = Context.getTypeSize(IVTy);
 
@@ -16680,6 +16681,13 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
   if (!AStmt)
     return StmtError();
 
+  if (const auto *DepthC =
+          OMPExecutableDirective::getSingleClause<OMPDepthClause>(Clauses)) {
+    Diag(DepthC->getBeginLoc(), diag::err_omp_clause_not_supported_yet)
+        << "depth" << getOpenMPDirectiveName(OMPD_fuse);
+    return StmtError();
+  }
+
   // Defer transformation in dependent contexts
   // The NumLoopNests argument is set to a placeholder 1 (even though
   // using looprange fuse could yield up to 3 top level loop nests)
@@ -17911,6 +17919,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPSimpleClause(
   case OMPC_safelen:
   case OMPC_simdlen:
   case OMPC_sizes:
+  case OMPC_depth:
   case OMPC_allocator:
   case OMPC_collapse:
   case OMPC_schedule:
@@ -18532,8 +18541,8 @@ OMPClause *SemaOpenMP::ActOnOpenMPDepthClause(Expr *DepthExpr,
     return nullptr;
   DepthExpr = DepthResult.get();
 
-  return OMPDepthClause::Create(getASTContext(), StartLoc, LParenLoc, EndLoc,
-                                DepthExpr);
+  return new (getASTContext())
+      OMPDepthClause(StartLoc, LParenLoc, EndLoc, DepthExpr);
 }
 
 OMPClause *SemaOpenMP::ActOnOpenMPLoopRangeClause(
@@ -18671,6 +18680,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPSingleExprWithArgClause(
   case OMPC_safelen:
   case OMPC_simdlen:
   case OMPC_sizes:
+  case OMPC_depth:
   case OMPC_allocator:
   case OMPC_collapse:
   case OMPC_proc_bind:
@@ -18959,6 +18969,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPClause(OpenMPClauseKind Kind,
   case OMPC_safelen:
   case OMPC_simdlen:
   case OMPC_sizes:
+  case OMPC_depth:
   case OMPC_allocator:
   case OMPC_collapse:
   case OMPC_schedule:
@@ -19643,6 +19654,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPVarListClause(OpenMPClauseKind Kind,
   case OMPC_safelen:
   case OMPC_simdlen:
   case OMPC_sizes:
+  case OMPC_depth:
   case OMPC_allocator:
   case OMPC_collapse:
   case OMPC_default:


From openmp-commits at lists.llvm.org  Tue Aug 18 04:33:07 2026
From: openmp-commits at lists.llvm.org (Amit Tiwari via Openmp-commits)
Date: Tue, 18 Aug 2026 04:33:07 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [Clang][OpenMP] Add
 `#pragma omp flatten` loop directive + `depth` clause (PR #206977)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206977@github.com>
Message-ID: <6a8442f3.b12cec6a.2c587b.2b74@mx.google.com>


https://github.com/loopacino updated https://github.com/llvm/llvm-project/pull/206977

>From 0fb5bd42f1e127faea963c359b1e3c69cb969c60 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 1 Jul 2026 09:06:58 -0400
Subject: [PATCH 01/14] add OMPFlattenDirective ast node

---
 clang/include/clang/AST/RecursiveASTVisitor.h |  3 +
 clang/include/clang/AST/StmtOpenMP.h          | 75 ++++++++++++++++++-
 clang/include/clang/Basic/StmtNodes.td        |  1 +
 clang/lib/AST/StmtOpenMP.cpp                  | 20 +++++
 clang/lib/AST/StmtPrinter.cpp                 |  5 ++
 clang/lib/AST/StmtProfile.cpp                 |  4 +
 clang/lib/Basic/OpenMPKinds.cpp               |  2 +-
 7 files changed, 108 insertions(+), 2 deletions(-)

diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index b000a34043696..e425f7a31f59c 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -3227,6 +3227,9 @@ DEF_TRAVERSE_STMT(OMPFuseDirective,
 DEF_TRAVERSE_STMT(OMPInterchangeDirective,
                   { TRY_TO(TraverseOMPExecutableDirective(S)); })
 
+DEF_TRAVERSE_STMT(OMPFlattenDirective,
+                  { TRY_TO(TraverseOMPExecutableDirective(S)); })
+
 DEF_TRAVERSE_STMT(OMPSplitDirective,
                   { TRY_TO(TraverseOMPExecutableDirective(S)); })
 
diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index dbc76e7df8ecd..1420331f37c1f 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -1041,7 +1041,7 @@ class OMPCanonicalLoopNestTransformationDirective
     Stmt::StmtClass C = T->getStmtClass();
     return C == OMPTileDirectiveClass || C == OMPUnrollDirectiveClass ||
            C == OMPReverseDirectiveClass || C == OMPInterchangeDirectiveClass ||
-           C == OMPStripeDirectiveClass;
+           C == OMPStripeDirectiveClass || C == OMPFlattenDirectiveClass;
   }
 };
 
@@ -5959,6 +5959,79 @@ class OMPInterchangeDirective final
   }
 };
 
+/// Represents the '#pragma omp flatten' loop transformation directive.
+///
+/// \code{c}
+///   #pragma omp flatten
+///   for (int i = 0; i < m; ++i)
+///     for (int j = 0; j < n; ++j)
+///       ..
+/// \endcode
+class OMPFlattenDirective final
+    : public OMPCanonicalLoopNestTransformationDirective {
+  friend class ASTStmtReader;
+  friend class OMPExecutableDirective;
+
+  /// Offsets of child members.
+  enum {
+    PreInitsOffset = 0,
+    TransformedStmtOffset,
+  };
+
+  explicit OMPFlattenDirective(SourceLocation StartLoc, SourceLocation EndLoc,
+                               unsigned NumLoops)
+      : OMPCanonicalLoopNestTransformationDirective(
+            OMPFlattenDirectiveClass, llvm::omp::OMPD_flatten, StartLoc, EndLoc,
+            NumLoops) {}
+
+  void setPreInits(Stmt *PreInits) {
+    Data->getChildren()[PreInitsOffset] = PreInits;
+  }
+
+  void setTransformedStmt(Stmt *S) {
+    Data->getChildren()[TransformedStmtOffset] = S;
+  }
+
+public:
+  /// Create a new AST node representation for '#pragma omp flatten'.
+  ///
+  /// \param C         Context of the AST.
+  /// \param StartLoc  Location of the introducer (e.g. the 'omp' token).
+  /// \param EndLoc    Location of the directive's end (e.g. the tok::eod).
+  /// \param Clauses   The directive's clauses.
+  /// \param NumLoops  Number of affected loops (the flatten depth, currently
+  ///                  always 2).
+  /// \param AssociatedStmt  The outermost associated loop.
+  /// \param TransformedStmt The flattened loop, or nullptr in dependent
+  ///                        contexts.
+  /// \param PreInits  Helper preinits statements for the loop nest.
+  static OMPFlattenDirective *
+  Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation EndLoc,
+         ArrayRef<OMPClause *> Clauses, unsigned NumLoops, Stmt *AssociatedStmt,
+         Stmt *TransformedStmt, Stmt *PreInits);
+
+  /// Build an empty '#pragma omp flatten' AST node for deserialization.
+  ///
+  /// \param C          Context of the AST.
+  /// \param NumClauses Number of clauses to allocate.
+  /// \param NumLoops   Number of associated loops to allocate.
+  static OMPFlattenDirective *
+  CreateEmpty(const ASTContext &C, unsigned NumClauses, unsigned NumLoops);
+
+  /// Gets the flattened loop after the transformation. This is the de-sugared
+  /// replacement or nullptr in dependent contexts.
+  Stmt *getTransformedStmt() const {
+    return Data->getChildren()[TransformedStmtOffset];
+  }
+
+  /// Return preinits statement.
+  Stmt *getPreInits() const { return Data->getChildren()[PreInitsOffset]; }
+
+  static bool classof(const Stmt *T) {
+    return T->getStmtClass() == OMPFlattenDirectiveClass;
+  }
+};
+
 /// The base class for all transformation directives of canonical loop
 /// sequences (currently only 'fuse')
 class OMPCanonicalLoopSequenceTransformationDirective
diff --git a/clang/include/clang/Basic/StmtNodes.td b/clang/include/clang/Basic/StmtNodes.td
index e166894ea024b..0d7a0c1490a1e 100644
--- a/clang/include/clang/Basic/StmtNodes.td
+++ b/clang/include/clang/Basic/StmtNodes.td
@@ -247,6 +247,7 @@ def OMPReverseDirective : StmtNode<OMPCanonicalLoopNestTransformationDirective>;
 def OMPSplitDirective : StmtNode<OMPCanonicalLoopNestTransformationDirective>;
 def OMPInterchangeDirective
     : StmtNode<OMPCanonicalLoopNestTransformationDirective>;
+def OMPFlattenDirective : StmtNode<OMPCanonicalLoopNestTransformationDirective>;
 def OMPCanonicalLoopSequenceTransformationDirective
     : StmtNode<OMPExecutableDirective, 1>;
 def OMPFuseDirective
diff --git a/clang/lib/AST/StmtOpenMP.cpp b/clang/lib/AST/StmtOpenMP.cpp
index 9d6b315effb41..477604ef3dafb 100644
--- a/clang/lib/AST/StmtOpenMP.cpp
+++ b/clang/lib/AST/StmtOpenMP.cpp
@@ -552,6 +552,26 @@ OMPInterchangeDirective::CreateEmpty(const ASTContext &C, unsigned NumClauses,
       SourceLocation(), SourceLocation(), NumLoops);
 }
 
+OMPFlattenDirective *OMPFlattenDirective::Create(
+    const ASTContext &C, SourceLocation StartLoc, SourceLocation EndLoc,
+    ArrayRef<OMPClause *> Clauses, unsigned NumLoops, Stmt *AssociatedStmt,
+    Stmt *TransformedStmt, Stmt *PreInits) {
+  OMPFlattenDirective *Dir = createDirective<OMPFlattenDirective>(
+      C, Clauses, AssociatedStmt, TransformedStmtOffset + 1, StartLoc, EndLoc,
+      NumLoops);
+  Dir->setTransformedStmt(TransformedStmt);
+  Dir->setPreInits(PreInits);
+  return Dir;
+}
+
+OMPFlattenDirective *OMPFlattenDirective::CreateEmpty(const ASTContext &C,
+                                                      unsigned NumClauses,
+                                                      unsigned NumLoops) {
+  return createEmptyDirective<OMPFlattenDirective>(
+      C, NumClauses, /*HasAssociatedStmt=*/true, TransformedStmtOffset + 1,
+      SourceLocation(), SourceLocation(), NumLoops);
+}
+
 OMPSplitDirective *
 OMPSplitDirective::Create(const ASTContext &C, SourceLocation StartLoc,
                           SourceLocation EndLoc, ArrayRef<OMPClause *> Clauses,
diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp
index 6c3294573e9d4..0bacb79ed86e3 100644
--- a/clang/lib/AST/StmtPrinter.cpp
+++ b/clang/lib/AST/StmtPrinter.cpp
@@ -800,6 +800,11 @@ void StmtPrinter::VisitOMPInterchangeDirective(OMPInterchangeDirective *Node) {
   PrintOMPExecutableDirective(Node);
 }
 
+void StmtPrinter::VisitOMPFlattenDirective(OMPFlattenDirective *Node) {
+  Indent() << "#pragma omp flatten";
+  PrintOMPExecutableDirective(Node);
+}
+
 void StmtPrinter::VisitOMPSplitDirective(OMPSplitDirective *Node) {
   Indent() << "#pragma omp split";
   PrintOMPExecutableDirective(Node);
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index 90eab530e0c2e..de7ed322d7424 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -1071,6 +1071,10 @@ void StmtProfiler::VisitOMPInterchangeDirective(
   VisitOMPCanonicalLoopNestTransformationDirective(S);
 }
 
+void StmtProfiler::VisitOMPFlattenDirective(const OMPFlattenDirective *S) {
+  VisitOMPCanonicalLoopNestTransformationDirective(S);
+}
+
 void StmtProfiler::VisitOMPSplitDirective(const OMPSplitDirective *S) {
   VisitOMPCanonicalLoopNestTransformationDirective(S);
 }
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index 675d86349c933..57e0347f8ea77 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -818,7 +818,7 @@ bool clang::isOpenMPCanonicalLoopNestTransformationDirective(
     OpenMPDirectiveKind DKind) {
   return DKind == OMPD_tile || DKind == OMPD_unroll || DKind == OMPD_reverse ||
          DKind == OMPD_split || DKind == OMPD_interchange ||
-         DKind == OMPD_stripe;
+         DKind == OMPD_stripe || DKind == OMPD_flatten;
 }
 
 bool clang::isOpenMPCanonicalLoopSequenceTransformationDirective(

>From 933007b6333584fb358adc4f89a61db1ae15d4cb Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:02:27 -0400
Subject: [PATCH 02/14] add OMPDepthClause ast node

---
 clang/include/clang/AST/OpenMPClause.h        | 68 +++++++++++++++++++
 clang/include/clang/AST/RecursiveASTVisitor.h |  6 ++
 clang/lib/AST/OpenMPClause.cpp                | 25 +++++++
 clang/lib/AST/StmtProfile.cpp                 |  5 ++
 clang/lib/Parse/ParseOpenMP.cpp               |  1 +
 llvm/include/llvm/Frontend/OpenMP/OMP.td      |  1 +
 6 files changed, 106 insertions(+)

diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 8ceafc4669297..50d1364a9726d 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -1380,6 +1380,74 @@ class OMPPartialClause final : public OMPClause {
   }
 };
 
+/// This represents the 'depth' clause on the '#pragma omp flatten' (and
+/// '#pragma omp fuse') loop-transformation directives.
+///
+/// \code
+/// #pragma omp flatten depth(3)
+/// \endcode
+/// In this example the 'flatten' directive has a 'depth' clause whose argument
+/// '3' specifies how many perfectly nested loops are combined into one.
+/// The argument must be a positive integer constant expression that evaluates
+/// to at most the loop nest depth of the associated loop nest.
+class OMPDepthClause final : public OMPClause {
+  friend class OMPClauseReader;
+
+  /// Location of '('.
+  SourceLocation LParenLoc;
+
+  /// The depth expression (number of loops to flatten).
+  Stmt *Depth = nullptr;
+
+  /// Build an empty clause.
+  explicit OMPDepthClause() : OMPClause(llvm::omp::OMPC_depth, {}, {}) {}
+
+  /// Set the depth expression.
+  void setDepth(Expr *E) { Depth = E; }
+
+  /// Sets the location of '('.
+  void setLParenLoc(SourceLocation Loc) { LParenLoc = Loc; }
+
+public:
+  /// Build an AST node for a 'depth' clause.
+  ///
+  /// \param C         Context of the AST.
+  /// \param StartLoc  Location of the 'depth' identifier.
+  /// \param LParenLoc Location of '('.
+  /// \param EndLoc    Location of ')'.
+  /// \param Depth     The depth expression.
+  static OMPDepthClause *Create(const ASTContext &C, SourceLocation StartLoc,
+                                SourceLocation LParenLoc, SourceLocation EndLoc,
+                                Expr *Depth);
+
+  /// Build an empty 'depth' AST node for deserialization.
+  ///
+  /// \param C     Context of the AST.
+  static OMPDepthClause *CreateEmpty(const ASTContext &C);
+
+  /// Returns the location of '('.
+  SourceLocation getLParenLoc() const { return LParenLoc; }
+
+  /// Returns the depth expression or nullptr if not set.
+  Expr *getDepth() const { return cast_or_null<Expr>(Depth); }
+
+  child_range children() { return child_range(&Depth, &Depth + 1); }
+  const_child_range children() const {
+    return const_child_range(&Depth, &Depth + 1);
+  }
+
+  child_range used_children() {
+    return child_range(child_iterator(), child_iterator());
+  }
+  const_child_range used_children() const {
+    return const_child_range(const_child_iterator(), const_child_iterator());
+  }
+
+  static bool classof(const OMPClause *T) {
+    return T->getClauseKind() == llvm::omp::OMPC_depth;
+  }
+};
+
 /// This represents 'collapse' clause in the '#pragma omp ...'
 /// directive.
 ///
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index e425f7a31f59c..da55e97fdfcb5 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -3568,6 +3568,12 @@ bool RecursiveASTVisitor<Derived>::VisitOMPPartialClause(OMPPartialClause *C) {
   return true;
 }
 
+template <typename Derived>
+bool RecursiveASTVisitor<Derived>::VisitOMPDepthClause(OMPDepthClause *C) {
+  TRY_TO(TraverseStmt(C->getDepth()));
+  return true;
+}
+
 template <typename Derived>
 bool
 RecursiveASTVisitor<Derived>::VisitOMPCollapseClause(OMPCollapseClause *C) {
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index ed00e80144c25..7b2f67a31cad9 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -1057,6 +1057,22 @@ OMPPartialClause *OMPPartialClause::CreateEmpty(const ASTContext &C) {
   return new (C) OMPPartialClause();
 }
 
+OMPDepthClause *OMPDepthClause::Create(const ASTContext &C,
+                                       SourceLocation StartLoc,
+                                       SourceLocation LParenLoc,
+                                       SourceLocation EndLoc, Expr *Depth) {
+  OMPDepthClause *Clause = CreateEmpty(C);
+  Clause->setLocStart(StartLoc);
+  Clause->setLParenLoc(LParenLoc);
+  Clause->setLocEnd(EndLoc);
+  Clause->setDepth(Depth);
+  return Clause;
+}
+
+OMPDepthClause *OMPDepthClause::CreateEmpty(const ASTContext &C) {
+  return new (C) OMPDepthClause();
+}
+
 OMPLoopRangeClause *
 OMPLoopRangeClause::Create(const ASTContext &C, SourceLocation StartLoc,
                            SourceLocation LParenLoc, SourceLocation FirstLoc,
@@ -2069,6 +2085,15 @@ void OMPClausePrinter::VisitOMPPermutationClause(OMPPermutationClause *Node) {
 
 void OMPClausePrinter::VisitOMPFullClause(OMPFullClause *Node) { OS << "full"; }
 
+void OMPClausePrinter::VisitOMPDepthClause(OMPDepthClause *Node) {
+  OS << "depth";
+  if (Expr *Depth = Node->getDepth()) {
+    OS << '(';
+    Depth->printPretty(OS, nullptr, Policy, 0);
+    OS << ')';
+  }
+}
+
 void OMPClausePrinter::VisitOMPPartialClause(OMPPartialClause *Node) {
   OS << "partial";
 
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index de7ed322d7424..838d6d02f7446 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -521,6 +521,11 @@ void OMPClauseProfiler::VisitOMPPartialClause(const OMPPartialClause *C) {
     Profiler->VisitExpr(Factor);
 }
 
+void OMPClauseProfiler::VisitOMPDepthClause(const OMPDepthClause *C) {
+  if (const Expr *Depth = C->getDepth())
+    Profiler->VisitExpr(Depth);
+}
+
 void OMPClauseProfiler::VisitOMPLoopRangeClause(const OMPLoopRangeClause *C) {
   if (const Expr *First = C->getFirst())
     Profiler->VisitExpr(First);
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index af52295df2d8b..952a517205381 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -3247,6 +3247,7 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
   case OMPC_nocontext:
   case OMPC_filter:
   case OMPC_partial:
+  case OMPC_depth:
   case OMPC_align:
   case OMPC_message:
   case OMPC_ompx_dyn_cgroup_mem:
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index 679a944fc4358..d49c5ee1a0db3 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -161,6 +161,7 @@ def OMPC_Depobj : Clause<[Spelling<"depobj">]> {
   let isImplicit = true;
 }
 def OMPC_Depth : Clause<[Spelling<"depth">]> {
+  let clangClass = "OMPDepthClause";
   let flangClass = "ScalarIntConstantExpr";
 }
 def OMPC_Destroy : Clause<[Spelling<"destroy">]> {

>From b63024e1e342cd168ef128a3f6a24909df2a82db Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:04:44 -0400
Subject: [PATCH 03/14] template-transform support for flatten directive &
 depth clause

---
 clang/lib/Sema/TreeTransform.h | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 3b99ff4bb9e23..865622ad50f1c 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -1804,6 +1804,14 @@ class TreeTransform {
                                                        LParenLoc, EndLoc);
   }
 
+  /// Build a new OpenMP 'depth' clause.
+  OMPClause *RebuildOMPDepthClause(Expr *Depth, SourceLocation StartLoc,
+                                   SourceLocation LParenLoc,
+                                   SourceLocation EndLoc) {
+    return getSema().OpenMP().ActOnOpenMPDepthClause(Depth, StartLoc, LParenLoc,
+                                                     EndLoc);
+  }
+
   OMPClause *
   RebuildOMPLoopRangeClause(Expr *First, Expr *Count, SourceLocation StartLoc,
                             SourceLocation LParenLoc, SourceLocation FirstLoc,
@@ -9791,6 +9799,17 @@ TreeTransform<Derived>::TransformOMPSplitDirective(OMPSplitDirective *D) {
   return Res;
 }
 
+template <typename Derived>
+StmtResult
+TreeTransform<Derived>::TransformOMPFlattenDirective(OMPFlattenDirective *D) {
+  DeclarationNameInfo DirName;
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      D->getDirectiveKind(), DirName, nullptr, D->getBeginLoc());
+  StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
+  return Res;
+}
+
 template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPFuseDirective(OMPFuseDirective *D) {
@@ -10717,6 +10736,20 @@ TreeTransform<Derived>::TransformOMPPartialClause(OMPPartialClause *C) {
                                  C->getEndLoc());
 }
 
+template <typename Derived>
+OMPClause *TreeTransform<Derived>::TransformOMPDepthClause(OMPDepthClause *C) {
+  ExprResult T = getDerived().TransformExpr(C->getDepth());
+  if (T.isInvalid())
+    return nullptr;
+  Expr *Depth = T.get();
+  bool Changed = Depth != C->getDepth();
+
+  if (!Changed && !getDerived().AlwaysRebuild())
+    return C;
+  return RebuildOMPDepthClause(Depth, C->getBeginLoc(), C->getLParenLoc(),
+                               C->getEndLoc());
+}
+
 template <typename Derived>
 OMPClause *
 TreeTransform<Derived>::TransformOMPLoopRangeClause(OMPLoopRangeClause *C) {

>From deafb222a7d251069c68a44fa267e58e750f1064 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:08:12 -0400
Subject: [PATCH 04/14] serialize flatten

---
 clang/include/clang/Serialization/ASTBitCodes.h |  1 +
 clang/lib/Serialization/ASTReaderStmt.cpp       | 11 +++++++++++
 clang/lib/Serialization/ASTWriterStmt.cpp       |  5 +++++
 3 files changed, 17 insertions(+)

diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h
index 279380de2f7fe..93a0e0a454b68 100644
--- a/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/clang/include/clang/Serialization/ASTBitCodes.h
@@ -1970,6 +1970,7 @@ enum StmtCode {
   STMT_OMP_REVERSE_DIRECTIVE,
   STMT_OMP_SPLIT_DIRECTIVE,
   STMT_OMP_INTERCHANGE_DIRECTIVE,
+  STMT_OMP_FLATTEN_DIRECTIVE,
   STMT_OMP_FUSE_DIRECTIVE,
   STMT_OMP_FOR_DIRECTIVE,
   STMT_OMP_FOR_SIMD_DIRECTIVE,
diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp
index 02ccf8d4d41c2..8c2debd6cdf32 100644
--- a/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -2537,6 +2537,10 @@ void ASTStmtReader::VisitOMPInterchangeDirective(OMPInterchangeDirective *D) {
   VisitOMPCanonicalLoopNestTransformationDirective(D);
 }
 
+void ASTStmtReader::VisitOMPFlattenDirective(OMPFlattenDirective *D) {
+  VisitOMPCanonicalLoopNestTransformationDirective(D);
+}
+
 void ASTStmtReader::VisitOMPSplitDirective(OMPSplitDirective *D) {
   VisitOMPCanonicalLoopNestTransformationDirective(D);
 }
@@ -3719,6 +3723,13 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) {
       break;
     }
 
+    case STMT_OMP_FLATTEN_DIRECTIVE: {
+      unsigned NumLoops = Record[ASTStmtReader::NumStmtFields];
+      unsigned NumClauses = Record[ASTStmtReader::NumStmtFields + 1];
+      S = OMPFlattenDirective::CreateEmpty(Context, NumClauses, NumLoops);
+      break;
+    }
+
     case STMT_OMP_FOR_DIRECTIVE: {
       unsigned CollapsedNum = Record[ASTStmtReader::NumStmtFields];
       unsigned NumClauses = Record[ASTStmtReader::NumStmtFields + 1];
diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp
index 4e9eadd730a56..dfe28a9b915e3 100644
--- a/clang/lib/Serialization/ASTWriterStmt.cpp
+++ b/clang/lib/Serialization/ASTWriterStmt.cpp
@@ -2547,6 +2547,11 @@ void ASTStmtWriter::VisitOMPInterchangeDirective(OMPInterchangeDirective *D) {
   Code = serialization::STMT_OMP_INTERCHANGE_DIRECTIVE;
 }
 
+void ASTStmtWriter::VisitOMPFlattenDirective(OMPFlattenDirective *D) {
+  VisitOMPCanonicalLoopNestTransformationDirective(D);
+  Code = serialization::STMT_OMP_FLATTEN_DIRECTIVE;
+}
+
 void ASTStmtWriter::VisitOMPSplitDirective(OMPSplitDirective *D) {
   VisitOMPCanonicalLoopNestTransformationDirective(D);
   Code = serialization::STMT_OMP_SPLIT_DIRECTIVE;

>From 3e2dfeb99230057b7cb9c484182ae33befbe1e41 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:09:03 -0400
Subject: [PATCH 05/14] serialize depth

---
 clang/lib/Serialization/ASTReader.cpp | 8 ++++++++
 clang/lib/Serialization/ASTWriter.cpp | 5 +++++
 2 files changed, 13 insertions(+)

diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index f8a6a38bb9b5c..4f51d79a52105 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -11495,6 +11495,9 @@ OMPClause *OMPClauseReader::readClause() {
   case llvm::omp::OMPC_partial:
     C = OMPPartialClause::CreateEmpty(Context);
     break;
+  case llvm::omp::OMPC_depth:
+    C = OMPDepthClause::CreateEmpty(Context);
+    break;
   case llvm::omp::OMPC_looprange:
     C = OMPLoopRangeClause::CreateEmpty(Context);
     break;
@@ -11924,6 +11927,11 @@ void OMPClauseReader::VisitOMPPartialClause(OMPPartialClause *C) {
   C->setLParenLoc(Record.readSourceLocation());
 }
 
+void OMPClauseReader::VisitOMPDepthClause(OMPDepthClause *C) {
+  C->setDepth(Record.readSubExpr());
+  C->setLParenLoc(Record.readSourceLocation());
+}
+
 void OMPClauseReader::VisitOMPLoopRangeClause(OMPLoopRangeClause *C) {
   C->setFirst(Record.readSubExpr());
   C->setCount(Record.readSubExpr());
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 357a7f7e95fa0..cbfeaa9e6638a 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -8106,6 +8106,11 @@ void OMPClauseWriter::VisitOMPPartialClause(OMPPartialClause *C) {
   Record.AddSourceLocation(C->getLParenLoc());
 }
 
+void OMPClauseWriter::VisitOMPDepthClause(OMPDepthClause *C) {
+  Record.AddStmt(C->getDepth());
+  Record.AddSourceLocation(C->getLParenLoc());
+}
+
 void OMPClauseWriter::VisitOMPLoopRangeClause(OMPLoopRangeClause *C) {
   Record.AddStmt(C->getFirst());
   Record.AddStmt(C->getCount());

>From 4ae746ca1eeaf11db10e859572a4500029d01cce Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:17:29 -0400
Subject: [PATCH 06/14] add analysis traversal support for flatten directive

---
 clang/lib/Sema/SemaExceptionSpec.cpp         | 1 +
 clang/lib/StaticAnalyzer/Core/ExprEngine.cpp | 1 +
 2 files changed, 2 insertions(+)

diff --git a/clang/lib/Sema/SemaExceptionSpec.cpp b/clang/lib/Sema/SemaExceptionSpec.cpp
index 40d530a1f3925..995b5b6010978 100644
--- a/clang/lib/Sema/SemaExceptionSpec.cpp
+++ b/clang/lib/Sema/SemaExceptionSpec.cpp
@@ -1508,6 +1508,7 @@ CanThrowResult Sema::canThrow(const Stmt *S) {
   case Stmt::OMPUnrollDirectiveClass:
   case Stmt::OMPReverseDirectiveClass:
   case Stmt::OMPInterchangeDirectiveClass:
+  case Stmt::OMPFlattenDirectiveClass:
   case Stmt::OMPSplitDirectiveClass:
   case Stmt::OMPFuseDirectiveClass:
   case Stmt::OMPSingleDirectiveClass:
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
index cfb294736ee02..3f666be00cc24 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
@@ -1783,6 +1783,7 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
     case Stmt::OMPStripeDirectiveClass:
     case Stmt::OMPTileDirectiveClass:
     case Stmt::OMPInterchangeDirectiveClass:
+    case Stmt::OMPFlattenDirectiveClass:
     case Stmt::OMPSplitDirectiveClass:
     case Stmt::OMPFuseDirectiveClass:
     case Stmt::OMPInteropDirectiveClass:

>From 415eba88ad0af63c552b4c5c2eabd660bcb29c02 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:21:25 -0400
Subject: [PATCH 07/14] codegen flatten directive

---
 clang/lib/CodeGen/CGStmt.cpp        | 3 +++
 clang/lib/CodeGen/CGStmtOpenMP.cpp  | 8 ++++++++
 clang/lib/CodeGen/CodeGenFunction.h | 1 +
 3 files changed, 12 insertions(+)

diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index 232094777f233..27f48cb63b553 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -236,6 +236,9 @@ void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {
   case Stmt::OMPInterchangeDirectiveClass:
     EmitOMPInterchangeDirective(cast<OMPInterchangeDirective>(*S));
     break;
+  case Stmt::OMPFlattenDirectiveClass:
+    EmitOMPFlattenDirective(cast<OMPFlattenDirective>(*S));
+    break;
   case Stmt::OMPFuseDirectiveClass:
     EmitOMPFuseDirective(cast<OMPFuseDirective>(*S));
     break;
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 95fd6694437fe..25da3430a23fe 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -218,6 +218,8 @@ class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
     } else if (const auto *Interchange =
                    dyn_cast<OMPInterchangeDirective>(&S)) {
       PreInits = Interchange->getPreInits();
+    } else if (const auto *Flatten = dyn_cast<OMPFlattenDirective>(&S)) {
+      PreInits = Flatten->getPreInits();
     } else {
       llvm_unreachable("Unknown loop-based directive kind.");
     }
@@ -3248,6 +3250,12 @@ void CodeGenFunction::EmitOMPInterchangeDirective(
   EmitStmt(S.getTransformedStmt());
 }
 
+void CodeGenFunction::EmitOMPFlattenDirective(const OMPFlattenDirective &S) {
+  // Emit the de-sugared statement.
+  OMPTransformDirectiveScopeRAII FlattenScope(*this, &S);
+  EmitStmt(S.getTransformedStmt());
+}
+
 void CodeGenFunction::EmitOMPFuseDirective(const OMPFuseDirective &S) {
   // Emit the de-sugared statement
   OMPTransformDirectiveScopeRAII FuseScope(*this, &S);
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 6d0718c243812..eea0163422cd3 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -3934,6 +3934,7 @@ class CodeGenFunction : public CodeGenTypeCache {
   void EmitOMPReverseDirective(const OMPReverseDirective &S);
   void EmitOMPSplitDirective(const OMPSplitDirective &S);
   void EmitOMPInterchangeDirective(const OMPInterchangeDirective &S);
+  void EmitOMPFlattenDirective(const OMPFlattenDirective &S);
   void EmitOMPFuseDirective(const OMPFuseDirective &S);
   void EmitOMPForDirective(const OMPForDirective &S);
   void EmitOMPForSimdDirective(const OMPForSimdDirective &S);

>From 814e2e9fe9bf4e28e4557999fbc16beec144cc39 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:24:42 -0400
Subject: [PATCH 08/14] add NYI CIR stub

---
 clang/lib/CIR/CodeGen/CIRGenFunction.h     | 1 +
 clang/lib/CIR/CodeGen/CIRGenStmt.cpp       | 2 ++
 clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp | 5 +++++
 3 files changed, 8 insertions(+)

diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.h b/clang/lib/CIR/CodeGen/CIRGenFunction.h
index 322355fde3957..8374583ffd4d0 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.h
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.h
@@ -2559,6 +2559,7 @@ class CIRGenFunction : public CIRGenTypeCache {
   mlir::LogicalResult emitOMPSplitDirective(const OMPSplitDirective &s);
   mlir::LogicalResult
   emitOMPInterchangeDirective(const OMPInterchangeDirective &s);
+  mlir::LogicalResult emitOMPFlattenDirective(const OMPFlattenDirective &s);
   mlir::LogicalResult emitOMPAssumeDirective(const OMPAssumeDirective &s);
   mlir::LogicalResult emitOMPMaskedDirective(const OMPMaskedDirective &s);
   mlir::LogicalResult emitOMPStripeDirective(const OMPStripeDirective &s);
diff --git a/clang/lib/CIR/CodeGen/CIRGenStmt.cpp b/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
index d3acac5801e74..33d425b22d61a 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
@@ -408,6 +408,8 @@ mlir::LogicalResult CIRGenFunction::emitStmt(const Stmt *s,
     return emitOMPSplitDirective(cast<OMPSplitDirective>(*s));
   case Stmt::OMPInterchangeDirectiveClass:
     return emitOMPInterchangeDirective(cast<OMPInterchangeDirective>(*s));
+  case Stmt::OMPFlattenDirectiveClass:
+    return emitOMPFlattenDirective(cast<OMPFlattenDirective>(*s));
   case Stmt::OMPAssumeDirectiveClass:
     return emitOMPAssumeDirective(cast<OMPAssumeDirective>(*s));
   case Stmt::OMPMaskedDirectiveClass:
diff --git a/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp b/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp
index 17a1fb8090f5c..4fffd17b25a16 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp
@@ -587,6 +587,11 @@ CIRGenFunction::emitOMPInterchangeDirective(const OMPInterchangeDirective &s) {
   return mlir::failure();
 }
 mlir::LogicalResult
+CIRGenFunction::emitOMPFlattenDirective(const OMPFlattenDirective &s) {
+  getCIRGenModule().errorNYI(s.getSourceRange(), "OpenMP OMPFlattenDirective");
+  return mlir::failure();
+}
+mlir::LogicalResult
 CIRGenFunction::emitOMPAssumeDirective(const OMPAssumeDirective &s) {
   getCIRGenModule().errorNYI(s.getSourceRange(), "OpenMP OMPAssumeDirective");
   return mlir::failure();

>From 0027ec6ab982a83766bd65f3e1bc45d9a671fc62 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:26:31 -0400
Subject: [PATCH 09/14] add libclang traversal support for depth clause

---
 clang/tools/libclang/CIndex.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index ac2fad38a1348..f8ab63e3e0dae 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2378,6 +2378,10 @@ void OMPClauseEnqueue::VisitOMPLoopRangeClause(const OMPLoopRangeClause *C) {
   Visitor->AddStmt(C->getCount());
 }
 
+void OMPClauseEnqueue::VisitOMPDepthClause(const OMPDepthClause *C) {
+  Visitor->AddStmt(C->getDepth());
+}
+
 void OMPClauseEnqueue::VisitOMPAllocatorClause(const OMPAllocatorClause *C) {
   Visitor->AddStmt(C->getAllocator());
 }

>From f5ac5f740c414310c8482f652f1bed936ae7cda3 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Fri, 10 Jul 2026 06:54:16 -0400
Subject: [PATCH 10/14] sema

---
 clang/include/clang/Sema/SemaOpenMP.h |  10 +
 clang/lib/Sema/SemaOpenMP.cpp         | 295 ++++++++++++++++++++++++++
 2 files changed, 305 insertions(+)

diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index 3621ce96b8724..7b086388416a0 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -470,6 +470,12 @@ class SemaOpenMP : public SemaBase {
                                              SourceLocation StartLoc,
                                              SourceLocation EndLoc);
 
+  /// Called on well-formed '#pragma omp flatten' after parsing of its
+  /// clauses and the associated statement.
+  StmtResult ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
+                                         Stmt *AStmt, SourceLocation StartLoc,
+                                         SourceLocation EndLoc);
+
   /// Called on well-formed '#pragma omp fuse' after parsing of its
   /// clauses and the associated statement.
   StmtResult ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
@@ -935,6 +941,10 @@ class SemaOpenMP : public SemaBase {
   OMPClause *ActOnOpenMPPartialClause(Expr *FactorExpr, SourceLocation StartLoc,
                                       SourceLocation LParenLoc,
                                       SourceLocation EndLoc);
+  /// Called on well-formed 'depth' clause.
+  OMPClause *ActOnOpenMPDepthClause(Expr *DepthExpr, SourceLocation StartLoc,
+                                    SourceLocation LParenLoc,
+                                    SourceLocation EndLoc);
   /// Called on well-formed 'collapse' clause.
   OMPClause *ActOnOpenMPCollapseClause(Expr *NumForLoops,
                                        SourceLocation StartLoc,
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 247a4a6ad9271..fcc256dec3c88 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -4629,6 +4629,7 @@ void SemaOpenMP::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind,
   case OMPD_reverse:
   case OMPD_split:
   case OMPD_interchange:
+  case OMPD_flatten:
   case OMPD_fuse:
   case OMPD_assume:
     break;
@@ -6475,6 +6476,10 @@ StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
     Res = ActOnOpenMPInterchangeDirective(ClausesWithImplicit, AStmt, StartLoc,
                                           EndLoc);
     break;
+  case OMPD_flatten:
+    Res = ActOnOpenMPFlattenDirective(ClausesWithImplicit, AStmt, StartLoc,
+                                      EndLoc);
+    break;
   case OMPD_fuse:
     Res =
         ActOnOpenMPFuseDirective(ClausesWithImplicit, AStmt, StartLoc, EndLoc);
@@ -16390,6 +16395,277 @@ StmtResult SemaOpenMP::ActOnOpenMPInterchangeDirective(
                                          buildPreInits(Context, PreInits));
 }
 
+/// Counts the perfectly nested canonical loop depth at \p AStmt. Returns
+/// std::nullopt if a loop-generating transformation prevents a static count.
+static std::optional<unsigned> getCanonicalLoopNestDepth(Stmt *AStmt) {
+  unsigned Depth = 0;
+  Stmt *CurStmt = AStmt ? AStmt->IgnoreContainers() : nullptr;
+  while (CurStmt) {
+    if (isa<OMPLoopTransformationDirective>(CurStmt))
+      return std::nullopt;
+    if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(CurStmt))
+      CurStmt = CanonLoop->getLoopStmt();
+    Stmt *Body = nullptr;
+    if (auto *For = dyn_cast<ForStmt>(CurStmt))
+      Body = For->getBody();
+    else if (auto *RangeFor = dyn_cast<CXXForRangeStmt>(CurStmt))
+      Body = RangeFor->getBody();
+    else
+      break;
+    ++Depth;
+    CurStmt = Body ? Body->IgnoreContainers() : nullptr;
+  }
+  return Depth;
+}
+
+StmtResult
+SemaOpenMP::ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
+                                        Stmt *AStmt, SourceLocation StartLoc,
+                                        SourceLocation EndLoc) {
+  ASTContext &Context = getASTContext();
+  DeclContext *CurContext = SemaRef.CurContext;
+  Scope *CurScope = SemaRef.getCurScope();
+
+  // Empty statement should only be possible if there already was an error.
+  if (!AStmt)
+    return StmtError();
+
+  // flatten without 'depth' clause combines two loops; 'depth(k)' selects k.
+  unsigned NumLoops = 2;
+  bool DepthIsValueDependent = false;
+  const auto *DepthClause =
+      OMPExecutableDirective::getSingleClause<OMPDepthClause>(Clauses);
+  if (DepthClause) {
+    Expr *DepthExpr = DepthClause->getDepth();
+    if (DepthExpr && DepthExpr->isValueDependent()) {
+      DepthIsValueDependent = true;
+    } else if (DepthExpr) {
+      Expr::EvalResult EvalResult;
+      if (DepthExpr->EvaluateAsInt(EvalResult, Context))
+        NumLoops = EvalResult.Val.getInt().getZExtValue();
+    }
+  }
+
+  // Report "expected k for loops, but found only n" when depth exceeds the
+  // perfect nest (same form as 'collapse'); skip if depth or nest is unknown.
+  if (DepthClause && !DepthIsValueDependent) {
+    if (std::optional<unsigned> NestDepth = getCanonicalLoopNestDepth(AStmt);
+        NestDepth && NumLoops > *NestDepth) {
+      Diag(AStmt->getBeginLoc(), diag::err_omp_not_for)
+          << /*expected N for loops form=*/1
+          << getOpenMPDirectiveName(OMPD_flatten) << NumLoops
+          << (*NestDepth > 0) << *NestDepth;
+      return StmtError();
+    }
+  }
+
+  // Defer when 'depth' is value-dependent (concrete k unknown until
+  // instantiation).
+  if (DepthIsValueDependent)
+    return OMPFlattenDirective::Create(Context, StartLoc, EndLoc, Clauses,
+                                       NumLoops, AStmt, nullptr, nullptr);
+
+  // Verify and diagnose loop nest.
+  SmallVector<OMPLoopBasedDirective::HelperExprs, 4> LoopHelpers(NumLoops);
+  Stmt *Body = nullptr;
+  SmallVector<SmallVector<Stmt *>, 4> OriginalInits;
+  if (!checkTransformableLoopNest(OMPD_flatten, AStmt, NumLoops, LoopHelpers,
+                                  Body, OriginalInits))
+    return StmtError();
+
+  // Delay flattening to when template is completely instantiated.
+  if (CurContext->isDependentContext())
+    return OMPFlattenDirective::Create(Context, StartLoc, EndLoc, Clauses,
+                                       NumLoops, AStmt, nullptr, nullptr);
+
+  assert(LoopHelpers.size() == NumLoops &&
+         "Expecting loop iteration space dimensionality to match number of "
+         "affected loops");
+  assert(OriginalInits.size() == NumLoops &&
+         "Expecting loop iteration space dimensionality to match number of "
+         "affected loops");
+
+  // Find the affected loops.
+  SmallVector<Stmt *> LoopStmts(NumLoops, nullptr);
+  collectLoopStmts(AStmt, LoopStmts);
+
+  // Collect pre-init statements in outer-to-inner order.
+  SmallVector<Stmt *> PreInits;
+  for (auto I : llvm::seq<int>(NumLoops)) {
+    OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
+    assert(LoopHelper.Counters.size() == 1 &&
+           "Single-dimensional loop iteration space expected");
+    addLoopPreInits(Context, LoopHelper, LoopStmts[I], OriginalInits[I],
+                    PreInits);
+  }
+
+  CaptureVars CopyTransformer(SemaRef);
+  auto MakeNumIterations = [&CopyTransformer,
+                            &LoopHelpers](unsigned I) -> Expr * {
+    return AssertSuccess(
+        CopyTransformer.TransformExpr(LoopHelpers[I].NumIterations));
+  };
+
+  OMPLoopBasedDirective::HelperExprs &OutermostHelper = LoopHelpers[0];
+  auto *OutermostCntVar = cast<DeclRefExpr>(OutermostHelper.Counters.front());
+  SourceLocation OrigVarLoc = OutermostCntVar->getExprLoc();
+  SourceLocation OrigVarLocBegin = OutermostCntVar->getBeginLoc();
+  SourceLocation OrigVarLocEnd = OutermostCntVar->getEndLoc();
+  SourceLocation CondLoc = OutermostHelper.Cond->getExprLoc();
+
+  // Product of trip counts; mirror 'collapse' IV-width selection to avoid
+  // overflow when several counts are multiplied.
+  auto BuildTripCount = [&](unsigned Bits) -> ExprResult {
+    ExprResult Product;
+    for (unsigned I = 0; I < NumLoops; ++I) {
+      ExprResult N = widenIterationCount(Bits, MakeNumIterations(I), SemaRef);
+      if (!N.isUsable())
+        return ExprError();
+      if (I == 0)
+        Product = N;
+      else
+        Product = SemaRef.BuildBinOp(CurScope, CondLoc, BO_Mul, Product.get(),
+                                     N.get());
+      if (!Product.isUsable())
+        return ExprError();
+    }
+    return Product;
+  };
+
+  bool AllCountsLessThan32Bits = true;
+  for (unsigned I = 0; I < NumLoops; ++I)
+    AllCountsLessThan32Bits &=
+        Context.getTypeSize(LoopHelpers[I].NumIterations->getType()) < 32;
+
+  ExprResult TripCount32 = BuildTripCount(/*Bits=*/32);
+  ExprResult TripCount64 = BuildTripCount(/*Bits=*/64);
+  if (!TripCount32.isUsable() || !TripCount64.isUsable())
+    return StmtError();
+
+  ExprResult TripCount = TripCount64;
+  if (Context.getTypeSize(TripCount32.get()->getType()) == 32 &&
+      (AllCountsLessThan32Bits || NumLoops == 1 ||
+       fitsInto(
+           /*Bits=*/32,
+           TripCount32.get()->getType()->hasSignedIntegerRepresentation(),
+           TripCount64.get(), SemaRef)))
+    TripCount = TripCount32;
+
+  QualType IVTy = TripCount.get()->getType();
+  uint64_t IVWidth = Context.getTypeSize(IVTy);
+
+  auto MakeNumIterationsInIVTy = [&](unsigned I) -> Expr * {
+    return AssertSuccess(SemaRef.PerformImplicitConversion(
+        MakeNumIterations(I), IVTy, AssignmentAction::Converting,
+        /*AllowExplicit=*/true));
+  };
+
+  // \code{.cpp}
+  //   for (auto .flatten.iv = 0; .flatten.iv < n0 * n1 * ...; ++.flatten.iv) {
+  //     .flatten.iv.0 = .flatten.iv / (n1 * ...);
+  //     i0 = ...;                                        // Updates[0]
+  //     .flatten.iv.1 = (.flatten.iv / ...) % n1;
+  //     i1 = ...;                                        // Updates[1]
+  //     ...
+  //     body(i0, i1, ...);
+  //   }
+  // \endcode
+  SmallString<64> FlattenedIVName(".flatten.iv");
+  VarDecl *FlattenedIVDecl = buildVarDecl(SemaRef, {}, IVTy, FlattenedIVName,
+                                          nullptr, OutermostCntVar);
+  auto MakeFlattenedRef = [&SemaRef = this->SemaRef, FlattenedIVDecl, IVTy,
+                           OrigVarLoc]() {
+    return buildDeclRefExpr(SemaRef, FlattenedIVDecl, IVTy, OrigVarLoc);
+  };
+
+  // For init-statement:
+  // \code{.cpp}
+  //   auto .flatten.iv = 0;
+  // \endcode
+  auto *Zero = IntegerLiteral::Create(Context, llvm::APInt::getZero(IVWidth),
+                                      IVTy, OrigVarLoc);
+  SemaRef.AddInitializerToDecl(FlattenedIVDecl, Zero, /*DirectInit=*/false);
+  StmtResult Init = new (Context)
+      DeclStmt(DeclGroupRef(FlattenedIVDecl), OrigVarLocBegin, OrigVarLocEnd);
+  if (!Init.isUsable())
+    return StmtError();
+
+  // For cond-expression:
+  // \code{.cpp}
+  //   .flatten.iv < n0 * n1 * ... * n(k-1)
+  // \endcode
+  ExprResult Cond = SemaRef.BuildBinOp(CurScope, CondLoc, BO_LT,
+                                       MakeFlattenedRef(), TripCount.get());
+  if (!Cond.isUsable())
+    return StmtError();
+
+  // For incr-statement:
+  // \code{.cpp}
+  //   ++.flatten.iv
+  // \endcode
+  ExprResult Incr =
+      SemaRef.BuildUnaryOp(CurScope, OutermostHelper.Inc->getExprLoc(),
+                           UO_PreInc, MakeFlattenedRef());
+  if (!Incr.isUsable())
+    return StmtError();
+
+  // Recover each logical iteration counter via mixed-radix div/mod; reuse the
+  // iteration variables from checkOpenMPLoop so Updates compute user counters.
+  SmallVector<Stmt *, 8> BodyStmts;
+  for (unsigned I = 0; I < NumLoops; ++I) {
+    OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
+    auto *IVRef = cast<DeclRefExpr>(LoopHelper.IterationVarRef);
+    auto *IVDecl = cast<VarDecl>(IVRef->getDecl());
+    std::string IVName = (".flatten.iv." + llvm::Twine(I)).str();
+    IVDecl->setDeclName(&SemaRef.PP.getIdentifierTable().get(IVName));
+
+    ExprResult Value = MakeFlattenedRef();
+    if (I + 1 < NumLoops) {
+      ExprResult Divisor = MakeNumIterationsInIVTy(I + 1);
+      for (unsigned J = I + 2; J < NumLoops; ++J) {
+        Divisor = SemaRef.BuildBinOp(CurScope, OrigVarLoc, BO_Mul,
+                                     Divisor.get(), MakeNumIterationsInIVTy(J));
+        if (!Divisor.isUsable())
+          return StmtError();
+      }
+      Value = SemaRef.BuildBinOp(CurScope, OrigVarLoc, BO_Div, Value.get(),
+                                 Divisor.get());
+      if (!Value.isUsable())
+        return StmtError();
+    }
+    if (I > 0) {
+      Value = SemaRef.BuildBinOp(CurScope, OrigVarLoc, BO_Rem, Value.get(),
+                                 MakeNumIterationsInIVTy(I));
+      if (!Value.isUsable())
+        return StmtError();
+    }
+
+    SemaRef.AddInitializerToDecl(IVDecl, Value.get(), /*DirectInit=*/false);
+    StmtResult IVStmt = new (Context)
+        DeclStmt(DeclGroupRef(IVDecl), OrigVarLocBegin, OrigVarLocEnd);
+    if (!IVStmt.isUsable())
+      return StmtError();
+
+    BodyStmts.push_back(IVStmt.get());
+    llvm::append_range(BodyStmts, LoopHelper.Updates);
+    if (auto *CXXFor = dyn_cast<CXXForRangeStmt>(LoopStmts[I]))
+      BodyStmts.push_back(CXXFor->getLoopVarStmt());
+  }
+  BodyStmts.push_back(Body);
+  auto *FlattenedBody =
+      CompoundStmt::Create(Context, BodyStmts, FPOptionsOverride(),
+                           Body->getBeginLoc(), Body->getEndLoc());
+
+  auto *FlattenedFor = new (Context) ForStmt(
+      Context, Init.get(), Cond.get(), nullptr, Incr.get(), FlattenedBody,
+      OutermostHelper.Init->getBeginLoc(), OutermostHelper.Init->getBeginLoc(),
+      OutermostHelper.Inc->getEndLoc());
+
+  return OMPFlattenDirective::Create(Context, StartLoc, EndLoc, Clauses,
+                                     NumLoops, AStmt, FlattenedFor,
+                                     buildPreInits(Context, PreInits));
+}
+
 StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
                                                 Stmt *AStmt,
                                                 SourceLocation StartLoc,
@@ -16920,6 +17196,9 @@ OMPClause *SemaOpenMP::ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind,
   case OMPC_partial:
     Res = ActOnOpenMPPartialClause(Expr, StartLoc, LParenLoc, EndLoc);
     break;
+  case OMPC_depth:
+    Res = ActOnOpenMPDepthClause(Expr, StartLoc, LParenLoc, EndLoc);
+    break;
   case OMPC_message:
     Res = ActOnOpenMPMessageClause(Expr, StartLoc, LParenLoc, EndLoc);
     break;
@@ -18241,6 +18520,22 @@ OMPClause *SemaOpenMP::ActOnOpenMPPartialClause(Expr *FactorExpr,
                                   FactorExpr);
 }
 
+OMPClause *SemaOpenMP::ActOnOpenMPDepthClause(Expr *DepthExpr,
+                                              SourceLocation StartLoc,
+                                              SourceLocation LParenLoc,
+                                              SourceLocation EndLoc) {
+  // The depth-expr must be a positive integer constant expression and
+  // not greater than the number of loops in the associated loop nest.
+  ExprResult DepthResult = VerifyPositiveIntegerConstantInClause(
+      DepthExpr, OMPC_depth, /*StrictlyPositive=*/true);
+  if (DepthResult.isInvalid())
+    return nullptr;
+  DepthExpr = DepthResult.get();
+
+  return OMPDepthClause::Create(getASTContext(), StartLoc, LParenLoc, EndLoc,
+                                DepthExpr);
+}
+
 OMPClause *SemaOpenMP::ActOnOpenMPLoopRangeClause(
     Expr *First, Expr *Count, SourceLocation StartLoc, SourceLocation LParenLoc,
     SourceLocation FirstLoc, SourceLocation CountLoc, SourceLocation EndLoc) {

>From 54be7347979638725cd4b6c421d11ec1cd618c43 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Mon, 27 Jul 2026 07:03:06 -0400
Subject: [PATCH 11/14] add libclang traversal support for flatten directive

---
 clang/tools/libclang/CIndex.cpp   | 7 +++++++
 clang/tools/libclang/CXCursor.cpp | 3 +++
 2 files changed, 10 insertions(+)

diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index f8ab63e3e0dae..9eb0ec8c21158 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2160,6 +2160,7 @@ class EnqueueVisitor : public ConstStmtVisitor<EnqueueVisitor, void>,
   void VisitOMPUnrollDirective(const OMPUnrollDirective *D);
   void VisitOMPReverseDirective(const OMPReverseDirective *D);
   void VisitOMPInterchangeDirective(const OMPInterchangeDirective *D);
+  void VisitOMPFlattenDirective(const OMPFlattenDirective *D);
   void VisitOMPCanonicalLoopSequenceTransformationDirective(
       const OMPCanonicalLoopSequenceTransformationDirective *D);
   void VisitOMPFuseDirective(const OMPFuseDirective *D);
@@ -3358,6 +3359,10 @@ void EnqueueVisitor::VisitOMPInterchangeDirective(
   VisitOMPCanonicalLoopNestTransformationDirective(D);
 }
 
+void EnqueueVisitor::VisitOMPFlattenDirective(const OMPFlattenDirective *D) {
+  VisitOMPCanonicalLoopNestTransformationDirective(D);
+}
+
 void EnqueueVisitor::VisitOMPCanonicalLoopSequenceTransformationDirective(
     const OMPCanonicalLoopSequenceTransformationDirective *D) {
   VisitOMPExecutableDirective(D);
@@ -6329,6 +6334,8 @@ CXString clang_getCursorKindSpelling(enum CXCursorKind Kind) {
     return cxstring::createRef("OMPReverseDirective");
   case CXCursor_OMPInterchangeDirective:
     return cxstring::createRef("OMPInterchangeDirective");
+  case CXCursor_OMPFlattenDirective:
+    return cxstring::createRef("OMPFlattenDirective");
   case CXCursor_OMPFuseDirective:
     return cxstring::createRef("OMPFuseDirective");
   case CXCursor_OMPSplitDirective:
diff --git a/clang/tools/libclang/CXCursor.cpp b/clang/tools/libclang/CXCursor.cpp
index b2feed578fc0f..97bbfd36558a9 100644
--- a/clang/tools/libclang/CXCursor.cpp
+++ b/clang/tools/libclang/CXCursor.cpp
@@ -703,6 +703,9 @@ CXCursor cxcursor::MakeCXCursor(const Stmt *S, const Decl *Parent,
   case Stmt::OMPInterchangeDirectiveClass:
     K = CXCursor_OMPInterchangeDirective;
     break;
+  case Stmt::OMPFlattenDirectiveClass:
+    K = CXCursor_OMPFlattenDirective;
+    break;
   case Stmt::OMPFuseDirectiveClass:
     K = CXCursor_OMPFuseDirective;
     break;

>From cdedb1045baf373937b0da858624a6ace14a3d21 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Mon, 27 Jul 2026 07:06:36 -0400
Subject: [PATCH 12/14] bindings

---
 clang/bindings/python/clang/cindex.py | 3 +++
 clang/include/clang-c/Index.h         | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/clang/bindings/python/clang/cindex.py b/clang/bindings/python/clang/cindex.py
index 24b737139dba8..ccad140e8c861 100644
--- a/clang/bindings/python/clang/cindex.py
+++ b/clang/bindings/python/clang/cindex.py
@@ -1456,6 +1456,9 @@ def is_unexposed(self):
     # OpenMP split directive.
     OMP_SPLIT_DIRECTIVE = 312
 
+    # OpenMP flatten directive.
+    OMP_FLATTEN_DIRECTIVE = 313
+
     # OpenACC Compute Construct.
     OPEN_ACC_COMPUTE_DIRECTIVE = 320
 
diff --git a/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h
index 8427236e0b444..f561fb5724897 100644
--- a/clang/include/clang-c/Index.h
+++ b/clang/include/clang-c/Index.h
@@ -2170,6 +2170,10 @@ enum CXCursorKind {
    */
   CXCursor_OMPSplitDirective = 312,
 
+  /** OpenMP flatten directive.
+   */
+  CXCursor_OMPFlattenDirective = 313,
+
   /** OpenACC Compute Construct.
    */
   CXCursor_OpenACCComputeConstruct = 320,

>From 94dee8fe1351f74760df0fb2d9903a291a5ccf66 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 5 Aug 2026 04:46:14 -0400
Subject: [PATCH 13/14] core tests-flatten and depth

---
 clang/test/OpenMP/flatten_codegen.cpp         |  65 ++++++++
 clang/test/OpenMP/flatten_depth_codegen.cpp   | 154 ++++++++++++++++++
 .../test/transform/flatten/depth-intfor.c     |  40 +++++
 .../runtime/test/transform/flatten/intfor.c   |  33 ++++
 .../test/transform/flatten/lit.local.cfg      |   5 +
 5 files changed, 297 insertions(+)
 create mode 100644 clang/test/OpenMP/flatten_codegen.cpp
 create mode 100644 clang/test/OpenMP/flatten_depth_codegen.cpp
 create mode 100644 openmp/runtime/test/transform/flatten/depth-intfor.c
 create mode 100644 openmp/runtime/test/transform/flatten/intfor.c
 create mode 100644 openmp/runtime/test/transform/flatten/lit.local.cfg

diff --git a/clang/test/OpenMP/flatten_codegen.cpp b/clang/test/OpenMP/flatten_codegen.cpp
new file mode 100644
index 0000000000000..f77b2fb00ce9b
--- /dev/null
+++ b/clang/test/OpenMP/flatten_codegen.cpp
@@ -0,0 +1,65 @@
+// Check code generation
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=60 -emit-llvm %s -o - | FileCheck %s
+
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+extern "C" void body(int, int);
+
+// CHECK-LABEL: define {{.*}}void @foo(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK:   store i64 0, ptr %.flatten.iv,
+// CHECK:   %[[COND_IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[MUL:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   icmp slt i64 %[[COND_IV]], %[[MUL]]
+// CHECK:   %[[DIV_IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[DIV:.+]] = sdiv i64 %[[DIV_IV]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   %[[REM_IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[REM:.+]] = srem i64 %[[REM_IV]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+// CHECK:   %[[I_VAL:.+]] = load i32, ptr %i,
+// CHECK:   %[[J_VAL:.+]] = load i32, ptr %j,
+// CHECK:   call void @body(i32{{.*}} %[[I_VAL]], i32{{.*}} %[[J_VAL]])
+// CHECK: for.inc:
+// CHECK:   %[[INC_IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[NEXT:.+]] = add nsw i64 %[[INC_IV]], 1
+// CHECK:   store i64 %[[NEXT]], ptr %.flatten.iv,
+// CHECK:   br label %for.cond
+extern "C" void foo(int n, int m) {
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+
+// CHECK-LABEL: define {{.*}}void @bar(
+// CHECK:   icmp slt i32 %{{.+}}, 63
+// CHECK:   sdiv i32 %{{.+}}, 9
+// CHECK:   srem i32 %{{.+}}, 9
+// CHECK:   call void @body(
+extern "C" void bar() {
+#pragma omp flatten
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+}
+
+// CHECK-LABEL: define {{.*}}void @baz(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   icmp slt i64 %{{.+}}, 10000000000
+// CHECK:   sdiv i64 %{{.+}}, 100000
+// CHECK:   srem i64 %{{.+}}, 100000
+// CHECK:   call void @body(
+extern "C" void baz() {
+#pragma omp flatten
+  for (int i = 0; i < 100000; ++i)
+    for (int j = 0; j < 100000; ++j)
+      body(i, j);
+}
+
+#endif
diff --git a/clang/test/OpenMP/flatten_depth_codegen.cpp b/clang/test/OpenMP/flatten_depth_codegen.cpp
new file mode 100644
index 0000000000000..77408fbcbdcf1
--- /dev/null
+++ b/clang/test/OpenMP/flatten_depth_codegen.cpp
@@ -0,0 +1,154 @@
+// Check code generation for the 'depth' clause on '#pragma omp flatten'.
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=61 -emit-llvm %s -o - | FileCheck %s
+
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+extern "C" void body(int, int, int);
+
+// CHECK-LABEL: define {{.*}}void @foo3(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK:   %.flatten.iv.2 = alloca i32
+// CHECK:   %[[CIV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[T01:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   %[[TALL:.+]] = mul nsw i64 %[[T01]], %{{.+}}
+// CHECK:   icmp slt i64 %[[CIV]], %[[TALL]]
+// CHECK:   %[[D0IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[D0M:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   %[[D0:.+]] = sdiv i64 %[[D0IV]], %[[D0M]]
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   %[[D1IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[D1:.+]] = sdiv i64 %[[D1IV]], %{{.+}}
+// CHECK:   %[[R1:.+]] = srem i64 %[[D1]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+// CHECK:   %[[R2IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[R2:.+]] = srem i64 %[[R2IV]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.2
+// CHECK:   call void @body(
+extern "C" void foo3(int n, int m, int p) {
+#pragma omp flatten depth(3)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        body(i, j, k);
+}
+
+// CHECK-LABEL: define {{.*}}void @foo2(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK:   %[[CIV2:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[M:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   icmp slt i64 %[[CIV2]], %[[M]]
+// CHECK:   sdiv i64 %{{.+}}, %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   srem i64 %{{.+}}, %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+extern "C" void foo2(int n, int m) {
+#pragma omp flatten depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j, 0);
+}
+
+// CHECK-LABEL: define {{.*}}void @foo1(
+// CHECK:   %.flatten.iv = alloca i32
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK: for.body:
+// CHECK:   %[[FV:.+]] = load i32, ptr %.flatten.iv,
+// CHECK:   store i32 %[[FV]], ptr %.flatten.iv.0,
+// CHECK-NOT: sdiv
+// CHECK-NOT: srem
+// CHECK:   call void @body(
+extern "C" void foo1(int n) {
+#pragma omp flatten depth(1)
+  for (int i = 0; i < n; ++i)
+    body(i, 0, 0);
+}
+
+// CHECK-LABEL: define {{.*}}void @foo2partial(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK-NOT: %.flatten.iv.2 = alloca
+// CHECK:   %[[CIV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[M:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   icmp slt i64 %[[CIV]], %[[M]]
+// CHECK:   sdiv i64 %{{.+}}, %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   srem i64 %{{.+}}, %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+// CHECK:   br label %for.cond{{.*}}
+// CHECK: for.cond{{.*}}:
+// CHECK:   icmp slt i32 %{{.+}}, 5
+extern "C" void foo2partial(int n, int m) {
+#pragma omp flatten depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < 5; ++k)
+        body(i, j, k);
+}
+
+// A value-dependent 'depth' (the argument is a non-type template parameter) is
+// deferred until instantiation, where the concrete depth drives the flattening.
+
+// CHECK-LABEL: define {{.*}}void @_Z11tmpl_depth3ILi3EEviii(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK:   %.flatten.iv.2 = alloca i32
+// CHECK:   %[[CIV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[T01:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   %[[TALL:.+]] = mul nsw i64 %[[T01]], %{{.+}}
+// CHECK:   icmp slt i64 %[[CIV]], %[[TALL]]
+// CHECK:   %[[D0IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[D0M:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   %[[D0:.+]] = sdiv i64 %[[D0IV]], %[[D0M]]
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   %[[D1IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[D1:.+]] = sdiv i64 %[[D1IV]], %{{.+}}
+// CHECK:   %[[R1:.+]] = srem i64 %[[D1]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+// CHECK:   %[[R2IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[R2:.+]] = srem i64 %[[R2IV]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.2
+// CHECK:   call void @body(
+template <int D>
+void tmpl_depth3(int n, int m, int p) {
+#pragma omp flatten depth(D)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        body(i, j, k);
+}
+
+// A value-dependent 'depth' that instantiates to depth(1) on a single loop is
+// well-formed: the shallower-than-default nest is accepted because validation is
+// deferred to instantiation. The single loop needs no div/mod recovery.
+
+// CHECK-LABEL: define {{.*}}void @_Z11tmpl_depth1ILi1EEvi(
+// CHECK:   %.flatten.iv = alloca i32
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK: for.body:
+// CHECK:   %[[FV:.+]] = load i32, ptr %.flatten.iv,
+// CHECK:   store i32 %[[FV]], ptr %.flatten.iv.0,
+// CHECK-NOT: sdiv
+// CHECK-NOT: srem
+// CHECK:   call void @body(
+template <int D>
+void tmpl_depth1(int n) {
+#pragma omp flatten depth(D)
+  for (int i = 0; i < n; ++i)
+    body(i, 0, 0);
+}
+
+void inst() {
+  tmpl_depth3<3>(1, 1, 1);
+  tmpl_depth1<1>(1);
+}
+
+#endif
diff --git a/openmp/runtime/test/transform/flatten/depth-intfor.c b/openmp/runtime/test/transform/flatten/depth-intfor.c
new file mode 100644
index 0000000000000..7211e02e25313
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/depth-intfor.c
@@ -0,0 +1,40 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+// 'depth(3)' fully flattens a three-deep perfectly nested loop into one loop.
+// This verifies that the flattened loop preserves the exact row-major
+// visitation order of the original nest, including non-unit steps and non-zero
+// start values.
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+int main() {
+  printf("do\n");
+#pragma omp flatten depth(3)
+  for (int i = 7; i < 11; i += 2)
+    for (int j = 1; j < 4; j += 1)
+      for (int k = 0; k < 2; ++k)
+        printf("i=%d j=%d k=%d\n", i, j, k);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: i=7 j=1 k=0
+// CHECK-NEXT: i=7 j=1 k=1
+// CHECK-NEXT: i=7 j=2 k=0
+// CHECK-NEXT: i=7 j=2 k=1
+// CHECK-NEXT: i=7 j=3 k=0
+// CHECK-NEXT: i=7 j=3 k=1
+// CHECK-NEXT: i=9 j=1 k=0
+// CHECK-NEXT: i=9 j=1 k=1
+// CHECK-NEXT: i=9 j=2 k=0
+// CHECK-NEXT: i=9 j=2 k=1
+// CHECK-NEXT: i=9 j=3 k=0
+// CHECK-NEXT: i=9 j=3 k=1
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/flatten/intfor.c b/openmp/runtime/test/transform/flatten/intfor.c
new file mode 100644
index 0000000000000..f937659fbebe1
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/intfor.c
@@ -0,0 +1,33 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+int main() {
+  printf("do\n");
+#pragma omp flatten
+  for (int i = 5; i < 11; i += 2)
+    for (int j = 1; j < 7; j += 2)
+      printf("i=%d j=%d\n", i, j);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// The flattened loop visits the original iteration space in the original
+// (row-major) order: for each outer iteration the inner loop runs fully.
+// CHECK:      do
+// CHECK-NEXT: i=5 j=1
+// CHECK-NEXT: i=5 j=3
+// CHECK-NEXT: i=5 j=5
+// CHECK-NEXT: i=7 j=1
+// CHECK-NEXT: i=7 j=3
+// CHECK-NEXT: i=7 j=5
+// CHECK-NEXT: i=9 j=1
+// CHECK-NEXT: i=9 j=3
+// CHECK-NEXT: i=9 j=5
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/flatten/lit.local.cfg b/openmp/runtime/test/transform/flatten/lit.local.cfg
new file mode 100644
index 0000000000000..e05bd627df1e9
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/lit.local.cfg
@@ -0,0 +1,5 @@
+# The flatten directive's depth clause requires OpenMP 6.1.
+for i, (pattern, replacement) in enumerate(config.substitutions):
+    if pattern == "%openmp_flags":
+        config.substitutions[i] = (pattern, replacement + " -fopenmp-version=61")
+        break

>From 667fe837a39dbc711dcc5288b736aa52fbe893b6 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 5 Aug 2026 04:48:45 -0400
Subject: [PATCH 14/14] TBD

---
 clang/docs/OpenMPSupport.rst                  |   2 +-
 clang/include/clang/ASTMatchers/ASTMatchers.h |  14 ++
 clang/lib/ASTMatchers/ASTMatchersInternal.cpp |   2 +
 clang/lib/ASTMatchers/Dynamic/Registry.cpp    |   1 +
 clang/test/Index/openmp-flatten.c             |  12 ++
 clang/test/OpenMP/flatten_ast_print.cpp       | 124 ++++++++++++++++++
 clang/test/OpenMP/flatten_codegen_for.cpp     |  67 ++++++++++
 clang/test/OpenMP/flatten_depth_ast_print.cpp |  74 +++++++++++
 clang/test/OpenMP/flatten_depth_messages.cpp  |  48 +++++++
 clang/test/OpenMP/flatten_messages.cpp        |  89 +++++++++++++
 .../test/OpenMP/flatten_serialize_module.cpp  |  25 ++++
 .../ASTMatchers/ASTMatchersNodeTest.cpp       |  22 ++++
 flatten-verify-tmp/01_default_depth2.cpp      |   8 ++
 flatten-verify-tmp/02_depth3_full.cpp         |   9 ++
 flatten-verify-tmp/03_depth2_partial.cpp      |  10 ++
 flatten-verify-tmp/04_depth1_identity.cpp     |   8 ++
 flatten-verify-tmp/05_rangefor_template.cpp   |  20 +++
 flatten-verify-tmp/README.md                  |  39 ++++++
 flatten-verify-tmp/end_result_check.cpp       |  82 ++++++++++++
 .../test/transform/flatten/depth-partial.c    |  45 +++++++
 openmp/runtime/test/transform/flatten/empty.c |  42 ++++++
 .../test/transform/flatten/foreach.cpp        |  31 +++++
 .../flatten/parallel-wsloop-foreach.cpp       |  32 +++++
 23 files changed, 805 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/Index/openmp-flatten.c
 create mode 100644 clang/test/OpenMP/flatten_ast_print.cpp
 create mode 100644 clang/test/OpenMP/flatten_codegen_for.cpp
 create mode 100644 clang/test/OpenMP/flatten_depth_ast_print.cpp
 create mode 100644 clang/test/OpenMP/flatten_depth_messages.cpp
 create mode 100644 clang/test/OpenMP/flatten_messages.cpp
 create mode 100644 clang/test/OpenMP/flatten_serialize_module.cpp
 create mode 100644 flatten-verify-tmp/01_default_depth2.cpp
 create mode 100644 flatten-verify-tmp/02_depth3_full.cpp
 create mode 100644 flatten-verify-tmp/03_depth2_partial.cpp
 create mode 100644 flatten-verify-tmp/04_depth1_identity.cpp
 create mode 100644 flatten-verify-tmp/05_rangefor_template.cpp
 create mode 100644 flatten-verify-tmp/README.md
 create mode 100644 flatten-verify-tmp/end_result_check.cpp
 create mode 100644 openmp/runtime/test/transform/flatten/depth-partial.c
 create mode 100644 openmp/runtime/test/transform/flatten/empty.c
 create mode 100644 openmp/runtime/test/transform/flatten/foreach.cpp
 create mode 100644 openmp/runtime/test/transform/flatten/parallel-wsloop-foreach.cpp

diff --git a/clang/docs/OpenMPSupport.rst b/clang/docs/OpenMPSupport.rst
index 0c21aa41ff269..c11a46c3bb6a6 100644
--- a/clang/docs/OpenMPSupport.rst
+++ b/clang/docs/OpenMPSupport.rst
@@ -666,7 +666,7 @@ implementation.
 +=============================================================+===========================+===========================+==========================================================================+
 | dyn_groupprivate clause                                     | :part:`partial`           | :part:`In Progress`       | C/C++: Host device support missing                                       |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
-| loop flatten transformation                                 | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
+| loop flatten transformation                                 | :part:`partial`           | :none:`unclaimed`         | Clang: depth clause supported (OpenMP 6.1); apply clause not yet supported|
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | loop grid/tile modifiers for sizes clause                   | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
diff --git a/clang/include/clang/ASTMatchers/ASTMatchers.h b/clang/include/clang/ASTMatchers/ASTMatchers.h
index bc0f35898a2c9..602bab9aa8cb8 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchers.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchers.h
@@ -8842,6 +8842,20 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt,
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, OMPSplitDirective>
     ompSplitDirective;
 
+/// Matches OpenMP ``flatten`` directive.
+///
+/// Given
+///
+/// \code
+///   #pragma omp flatten
+///   for (int i = 0; i < m; ++i)
+///     for (int j = 0; j < n; ++j) {}
+/// \endcode
+///
+/// ``ompFlattenDirective()`` matches the flatten directive.
+extern const internal::VariadicDynCastAllOfMatcher<Stmt, OMPFlattenDirective>
+    ompFlattenDirective;
+
 /// Matches OpenMP ``counts`` clause used by ``#pragma omp split``.
 ///
 /// Given ``#pragma omp split counts(1, 2, omp_fill)``, ``ompCountsClause()``
diff --git a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
index 5cbf134620e34..e2c0c25518b70 100644
--- a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
+++ b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
@@ -1141,6 +1141,8 @@ const internal::VariadicDynCastAllOfMatcher<Stmt, OMPTargetUpdateDirective>
     ompTargetUpdateDirective;
 const internal::VariadicDynCastAllOfMatcher<Stmt, OMPSplitDirective>
     ompSplitDirective;
+const internal::VariadicDynCastAllOfMatcher<Stmt, OMPFlattenDirective>
+    ompFlattenDirective;
 const internal::VariadicDynCastAllOfMatcher<OMPClause, OMPCountsClause>
     ompCountsClause;
 const internal::VariadicDynCastAllOfMatcher<OMPClause, OMPDefaultClause>
diff --git a/clang/lib/ASTMatchers/Dynamic/Registry.cpp b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
index a04070971f0eb..a6252363e08ce 100644
--- a/clang/lib/ASTMatchers/Dynamic/Registry.cpp
+++ b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
@@ -534,6 +534,7 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(ompFromClause);
   REGISTER_MATCHER(ompToClause);
   REGISTER_MATCHER(ompExecutableDirective);
+  REGISTER_MATCHER(ompFlattenDirective);
   REGISTER_MATCHER(ompSplitDirective);
   REGISTER_MATCHER(ompTargetUpdateDirective);
   REGISTER_MATCHER(on);
diff --git a/clang/test/Index/openmp-flatten.c b/clang/test/Index/openmp-flatten.c
new file mode 100644
index 0000000000000..74bc3cda3ef1a
--- /dev/null
+++ b/clang/test/Index/openmp-flatten.c
@@ -0,0 +1,12 @@
+// RUN: c-index-test -test-load-source local %s -fopenmp=libomp -fopenmp-version=60 | FileCheck %s
+
+void test(void) {
+#pragma omp flatten
+  for (int i = 0; i < 20; i += 1)
+    for (int j = 0; j < 30; j += 1)
+      ;
+}
+
+// CHECK: openmp-flatten.c:4:1: OMPFlattenDirective= Extent=[4:1 - 4:20]
+// CHECK: openmp-flatten.c:5:3: ForStmt= Extent=[5:3 - 7:8]
+// CHECK: openmp-flatten.c:6:5: ForStmt= Extent=[6:5 - 7:8]
diff --git a/clang/test/OpenMP/flatten_ast_print.cpp b/clang/test/OpenMP/flatten_ast_print.cpp
new file mode 100644
index 0000000000000..e1115bf112c40
--- /dev/null
+++ b/clang/test/OpenMP/flatten_ast_print.cpp
@@ -0,0 +1,124 @@
+// Check no warnings/errors
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -fsyntax-only -verify %s
+// expected-no-diagnostics
+
+// Check AST and unparsing
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -ast-dump  %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -ast-print %s | FileCheck %s --check-prefix=PRINT
+
+// Check same results after serialization round-trip
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -emit-pch -o %t %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -include-pch %t -ast-dump-all %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -include-pch %t -ast-print    %s | FileCheck %s --check-prefix=PRINT
+
+#ifndef HEADER
+#define HEADER
+
+// placeholder for loop body code.
+extern "C" void body(...);
+
+// PRINT-LABEL: void foo1(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo1
+void foo1() {
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (int i = 0; i < 7; i += 1)
+  // DUMP-NEXT: ForStmt
+  for (int i = 0; i < 7; i += 1)
+    // PRINT: for (int j = 0; j < 9; j += 1)
+    // DUMP:  ForStmt
+    for (int j = 0; j < 9; j += 1)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// PRINT-LABEL: void foo2(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo2
+void foo2(int start1, int end1, int step1, int start2, int end2, int step2) {
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (int i = start1; i < end1; i += step1)
+  // DUMP-NEXT: ForStmt
+  for (int i = start1; i < end1; i += step1)
+    // PRINT: for (int j = start2; j < end2; j += step2)
+    // DUMP:  ForStmt
+    for (int j = start2; j < end2; j += step2)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// PRINT-LABEL: void foo3(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo3
+void foo3() {
+  // Verify that a flattened loop can itself be the operand of another loop
+  // transformation. The outer reverse consumes the loop generated by flatten.
+  // PRINT:     #pragma omp reverse
+  // DUMP:      OMPReverseDirective
+  #pragma omp reverse
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (int i = 0; i < 13; i += 1)
+  // DUMP-NEXT: ForStmt
+  for (int i = 0; i < 13; i += 1)
+    // PRINT: for (int j = 0; j < 17; j += 1)
+    // DUMP:  ForStmt
+    for (int j = 0; j < 17; j += 1)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// Range-based for loops are supported as affected loops.
+// PRINT-LABEL: void foo4(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo4
+struct Range {
+  int *begin();
+  int *end();
+};
+void foo4(Range r1, Range r2) {
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (int i : r1)
+  // DUMP-NEXT: CXXForRangeStmt
+  for (int i : r1)
+    // PRINT: for (int j : r2)
+    // DUMP:  CXXForRangeStmt
+    for (int j : r2)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// The directive is instantiated together with its enclosing template.
+// PRINT-LABEL: template <typename T, int LO, int HI> void foo5()
+// DUMP-LABEL:  FunctionTemplateDecl {{.*}} foo5
+template <typename T, int LO, int HI>
+void foo5() {
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (T i = LO; i < HI; ++i)
+  // DUMP-NEXT: ForStmt
+  for (T i = LO; i < HI; ++i)
+    // PRINT: for (T j = LO; j < HI; ++j)
+    // DUMP:  ForStmt
+    for (T j = LO; j < HI; ++j)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// PRINT-LABEL: template<> void foo5<int, 0, 8>()
+// DUMP-LABEL:  FunctionDecl {{.*}} foo5 'void ()' implicit_instantiation
+// DUMP:        OMPFlattenDirective
+void inst() {
+  foo5<int, 0, 8>();
+}
+
+#endif
diff --git a/clang/test/OpenMP/flatten_codegen_for.cpp b/clang/test/OpenMP/flatten_codegen_for.cpp
new file mode 100644
index 0000000000000..d64128bd25d78
--- /dev/null
+++ b/clang/test/OpenMP/flatten_codegen_for.cpp
@@ -0,0 +1,67 @@
+// Check that the loop generated by 'flatten' can be consumed by a loop
+// associated directive ('omp for' / 'omp parallel for'), and that signed and
+// unsigned induction variables generate signed/unsigned division.
+
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=60 -emit-llvm %s -o - | FileCheck %s
+
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+extern "C" void body(int, int);
+extern "C" void ubody(unsigned, unsigned);
+
+// 'omp for' worksharing over the flattened iteration space: the flattened loop
+// uses a 64-bit induction variable for the variable-bound product trip count,
+// so the worksharing loop is distributed with the 64-bit static-init call and
+// the body recovers the logical counters via signed division/remainder.
+//
+// CHECK-LABEL: define {{.*}}void @wsfor(
+// CHECK:   mul nsw i64
+// CHECK:   call void @__kmpc_for_static_init_8(
+// CHECK:   sdiv i64
+// CHECK:   srem i64
+// CHECK:   call void @body(
+// CHECK:   call void @__kmpc_for_static_fini(
+extern "C" void wsfor(int n, int m) {
+#pragma omp for
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+
+// 'omp parallel for' outlines the flattened loop and worksharing-distributes
+// it.
+//
+// CHECK-LABEL: define {{.*}}void @pfor(
+// CHECK:   call {{.*}}@__kmpc_fork_call(
+// CHECK-LABEL: define internal void @pfor.omp_outlined(
+// CHECK:   call void @__kmpc_for_static_init_8(
+// CHECK:   sdiv i64
+// CHECK:   srem i64
+// CHECK:   call void @body(
+extern "C" void pfor(int n, int m) {
+#pragma omp parallel for
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+
+// Unsigned induction variables produce unsigned division/remainder.
+//
+// CHECK-LABEL: define {{.*}}void @ufor(
+// CHECK:   icmp ult i32 %{{.+}}, 63
+// CHECK:   udiv i32 %{{.+}}, 9
+// CHECK:   urem i32 %{{.+}}, 9
+// CHECK:   call void @ubody(
+extern "C" void ufor() {
+#pragma omp flatten
+  for (unsigned i = 0; i < 7u; ++i)
+    for (unsigned j = 0; j < 9u; ++j)
+      ubody(i, j);
+}
+
+#endif
diff --git a/clang/test/OpenMP/flatten_depth_ast_print.cpp b/clang/test/OpenMP/flatten_depth_ast_print.cpp
new file mode 100644
index 0000000000000..ceaaa68e1f79a
--- /dev/null
+++ b/clang/test/OpenMP/flatten_depth_ast_print.cpp
@@ -0,0 +1,74 @@
+// Check no warnings/errors
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -fsyntax-only -verify %s
+// expected-no-diagnostics
+
+// Check AST and unparsing
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-dump  %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-print %s | FileCheck %s --check-prefix=PRINT
+
+// Check same results after serialization round-trip
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -emit-pch -o %t %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -include-pch %t -ast-dump-all %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -include-pch %t -ast-print    %s | FileCheck %s --check-prefix=PRINT
+
+#ifndef HEADER
+#define HEADER
+
+// placeholder for loop body code.
+extern "C" void body(...);
+
+// The depth clause round-trips through -ast-print and PCH serialization.
+// PRINT-LABEL: void foo_depth3(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo_depth3
+void foo_depth3(int n, int m, int p) {
+  // PRINT:     #pragma omp flatten depth(3)
+  // DUMP:      OMPFlattenDirective
+  // DUMP-NEXT: OMPDepthClause
+  #pragma omp flatten depth(3)
+  // PRINT: for (int i = 0; i < n; ++i)
+  // DUMP:  ForStmt
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        // PRINT: body(i, j, k);
+        // DUMP:  CallExpr
+        body(i, j, k);
+}
+
+// depth(2) is accepted and printed like any other depth argument.
+// PRINT-LABEL: void foo_depth2(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo_depth2
+void foo_depth2(int n, int m) {
+  // PRINT:     #pragma omp flatten depth(2)
+  // DUMP:      OMPFlattenDirective
+  // DUMP-NEXT: OMPDepthClause
+  #pragma omp flatten depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+
+// The depth clause is instantiated together with its enclosing template.
+// PRINT-LABEL: template <typename T> void foo_tmpl()
+// DUMP-LABEL:  FunctionTemplateDecl {{.*}} foo_tmpl
+template <typename T>
+void foo_tmpl() {
+  // PRINT:     #pragma omp flatten depth(3)
+  // DUMP:      OMPFlattenDirective
+  // DUMP-NEXT: OMPDepthClause
+  #pragma omp flatten depth(3)
+  for (T i = 0; i < 8; ++i)
+    for (T j = 0; j < 8; ++j)
+      for (T k = 0; k < 8; ++k)
+        body(i, j, k);
+}
+
+// PRINT-LABEL: template<> void foo_tmpl<int>()
+// DUMP-LABEL:  FunctionDecl {{.*}} foo_tmpl 'void ()' implicit_instantiation
+// DUMP:        OMPFlattenDirective
+// DUMP-NEXT:   OMPDepthClause
+void inst() {
+  foo_tmpl<int>();
+}
+
+#endif
diff --git a/clang/test/OpenMP/flatten_depth_messages.cpp b/clang/test/OpenMP/flatten_depth_messages.cpp
new file mode 100644
index 0000000000000..16f60318d0d06
--- /dev/null
+++ b/clang/test/OpenMP/flatten_depth_messages.cpp
@@ -0,0 +1,48 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=61 -fsyntax-only -Wuninitialized -verify %s
+
+extern "C" void body(...);
+
+// expected-note at +1 {{declared here}}
+void func(int n) {
+
+  // The depth argument must be a positive integer constant expression.
+  // expected-error at +1 {{argument to 'depth' clause must be a strictly positive integer value}}
+  #pragma omp flatten depth(0)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // expected-error at +1 {{argument to 'depth' clause must be a strictly positive integer value}}
+  #pragma omp flatten depth(-1)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // A non-constant argument is rejected.
+  // expected-error at +2 {{expression is not an integral constant expression}}
+  // expected-note at +1 {{function parameter 'n' with unknown value cannot be used in a constant expression}}
+  #pragma omp flatten depth(n)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // At most one 'depth' clause may appear on the directive.
+  // expected-error at +1 {{directive '#pragma omp flatten' cannot contain more than one 'depth' clause}}
+  #pragma omp flatten depth(2) depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // The depth must be at most the loop nest depth: depth(3) requires three
+  // perfectly nested loops, but only two are present here.
+  #pragma omp flatten depth(3)
+  // expected-error at +1 {{expected 3 for loops after '#pragma omp flatten', but found only 2}}
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // depth(1) flattens a single loop and is well-formed (no diagnostic).
+  #pragma omp flatten depth(1)
+  for (int i = 0; i < n; ++i)
+    body(i);
+}
diff --git a/clang/test/OpenMP/flatten_messages.cpp b/clang/test/OpenMP/flatten_messages.cpp
new file mode 100644
index 0000000000000..a022e0e8999d2
--- /dev/null
+++ b/clang/test/OpenMP/flatten_messages.cpp
@@ -0,0 +1,89 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=60 -fsyntax-only -Wuninitialized -verify %s
+
+extern "C" void body(...);
+
+void func(int n) {
+
+  // The associated statement must be a for loop.
+  // expected-error at +2 {{statement after '#pragma omp flatten' must be a for loop}}
+  #pragma omp flatten
+  ;
+
+  // A non-loop statement is rejected as well.
+  // expected-error at +2 {{statement after '#pragma omp flatten' must be a for loop}}
+  #pragma omp flatten
+  int b = 0;
+
+  // A single loop is not enough: flatten combines two perfectly nested loops,
+  // so the body of the outer loop must itself be a for loop.
+  #pragma omp flatten
+  for (int i = 0; i < 7; ++i)
+    // expected-error at +1 {{statement after '#pragma omp flatten' must be a for loop}}
+    ;
+
+  // The associated statement of a directive is not a for loop.
+  // expected-error at +2 {{statement after '#pragma omp flatten' must be a for loop}}
+  #pragma omp flatten
+  #pragma omp for
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 7; ++j)
+      body(i, j);
+
+  {
+    // expected-error at +2 {{expected statement}}
+    #pragma omp flatten
+  }
+
+  // The loops must be perfectly nested: no code is allowed between them.
+  #pragma omp flatten
+  for (int i = 0; i < n; ++i) {
+    int x = 0;
+    // expected-error at -2 {{statement after '#pragma omp flatten' must be a for loop}}
+    for (int j = 0; j < n; ++j)
+      body(i, j, x);
+  }
+
+  // Each affected loop must be in OpenMP canonical form.
+  #pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    // expected-error at +1 {{condition of OpenMP for loop must be a relational comparison ('<', '<=', '>', '>=', or '!=') of loop variable 'j'}}
+    for (int j = 0; j / 3 < n; ++j)
+      body(i, j);
+
+  // The affected loops must be rectangular: an inner bound may not depend on an
+  // outer loop counter.
+  #pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    // expected-error at +1 {{expected loop invariant expression}}
+    for (int j = i; j < n; ++j)
+      body(i, j);
+
+  // The 'sizes' clause is not allowed on 'flatten'.
+  // expected-error at +1 {{unexpected OpenMP clause 'sizes' in directive '#pragma omp flatten'}}
+  #pragma omp flatten sizes(2)
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+
+  // The 'permutation' clause is not allowed on 'flatten'.
+  // expected-error at +1 {{unexpected OpenMP clause 'permutation' in directive '#pragma omp flatten'}}
+  #pragma omp flatten permutation(2, 1)
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+
+  // Tokens after the directive name are ignored with a warning.
+  // expected-warning at +1 {{extra tokens at the end of '#pragma omp flatten' are ignored}}
+  #pragma omp flatten foo
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+
+  // The 'depth' clause is only available from OpenMP 6.1; it is rejected here
+  // under -fopenmp-version=60.
+  // expected-error at +1 {{unexpected OpenMP clause 'depth' in directive '#pragma omp flatten'}}
+  #pragma omp flatten depth(2)
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+}
diff --git a/clang/test/OpenMP/flatten_serialize_module.cpp b/clang/test/OpenMP/flatten_serialize_module.cpp
new file mode 100644
index 0000000000000..5d3420500b282
--- /dev/null
+++ b/clang/test/OpenMP/flatten_serialize_module.cpp
@@ -0,0 +1,25 @@
+// C++20 module interface with `#pragma omp flatten` — emit BMI + import; AST retains directive.
+//
+// RUN: rm -rf %t && split-file %s %t && cd %t
+// RUN: %clang_cc1 -std=c++20 -fopenmp -fopenmp-version=60 -triple x86_64-unknown-linux-gnu %t/FlattenMod.cppm -emit-module-interface -o %t/FlattenMod.pcm
+// RUN: %clang_cc1 -std=c++20 -fopenmp -fopenmp-version=60 -triple x86_64-unknown-linux-gnu %t/UseFlattenMod.cpp -fmodule-file=FlattenMod=%t/FlattenMod.pcm -ast-dump-all | FileCheck %t/FlattenMod.cppm
+
+// expected-no-diagnostics
+
+//--- FlattenMod.cppm
+module;
+export module FlattenMod;
+
+export void flattenfoo(int n, int m) {
+// CHECK: OMPFlattenDirective
+// CHECK: ForStmt
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j) {
+    }
+}
+
+//--- UseFlattenMod.cpp
+import FlattenMod;
+
+void g(void) { flattenfoo(10, 20); }
diff --git a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
index 4190d4703e37d..34c0fd27821d1 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
@@ -3126,6 +3126,28 @@ void f() {
   EXPECT_TRUE(notMatchesWithOpenMP60(ParallelOnly, Matcher));
 }
 
+// OpenMP 6 flatten directive
+TEST(ASTMatchersTestOpenMP, OMPFlattenDirective) {
+  auto Matcher = stmt(ompFlattenDirective());
+
+  StringRef FlattenOk = R"(
+void f() {
+#pragma omp flatten
+  for (int i = 0; i < 10; ++i)
+    for (int j = 0; j < 10; ++j) {}
+}
+)";
+  EXPECT_TRUE(matchesWithOpenMP60(FlattenOk, Matcher));
+
+  StringRef ParallelOnly = R"(
+void f() {
+#pragma omp parallel
+  ;
+}
+)";
+  EXPECT_TRUE(notMatchesWithOpenMP60(ParallelOnly, Matcher));
+}
+
 TEST(ASTMatchersTestOpenMP, OMPSplitDirective_HasCountsClause) {
   auto Matcher = stmt(ompSplitDirective(hasAnyClause(ompCountsClause())));
 
diff --git a/flatten-verify-tmp/01_default_depth2.cpp b/flatten-verify-tmp/01_default_depth2.cpp
new file mode 100644
index 0000000000000..900d0263622cd
--- /dev/null
+++ b/flatten-verify-tmp/01_default_depth2.cpp
@@ -0,0 +1,8 @@
+// Default flatten: combine 2 perfectly nested loops (depth defaults to 2).
+extern "C" void body(int, int);
+void t(int n, int m) {
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
diff --git a/flatten-verify-tmp/02_depth3_full.cpp b/flatten-verify-tmp/02_depth3_full.cpp
new file mode 100644
index 0000000000000..ba9f0b7ed8808
--- /dev/null
+++ b/flatten-verify-tmp/02_depth3_full.cpp
@@ -0,0 +1,9 @@
+// depth(3): fully flatten a three-deep perfect nest (OpenMP 6.1).
+extern "C" void body(int, int, int);
+void t(int n, int m, int p) {
+#pragma omp flatten depth(3)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        body(i, j, k);
+}
diff --git a/flatten-verify-tmp/03_depth2_partial.cpp b/flatten-verify-tmp/03_depth2_partial.cpp
new file mode 100644
index 0000000000000..35b9dc83834fd
--- /dev/null
+++ b/flatten-verify-tmp/03_depth2_partial.cpp
@@ -0,0 +1,10 @@
+// depth(2) on a three-deep nest: only the outer two loops are combined; the
+// innermost loop must remain as ordinary body of the flattened loop.
+extern "C" void body(int, int, int);
+void t(int n, int m, int p) {
+#pragma omp flatten depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        body(i, j, k);
+}
diff --git a/flatten-verify-tmp/04_depth1_identity.cpp b/flatten-verify-tmp/04_depth1_identity.cpp
new file mode 100644
index 0000000000000..ae62a11312fe8
--- /dev/null
+++ b/flatten-verify-tmp/04_depth1_identity.cpp
@@ -0,0 +1,8 @@
+// depth(1): identity transform. A single loop is "combined"; no mixed-radix
+// decomposition (no srem), the flattened counter maps straight to the loop var.
+extern "C" void body(int);
+void t(int n) {
+#pragma omp flatten depth(1)
+  for (int i = 0; i < n; ++i)
+    body(i);
+}
diff --git a/flatten-verify-tmp/05_rangefor_template.cpp b/flatten-verify-tmp/05_rangefor_template.cpp
new file mode 100644
index 0000000000000..89928c34a8a15
--- /dev/null
+++ b/flatten-verify-tmp/05_rangefor_template.cpp
@@ -0,0 +1,20 @@
+// Range-for (CXXForRangeStmt) nest + a function template instantiated with a
+// dependent depth(K). Exercises the __begin/__end pre-inits and the deferred
+// (dependent-context) desugaring path.
+#include <stdlibc++.h>
+extern "C" void body(int, int);
+
+void ranges(std::vector<int> &a, std::vector<int> &b) {
+#pragma omp flatten
+  for (int i : a)
+    for (int j : b)
+      body(i, j);
+}
+
+template <int K> void tmpl(int n, int m) {
+#pragma omp flatten depth(K)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+template void tmpl<2>(int, int);
diff --git a/flatten-verify-tmp/README.md b/flatten-verify-tmp/README.md
new file mode 100644
index 0000000000000..f48f4d06cfec0
--- /dev/null
+++ b/flatten-verify-tmp/README.md
@@ -0,0 +1,39 @@
+# Temporary flatten + depth verification cases
+
+Throwaway sandbox for eyeballing the flatten desugaring (AST / IR / runtime).
+Not part of any PR — delete freely. The committed regression tests live under
+`clang/test/OpenMP/flatten*` and `openmp/runtime/test/transform/flatten/*`.
+
+`CLANG=build/bin/clang`, `CLANGXX=build/bin/clang++`, run from the repo root.
+
+| File | Case | What to look for |
+|------|------|------------------|
+| `01_default_depth2.cpp` | default (no clause) = depth 2 | i64 `.flatten.iv`, `mul nsw`, `sdiv`+`srem` |
+| `02_depth3_full.cpp` | `depth(3)` full flatten | 3 `.flatten.iv.N`, two-level mixed radix |
+| `03_depth2_partial.cpp` | `depth(2)` on 3-deep nest | 2 flatten IVs, **inner `%k` loop stays** |
+| `04_depth1_identity.cpp` | `depth(1)` identity | i32 IV, **no `srem`**, direct store |
+| `05_rangefor_template.cpp` | range-for + template `depth(K)` | ast-print round-trips; instantiation |
+| `end_result_check.cpp` | runtime semantic check | prints `OK`, exit 0 (order+set preserved) |
+
+## Commands
+
+AST (round-trip original loops) and the desugared node:
+```
+$CLANG -cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-print FILE
+$CLANG -cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-dump  FILE
+```
+
+IR (the div/mod arithmetic that implements the transform):
+```
+$CLANG -cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -emit-llvm -o - FILE
+```
+
+Range-for / template need the driver (libstdc++ + template instantiation):
+```
+$CLANGXX -fopenmp -fopenmp-version=61 -Xclang -ast-print -fsyntax-only 05_rangefor_template.cpp
+```
+
+Runtime end-result (must print OK, exit 0):
+```
+$CLANGXX -fopenmp -fopenmp-version=61 -O0 end_result_check.cpp -o /tmp/erc && /tmp/erc
+```
diff --git a/flatten-verify-tmp/end_result_check.cpp b/flatten-verify-tmp/end_result_check.cpp
new file mode 100644
index 0000000000000..3aa8850829058
--- /dev/null
+++ b/flatten-verify-tmp/end_result_check.cpp
@@ -0,0 +1,82 @@
+// End-result (semantic) verification: flatten must preserve BOTH the set of
+// iterations and their row-major visitation ORDER. We record the sequence of
+// visited tuples from a flattened nest and compare it against the sequence from
+// an identical non-flattened reference nest. Exit code 0 == identical.
+//
+// flatten is a pure frontend AST desugaring to an ordinary loop, so this runs
+// as a normal program (build with: clang++ -fopenmp -fopenmp-version=61).
+#include <cassert>
+#include <cstdio>
+#include <vector>
+
+using Seq = std::vector<long>;
+
+static void rec(Seq &s, int i, int j, int k = -1) {
+  s.push_back(i);
+  s.push_back(j);
+  if (k >= 0)
+    s.push_back(k);
+}
+
+int main() {
+  const int n = 3, m = 4, p = 2;
+
+  // --- depth(2) default over 2 loops ---
+  {
+    Seq flat, ref;
+#pragma omp flatten
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        rec(flat, i, j);
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        rec(ref, i, j);
+    assert(flat == ref && "depth(2) order/set mismatch");
+  }
+
+  // --- depth(3) full flatten over 3 loops ---
+  {
+    Seq flat, ref;
+#pragma omp flatten depth(3)
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        for (int k = 0; k < p; ++k)
+          rec(flat, i, j, k);
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        for (int k = 0; k < p; ++k)
+          rec(ref, i, j, k);
+    assert(flat == ref && "depth(3) order/set mismatch");
+  }
+
+  // --- depth(2) partial: inner k-loop stays intact ---
+  {
+    Seq flat, ref;
+#pragma omp flatten depth(2)
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        for (int k = 0; k < p; ++k)
+          rec(flat, i, j, k);
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        for (int k = 0; k < p; ++k)
+          rec(ref, i, j, k);
+    assert(flat == ref && "depth(2)-partial order/set mismatch");
+  }
+
+  // --- non-unit step / non-zero start must survive ---
+  {
+    Seq flat, ref;
+#pragma omp flatten
+    for (int i = 5; i < 12; i += 2)
+      for (int j = 1; j < 7; j += 3)
+        rec(flat, i, j);
+    for (int i = 5; i < 12; i += 2)
+      for (int j = 1; j < 7; j += 3)
+        rec(ref, i, j);
+    assert(flat == ref && "stride/offset mismatch");
+  }
+
+  std::puts("flatten end-result preserved: OK");
+  return 0;
+}
diff --git a/openmp/runtime/test/transform/flatten/depth-partial.c b/openmp/runtime/test/transform/flatten/depth-partial.c
new file mode 100644
index 0000000000000..2fda3a04cbd84
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/depth-partial.c
@@ -0,0 +1,45 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+// A depth(2) clause on a three-deep loop nest flattens only the outermost two
+// loops; the innermost loop is left untouched and executes normally inside the
+// flattened loop body.
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+int main() {
+  printf("do\n");
+#pragma omp flatten depth(2)
+  for (int i = 5; i < 11; i += 2)
+    for (int j = 1; j < 7; j += 2)
+      for (int k = 0; k < 2; ++k)
+        printf("i=%d j=%d k=%d\n", i, j, k);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: i=5 j=1 k=0
+// CHECK-NEXT: i=5 j=1 k=1
+// CHECK-NEXT: i=5 j=3 k=0
+// CHECK-NEXT: i=5 j=3 k=1
+// CHECK-NEXT: i=5 j=5 k=0
+// CHECK-NEXT: i=5 j=5 k=1
+// CHECK-NEXT: i=7 j=1 k=0
+// CHECK-NEXT: i=7 j=1 k=1
+// CHECK-NEXT: i=7 j=3 k=0
+// CHECK-NEXT: i=7 j=3 k=1
+// CHECK-NEXT: i=7 j=5 k=0
+// CHECK-NEXT: i=7 j=5 k=1
+// CHECK-NEXT: i=9 j=1 k=0
+// CHECK-NEXT: i=9 j=1 k=1
+// CHECK-NEXT: i=9 j=3 k=0
+// CHECK-NEXT: i=9 j=3 k=1
+// CHECK-NEXT: i=9 j=5 k=0
+// CHECK-NEXT: i=9 j=5 k=1
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/flatten/empty.c b/openmp/runtime/test/transform/flatten/empty.c
new file mode 100644
index 0000000000000..ecf1e8ef622f2
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/empty.c
@@ -0,0 +1,42 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+// The bounds are runtime values so that an empty iteration space does not
+// trigger a compile-time division-by-zero diagnostic for the (never executed)
+// induction-variable recovery.
+static void flatten(int n, int m) {
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      printf("i=%d j=%d\n", i, j);
+}
+
+int main() {
+  printf("empty-inner-begin\n");
+  flatten(3, 0);
+  printf("empty-inner-end\n");
+
+  printf("empty-outer-begin\n");
+  flatten(0, 3);
+  printf("empty-outer-end\n");
+
+  printf("single-begin\n");
+  flatten(1, 1);
+  printf("single-end\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      empty-inner-begin
+// CHECK-NEXT: empty-inner-end
+// CHECK-NEXT: empty-outer-begin
+// CHECK-NEXT: empty-outer-end
+// CHECK-NEXT: single-begin
+// CHECK-NEXT: i=0 j=0
+// CHECK-NEXT: single-end
diff --git a/openmp/runtime/test/transform/flatten/foreach.cpp b/openmp/runtime/test/transform/flatten/foreach.cpp
new file mode 100644
index 0000000000000..14997f7c4051d
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/foreach.cpp
@@ -0,0 +1,31 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdio>
+#include <vector>
+
+int main() {
+  printf("do\n");
+  std::vector<int> outer{10, 20};
+  std::vector<int> inner{1, 2, 3};
+#pragma omp flatten
+  for (int a : outer)
+    for (int b : inner)
+      printf("a=%d b=%d\n", a, b);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: a=10 b=1
+// CHECK-NEXT: a=10 b=2
+// CHECK-NEXT: a=10 b=3
+// CHECK-NEXT: a=20 b=1
+// CHECK-NEXT: a=20 b=2
+// CHECK-NEXT: a=20 b=3
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/flatten/parallel-wsloop-foreach.cpp b/openmp/runtime/test/transform/flatten/parallel-wsloop-foreach.cpp
new file mode 100644
index 0000000000000..7ddcde2fd622c
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/parallel-wsloop-foreach.cpp
@@ -0,0 +1,32 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdio>
+
+int main() {
+  printf("do\n");
+  // The loop generated by 'flatten' is the canonical loop associated with the
+  // worksharing 'omp for'. With a single thread the iteration order is the
+  // original row-major order.
+#pragma omp parallel for num_threads(1)
+#pragma omp flatten
+  for (int i = 0; i < 2; ++i)
+    for (int j = 0; j < 3; ++j)
+      printf("i=%d j=%d\n", i, j);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: i=0 j=0
+// CHECK-NEXT: i=0 j=1
+// CHECK-NEXT: i=0 j=2
+// CHECK-NEXT: i=1 j=0
+// CHECK-NEXT: i=1 j=1
+// CHECK-NEXT: i=1 j=2
+// CHECK-NEXT: done


From openmp-commits at lists.llvm.org  Tue Aug 18 04:34:08 2026
From: openmp-commits at lists.llvm.org (Amit Tiwari via Openmp-commits)
Date: Tue, 18 Aug 2026 04:34:08 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [Clang][OpenMP] Add
 `#pragma omp flatten` loop directive + `depth` clause (PR #206977)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206977@github.com>
Message-ID: <6a844330.6904917c.2bc6c7.601e@mx.google.com>


https://github.com/loopacino updated https://github.com/llvm/llvm-project/pull/206977

>From 0fb5bd42f1e127faea963c359b1e3c69cb969c60 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 1 Jul 2026 09:06:58 -0400
Subject: [PATCH 01/16] add OMPFlattenDirective ast node

---
 clang/include/clang/AST/RecursiveASTVisitor.h |  3 +
 clang/include/clang/AST/StmtOpenMP.h          | 75 ++++++++++++++++++-
 clang/include/clang/Basic/StmtNodes.td        |  1 +
 clang/lib/AST/StmtOpenMP.cpp                  | 20 +++++
 clang/lib/AST/StmtPrinter.cpp                 |  5 ++
 clang/lib/AST/StmtProfile.cpp                 |  4 +
 clang/lib/Basic/OpenMPKinds.cpp               |  2 +-
 7 files changed, 108 insertions(+), 2 deletions(-)

diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index b000a34043696..e425f7a31f59c 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -3227,6 +3227,9 @@ DEF_TRAVERSE_STMT(OMPFuseDirective,
 DEF_TRAVERSE_STMT(OMPInterchangeDirective,
                   { TRY_TO(TraverseOMPExecutableDirective(S)); })
 
+DEF_TRAVERSE_STMT(OMPFlattenDirective,
+                  { TRY_TO(TraverseOMPExecutableDirective(S)); })
+
 DEF_TRAVERSE_STMT(OMPSplitDirective,
                   { TRY_TO(TraverseOMPExecutableDirective(S)); })
 
diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index dbc76e7df8ecd..1420331f37c1f 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -1041,7 +1041,7 @@ class OMPCanonicalLoopNestTransformationDirective
     Stmt::StmtClass C = T->getStmtClass();
     return C == OMPTileDirectiveClass || C == OMPUnrollDirectiveClass ||
            C == OMPReverseDirectiveClass || C == OMPInterchangeDirectiveClass ||
-           C == OMPStripeDirectiveClass;
+           C == OMPStripeDirectiveClass || C == OMPFlattenDirectiveClass;
   }
 };
 
@@ -5959,6 +5959,79 @@ class OMPInterchangeDirective final
   }
 };
 
+/// Represents the '#pragma omp flatten' loop transformation directive.
+///
+/// \code{c}
+///   #pragma omp flatten
+///   for (int i = 0; i < m; ++i)
+///     for (int j = 0; j < n; ++j)
+///       ..
+/// \endcode
+class OMPFlattenDirective final
+    : public OMPCanonicalLoopNestTransformationDirective {
+  friend class ASTStmtReader;
+  friend class OMPExecutableDirective;
+
+  /// Offsets of child members.
+  enum {
+    PreInitsOffset = 0,
+    TransformedStmtOffset,
+  };
+
+  explicit OMPFlattenDirective(SourceLocation StartLoc, SourceLocation EndLoc,
+                               unsigned NumLoops)
+      : OMPCanonicalLoopNestTransformationDirective(
+            OMPFlattenDirectiveClass, llvm::omp::OMPD_flatten, StartLoc, EndLoc,
+            NumLoops) {}
+
+  void setPreInits(Stmt *PreInits) {
+    Data->getChildren()[PreInitsOffset] = PreInits;
+  }
+
+  void setTransformedStmt(Stmt *S) {
+    Data->getChildren()[TransformedStmtOffset] = S;
+  }
+
+public:
+  /// Create a new AST node representation for '#pragma omp flatten'.
+  ///
+  /// \param C         Context of the AST.
+  /// \param StartLoc  Location of the introducer (e.g. the 'omp' token).
+  /// \param EndLoc    Location of the directive's end (e.g. the tok::eod).
+  /// \param Clauses   The directive's clauses.
+  /// \param NumLoops  Number of affected loops (the flatten depth, currently
+  ///                  always 2).
+  /// \param AssociatedStmt  The outermost associated loop.
+  /// \param TransformedStmt The flattened loop, or nullptr in dependent
+  ///                        contexts.
+  /// \param PreInits  Helper preinits statements for the loop nest.
+  static OMPFlattenDirective *
+  Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation EndLoc,
+         ArrayRef<OMPClause *> Clauses, unsigned NumLoops, Stmt *AssociatedStmt,
+         Stmt *TransformedStmt, Stmt *PreInits);
+
+  /// Build an empty '#pragma omp flatten' AST node for deserialization.
+  ///
+  /// \param C          Context of the AST.
+  /// \param NumClauses Number of clauses to allocate.
+  /// \param NumLoops   Number of associated loops to allocate.
+  static OMPFlattenDirective *
+  CreateEmpty(const ASTContext &C, unsigned NumClauses, unsigned NumLoops);
+
+  /// Gets the flattened loop after the transformation. This is the de-sugared
+  /// replacement or nullptr in dependent contexts.
+  Stmt *getTransformedStmt() const {
+    return Data->getChildren()[TransformedStmtOffset];
+  }
+
+  /// Return preinits statement.
+  Stmt *getPreInits() const { return Data->getChildren()[PreInitsOffset]; }
+
+  static bool classof(const Stmt *T) {
+    return T->getStmtClass() == OMPFlattenDirectiveClass;
+  }
+};
+
 /// The base class for all transformation directives of canonical loop
 /// sequences (currently only 'fuse')
 class OMPCanonicalLoopSequenceTransformationDirective
diff --git a/clang/include/clang/Basic/StmtNodes.td b/clang/include/clang/Basic/StmtNodes.td
index e166894ea024b..0d7a0c1490a1e 100644
--- a/clang/include/clang/Basic/StmtNodes.td
+++ b/clang/include/clang/Basic/StmtNodes.td
@@ -247,6 +247,7 @@ def OMPReverseDirective : StmtNode<OMPCanonicalLoopNestTransformationDirective>;
 def OMPSplitDirective : StmtNode<OMPCanonicalLoopNestTransformationDirective>;
 def OMPInterchangeDirective
     : StmtNode<OMPCanonicalLoopNestTransformationDirective>;
+def OMPFlattenDirective : StmtNode<OMPCanonicalLoopNestTransformationDirective>;
 def OMPCanonicalLoopSequenceTransformationDirective
     : StmtNode<OMPExecutableDirective, 1>;
 def OMPFuseDirective
diff --git a/clang/lib/AST/StmtOpenMP.cpp b/clang/lib/AST/StmtOpenMP.cpp
index 9d6b315effb41..477604ef3dafb 100644
--- a/clang/lib/AST/StmtOpenMP.cpp
+++ b/clang/lib/AST/StmtOpenMP.cpp
@@ -552,6 +552,26 @@ OMPInterchangeDirective::CreateEmpty(const ASTContext &C, unsigned NumClauses,
       SourceLocation(), SourceLocation(), NumLoops);
 }
 
+OMPFlattenDirective *OMPFlattenDirective::Create(
+    const ASTContext &C, SourceLocation StartLoc, SourceLocation EndLoc,
+    ArrayRef<OMPClause *> Clauses, unsigned NumLoops, Stmt *AssociatedStmt,
+    Stmt *TransformedStmt, Stmt *PreInits) {
+  OMPFlattenDirective *Dir = createDirective<OMPFlattenDirective>(
+      C, Clauses, AssociatedStmt, TransformedStmtOffset + 1, StartLoc, EndLoc,
+      NumLoops);
+  Dir->setTransformedStmt(TransformedStmt);
+  Dir->setPreInits(PreInits);
+  return Dir;
+}
+
+OMPFlattenDirective *OMPFlattenDirective::CreateEmpty(const ASTContext &C,
+                                                      unsigned NumClauses,
+                                                      unsigned NumLoops) {
+  return createEmptyDirective<OMPFlattenDirective>(
+      C, NumClauses, /*HasAssociatedStmt=*/true, TransformedStmtOffset + 1,
+      SourceLocation(), SourceLocation(), NumLoops);
+}
+
 OMPSplitDirective *
 OMPSplitDirective::Create(const ASTContext &C, SourceLocation StartLoc,
                           SourceLocation EndLoc, ArrayRef<OMPClause *> Clauses,
diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp
index 6c3294573e9d4..0bacb79ed86e3 100644
--- a/clang/lib/AST/StmtPrinter.cpp
+++ b/clang/lib/AST/StmtPrinter.cpp
@@ -800,6 +800,11 @@ void StmtPrinter::VisitOMPInterchangeDirective(OMPInterchangeDirective *Node) {
   PrintOMPExecutableDirective(Node);
 }
 
+void StmtPrinter::VisitOMPFlattenDirective(OMPFlattenDirective *Node) {
+  Indent() << "#pragma omp flatten";
+  PrintOMPExecutableDirective(Node);
+}
+
 void StmtPrinter::VisitOMPSplitDirective(OMPSplitDirective *Node) {
   Indent() << "#pragma omp split";
   PrintOMPExecutableDirective(Node);
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index 90eab530e0c2e..de7ed322d7424 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -1071,6 +1071,10 @@ void StmtProfiler::VisitOMPInterchangeDirective(
   VisitOMPCanonicalLoopNestTransformationDirective(S);
 }
 
+void StmtProfiler::VisitOMPFlattenDirective(const OMPFlattenDirective *S) {
+  VisitOMPCanonicalLoopNestTransformationDirective(S);
+}
+
 void StmtProfiler::VisitOMPSplitDirective(const OMPSplitDirective *S) {
   VisitOMPCanonicalLoopNestTransformationDirective(S);
 }
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index 675d86349c933..57e0347f8ea77 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -818,7 +818,7 @@ bool clang::isOpenMPCanonicalLoopNestTransformationDirective(
     OpenMPDirectiveKind DKind) {
   return DKind == OMPD_tile || DKind == OMPD_unroll || DKind == OMPD_reverse ||
          DKind == OMPD_split || DKind == OMPD_interchange ||
-         DKind == OMPD_stripe;
+         DKind == OMPD_stripe || DKind == OMPD_flatten;
 }
 
 bool clang::isOpenMPCanonicalLoopSequenceTransformationDirective(

>From 933007b6333584fb358adc4f89a61db1ae15d4cb Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:02:27 -0400
Subject: [PATCH 02/16] add OMPDepthClause ast node

---
 clang/include/clang/AST/OpenMPClause.h        | 68 +++++++++++++++++++
 clang/include/clang/AST/RecursiveASTVisitor.h |  6 ++
 clang/lib/AST/OpenMPClause.cpp                | 25 +++++++
 clang/lib/AST/StmtProfile.cpp                 |  5 ++
 clang/lib/Parse/ParseOpenMP.cpp               |  1 +
 llvm/include/llvm/Frontend/OpenMP/OMP.td      |  1 +
 6 files changed, 106 insertions(+)

diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 8ceafc4669297..50d1364a9726d 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -1380,6 +1380,74 @@ class OMPPartialClause final : public OMPClause {
   }
 };
 
+/// This represents the 'depth' clause on the '#pragma omp flatten' (and
+/// '#pragma omp fuse') loop-transformation directives.
+///
+/// \code
+/// #pragma omp flatten depth(3)
+/// \endcode
+/// In this example the 'flatten' directive has a 'depth' clause whose argument
+/// '3' specifies how many perfectly nested loops are combined into one.
+/// The argument must be a positive integer constant expression that evaluates
+/// to at most the loop nest depth of the associated loop nest.
+class OMPDepthClause final : public OMPClause {
+  friend class OMPClauseReader;
+
+  /// Location of '('.
+  SourceLocation LParenLoc;
+
+  /// The depth expression (number of loops to flatten).
+  Stmt *Depth = nullptr;
+
+  /// Build an empty clause.
+  explicit OMPDepthClause() : OMPClause(llvm::omp::OMPC_depth, {}, {}) {}
+
+  /// Set the depth expression.
+  void setDepth(Expr *E) { Depth = E; }
+
+  /// Sets the location of '('.
+  void setLParenLoc(SourceLocation Loc) { LParenLoc = Loc; }
+
+public:
+  /// Build an AST node for a 'depth' clause.
+  ///
+  /// \param C         Context of the AST.
+  /// \param StartLoc  Location of the 'depth' identifier.
+  /// \param LParenLoc Location of '('.
+  /// \param EndLoc    Location of ')'.
+  /// \param Depth     The depth expression.
+  static OMPDepthClause *Create(const ASTContext &C, SourceLocation StartLoc,
+                                SourceLocation LParenLoc, SourceLocation EndLoc,
+                                Expr *Depth);
+
+  /// Build an empty 'depth' AST node for deserialization.
+  ///
+  /// \param C     Context of the AST.
+  static OMPDepthClause *CreateEmpty(const ASTContext &C);
+
+  /// Returns the location of '('.
+  SourceLocation getLParenLoc() const { return LParenLoc; }
+
+  /// Returns the depth expression or nullptr if not set.
+  Expr *getDepth() const { return cast_or_null<Expr>(Depth); }
+
+  child_range children() { return child_range(&Depth, &Depth + 1); }
+  const_child_range children() const {
+    return const_child_range(&Depth, &Depth + 1);
+  }
+
+  child_range used_children() {
+    return child_range(child_iterator(), child_iterator());
+  }
+  const_child_range used_children() const {
+    return const_child_range(const_child_iterator(), const_child_iterator());
+  }
+
+  static bool classof(const OMPClause *T) {
+    return T->getClauseKind() == llvm::omp::OMPC_depth;
+  }
+};
+
 /// This represents 'collapse' clause in the '#pragma omp ...'
 /// directive.
 ///
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index e425f7a31f59c..da55e97fdfcb5 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -3568,6 +3568,12 @@ bool RecursiveASTVisitor<Derived>::VisitOMPPartialClause(OMPPartialClause *C) {
   return true;
 }
 
+template <typename Derived>
+bool RecursiveASTVisitor<Derived>::VisitOMPDepthClause(OMPDepthClause *C) {
+  TRY_TO(TraverseStmt(C->getDepth()));
+  return true;
+}
+
 template <typename Derived>
 bool
 RecursiveASTVisitor<Derived>::VisitOMPCollapseClause(OMPCollapseClause *C) {
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index ed00e80144c25..7b2f67a31cad9 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -1057,6 +1057,22 @@ OMPPartialClause *OMPPartialClause::CreateEmpty(const ASTContext &C) {
   return new (C) OMPPartialClause();
 }
 
+OMPDepthClause *OMPDepthClause::Create(const ASTContext &C,
+                                       SourceLocation StartLoc,
+                                       SourceLocation LParenLoc,
+                                       SourceLocation EndLoc, Expr *Depth) {
+  OMPDepthClause *Clause = CreateEmpty(C);
+  Clause->setLocStart(StartLoc);
+  Clause->setLParenLoc(LParenLoc);
+  Clause->setLocEnd(EndLoc);
+  Clause->setDepth(Depth);
+  return Clause;
+}
+
+OMPDepthClause *OMPDepthClause::CreateEmpty(const ASTContext &C) {
+  return new (C) OMPDepthClause();
+}
+
 OMPLoopRangeClause *
 OMPLoopRangeClause::Create(const ASTContext &C, SourceLocation StartLoc,
                            SourceLocation LParenLoc, SourceLocation FirstLoc,
@@ -2069,6 +2085,15 @@ void OMPClausePrinter::VisitOMPPermutationClause(OMPPermutationClause *Node) {
 
 void OMPClausePrinter::VisitOMPFullClause(OMPFullClause *Node) { OS << "full"; }
 
+void OMPClausePrinter::VisitOMPDepthClause(OMPDepthClause *Node) {
+  OS << "depth";
+  if (Expr *Depth = Node->getDepth()) {
+    OS << '(';
+    Depth->printPretty(OS, nullptr, Policy, 0);
+    OS << ')';
+  }
+}
+
 void OMPClausePrinter::VisitOMPPartialClause(OMPPartialClause *Node) {
   OS << "partial";
 
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index de7ed322d7424..838d6d02f7446 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -521,6 +521,11 @@ void OMPClauseProfiler::VisitOMPPartialClause(const OMPPartialClause *C) {
     Profiler->VisitExpr(Factor);
 }
 
+void OMPClauseProfiler::VisitOMPDepthClause(const OMPDepthClause *C) {
+  if (const Expr *Depth = C->getDepth())
+    Profiler->VisitExpr(Depth);
+}
+
 void OMPClauseProfiler::VisitOMPLoopRangeClause(const OMPLoopRangeClause *C) {
   if (const Expr *First = C->getFirst())
     Profiler->VisitExpr(First);
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index af52295df2d8b..952a517205381 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -3247,6 +3247,7 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
   case OMPC_nocontext:
   case OMPC_filter:
   case OMPC_partial:
+  case OMPC_depth:
   case OMPC_align:
   case OMPC_message:
   case OMPC_ompx_dyn_cgroup_mem:
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index 679a944fc4358..d49c5ee1a0db3 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -161,6 +161,7 @@ def OMPC_Depobj : Clause<[Spelling<"depobj">]> {
   let isImplicit = true;
 }
 def OMPC_Depth : Clause<[Spelling<"depth">]> {
+  let clangClass = "OMPDepthClause";
   let flangClass = "ScalarIntConstantExpr";
 }
 def OMPC_Destroy : Clause<[Spelling<"destroy">]> {

>From b63024e1e342cd168ef128a3f6a24909df2a82db Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:04:44 -0400
Subject: [PATCH 03/16] template-transform support for flatten directive &
 depth clause

---
 clang/lib/Sema/TreeTransform.h | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 3b99ff4bb9e23..865622ad50f1c 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -1804,6 +1804,14 @@ class TreeTransform {
                                                        LParenLoc, EndLoc);
   }
 
+  /// Build a new OpenMP 'depth' clause.
+  OMPClause *RebuildOMPDepthClause(Expr *Depth, SourceLocation StartLoc,
+                                   SourceLocation LParenLoc,
+                                   SourceLocation EndLoc) {
+    return getSema().OpenMP().ActOnOpenMPDepthClause(Depth, StartLoc, LParenLoc,
+                                                     EndLoc);
+  }
+
   OMPClause *
   RebuildOMPLoopRangeClause(Expr *First, Expr *Count, SourceLocation StartLoc,
                             SourceLocation LParenLoc, SourceLocation FirstLoc,
@@ -9791,6 +9799,17 @@ TreeTransform<Derived>::TransformOMPSplitDirective(OMPSplitDirective *D) {
   return Res;
 }
 
+template <typename Derived>
+StmtResult
+TreeTransform<Derived>::TransformOMPFlattenDirective(OMPFlattenDirective *D) {
+  DeclarationNameInfo DirName;
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      D->getDirectiveKind(), DirName, nullptr, D->getBeginLoc());
+  StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
+  return Res;
+}
+
 template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPFuseDirective(OMPFuseDirective *D) {
@@ -10717,6 +10736,20 @@ TreeTransform<Derived>::TransformOMPPartialClause(OMPPartialClause *C) {
                                  C->getEndLoc());
 }
 
+template <typename Derived>
+OMPClause *TreeTransform<Derived>::TransformOMPDepthClause(OMPDepthClause *C) {
+  ExprResult T = getDerived().TransformExpr(C->getDepth());
+  if (T.isInvalid())
+    return nullptr;
+  Expr *Depth = T.get();
+  bool Changed = Depth != C->getDepth();
+
+  if (!Changed && !getDerived().AlwaysRebuild())
+    return C;
+  return RebuildOMPDepthClause(Depth, C->getBeginLoc(), C->getLParenLoc(),
+                               C->getEndLoc());
+}
+
 template <typename Derived>
 OMPClause *
 TreeTransform<Derived>::TransformOMPLoopRangeClause(OMPLoopRangeClause *C) {

>From deafb222a7d251069c68a44fa267e58e750f1064 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:08:12 -0400
Subject: [PATCH 04/16] serialize flatten

---
 clang/include/clang/Serialization/ASTBitCodes.h |  1 +
 clang/lib/Serialization/ASTReaderStmt.cpp       | 11 +++++++++++
 clang/lib/Serialization/ASTWriterStmt.cpp       |  5 +++++
 3 files changed, 17 insertions(+)

diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h
index 279380de2f7fe..93a0e0a454b68 100644
--- a/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/clang/include/clang/Serialization/ASTBitCodes.h
@@ -1970,6 +1970,7 @@ enum StmtCode {
   STMT_OMP_REVERSE_DIRECTIVE,
   STMT_OMP_SPLIT_DIRECTIVE,
   STMT_OMP_INTERCHANGE_DIRECTIVE,
+  STMT_OMP_FLATTEN_DIRECTIVE,
   STMT_OMP_FUSE_DIRECTIVE,
   STMT_OMP_FOR_DIRECTIVE,
   STMT_OMP_FOR_SIMD_DIRECTIVE,
diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp
index 02ccf8d4d41c2..8c2debd6cdf32 100644
--- a/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -2537,6 +2537,10 @@ void ASTStmtReader::VisitOMPInterchangeDirective(OMPInterchangeDirective *D) {
   VisitOMPCanonicalLoopNestTransformationDirective(D);
 }
 
+void ASTStmtReader::VisitOMPFlattenDirective(OMPFlattenDirective *D) {
+  VisitOMPCanonicalLoopNestTransformationDirective(D);
+}
+
 void ASTStmtReader::VisitOMPSplitDirective(OMPSplitDirective *D) {
   VisitOMPCanonicalLoopNestTransformationDirective(D);
 }
@@ -3719,6 +3723,13 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) {
       break;
     }
 
+    case STMT_OMP_FLATTEN_DIRECTIVE: {
+      unsigned NumLoops = Record[ASTStmtReader::NumStmtFields];
+      unsigned NumClauses = Record[ASTStmtReader::NumStmtFields + 1];
+      S = OMPFlattenDirective::CreateEmpty(Context, NumClauses, NumLoops);
+      break;
+    }
+
     case STMT_OMP_FOR_DIRECTIVE: {
       unsigned CollapsedNum = Record[ASTStmtReader::NumStmtFields];
       unsigned NumClauses = Record[ASTStmtReader::NumStmtFields + 1];
diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp
index 4e9eadd730a56..dfe28a9b915e3 100644
--- a/clang/lib/Serialization/ASTWriterStmt.cpp
+++ b/clang/lib/Serialization/ASTWriterStmt.cpp
@@ -2547,6 +2547,11 @@ void ASTStmtWriter::VisitOMPInterchangeDirective(OMPInterchangeDirective *D) {
   Code = serialization::STMT_OMP_INTERCHANGE_DIRECTIVE;
 }
 
+void ASTStmtWriter::VisitOMPFlattenDirective(OMPFlattenDirective *D) {
+  VisitOMPCanonicalLoopNestTransformationDirective(D);
+  Code = serialization::STMT_OMP_FLATTEN_DIRECTIVE;
+}
+
 void ASTStmtWriter::VisitOMPSplitDirective(OMPSplitDirective *D) {
   VisitOMPCanonicalLoopNestTransformationDirective(D);
   Code = serialization::STMT_OMP_SPLIT_DIRECTIVE;

>From 3e2dfeb99230057b7cb9c484182ae33befbe1e41 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:09:03 -0400
Subject: [PATCH 05/16] serialize depth

---
 clang/lib/Serialization/ASTReader.cpp | 8 ++++++++
 clang/lib/Serialization/ASTWriter.cpp | 5 +++++
 2 files changed, 13 insertions(+)

diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index f8a6a38bb9b5c..4f51d79a52105 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -11495,6 +11495,9 @@ OMPClause *OMPClauseReader::readClause() {
   case llvm::omp::OMPC_partial:
     C = OMPPartialClause::CreateEmpty(Context);
     break;
+  case llvm::omp::OMPC_depth:
+    C = OMPDepthClause::CreateEmpty(Context);
+    break;
   case llvm::omp::OMPC_looprange:
     C = OMPLoopRangeClause::CreateEmpty(Context);
     break;
@@ -11924,6 +11927,11 @@ void OMPClauseReader::VisitOMPPartialClause(OMPPartialClause *C) {
   C->setLParenLoc(Record.readSourceLocation());
 }
 
+void OMPClauseReader::VisitOMPDepthClause(OMPDepthClause *C) {
+  C->setDepth(Record.readSubExpr());
+  C->setLParenLoc(Record.readSourceLocation());
+}
+
 void OMPClauseReader::VisitOMPLoopRangeClause(OMPLoopRangeClause *C) {
   C->setFirst(Record.readSubExpr());
   C->setCount(Record.readSubExpr());
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 357a7f7e95fa0..cbfeaa9e6638a 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -8106,6 +8106,11 @@ void OMPClauseWriter::VisitOMPPartialClause(OMPPartialClause *C) {
   Record.AddSourceLocation(C->getLParenLoc());
 }
 
+void OMPClauseWriter::VisitOMPDepthClause(OMPDepthClause *C) {
+  Record.AddStmt(C->getDepth());
+  Record.AddSourceLocation(C->getLParenLoc());
+}
+
 void OMPClauseWriter::VisitOMPLoopRangeClause(OMPLoopRangeClause *C) {
   Record.AddStmt(C->getFirst());
   Record.AddStmt(C->getCount());

>From 4ae746ca1eeaf11db10e859572a4500029d01cce Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:17:29 -0400
Subject: [PATCH 06/16] add analysis traversal support for flatten directive

---
 clang/lib/Sema/SemaExceptionSpec.cpp         | 1 +
 clang/lib/StaticAnalyzer/Core/ExprEngine.cpp | 1 +
 2 files changed, 2 insertions(+)

diff --git a/clang/lib/Sema/SemaExceptionSpec.cpp b/clang/lib/Sema/SemaExceptionSpec.cpp
index 40d530a1f3925..995b5b6010978 100644
--- a/clang/lib/Sema/SemaExceptionSpec.cpp
+++ b/clang/lib/Sema/SemaExceptionSpec.cpp
@@ -1508,6 +1508,7 @@ CanThrowResult Sema::canThrow(const Stmt *S) {
   case Stmt::OMPUnrollDirectiveClass:
   case Stmt::OMPReverseDirectiveClass:
   case Stmt::OMPInterchangeDirectiveClass:
+  case Stmt::OMPFlattenDirectiveClass:
   case Stmt::OMPSplitDirectiveClass:
   case Stmt::OMPFuseDirectiveClass:
   case Stmt::OMPSingleDirectiveClass:
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
index cfb294736ee02..3f666be00cc24 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
@@ -1783,6 +1783,7 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
     case Stmt::OMPStripeDirectiveClass:
     case Stmt::OMPTileDirectiveClass:
     case Stmt::OMPInterchangeDirectiveClass:
+    case Stmt::OMPFlattenDirectiveClass:
     case Stmt::OMPSplitDirectiveClass:
     case Stmt::OMPFuseDirectiveClass:
     case Stmt::OMPInteropDirectiveClass:

>From 415eba88ad0af63c552b4c5c2eabd660bcb29c02 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:21:25 -0400
Subject: [PATCH 07/16] codegen flatten directive

---
 clang/lib/CodeGen/CGStmt.cpp        | 3 +++
 clang/lib/CodeGen/CGStmtOpenMP.cpp  | 8 ++++++++
 clang/lib/CodeGen/CodeGenFunction.h | 1 +
 3 files changed, 12 insertions(+)

diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index 232094777f233..27f48cb63b553 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -236,6 +236,9 @@ void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {
   case Stmt::OMPInterchangeDirectiveClass:
     EmitOMPInterchangeDirective(cast<OMPInterchangeDirective>(*S));
     break;
+  case Stmt::OMPFlattenDirectiveClass:
+    EmitOMPFlattenDirective(cast<OMPFlattenDirective>(*S));
+    break;
   case Stmt::OMPFuseDirectiveClass:
     EmitOMPFuseDirective(cast<OMPFuseDirective>(*S));
     break;
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 95fd6694437fe..25da3430a23fe 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -218,6 +218,8 @@ class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
     } else if (const auto *Interchange =
                    dyn_cast<OMPInterchangeDirective>(&S)) {
       PreInits = Interchange->getPreInits();
+    } else if (const auto *Flatten = dyn_cast<OMPFlattenDirective>(&S)) {
+      PreInits = Flatten->getPreInits();
     } else {
       llvm_unreachable("Unknown loop-based directive kind.");
     }
@@ -3248,6 +3250,12 @@ void CodeGenFunction::EmitOMPInterchangeDirective(
   EmitStmt(S.getTransformedStmt());
 }
 
+void CodeGenFunction::EmitOMPFlattenDirective(const OMPFlattenDirective &S) {
+  // Emit the de-sugared statement.
+  OMPTransformDirectiveScopeRAII FlattenScope(*this, &S);
+  EmitStmt(S.getTransformedStmt());
+}
+
 void CodeGenFunction::EmitOMPFuseDirective(const OMPFuseDirective &S) {
   // Emit the de-sugared statement
   OMPTransformDirectiveScopeRAII FuseScope(*this, &S);
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 6d0718c243812..eea0163422cd3 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -3934,6 +3934,7 @@ class CodeGenFunction : public CodeGenTypeCache {
   void EmitOMPReverseDirective(const OMPReverseDirective &S);
   void EmitOMPSplitDirective(const OMPSplitDirective &S);
   void EmitOMPInterchangeDirective(const OMPInterchangeDirective &S);
+  void EmitOMPFlattenDirective(const OMPFlattenDirective &S);
   void EmitOMPFuseDirective(const OMPFuseDirective &S);
   void EmitOMPForDirective(const OMPForDirective &S);
   void EmitOMPForSimdDirective(const OMPForSimdDirective &S);

>From 814e2e9fe9bf4e28e4557999fbc16beec144cc39 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:24:42 -0400
Subject: [PATCH 08/16] add NYI CIR stub

---
 clang/lib/CIR/CodeGen/CIRGenFunction.h     | 1 +
 clang/lib/CIR/CodeGen/CIRGenStmt.cpp       | 2 ++
 clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp | 5 +++++
 3 files changed, 8 insertions(+)

diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.h b/clang/lib/CIR/CodeGen/CIRGenFunction.h
index 322355fde3957..8374583ffd4d0 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.h
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.h
@@ -2559,6 +2559,7 @@ class CIRGenFunction : public CIRGenTypeCache {
   mlir::LogicalResult emitOMPSplitDirective(const OMPSplitDirective &s);
   mlir::LogicalResult
   emitOMPInterchangeDirective(const OMPInterchangeDirective &s);
+  mlir::LogicalResult emitOMPFlattenDirective(const OMPFlattenDirective &s);
   mlir::LogicalResult emitOMPAssumeDirective(const OMPAssumeDirective &s);
   mlir::LogicalResult emitOMPMaskedDirective(const OMPMaskedDirective &s);
   mlir::LogicalResult emitOMPStripeDirective(const OMPStripeDirective &s);
diff --git a/clang/lib/CIR/CodeGen/CIRGenStmt.cpp b/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
index d3acac5801e74..33d425b22d61a 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
@@ -408,6 +408,8 @@ mlir::LogicalResult CIRGenFunction::emitStmt(const Stmt *s,
     return emitOMPSplitDirective(cast<OMPSplitDirective>(*s));
   case Stmt::OMPInterchangeDirectiveClass:
     return emitOMPInterchangeDirective(cast<OMPInterchangeDirective>(*s));
+  case Stmt::OMPFlattenDirectiveClass:
+    return emitOMPFlattenDirective(cast<OMPFlattenDirective>(*s));
   case Stmt::OMPAssumeDirectiveClass:
     return emitOMPAssumeDirective(cast<OMPAssumeDirective>(*s));
   case Stmt::OMPMaskedDirectiveClass:
diff --git a/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp b/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp
index 17a1fb8090f5c..4fffd17b25a16 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp
@@ -587,6 +587,11 @@ CIRGenFunction::emitOMPInterchangeDirective(const OMPInterchangeDirective &s) {
   return mlir::failure();
 }
 mlir::LogicalResult
+CIRGenFunction::emitOMPFlattenDirective(const OMPFlattenDirective &s) {
+  getCIRGenModule().errorNYI(s.getSourceRange(), "OpenMP OMPFlattenDirective");
+  return mlir::failure();
+}
+mlir::LogicalResult
 CIRGenFunction::emitOMPAssumeDirective(const OMPAssumeDirective &s) {
   getCIRGenModule().errorNYI(s.getSourceRange(), "OpenMP OMPAssumeDirective");
   return mlir::failure();

>From 0027ec6ab982a83766bd65f3e1bc45d9a671fc62 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:26:31 -0400
Subject: [PATCH 09/16] add libclang traversal support for depth clause

---
 clang/tools/libclang/CIndex.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index ac2fad38a1348..f8ab63e3e0dae 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2378,6 +2378,10 @@ void OMPClauseEnqueue::VisitOMPLoopRangeClause(const OMPLoopRangeClause *C) {
   Visitor->AddStmt(C->getCount());
 }
 
+void OMPClauseEnqueue::VisitOMPDepthClause(const OMPDepthClause *C) {
+  Visitor->AddStmt(C->getDepth());
+}
+
 void OMPClauseEnqueue::VisitOMPAllocatorClause(const OMPAllocatorClause *C) {
   Visitor->AddStmt(C->getAllocator());
 }

>From f5ac5f740c414310c8482f652f1bed936ae7cda3 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Fri, 10 Jul 2026 06:54:16 -0400
Subject: [PATCH 10/16] sema

---
 clang/include/clang/Sema/SemaOpenMP.h |  10 +
 clang/lib/Sema/SemaOpenMP.cpp         | 295 ++++++++++++++++++++++++++
 2 files changed, 305 insertions(+)

diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index 3621ce96b8724..7b086388416a0 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -470,6 +470,12 @@ class SemaOpenMP : public SemaBase {
                                              SourceLocation StartLoc,
                                              SourceLocation EndLoc);
 
+  /// Called on well-formed '#pragma omp flatten' after parsing of its
+  /// clauses and the associated statement.
+  StmtResult ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
+                                         Stmt *AStmt, SourceLocation StartLoc,
+                                         SourceLocation EndLoc);
+
   /// Called on well-formed '#pragma omp fuse' after parsing of its
   /// clauses and the associated statement.
   StmtResult ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
@@ -935,6 +941,10 @@ class SemaOpenMP : public SemaBase {
   OMPClause *ActOnOpenMPPartialClause(Expr *FactorExpr, SourceLocation StartLoc,
                                       SourceLocation LParenLoc,
                                       SourceLocation EndLoc);
+  /// Called on well-formed 'depth' clause.
+  OMPClause *ActOnOpenMPDepthClause(Expr *DepthExpr, SourceLocation StartLoc,
+                                    SourceLocation LParenLoc,
+                                    SourceLocation EndLoc);
   /// Called on well-formed 'collapse' clause.
   OMPClause *ActOnOpenMPCollapseClause(Expr *NumForLoops,
                                        SourceLocation StartLoc,
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 247a4a6ad9271..fcc256dec3c88 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -4629,6 +4629,7 @@ void SemaOpenMP::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind,
   case OMPD_reverse:
   case OMPD_split:
   case OMPD_interchange:
+  case OMPD_flatten:
   case OMPD_fuse:
   case OMPD_assume:
     break;
@@ -6475,6 +6476,10 @@ StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
     Res = ActOnOpenMPInterchangeDirective(ClausesWithImplicit, AStmt, StartLoc,
                                           EndLoc);
     break;
+  case OMPD_flatten:
+    Res = ActOnOpenMPFlattenDirective(ClausesWithImplicit, AStmt, StartLoc,
+                                      EndLoc);
+    break;
   case OMPD_fuse:
     Res =
         ActOnOpenMPFuseDirective(ClausesWithImplicit, AStmt, StartLoc, EndLoc);
@@ -16390,6 +16395,277 @@ StmtResult SemaOpenMP::ActOnOpenMPInterchangeDirective(
                                          buildPreInits(Context, PreInits));
 }
 
+/// Counts the perfectly nested canonical loop depth at \p AStmt. Returns
+/// std::nullopt if a loop-generating transformation prevents a static count.
+static std::optional<unsigned> getCanonicalLoopNestDepth(Stmt *AStmt) {
+  unsigned Depth = 0;
+  Stmt *CurStmt = AStmt ? AStmt->IgnoreContainers() : nullptr;
+  while (CurStmt) {
+    if (isa<OMPLoopTransformationDirective>(CurStmt))
+      return std::nullopt;
+    if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(CurStmt))
+      CurStmt = CanonLoop->getLoopStmt();
+    Stmt *Body = nullptr;
+    if (auto *For = dyn_cast<ForStmt>(CurStmt))
+      Body = For->getBody();
+    else if (auto *RangeFor = dyn_cast<CXXForRangeStmt>(CurStmt))
+      Body = RangeFor->getBody();
+    else
+      break;
+    ++Depth;
+    CurStmt = Body ? Body->IgnoreContainers() : nullptr;
+  }
+  return Depth;
+}
+
+StmtResult
+SemaOpenMP::ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
+                                        Stmt *AStmt, SourceLocation StartLoc,
+                                        SourceLocation EndLoc) {
+  ASTContext &Context = getASTContext();
+  DeclContext *CurContext = SemaRef.CurContext;
+  Scope *CurScope = SemaRef.getCurScope();
+
+  // Empty statement should only be possible if there already was an error.
+  if (!AStmt)
+    return StmtError();
+
+  // flatten without 'depth' clause combines two loops; 'depth(k)' selects k.
+  unsigned NumLoops = 2;
+  bool DepthIsValueDependent = false;
+  const auto *DepthClause =
+      OMPExecutableDirective::getSingleClause<OMPDepthClause>(Clauses);
+  if (DepthClause) {
+    Expr *DepthExpr = DepthClause->getDepth();
+    if (DepthExpr && DepthExpr->isValueDependent()) {
+      DepthIsValueDependent = true;
+    } else if (DepthExpr) {
+      Expr::EvalResult EvalResult;
+      if (DepthExpr->EvaluateAsInt(EvalResult, Context))
+        NumLoops = EvalResult.Val.getInt().getZExtValue();
+    }
+  }
+
+  // Report "expected k for loops, but found only n" when depth exceeds the
+  // perfect nest (same form as 'collapse'); skip if depth or nest is unknown.
+  if (DepthClause && !DepthIsValueDependent) {
+    if (std::optional<unsigned> NestDepth = getCanonicalLoopNestDepth(AStmt);
+        NestDepth && NumLoops > *NestDepth) {
+      Diag(AStmt->getBeginLoc(), diag::err_omp_not_for)
+          << /*expected N for loops form=*/1
+          << getOpenMPDirectiveName(OMPD_flatten) << NumLoops
+          << (*NestDepth > 0) << *NestDepth;
+      return StmtError();
+    }
+  }
+
+  // Defer when 'depth' is value-dependent (concrete k unknown until
+  // instantiation).
+  if (DepthIsValueDependent)
+    return OMPFlattenDirective::Create(Context, StartLoc, EndLoc, Clauses,
+                                       NumLoops, AStmt, nullptr, nullptr);
+
+  // Verify and diagnose loop nest.
+  SmallVector<OMPLoopBasedDirective::HelperExprs, 4> LoopHelpers(NumLoops);
+  Stmt *Body = nullptr;
+  SmallVector<SmallVector<Stmt *>, 4> OriginalInits;
+  if (!checkTransformableLoopNest(OMPD_flatten, AStmt, NumLoops, LoopHelpers,
+                                  Body, OriginalInits))
+    return StmtError();
+
+  // Delay flattening to when template is completely instantiated.
+  if (CurContext->isDependentContext())
+    return OMPFlattenDirective::Create(Context, StartLoc, EndLoc, Clauses,
+                                       NumLoops, AStmt, nullptr, nullptr);
+
+  assert(LoopHelpers.size() == NumLoops &&
+         "Expecting loop iteration space dimensionality to match number of "
+         "affected loops");
+  assert(OriginalInits.size() == NumLoops &&
+         "Expecting loop iteration space dimensionality to match number of "
+         "affected loops");
+
+  // Find the affected loops.
+  SmallVector<Stmt *> LoopStmts(NumLoops, nullptr);
+  collectLoopStmts(AStmt, LoopStmts);
+
+  // Collect pre-init statements in outer-to-inner order.
+  SmallVector<Stmt *> PreInits;
+  for (auto I : llvm::seq<int>(NumLoops)) {
+    OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
+    assert(LoopHelper.Counters.size() == 1 &&
+           "Single-dimensional loop iteration space expected");
+    addLoopPreInits(Context, LoopHelper, LoopStmts[I], OriginalInits[I],
+                    PreInits);
+  }
+
+  CaptureVars CopyTransformer(SemaRef);
+  auto MakeNumIterations = [&CopyTransformer,
+                            &LoopHelpers](unsigned I) -> Expr * {
+    return AssertSuccess(
+        CopyTransformer.TransformExpr(LoopHelpers[I].NumIterations));
+  };
+
+  OMPLoopBasedDirective::HelperExprs &OutermostHelper = LoopHelpers[0];
+  auto *OutermostCntVar = cast<DeclRefExpr>(OutermostHelper.Counters.front());
+  SourceLocation OrigVarLoc = OutermostCntVar->getExprLoc();
+  SourceLocation OrigVarLocBegin = OutermostCntVar->getBeginLoc();
+  SourceLocation OrigVarLocEnd = OutermostCntVar->getEndLoc();
+  SourceLocation CondLoc = OutermostHelper.Cond->getExprLoc();
+
+  // Product of trip counts; mirror 'collapse' IV-width selection to avoid
+  // overflow when several counts are multiplied.
+  auto BuildTripCount = [&](unsigned Bits) -> ExprResult {
+    ExprResult Product;
+    for (unsigned I = 0; I < NumLoops; ++I) {
+      ExprResult N = widenIterationCount(Bits, MakeNumIterations(I), SemaRef);
+      if (!N.isUsable())
+        return ExprError();
+      if (I == 0)
+        Product = N;
+      else
+        Product = SemaRef.BuildBinOp(CurScope, CondLoc, BO_Mul, Product.get(),
+                                     N.get());
+      if (!Product.isUsable())
+        return ExprError();
+    }
+    return Product;
+  };
+
+  bool AllCountsLessThan32Bits = true;
+  for (unsigned I = 0; I < NumLoops; ++I)
+    AllCountsLessThan32Bits &=
+        Context.getTypeSize(LoopHelpers[I].NumIterations->getType()) < 32;
+
+  ExprResult TripCount32 = BuildTripCount(/*Bits=*/32);
+  ExprResult TripCount64 = BuildTripCount(/*Bits=*/64);
+  if (!TripCount32.isUsable() || !TripCount64.isUsable())
+    return StmtError();
+
+  ExprResult TripCount = TripCount64;
+  if (Context.getTypeSize(TripCount32.get()->getType()) == 32 &&
+      (AllCountsLessThan32Bits || NumLoops == 1 ||
+       fitsInto(
+           /*Bits=*/32,
+           TripCount32.get()->getType()->hasSignedIntegerRepresentation(),
+           TripCount64.get(), SemaRef)))
+    TripCount = TripCount32;
+
+  QualType IVTy = TripCount.get()->getType();
+  uint64_t IVWidth = Context.getTypeSize(IVTy);
+
+  auto MakeNumIterationsInIVTy = [&](unsigned I) -> Expr * {
+    return AssertSuccess(SemaRef.PerformImplicitConversion(
+        MakeNumIterations(I), IVTy, AssignmentAction::Converting,
+        /*AllowExplicit=*/true));
+  };
+
+  // \code{.cpp}
+  //   for (auto .flatten.iv = 0; .flatten.iv < n0 * n1 * ...; ++.flatten.iv) {
+  //     .flatten.iv.0 = .flatten.iv / (n1 * ...);
+  //     i0 = ...;                                        // Updates[0]
+  //     .flatten.iv.1 = (.flatten.iv / ...) % n1;
+  //     i1 = ...;                                        // Updates[1]
+  //     ...
+  //     body(i0, i1, ...);
+  //   }
+  // \endcode
+  SmallString<64> FlattenedIVName(".flatten.iv");
+  VarDecl *FlattenedIVDecl = buildVarDecl(SemaRef, {}, IVTy, FlattenedIVName,
+                                          nullptr, OutermostCntVar);
+  auto MakeFlattenedRef = [&SemaRef = this->SemaRef, FlattenedIVDecl, IVTy,
+                           OrigVarLoc]() {
+    return buildDeclRefExpr(SemaRef, FlattenedIVDecl, IVTy, OrigVarLoc);
+  };
+
+  // For init-statement:
+  // \code{.cpp}
+  //   auto .flatten.iv = 0;
+  // \endcode
+  auto *Zero = IntegerLiteral::Create(Context, llvm::APInt::getZero(IVWidth),
+                                      IVTy, OrigVarLoc);
+  SemaRef.AddInitializerToDecl(FlattenedIVDecl, Zero, /*DirectInit=*/false);
+  StmtResult Init = new (Context)
+      DeclStmt(DeclGroupRef(FlattenedIVDecl), OrigVarLocBegin, OrigVarLocEnd);
+  if (!Init.isUsable())
+    return StmtError();
+
+  // For cond-expression:
+  // \code{.cpp}
+  //   .flatten.iv < n0 * n1 * ... * n(k-1)
+  // \endcode
+  ExprResult Cond = SemaRef.BuildBinOp(CurScope, CondLoc, BO_LT,
+                                       MakeFlattenedRef(), TripCount.get());
+  if (!Cond.isUsable())
+    return StmtError();
+
+  // For incr-statement:
+  // \code{.cpp}
+  //   ++.flatten.iv
+  // \endcode
+  ExprResult Incr =
+      SemaRef.BuildUnaryOp(CurScope, OutermostHelper.Inc->getExprLoc(),
+                           UO_PreInc, MakeFlattenedRef());
+  if (!Incr.isUsable())
+    return StmtError();
+
+  // Recover each logical iteration counter via mixed-radix div/mod; reuse the
+  // iteration variables from checkOpenMPLoop so Updates compute user counters.
+  SmallVector<Stmt *, 8> BodyStmts;
+  for (unsigned I = 0; I < NumLoops; ++I) {
+    OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
+    auto *IVRef = cast<DeclRefExpr>(LoopHelper.IterationVarRef);
+    auto *IVDecl = cast<VarDecl>(IVRef->getDecl());
+    std::string IVName = (".flatten.iv." + llvm::Twine(I)).str();
+    IVDecl->setDeclName(&SemaRef.PP.getIdentifierTable().get(IVName));
+
+    ExprResult Value = MakeFlattenedRef();
+    if (I + 1 < NumLoops) {
+      ExprResult Divisor = MakeNumIterationsInIVTy(I + 1);
+      for (unsigned J = I + 2; J < NumLoops; ++J) {
+        Divisor = SemaRef.BuildBinOp(CurScope, OrigVarLoc, BO_Mul,
+                                     Divisor.get(), MakeNumIterationsInIVTy(J));
+        if (!Divisor.isUsable())
+          return StmtError();
+      }
+      Value = SemaRef.BuildBinOp(CurScope, OrigVarLoc, BO_Div, Value.get(),
+                                 Divisor.get());
+      if (!Value.isUsable())
+        return StmtError();
+    }
+    if (I > 0) {
+      Value = SemaRef.BuildBinOp(CurScope, OrigVarLoc, BO_Rem, Value.get(),
+                                 MakeNumIterationsInIVTy(I));
+      if (!Value.isUsable())
+        return StmtError();
+    }
+
+    SemaRef.AddInitializerToDecl(IVDecl, Value.get(), /*DirectInit=*/false);
+    StmtResult IVStmt = new (Context)
+        DeclStmt(DeclGroupRef(IVDecl), OrigVarLocBegin, OrigVarLocEnd);
+    if (!IVStmt.isUsable())
+      return StmtError();
+
+    BodyStmts.push_back(IVStmt.get());
+    llvm::append_range(BodyStmts, LoopHelper.Updates);
+    if (auto *CXXFor = dyn_cast<CXXForRangeStmt>(LoopStmts[I]))
+      BodyStmts.push_back(CXXFor->getLoopVarStmt());
+  }
+  BodyStmts.push_back(Body);
+  auto *FlattenedBody =
+      CompoundStmt::Create(Context, BodyStmts, FPOptionsOverride(),
+                           Body->getBeginLoc(), Body->getEndLoc());
+
+  auto *FlattenedFor = new (Context) ForStmt(
+      Context, Init.get(), Cond.get(), nullptr, Incr.get(), FlattenedBody,
+      OutermostHelper.Init->getBeginLoc(), OutermostHelper.Init->getBeginLoc(),
+      OutermostHelper.Inc->getEndLoc());
+
+  return OMPFlattenDirective::Create(Context, StartLoc, EndLoc, Clauses,
+                                     NumLoops, AStmt, FlattenedFor,
+                                     buildPreInits(Context, PreInits));
+}
+
 StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
                                                 Stmt *AStmt,
                                                 SourceLocation StartLoc,
@@ -16920,6 +17196,9 @@ OMPClause *SemaOpenMP::ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind,
   case OMPC_partial:
     Res = ActOnOpenMPPartialClause(Expr, StartLoc, LParenLoc, EndLoc);
     break;
+  case OMPC_depth:
+    Res = ActOnOpenMPDepthClause(Expr, StartLoc, LParenLoc, EndLoc);
+    break;
   case OMPC_message:
     Res = ActOnOpenMPMessageClause(Expr, StartLoc, LParenLoc, EndLoc);
     break;
@@ -18241,6 +18520,22 @@ OMPClause *SemaOpenMP::ActOnOpenMPPartialClause(Expr *FactorExpr,
                                   FactorExpr);
 }
 
+OMPClause *SemaOpenMP::ActOnOpenMPDepthClause(Expr *DepthExpr,
+                                              SourceLocation StartLoc,
+                                              SourceLocation LParenLoc,
+                                              SourceLocation EndLoc) {
+  // The depth-expr must be a positive integer constant expression and
+  // not greater than the number of loops in the associated loop nest.
+  ExprResult DepthResult = VerifyPositiveIntegerConstantInClause(
+      DepthExpr, OMPC_depth, /*StrictlyPositive=*/true);
+  if (DepthResult.isInvalid())
+    return nullptr;
+  DepthExpr = DepthResult.get();
+
+  return OMPDepthClause::Create(getASTContext(), StartLoc, LParenLoc, EndLoc,
+                                DepthExpr);
+}
+
 OMPClause *SemaOpenMP::ActOnOpenMPLoopRangeClause(
     Expr *First, Expr *Count, SourceLocation StartLoc, SourceLocation LParenLoc,
     SourceLocation FirstLoc, SourceLocation CountLoc, SourceLocation EndLoc) {

>From 54be7347979638725cd4b6c421d11ec1cd618c43 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Mon, 27 Jul 2026 07:03:06 -0400
Subject: [PATCH 11/16] add libclang traversal support for flatten directive

---
 clang/tools/libclang/CIndex.cpp   | 7 +++++++
 clang/tools/libclang/CXCursor.cpp | 3 +++
 2 files changed, 10 insertions(+)

diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index f8ab63e3e0dae..9eb0ec8c21158 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2160,6 +2160,7 @@ class EnqueueVisitor : public ConstStmtVisitor<EnqueueVisitor, void>,
   void VisitOMPUnrollDirective(const OMPUnrollDirective *D);
   void VisitOMPReverseDirective(const OMPReverseDirective *D);
   void VisitOMPInterchangeDirective(const OMPInterchangeDirective *D);
+  void VisitOMPFlattenDirective(const OMPFlattenDirective *D);
   void VisitOMPCanonicalLoopSequenceTransformationDirective(
       const OMPCanonicalLoopSequenceTransformationDirective *D);
   void VisitOMPFuseDirective(const OMPFuseDirective *D);
@@ -3358,6 +3359,10 @@ void EnqueueVisitor::VisitOMPInterchangeDirective(
   VisitOMPCanonicalLoopNestTransformationDirective(D);
 }
 
+void EnqueueVisitor::VisitOMPFlattenDirective(const OMPFlattenDirective *D) {
+  VisitOMPCanonicalLoopNestTransformationDirective(D);
+}
+
 void EnqueueVisitor::VisitOMPCanonicalLoopSequenceTransformationDirective(
     const OMPCanonicalLoopSequenceTransformationDirective *D) {
   VisitOMPExecutableDirective(D);
@@ -6329,6 +6334,8 @@ CXString clang_getCursorKindSpelling(enum CXCursorKind Kind) {
     return cxstring::createRef("OMPReverseDirective");
   case CXCursor_OMPInterchangeDirective:
     return cxstring::createRef("OMPInterchangeDirective");
+  case CXCursor_OMPFlattenDirective:
+    return cxstring::createRef("OMPFlattenDirective");
   case CXCursor_OMPFuseDirective:
     return cxstring::createRef("OMPFuseDirective");
   case CXCursor_OMPSplitDirective:
diff --git a/clang/tools/libclang/CXCursor.cpp b/clang/tools/libclang/CXCursor.cpp
index b2feed578fc0f..97bbfd36558a9 100644
--- a/clang/tools/libclang/CXCursor.cpp
+++ b/clang/tools/libclang/CXCursor.cpp
@@ -703,6 +703,9 @@ CXCursor cxcursor::MakeCXCursor(const Stmt *S, const Decl *Parent,
   case Stmt::OMPInterchangeDirectiveClass:
     K = CXCursor_OMPInterchangeDirective;
     break;
+  case Stmt::OMPFlattenDirectiveClass:
+    K = CXCursor_OMPFlattenDirective;
+    break;
   case Stmt::OMPFuseDirectiveClass:
     K = CXCursor_OMPFuseDirective;
     break;

>From cdedb1045baf373937b0da858624a6ace14a3d21 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Mon, 27 Jul 2026 07:06:36 -0400
Subject: [PATCH 12/16] bindings

---
 clang/bindings/python/clang/cindex.py | 3 +++
 clang/include/clang-c/Index.h         | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/clang/bindings/python/clang/cindex.py b/clang/bindings/python/clang/cindex.py
index 24b737139dba8..ccad140e8c861 100644
--- a/clang/bindings/python/clang/cindex.py
+++ b/clang/bindings/python/clang/cindex.py
@@ -1456,6 +1456,9 @@ def is_unexposed(self):
     # OpenMP split directive.
     OMP_SPLIT_DIRECTIVE = 312
 
+    # OpenMP flatten directive.
+    OMP_FLATTEN_DIRECTIVE = 313
+
     # OpenACC Compute Construct.
     OPEN_ACC_COMPUTE_DIRECTIVE = 320
 
diff --git a/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h
index 8427236e0b444..f561fb5724897 100644
--- a/clang/include/clang-c/Index.h
+++ b/clang/include/clang-c/Index.h
@@ -2170,6 +2170,10 @@ enum CXCursorKind {
    */
   CXCursor_OMPSplitDirective = 312,
 
+  /** OpenMP flatten directive.
+   */
+  CXCursor_OMPFlattenDirective = 313,
+
   /** OpenACC Compute Construct.
    */
   CXCursor_OpenACCComputeConstruct = 320,

>From 94dee8fe1351f74760df0fb2d9903a291a5ccf66 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 5 Aug 2026 04:46:14 -0400
Subject: [PATCH 13/16] core tests-flatten and depth

---
 clang/test/OpenMP/flatten_codegen.cpp         |  65 ++++++++
 clang/test/OpenMP/flatten_depth_codegen.cpp   | 154 ++++++++++++++++++
 .../test/transform/flatten/depth-intfor.c     |  40 +++++
 .../runtime/test/transform/flatten/intfor.c   |  33 ++++
 .../test/transform/flatten/lit.local.cfg      |   5 +
 5 files changed, 297 insertions(+)
 create mode 100644 clang/test/OpenMP/flatten_codegen.cpp
 create mode 100644 clang/test/OpenMP/flatten_depth_codegen.cpp
 create mode 100644 openmp/runtime/test/transform/flatten/depth-intfor.c
 create mode 100644 openmp/runtime/test/transform/flatten/intfor.c
 create mode 100644 openmp/runtime/test/transform/flatten/lit.local.cfg

diff --git a/clang/test/OpenMP/flatten_codegen.cpp b/clang/test/OpenMP/flatten_codegen.cpp
new file mode 100644
index 0000000000000..f77b2fb00ce9b
--- /dev/null
+++ b/clang/test/OpenMP/flatten_codegen.cpp
@@ -0,0 +1,65 @@
+// Check code generation
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=60 -emit-llvm %s -o - | FileCheck %s
+
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+extern "C" void body(int, int);
+
+// CHECK-LABEL: define {{.*}}void @foo(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK:   store i64 0, ptr %.flatten.iv,
+// CHECK:   %[[COND_IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[MUL:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   icmp slt i64 %[[COND_IV]], %[[MUL]]
+// CHECK:   %[[DIV_IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[DIV:.+]] = sdiv i64 %[[DIV_IV]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   %[[REM_IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[REM:.+]] = srem i64 %[[REM_IV]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+// CHECK:   %[[I_VAL:.+]] = load i32, ptr %i,
+// CHECK:   %[[J_VAL:.+]] = load i32, ptr %j,
+// CHECK:   call void @body(i32{{.*}} %[[I_VAL]], i32{{.*}} %[[J_VAL]])
+// CHECK: for.inc:
+// CHECK:   %[[INC_IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[NEXT:.+]] = add nsw i64 %[[INC_IV]], 1
+// CHECK:   store i64 %[[NEXT]], ptr %.flatten.iv,
+// CHECK:   br label %for.cond
+extern "C" void foo(int n, int m) {
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+
+// CHECK-LABEL: define {{.*}}void @bar(
+// CHECK:   icmp slt i32 %{{.+}}, 63
+// CHECK:   sdiv i32 %{{.+}}, 9
+// CHECK:   srem i32 %{{.+}}, 9
+// CHECK:   call void @body(
+extern "C" void bar() {
+#pragma omp flatten
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+}
+
+// CHECK-LABEL: define {{.*}}void @baz(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   icmp slt i64 %{{.+}}, 10000000000
+// CHECK:   sdiv i64 %{{.+}}, 100000
+// CHECK:   srem i64 %{{.+}}, 100000
+// CHECK:   call void @body(
+extern "C" void baz() {
+#pragma omp flatten
+  for (int i = 0; i < 100000; ++i)
+    for (int j = 0; j < 100000; ++j)
+      body(i, j);
+}
+
+#endif
diff --git a/clang/test/OpenMP/flatten_depth_codegen.cpp b/clang/test/OpenMP/flatten_depth_codegen.cpp
new file mode 100644
index 0000000000000..77408fbcbdcf1
--- /dev/null
+++ b/clang/test/OpenMP/flatten_depth_codegen.cpp
@@ -0,0 +1,154 @@
+// Check code generation for the 'depth' clause on '#pragma omp flatten'.
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=61 -emit-llvm %s -o - | FileCheck %s
+
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+extern "C" void body(int, int, int);
+
+// CHECK-LABEL: define {{.*}}void @foo3(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK:   %.flatten.iv.2 = alloca i32
+// CHECK:   %[[CIV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[T01:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   %[[TALL:.+]] = mul nsw i64 %[[T01]], %{{.+}}
+// CHECK:   icmp slt i64 %[[CIV]], %[[TALL]]
+// CHECK:   %[[D0IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[D0M:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   %[[D0:.+]] = sdiv i64 %[[D0IV]], %[[D0M]]
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   %[[D1IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[D1:.+]] = sdiv i64 %[[D1IV]], %{{.+}}
+// CHECK:   %[[R1:.+]] = srem i64 %[[D1]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+// CHECK:   %[[R2IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[R2:.+]] = srem i64 %[[R2IV]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.2
+// CHECK:   call void @body(
+extern "C" void foo3(int n, int m, int p) {
+#pragma omp flatten depth(3)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        body(i, j, k);
+}
+
+// CHECK-LABEL: define {{.*}}void @foo2(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK:   %[[CIV2:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[M:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   icmp slt i64 %[[CIV2]], %[[M]]
+// CHECK:   sdiv i64 %{{.+}}, %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   srem i64 %{{.+}}, %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+extern "C" void foo2(int n, int m) {
+#pragma omp flatten depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j, 0);
+}
+
+// CHECK-LABEL: define {{.*}}void @foo1(
+// CHECK:   %.flatten.iv = alloca i32
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK: for.body:
+// CHECK:   %[[FV:.+]] = load i32, ptr %.flatten.iv,
+// CHECK:   store i32 %[[FV]], ptr %.flatten.iv.0,
+// CHECK-NOT: sdiv
+// CHECK-NOT: srem
+// CHECK:   call void @body(
+extern "C" void foo1(int n) {
+#pragma omp flatten depth(1)
+  for (int i = 0; i < n; ++i)
+    body(i, 0, 0);
+}
+
+// CHECK-LABEL: define {{.*}}void @foo2partial(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK-NOT: %.flatten.iv.2 = alloca
+// CHECK:   %[[CIV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[M:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   icmp slt i64 %[[CIV]], %[[M]]
+// CHECK:   sdiv i64 %{{.+}}, %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   srem i64 %{{.+}}, %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+// CHECK:   br label %for.cond{{.*}}
+// CHECK: for.cond{{.*}}:
+// CHECK:   icmp slt i32 %{{.+}}, 5
+extern "C" void foo2partial(int n, int m) {
+#pragma omp flatten depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < 5; ++k)
+        body(i, j, k);
+}
+
+// A value-dependent 'depth' (the argument is a non-type template parameter) is
+// deferred until instantiation, where the concrete depth drives the flattening.
+
+// CHECK-LABEL: define {{.*}}void @_Z11tmpl_depth3ILi3EEviii(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK:   %.flatten.iv.2 = alloca i32
+// CHECK:   %[[CIV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[T01:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   %[[TALL:.+]] = mul nsw i64 %[[T01]], %{{.+}}
+// CHECK:   icmp slt i64 %[[CIV]], %[[TALL]]
+// CHECK:   %[[D0IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[D0M:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   %[[D0:.+]] = sdiv i64 %[[D0IV]], %[[D0M]]
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   %[[D1IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[D1:.+]] = sdiv i64 %[[D1IV]], %{{.+}}
+// CHECK:   %[[R1:.+]] = srem i64 %[[D1]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+// CHECK:   %[[R2IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[R2:.+]] = srem i64 %[[R2IV]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.2
+// CHECK:   call void @body(
+template <int D>
+void tmpl_depth3(int n, int m, int p) {
+#pragma omp flatten depth(D)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        body(i, j, k);
+}
+
+// A value-dependent 'depth' that instantiates to depth(1) on a single loop is
+// well-formed: the shallower-than-default nest is accepted because validation is
+// deferred to instantiation. The single loop needs no div/mod recovery.
+
+// CHECK-LABEL: define {{.*}}void @_Z11tmpl_depth1ILi1EEvi(
+// CHECK:   %.flatten.iv = alloca i32
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK: for.body:
+// CHECK:   %[[FV:.+]] = load i32, ptr %.flatten.iv,
+// CHECK:   store i32 %[[FV]], ptr %.flatten.iv.0,
+// CHECK-NOT: sdiv
+// CHECK-NOT: srem
+// CHECK:   call void @body(
+template <int D>
+void tmpl_depth1(int n) {
+#pragma omp flatten depth(D)
+  for (int i = 0; i < n; ++i)
+    body(i, 0, 0);
+}
+
+void inst() {
+  tmpl_depth3<3>(1, 1, 1);
+  tmpl_depth1<1>(1);
+}
+
+#endif
diff --git a/openmp/runtime/test/transform/flatten/depth-intfor.c b/openmp/runtime/test/transform/flatten/depth-intfor.c
new file mode 100644
index 0000000000000..7211e02e25313
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/depth-intfor.c
@@ -0,0 +1,40 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+// 'depth(3)' fully flattens a three-deep perfectly nested loop into one loop.
+// This verifies that the flattened loop preserves the exact row-major
+// visitation order of the original nest, including non-unit steps and non-zero
+// start values.
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+int main() {
+  printf("do\n");
+#pragma omp flatten depth(3)
+  for (int i = 7; i < 11; i += 2)
+    for (int j = 1; j < 4; j += 1)
+      for (int k = 0; k < 2; ++k)
+        printf("i=%d j=%d k=%d\n", i, j, k);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: i=7 j=1 k=0
+// CHECK-NEXT: i=7 j=1 k=1
+// CHECK-NEXT: i=7 j=2 k=0
+// CHECK-NEXT: i=7 j=2 k=1
+// CHECK-NEXT: i=7 j=3 k=0
+// CHECK-NEXT: i=7 j=3 k=1
+// CHECK-NEXT: i=9 j=1 k=0
+// CHECK-NEXT: i=9 j=1 k=1
+// CHECK-NEXT: i=9 j=2 k=0
+// CHECK-NEXT: i=9 j=2 k=1
+// CHECK-NEXT: i=9 j=3 k=0
+// CHECK-NEXT: i=9 j=3 k=1
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/flatten/intfor.c b/openmp/runtime/test/transform/flatten/intfor.c
new file mode 100644
index 0000000000000..f937659fbebe1
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/intfor.c
@@ -0,0 +1,33 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+int main() {
+  printf("do\n");
+#pragma omp flatten
+  for (int i = 5; i < 11; i += 2)
+    for (int j = 1; j < 7; j += 2)
+      printf("i=%d j=%d\n", i, j);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// The flattened loop visits the original iteration space in the original
+// (row-major) order: for each outer iteration the inner loop runs fully.
+// CHECK:      do
+// CHECK-NEXT: i=5 j=1
+// CHECK-NEXT: i=5 j=3
+// CHECK-NEXT: i=5 j=5
+// CHECK-NEXT: i=7 j=1
+// CHECK-NEXT: i=7 j=3
+// CHECK-NEXT: i=7 j=5
+// CHECK-NEXT: i=9 j=1
+// CHECK-NEXT: i=9 j=3
+// CHECK-NEXT: i=9 j=5
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/flatten/lit.local.cfg b/openmp/runtime/test/transform/flatten/lit.local.cfg
new file mode 100644
index 0000000000000..e05bd627df1e9
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/lit.local.cfg
@@ -0,0 +1,5 @@
+# The flatten directive's depth clause requires OpenMP 6.1.
+for i, (pattern, replacement) in enumerate(config.substitutions):
+    if pattern == "%openmp_flags":
+        config.substitutions[i] = (pattern, replacement + " -fopenmp-version=61")
+        break

>From 667fe837a39dbc711dcc5288b736aa52fbe893b6 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 5 Aug 2026 04:48:45 -0400
Subject: [PATCH 14/16] TBD

---
 clang/docs/OpenMPSupport.rst                  |   2 +-
 clang/include/clang/ASTMatchers/ASTMatchers.h |  14 ++
 clang/lib/ASTMatchers/ASTMatchersInternal.cpp |   2 +
 clang/lib/ASTMatchers/Dynamic/Registry.cpp    |   1 +
 clang/test/Index/openmp-flatten.c             |  12 ++
 clang/test/OpenMP/flatten_ast_print.cpp       | 124 ++++++++++++++++++
 clang/test/OpenMP/flatten_codegen_for.cpp     |  67 ++++++++++
 clang/test/OpenMP/flatten_depth_ast_print.cpp |  74 +++++++++++
 clang/test/OpenMP/flatten_depth_messages.cpp  |  48 +++++++
 clang/test/OpenMP/flatten_messages.cpp        |  89 +++++++++++++
 .../test/OpenMP/flatten_serialize_module.cpp  |  25 ++++
 .../ASTMatchers/ASTMatchersNodeTest.cpp       |  22 ++++
 flatten-verify-tmp/01_default_depth2.cpp      |   8 ++
 flatten-verify-tmp/02_depth3_full.cpp         |   9 ++
 flatten-verify-tmp/03_depth2_partial.cpp      |  10 ++
 flatten-verify-tmp/04_depth1_identity.cpp     |   8 ++
 flatten-verify-tmp/05_rangefor_template.cpp   |  20 +++
 flatten-verify-tmp/README.md                  |  39 ++++++
 flatten-verify-tmp/end_result_check.cpp       |  82 ++++++++++++
 .../test/transform/flatten/depth-partial.c    |  45 +++++++
 openmp/runtime/test/transform/flatten/empty.c |  42 ++++++
 .../test/transform/flatten/foreach.cpp        |  31 +++++
 .../flatten/parallel-wsloop-foreach.cpp       |  32 +++++
 23 files changed, 805 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/Index/openmp-flatten.c
 create mode 100644 clang/test/OpenMP/flatten_ast_print.cpp
 create mode 100644 clang/test/OpenMP/flatten_codegen_for.cpp
 create mode 100644 clang/test/OpenMP/flatten_depth_ast_print.cpp
 create mode 100644 clang/test/OpenMP/flatten_depth_messages.cpp
 create mode 100644 clang/test/OpenMP/flatten_messages.cpp
 create mode 100644 clang/test/OpenMP/flatten_serialize_module.cpp
 create mode 100644 flatten-verify-tmp/01_default_depth2.cpp
 create mode 100644 flatten-verify-tmp/02_depth3_full.cpp
 create mode 100644 flatten-verify-tmp/03_depth2_partial.cpp
 create mode 100644 flatten-verify-tmp/04_depth1_identity.cpp
 create mode 100644 flatten-verify-tmp/05_rangefor_template.cpp
 create mode 100644 flatten-verify-tmp/README.md
 create mode 100644 flatten-verify-tmp/end_result_check.cpp
 create mode 100644 openmp/runtime/test/transform/flatten/depth-partial.c
 create mode 100644 openmp/runtime/test/transform/flatten/empty.c
 create mode 100644 openmp/runtime/test/transform/flatten/foreach.cpp
 create mode 100644 openmp/runtime/test/transform/flatten/parallel-wsloop-foreach.cpp

diff --git a/clang/docs/OpenMPSupport.rst b/clang/docs/OpenMPSupport.rst
index 0c21aa41ff269..c11a46c3bb6a6 100644
--- a/clang/docs/OpenMPSupport.rst
+++ b/clang/docs/OpenMPSupport.rst
@@ -666,7 +666,7 @@ implementation.
 +=============================================================+===========================+===========================+==========================================================================+
 | dyn_groupprivate clause                                     | :part:`partial`           | :part:`In Progress`       | C/C++: Host device support missing                                       |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
-| loop flatten transformation                                 | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
+| loop flatten transformation                                 | :part:`partial`           | :none:`unclaimed`         | Clang: depth clause supported (OpenMP 6.1); apply clause not yet supported|
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | loop grid/tile modifiers for sizes clause                   | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
diff --git a/clang/include/clang/ASTMatchers/ASTMatchers.h b/clang/include/clang/ASTMatchers/ASTMatchers.h
index bc0f35898a2c9..602bab9aa8cb8 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchers.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchers.h
@@ -8842,6 +8842,20 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt,
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, OMPSplitDirective>
     ompSplitDirective;
 
+/// Matches OpenMP ``flatten`` directive.
+///
+/// Given
+///
+/// \code
+///   #pragma omp flatten
+///   for (int i = 0; i < m; ++i)
+///     for (int j = 0; j < n; ++j) {}
+/// \endcode
+///
+/// ``ompFlattenDirective()`` matches the flatten directive.
+extern const internal::VariadicDynCastAllOfMatcher<Stmt, OMPFlattenDirective>
+    ompFlattenDirective;
+
 /// Matches OpenMP ``counts`` clause used by ``#pragma omp split``.
 ///
 /// Given ``#pragma omp split counts(1, 2, omp_fill)``, ``ompCountsClause()``
diff --git a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
index 5cbf134620e34..e2c0c25518b70 100644
--- a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
+++ b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
@@ -1141,6 +1141,8 @@ const internal::VariadicDynCastAllOfMatcher<Stmt, OMPTargetUpdateDirective>
     ompTargetUpdateDirective;
 const internal::VariadicDynCastAllOfMatcher<Stmt, OMPSplitDirective>
     ompSplitDirective;
+const internal::VariadicDynCastAllOfMatcher<Stmt, OMPFlattenDirective>
+    ompFlattenDirective;
 const internal::VariadicDynCastAllOfMatcher<OMPClause, OMPCountsClause>
     ompCountsClause;
 const internal::VariadicDynCastAllOfMatcher<OMPClause, OMPDefaultClause>
diff --git a/clang/lib/ASTMatchers/Dynamic/Registry.cpp b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
index a04070971f0eb..a6252363e08ce 100644
--- a/clang/lib/ASTMatchers/Dynamic/Registry.cpp
+++ b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
@@ -534,6 +534,7 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(ompFromClause);
   REGISTER_MATCHER(ompToClause);
   REGISTER_MATCHER(ompExecutableDirective);
+  REGISTER_MATCHER(ompFlattenDirective);
   REGISTER_MATCHER(ompSplitDirective);
   REGISTER_MATCHER(ompTargetUpdateDirective);
   REGISTER_MATCHER(on);
diff --git a/clang/test/Index/openmp-flatten.c b/clang/test/Index/openmp-flatten.c
new file mode 100644
index 0000000000000..74bc3cda3ef1a
--- /dev/null
+++ b/clang/test/Index/openmp-flatten.c
@@ -0,0 +1,12 @@
+// RUN: c-index-test -test-load-source local %s -fopenmp=libomp -fopenmp-version=60 | FileCheck %s
+
+void test(void) {
+#pragma omp flatten
+  for (int i = 0; i < 20; i += 1)
+    for (int j = 0; j < 30; j += 1)
+      ;
+}
+
+// CHECK: openmp-flatten.c:4:1: OMPFlattenDirective= Extent=[4:1 - 4:20]
+// CHECK: openmp-flatten.c:5:3: ForStmt= Extent=[5:3 - 7:8]
+// CHECK: openmp-flatten.c:6:5: ForStmt= Extent=[6:5 - 7:8]
diff --git a/clang/test/OpenMP/flatten_ast_print.cpp b/clang/test/OpenMP/flatten_ast_print.cpp
new file mode 100644
index 0000000000000..e1115bf112c40
--- /dev/null
+++ b/clang/test/OpenMP/flatten_ast_print.cpp
@@ -0,0 +1,124 @@
+// Check no warnings/errors
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -fsyntax-only -verify %s
+// expected-no-diagnostics
+
+// Check AST and unparsing
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -ast-dump  %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -ast-print %s | FileCheck %s --check-prefix=PRINT
+
+// Check same results after serialization round-trip
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -emit-pch -o %t %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -include-pch %t -ast-dump-all %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -include-pch %t -ast-print    %s | FileCheck %s --check-prefix=PRINT
+
+#ifndef HEADER
+#define HEADER
+
+// placeholder for loop body code.
+extern "C" void body(...);
+
+// PRINT-LABEL: void foo1(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo1
+void foo1() {
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (int i = 0; i < 7; i += 1)
+  // DUMP-NEXT: ForStmt
+  for (int i = 0; i < 7; i += 1)
+    // PRINT: for (int j = 0; j < 9; j += 1)
+    // DUMP:  ForStmt
+    for (int j = 0; j < 9; j += 1)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// PRINT-LABEL: void foo2(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo2
+void foo2(int start1, int end1, int step1, int start2, int end2, int step2) {
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (int i = start1; i < end1; i += step1)
+  // DUMP-NEXT: ForStmt
+  for (int i = start1; i < end1; i += step1)
+    // PRINT: for (int j = start2; j < end2; j += step2)
+    // DUMP:  ForStmt
+    for (int j = start2; j < end2; j += step2)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// PRINT-LABEL: void foo3(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo3
+void foo3() {
+  // Verify that a flattened loop can itself be the operand of another loop
+  // transformation. The outer reverse consumes the loop generated by flatten.
+  // PRINT:     #pragma omp reverse
+  // DUMP:      OMPReverseDirective
+  #pragma omp reverse
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (int i = 0; i < 13; i += 1)
+  // DUMP-NEXT: ForStmt
+  for (int i = 0; i < 13; i += 1)
+    // PRINT: for (int j = 0; j < 17; j += 1)
+    // DUMP:  ForStmt
+    for (int j = 0; j < 17; j += 1)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// Range-based for loops are supported as affected loops.
+// PRINT-LABEL: void foo4(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo4
+struct Range {
+  int *begin();
+  int *end();
+};
+void foo4(Range r1, Range r2) {
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (int i : r1)
+  // DUMP-NEXT: CXXForRangeStmt
+  for (int i : r1)
+    // PRINT: for (int j : r2)
+    // DUMP:  CXXForRangeStmt
+    for (int j : r2)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// The directive is instantiated together with its enclosing template.
+// PRINT-LABEL: template <typename T, int LO, int HI> void foo5()
+// DUMP-LABEL:  FunctionTemplateDecl {{.*}} foo5
+template <typename T, int LO, int HI>
+void foo5() {
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (T i = LO; i < HI; ++i)
+  // DUMP-NEXT: ForStmt
+  for (T i = LO; i < HI; ++i)
+    // PRINT: for (T j = LO; j < HI; ++j)
+    // DUMP:  ForStmt
+    for (T j = LO; j < HI; ++j)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// PRINT-LABEL: template<> void foo5<int, 0, 8>()
+// DUMP-LABEL:  FunctionDecl {{.*}} foo5 'void ()' implicit_instantiation
+// DUMP:        OMPFlattenDirective
+void inst() {
+  foo5<int, 0, 8>();
+}
+
+#endif
diff --git a/clang/test/OpenMP/flatten_codegen_for.cpp b/clang/test/OpenMP/flatten_codegen_for.cpp
new file mode 100644
index 0000000000000..d64128bd25d78
--- /dev/null
+++ b/clang/test/OpenMP/flatten_codegen_for.cpp
@@ -0,0 +1,67 @@
+// Check that the loop generated by 'flatten' can be consumed by a loop
+// associated directive ('omp for' / 'omp parallel for'), and that signed and
+// unsigned induction variables generate signed/unsigned division.
+
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=60 -emit-llvm %s -o - | FileCheck %s
+
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+extern "C" void body(int, int);
+extern "C" void ubody(unsigned, unsigned);
+
+// 'omp for' worksharing over the flattened iteration space: the flattened loop
+// uses a 64-bit induction variable for the variable-bound product trip count,
+// so the worksharing loop is distributed with the 64-bit static-init call and
+// the body recovers the logical counters via signed division/remainder.
+//
+// CHECK-LABEL: define {{.*}}void @wsfor(
+// CHECK:   mul nsw i64
+// CHECK:   call void @__kmpc_for_static_init_8(
+// CHECK:   sdiv i64
+// CHECK:   srem i64
+// CHECK:   call void @body(
+// CHECK:   call void @__kmpc_for_static_fini(
+extern "C" void wsfor(int n, int m) {
+#pragma omp for
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+
+// 'omp parallel for' outlines the flattened loop and worksharing-distributes
+// it.
+//
+// CHECK-LABEL: define {{.*}}void @pfor(
+// CHECK:   call {{.*}}@__kmpc_fork_call(
+// CHECK-LABEL: define internal void @pfor.omp_outlined(
+// CHECK:   call void @__kmpc_for_static_init_8(
+// CHECK:   sdiv i64
+// CHECK:   srem i64
+// CHECK:   call void @body(
+extern "C" void pfor(int n, int m) {
+#pragma omp parallel for
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+
+// Unsigned induction variables produce unsigned division/remainder.
+//
+// CHECK-LABEL: define {{.*}}void @ufor(
+// CHECK:   icmp ult i32 %{{.+}}, 63
+// CHECK:   udiv i32 %{{.+}}, 9
+// CHECK:   urem i32 %{{.+}}, 9
+// CHECK:   call void @ubody(
+extern "C" void ufor() {
+#pragma omp flatten
+  for (unsigned i = 0; i < 7u; ++i)
+    for (unsigned j = 0; j < 9u; ++j)
+      ubody(i, j);
+}
+
+#endif
diff --git a/clang/test/OpenMP/flatten_depth_ast_print.cpp b/clang/test/OpenMP/flatten_depth_ast_print.cpp
new file mode 100644
index 0000000000000..ceaaa68e1f79a
--- /dev/null
+++ b/clang/test/OpenMP/flatten_depth_ast_print.cpp
@@ -0,0 +1,74 @@
+// Check no warnings/errors
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -fsyntax-only -verify %s
+// expected-no-diagnostics
+
+// Check AST and unparsing
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-dump  %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-print %s | FileCheck %s --check-prefix=PRINT
+
+// Check same results after serialization round-trip
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -emit-pch -o %t %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -include-pch %t -ast-dump-all %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -include-pch %t -ast-print    %s | FileCheck %s --check-prefix=PRINT
+
+#ifndef HEADER
+#define HEADER
+
+// placeholder for loop body code.
+extern "C" void body(...);
+
+// The depth clause round-trips through -ast-print and PCH serialization.
+// PRINT-LABEL: void foo_depth3(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo_depth3
+void foo_depth3(int n, int m, int p) {
+  // PRINT:     #pragma omp flatten depth(3)
+  // DUMP:      OMPFlattenDirective
+  // DUMP-NEXT: OMPDepthClause
+  #pragma omp flatten depth(3)
+  // PRINT: for (int i = 0; i < n; ++i)
+  // DUMP:  ForStmt
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        // PRINT: body(i, j, k);
+        // DUMP:  CallExpr
+        body(i, j, k);
+}
+
+// depth(2) is accepted and printed like any other depth argument.
+// PRINT-LABEL: void foo_depth2(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo_depth2
+void foo_depth2(int n, int m) {
+  // PRINT:     #pragma omp flatten depth(2)
+  // DUMP:      OMPFlattenDirective
+  // DUMP-NEXT: OMPDepthClause
+  #pragma omp flatten depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+
+// The depth clause is instantiated together with its enclosing template.
+// PRINT-LABEL: template <typename T> void foo_tmpl()
+// DUMP-LABEL:  FunctionTemplateDecl {{.*}} foo_tmpl
+template <typename T>
+void foo_tmpl() {
+  // PRINT:     #pragma omp flatten depth(3)
+  // DUMP:      OMPFlattenDirective
+  // DUMP-NEXT: OMPDepthClause
+  #pragma omp flatten depth(3)
+  for (T i = 0; i < 8; ++i)
+    for (T j = 0; j < 8; ++j)
+      for (T k = 0; k < 8; ++k)
+        body(i, j, k);
+}
+
+// PRINT-LABEL: template<> void foo_tmpl<int>()
+// DUMP-LABEL:  FunctionDecl {{.*}} foo_tmpl 'void ()' implicit_instantiation
+// DUMP:        OMPFlattenDirective
+// DUMP-NEXT:   OMPDepthClause
+void inst() {
+  foo_tmpl<int>();
+}
+
+#endif
diff --git a/clang/test/OpenMP/flatten_depth_messages.cpp b/clang/test/OpenMP/flatten_depth_messages.cpp
new file mode 100644
index 0000000000000..16f60318d0d06
--- /dev/null
+++ b/clang/test/OpenMP/flatten_depth_messages.cpp
@@ -0,0 +1,48 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=61 -fsyntax-only -Wuninitialized -verify %s
+
+extern "C" void body(...);
+
+// expected-note at +1 {{declared here}}
+void func(int n) {
+
+  // The depth argument must be a positive integer constant expression.
+  // expected-error at +1 {{argument to 'depth' clause must be a strictly positive integer value}}
+  #pragma omp flatten depth(0)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // expected-error at +1 {{argument to 'depth' clause must be a strictly positive integer value}}
+  #pragma omp flatten depth(-1)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // A non-constant argument is rejected.
+  // expected-error at +2 {{expression is not an integral constant expression}}
+  // expected-note at +1 {{function parameter 'n' with unknown value cannot be used in a constant expression}}
+  #pragma omp flatten depth(n)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // At most one 'depth' clause may appear on the directive.
+  // expected-error at +1 {{directive '#pragma omp flatten' cannot contain more than one 'depth' clause}}
+  #pragma omp flatten depth(2) depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // The depth must be at most the loop nest depth: depth(3) requires three
+  // perfectly nested loops, but only two are present here.
+  #pragma omp flatten depth(3)
+  // expected-error at +1 {{expected 3 for loops after '#pragma omp flatten', but found only 2}}
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // depth(1) flattens a single loop and is well-formed (no diagnostic).
+  #pragma omp flatten depth(1)
+  for (int i = 0; i < n; ++i)
+    body(i);
+}
diff --git a/clang/test/OpenMP/flatten_messages.cpp b/clang/test/OpenMP/flatten_messages.cpp
new file mode 100644
index 0000000000000..a022e0e8999d2
--- /dev/null
+++ b/clang/test/OpenMP/flatten_messages.cpp
@@ -0,0 +1,89 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=60 -fsyntax-only -Wuninitialized -verify %s
+
+extern "C" void body(...);
+
+void func(int n) {
+
+  // The associated statement must be a for loop.
+  // expected-error at +2 {{statement after '#pragma omp flatten' must be a for loop}}
+  #pragma omp flatten
+  ;
+
+  // A non-loop statement is rejected as well.
+  // expected-error at +2 {{statement after '#pragma omp flatten' must be a for loop}}
+  #pragma omp flatten
+  int b = 0;
+
+  // A single loop is not enough: flatten combines two perfectly nested loops,
+  // so the body of the outer loop must itself be a for loop.
+  #pragma omp flatten
+  for (int i = 0; i < 7; ++i)
+    // expected-error at +1 {{statement after '#pragma omp flatten' must be a for loop}}
+    ;
+
+  // The associated statement of a directive is not a for loop.
+  // expected-error at +2 {{statement after '#pragma omp flatten' must be a for loop}}
+  #pragma omp flatten
+  #pragma omp for
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 7; ++j)
+      body(i, j);
+
+  {
+    // expected-error at +2 {{expected statement}}
+    #pragma omp flatten
+  }
+
+  // The loops must be perfectly nested: no code is allowed between them.
+  #pragma omp flatten
+  for (int i = 0; i < n; ++i) {
+    int x = 0;
+    // expected-error at -2 {{statement after '#pragma omp flatten' must be a for loop}}
+    for (int j = 0; j < n; ++j)
+      body(i, j, x);
+  }
+
+  // Each affected loop must be in OpenMP canonical form.
+  #pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    // expected-error at +1 {{condition of OpenMP for loop must be a relational comparison ('<', '<=', '>', '>=', or '!=') of loop variable 'j'}}
+    for (int j = 0; j / 3 < n; ++j)
+      body(i, j);
+
+  // The affected loops must be rectangular: an inner bound may not depend on an
+  // outer loop counter.
+  #pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    // expected-error at +1 {{expected loop invariant expression}}
+    for (int j = i; j < n; ++j)
+      body(i, j);
+
+  // The 'sizes' clause is not allowed on 'flatten'.
+  // expected-error at +1 {{unexpected OpenMP clause 'sizes' in directive '#pragma omp flatten'}}
+  #pragma omp flatten sizes(2)
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+
+  // The 'permutation' clause is not allowed on 'flatten'.
+  // expected-error at +1 {{unexpected OpenMP clause 'permutation' in directive '#pragma omp flatten'}}
+  #pragma omp flatten permutation(2, 1)
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+
+  // Tokens after the directive name are ignored with a warning.
+  // expected-warning at +1 {{extra tokens at the end of '#pragma omp flatten' are ignored}}
+  #pragma omp flatten foo
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+
+  // The 'depth' clause is only available from OpenMP 6.1; it is rejected here
+  // under -fopenmp-version=60.
+  // expected-error at +1 {{unexpected OpenMP clause 'depth' in directive '#pragma omp flatten'}}
+  #pragma omp flatten depth(2)
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+}
diff --git a/clang/test/OpenMP/flatten_serialize_module.cpp b/clang/test/OpenMP/flatten_serialize_module.cpp
new file mode 100644
index 0000000000000..5d3420500b282
--- /dev/null
+++ b/clang/test/OpenMP/flatten_serialize_module.cpp
@@ -0,0 +1,25 @@
+// C++20 module interface with `#pragma omp flatten` — emit BMI + import; AST retains directive.
+//
+// RUN: rm -rf %t && split-file %s %t && cd %t
+// RUN: %clang_cc1 -std=c++20 -fopenmp -fopenmp-version=60 -triple x86_64-unknown-linux-gnu %t/FlattenMod.cppm -emit-module-interface -o %t/FlattenMod.pcm
+// RUN: %clang_cc1 -std=c++20 -fopenmp -fopenmp-version=60 -triple x86_64-unknown-linux-gnu %t/UseFlattenMod.cpp -fmodule-file=FlattenMod=%t/FlattenMod.pcm -ast-dump-all | FileCheck %t/FlattenMod.cppm
+
+// expected-no-diagnostics
+
+//--- FlattenMod.cppm
+module;
+export module FlattenMod;
+
+export void flattenfoo(int n, int m) {
+// CHECK: OMPFlattenDirective
+// CHECK: ForStmt
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j) {
+    }
+}
+
+//--- UseFlattenMod.cpp
+import FlattenMod;
+
+void g(void) { flattenfoo(10, 20); }
diff --git a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
index 4190d4703e37d..34c0fd27821d1 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
@@ -3126,6 +3126,28 @@ void f() {
   EXPECT_TRUE(notMatchesWithOpenMP60(ParallelOnly, Matcher));
 }
 
+// OpenMP 6 flatten directive
+TEST(ASTMatchersTestOpenMP, OMPFlattenDirective) {
+  auto Matcher = stmt(ompFlattenDirective());
+
+  StringRef FlattenOk = R"(
+void f() {
+#pragma omp flatten
+  for (int i = 0; i < 10; ++i)
+    for (int j = 0; j < 10; ++j) {}
+}
+)";
+  EXPECT_TRUE(matchesWithOpenMP60(FlattenOk, Matcher));
+
+  StringRef ParallelOnly = R"(
+void f() {
+#pragma omp parallel
+  ;
+}
+)";
+  EXPECT_TRUE(notMatchesWithOpenMP60(ParallelOnly, Matcher));
+}
+
 TEST(ASTMatchersTestOpenMP, OMPSplitDirective_HasCountsClause) {
   auto Matcher = stmt(ompSplitDirective(hasAnyClause(ompCountsClause())));
 
diff --git a/flatten-verify-tmp/01_default_depth2.cpp b/flatten-verify-tmp/01_default_depth2.cpp
new file mode 100644
index 0000000000000..900d0263622cd
--- /dev/null
+++ b/flatten-verify-tmp/01_default_depth2.cpp
@@ -0,0 +1,8 @@
+// Default flatten: combine 2 perfectly nested loops (depth defaults to 2).
+extern "C" void body(int, int);
+void t(int n, int m) {
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
diff --git a/flatten-verify-tmp/02_depth3_full.cpp b/flatten-verify-tmp/02_depth3_full.cpp
new file mode 100644
index 0000000000000..ba9f0b7ed8808
--- /dev/null
+++ b/flatten-verify-tmp/02_depth3_full.cpp
@@ -0,0 +1,9 @@
+// depth(3): fully flatten a three-deep perfect nest (OpenMP 6.1).
+extern "C" void body(int, int, int);
+void t(int n, int m, int p) {
+#pragma omp flatten depth(3)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        body(i, j, k);
+}
diff --git a/flatten-verify-tmp/03_depth2_partial.cpp b/flatten-verify-tmp/03_depth2_partial.cpp
new file mode 100644
index 0000000000000..35b9dc83834fd
--- /dev/null
+++ b/flatten-verify-tmp/03_depth2_partial.cpp
@@ -0,0 +1,10 @@
+// depth(2) on a three-deep nest: only the outer two loops are combined; the
+// innermost loop must remain as ordinary body of the flattened loop.
+extern "C" void body(int, int, int);
+void t(int n, int m, int p) {
+#pragma omp flatten depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        body(i, j, k);
+}
diff --git a/flatten-verify-tmp/04_depth1_identity.cpp b/flatten-verify-tmp/04_depth1_identity.cpp
new file mode 100644
index 0000000000000..ae62a11312fe8
--- /dev/null
+++ b/flatten-verify-tmp/04_depth1_identity.cpp
@@ -0,0 +1,8 @@
+// depth(1): identity transform. A single loop is "combined"; no mixed-radix
+// decomposition (no srem), the flattened counter maps straight to the loop var.
+extern "C" void body(int);
+void t(int n) {
+#pragma omp flatten depth(1)
+  for (int i = 0; i < n; ++i)
+    body(i);
+}
diff --git a/flatten-verify-tmp/05_rangefor_template.cpp b/flatten-verify-tmp/05_rangefor_template.cpp
new file mode 100644
index 0000000000000..89928c34a8a15
--- /dev/null
+++ b/flatten-verify-tmp/05_rangefor_template.cpp
@@ -0,0 +1,20 @@
+// Range-for (CXXForRangeStmt) nest + a function template instantiated with a
+// dependent depth(K). Exercises the __begin/__end pre-inits and the deferred
+// (dependent-context) desugaring path.
+#include <stdlibc++.h>
+extern "C" void body(int, int);
+
+void ranges(std::vector<int> &a, std::vector<int> &b) {
+#pragma omp flatten
+  for (int i : a)
+    for (int j : b)
+      body(i, j);
+}
+
+template <int K> void tmpl(int n, int m) {
+#pragma omp flatten depth(K)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+template void tmpl<2>(int, int);
diff --git a/flatten-verify-tmp/README.md b/flatten-verify-tmp/README.md
new file mode 100644
index 0000000000000..f48f4d06cfec0
--- /dev/null
+++ b/flatten-verify-tmp/README.md
@@ -0,0 +1,39 @@
+# Temporary flatten + depth verification cases
+
+Throwaway sandbox for eyeballing the flatten desugaring (AST / IR / runtime).
+Not part of any PR — delete freely. The committed regression tests live under
+`clang/test/OpenMP/flatten*` and `openmp/runtime/test/transform/flatten/*`.
+
+`CLANG=build/bin/clang`, `CLANGXX=build/bin/clang++`, run from the repo root.
+
+| File | Case | What to look for |
+|------|------|------------------|
+| `01_default_depth2.cpp` | default (no clause) = depth 2 | i64 `.flatten.iv`, `mul nsw`, `sdiv`+`srem` |
+| `02_depth3_full.cpp` | `depth(3)` full flatten | 3 `.flatten.iv.N`, two-level mixed radix |
+| `03_depth2_partial.cpp` | `depth(2)` on 3-deep nest | 2 flatten IVs, **inner `%k` loop stays** |
+| `04_depth1_identity.cpp` | `depth(1)` identity | i32 IV, **no `srem`**, direct store |
+| `05_rangefor_template.cpp` | range-for + template `depth(K)` | ast-print round-trips; instantiation |
+| `end_result_check.cpp` | runtime semantic check | prints `OK`, exit 0 (order+set preserved) |
+
+## Commands
+
+AST (round-trip original loops) and the desugared node:
+```
+$CLANG -cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-print FILE
+$CLANG -cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-dump  FILE
+```
+
+IR (the div/mod arithmetic that implements the transform):
+```
+$CLANG -cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -emit-llvm -o - FILE
+```
+
+Range-for / template need the driver (libstdc++ + template instantiation):
+```
+$CLANGXX -fopenmp -fopenmp-version=61 -Xclang -ast-print -fsyntax-only 05_rangefor_template.cpp
+```
+
+Runtime end-result (must print OK, exit 0):
+```
+$CLANGXX -fopenmp -fopenmp-version=61 -O0 end_result_check.cpp -o /tmp/erc && /tmp/erc
+```
diff --git a/flatten-verify-tmp/end_result_check.cpp b/flatten-verify-tmp/end_result_check.cpp
new file mode 100644
index 0000000000000..3aa8850829058
--- /dev/null
+++ b/flatten-verify-tmp/end_result_check.cpp
@@ -0,0 +1,82 @@
+// End-result (semantic) verification: flatten must preserve BOTH the set of
+// iterations and their row-major visitation ORDER. We record the sequence of
+// visited tuples from a flattened nest and compare it against the sequence from
+// an identical non-flattened reference nest. Exit code 0 == identical.
+//
+// flatten is a pure frontend AST desugaring to an ordinary loop, so this runs
+// as a normal program (build with: clang++ -fopenmp -fopenmp-version=61).
+#include <cassert>
+#include <cstdio>
+#include <vector>
+
+using Seq = std::vector<long>;
+
+static void rec(Seq &s, int i, int j, int k = -1) {
+  s.push_back(i);
+  s.push_back(j);
+  if (k >= 0)
+    s.push_back(k);
+}
+
+int main() {
+  const int n = 3, m = 4, p = 2;
+
+  // --- depth(2) default over 2 loops ---
+  {
+    Seq flat, ref;
+#pragma omp flatten
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        rec(flat, i, j);
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        rec(ref, i, j);
+    assert(flat == ref && "depth(2) order/set mismatch");
+  }
+
+  // --- depth(3) full flatten over 3 loops ---
+  {
+    Seq flat, ref;
+#pragma omp flatten depth(3)
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        for (int k = 0; k < p; ++k)
+          rec(flat, i, j, k);
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        for (int k = 0; k < p; ++k)
+          rec(ref, i, j, k);
+    assert(flat == ref && "depth(3) order/set mismatch");
+  }
+
+  // --- depth(2) partial: inner k-loop stays intact ---
+  {
+    Seq flat, ref;
+#pragma omp flatten depth(2)
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        for (int k = 0; k < p; ++k)
+          rec(flat, i, j, k);
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        for (int k = 0; k < p; ++k)
+          rec(ref, i, j, k);
+    assert(flat == ref && "depth(2)-partial order/set mismatch");
+  }
+
+  // --- non-unit step / non-zero start must survive ---
+  {
+    Seq flat, ref;
+#pragma omp flatten
+    for (int i = 5; i < 12; i += 2)
+      for (int j = 1; j < 7; j += 3)
+        rec(flat, i, j);
+    for (int i = 5; i < 12; i += 2)
+      for (int j = 1; j < 7; j += 3)
+        rec(ref, i, j);
+    assert(flat == ref && "stride/offset mismatch");
+  }
+
+  std::puts("flatten end-result preserved: OK");
+  return 0;
+}
diff --git a/openmp/runtime/test/transform/flatten/depth-partial.c b/openmp/runtime/test/transform/flatten/depth-partial.c
new file mode 100644
index 0000000000000..2fda3a04cbd84
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/depth-partial.c
@@ -0,0 +1,45 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+// A depth(2) clause on a three-deep loop nest flattens only the outermost two
+// loops; the innermost loop is left untouched and executes normally inside the
+// flattened loop body.
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+int main() {
+  printf("do\n");
+#pragma omp flatten depth(2)
+  for (int i = 5; i < 11; i += 2)
+    for (int j = 1; j < 7; j += 2)
+      for (int k = 0; k < 2; ++k)
+        printf("i=%d j=%d k=%d\n", i, j, k);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: i=5 j=1 k=0
+// CHECK-NEXT: i=5 j=1 k=1
+// CHECK-NEXT: i=5 j=3 k=0
+// CHECK-NEXT: i=5 j=3 k=1
+// CHECK-NEXT: i=5 j=5 k=0
+// CHECK-NEXT: i=5 j=5 k=1
+// CHECK-NEXT: i=7 j=1 k=0
+// CHECK-NEXT: i=7 j=1 k=1
+// CHECK-NEXT: i=7 j=3 k=0
+// CHECK-NEXT: i=7 j=3 k=1
+// CHECK-NEXT: i=7 j=5 k=0
+// CHECK-NEXT: i=7 j=5 k=1
+// CHECK-NEXT: i=9 j=1 k=0
+// CHECK-NEXT: i=9 j=1 k=1
+// CHECK-NEXT: i=9 j=3 k=0
+// CHECK-NEXT: i=9 j=3 k=1
+// CHECK-NEXT: i=9 j=5 k=0
+// CHECK-NEXT: i=9 j=5 k=1
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/flatten/empty.c b/openmp/runtime/test/transform/flatten/empty.c
new file mode 100644
index 0000000000000..ecf1e8ef622f2
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/empty.c
@@ -0,0 +1,42 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+// The bounds are runtime values so that an empty iteration space does not
+// trigger a compile-time division-by-zero diagnostic for the (never executed)
+// induction-variable recovery.
+static void flatten(int n, int m) {
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      printf("i=%d j=%d\n", i, j);
+}
+
+int main() {
+  printf("empty-inner-begin\n");
+  flatten(3, 0);
+  printf("empty-inner-end\n");
+
+  printf("empty-outer-begin\n");
+  flatten(0, 3);
+  printf("empty-outer-end\n");
+
+  printf("single-begin\n");
+  flatten(1, 1);
+  printf("single-end\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      empty-inner-begin
+// CHECK-NEXT: empty-inner-end
+// CHECK-NEXT: empty-outer-begin
+// CHECK-NEXT: empty-outer-end
+// CHECK-NEXT: single-begin
+// CHECK-NEXT: i=0 j=0
+// CHECK-NEXT: single-end
diff --git a/openmp/runtime/test/transform/flatten/foreach.cpp b/openmp/runtime/test/transform/flatten/foreach.cpp
new file mode 100644
index 0000000000000..14997f7c4051d
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/foreach.cpp
@@ -0,0 +1,31 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdio>
+#include <vector>
+
+int main() {
+  printf("do\n");
+  std::vector<int> outer{10, 20};
+  std::vector<int> inner{1, 2, 3};
+#pragma omp flatten
+  for (int a : outer)
+    for (int b : inner)
+      printf("a=%d b=%d\n", a, b);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: a=10 b=1
+// CHECK-NEXT: a=10 b=2
+// CHECK-NEXT: a=10 b=3
+// CHECK-NEXT: a=20 b=1
+// CHECK-NEXT: a=20 b=2
+// CHECK-NEXT: a=20 b=3
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/flatten/parallel-wsloop-foreach.cpp b/openmp/runtime/test/transform/flatten/parallel-wsloop-foreach.cpp
new file mode 100644
index 0000000000000..7ddcde2fd622c
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/parallel-wsloop-foreach.cpp
@@ -0,0 +1,32 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdio>
+
+int main() {
+  printf("do\n");
+  // The loop generated by 'flatten' is the canonical loop associated with the
+  // worksharing 'omp for'. With a single thread the iteration order is the
+  // original row-major order.
+#pragma omp parallel for num_threads(1)
+#pragma omp flatten
+  for (int i = 0; i < 2; ++i)
+    for (int j = 0; j < 3; ++j)
+      printf("i=%d j=%d\n", i, j);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: i=0 j=0
+// CHECK-NEXT: i=0 j=1
+// CHECK-NEXT: i=0 j=2
+// CHECK-NEXT: i=1 j=0
+// CHECK-NEXT: i=1 j=1
+// CHECK-NEXT: i=1 j=2
+// CHECK-NEXT: done

>From 267ca5079c57bc299a056eb5a7d414c74c6c04f4 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Tue, 18 Aug 2026 07:15:20 -0400
Subject: [PATCH 15/16] conflict-resolve

---
 clang/bindings/python/clang/cindex.py | 5 ++++-
 clang/include/clang-c/Index.h         | 6 +++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/clang/bindings/python/clang/cindex.py b/clang/bindings/python/clang/cindex.py
index ccad140e8c861..56e20fdfad781 100644
--- a/clang/bindings/python/clang/cindex.py
+++ b/clang/bindings/python/clang/cindex.py
@@ -1456,8 +1456,11 @@ def is_unexposed(self):
     # OpenMP split directive.
     OMP_SPLIT_DIRECTIVE = 312
 
+    # OpenMP ordered-blockassoc directive.
+    OMP_ORDERED_BLOCK_ASSOC_DIRECTIVE = 313
+
     # OpenMP flatten directive.
-    OMP_FLATTEN_DIRECTIVE = 313
+    OMP_FLATTEN_DIRECTIVE = 314
 
     # OpenACC Compute Construct.
     OPEN_ACC_COMPUTE_DIRECTIVE = 320
diff --git a/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h
index f561fb5724897..4c4cf392f172e 100644
--- a/clang/include/clang-c/Index.h
+++ b/clang/include/clang-c/Index.h
@@ -2170,9 +2170,13 @@ enum CXCursorKind {
    */
   CXCursor_OMPSplitDirective = 312,
 
+  /** OpenMP ordered-blockassoc directive.
+   */
+  CXCursor_OMPOrderedBlockAssocDirective = 313,
+
   /** OpenMP flatten directive.
    */
-  CXCursor_OMPFlattenDirective = 313,
+  CXCursor_OMPFlattenDirective = 314,
 
   /** OpenACC Compute Construct.
    */

>From 4788e3e9a56751f93a2268d6de146bc9f2bcd34d Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Tue, 18 Aug 2026 07:17:36 -0400
Subject: [PATCH 16/16] revised

---
 clang/include/clang/AST/OpenMPClause.h        |   8 +-
 clang/include/clang/AST/StmtOpenMP.h          |   4 +-
 .../clang/Basic/DiagnosticSemaKinds.td        |   6 +
 clang/lib/AST/OpenMPClause.cpp                |  12 --
 clang/lib/Sema/SemaOpenMP.cpp                 | 112 ++++++++++--------
 5 files changed, 74 insertions(+), 68 deletions(-)

diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 50d1364a9726d..05ccf4db5752b 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -1411,14 +1411,14 @@ class OMPDepthClause final : public OMPClause {
 public:
   /// Build an AST node for a 'depth' clause.
   ///
-  /// \param C         Context of the AST.
   /// \param StartLoc  Location of the 'depth' identifier.
   /// \param LParenLoc Location of '('.
   /// \param EndLoc    Location of ')'.
   /// \param Depth     The depth expression.
-  static OMPDepthClause *Create(const ASTContext &C, SourceLocation StartLoc,
-                                SourceLocation LParenLoc, SourceLocation EndLoc,
-                                Expr *Depth);
+  OMPDepthClause(SourceLocation StartLoc, SourceLocation LParenLoc,
+                 SourceLocation EndLoc, Expr *Depth)
+      : OMPClause(llvm::omp::OMPC_depth, StartLoc, EndLoc),
+        LParenLoc(LParenLoc), Depth(Depth) {}
 
   /// Build an empty 'depth' AST node for deserialization.
   ///
diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index 1420331f37c1f..4ae6f364dced3 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -5999,8 +5999,8 @@ class OMPFlattenDirective final
   /// \param StartLoc  Location of the introducer (e.g. the 'omp' token).
   /// \param EndLoc    Location of the directive's end (e.g. the tok::eod).
   /// \param Clauses   The directive's clauses.
-  /// \param NumLoops  Number of affected loops (the flatten depth, currently
-  ///                  always 2).
+  /// \param NumLoops  Number of affected loops (the flatten depth: the
+  ///                  argument of the 'depth' clause, or 2 if omitted).
   /// \param AssociatedStmt  The outermost associated loop.
   /// \param TransformedStmt The flattened loop, or nullptr in dependent
   ///                        contexts.
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 7e20630708312..fc44b8d15239e 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12172,6 +12172,12 @@ def err_omp_invalid_looprange
 def warn_omp_redundant_fusion : Warning<"looprange clause selects a single "
                                         "loop, resulting in redundant fusion">,
                                 InGroup<OpenMPClauses>;
+def warn_omp_flatten_omitted_depth : Warning<
+  "'flatten' without a 'depth' clause only combines 2 loops, but 3 or more "
+  "loops are perfectly nested">,
+  InGroup<OpenMPClauses>;
+def err_omp_clause_not_supported_yet : Error<
+  "'%0' clause is not yet supported on '#pragma omp %1' directive">;
 def err_omp_not_for : Error<
   "%select{statement after '#pragma omp %1' must be a for loop|"
   "expected %2 for loops after '#pragma omp %1'%select{|, but found only %4}3}0">;
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index 7b2f67a31cad9..7311e404b66c3 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -1057,18 +1057,6 @@ OMPPartialClause *OMPPartialClause::CreateEmpty(const ASTContext &C) {
   return new (C) OMPPartialClause();
 }
 
-OMPDepthClause *OMPDepthClause::Create(const ASTContext &C,
-                                       SourceLocation StartLoc,
-                                       SourceLocation LParenLoc,
-                                       SourceLocation EndLoc, Expr *Depth) {
-  OMPDepthClause *Clause = CreateEmpty(C);
-  Clause->setLocStart(StartLoc);
-  Clause->setLParenLoc(LParenLoc);
-  Clause->setLocEnd(EndLoc);
-  Clause->setDepth(Depth);
-  return Clause;
-}
-
 OMPDepthClause *OMPDepthClause::CreateEmpty(const ASTContext &C) {
   return new (C) OMPDepthClause();
 }
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index fcc256dec3c88..caaba5a3be9fa 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -6839,6 +6839,7 @@ StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
       case OMPC_safelen:
       case OMPC_simdlen:
       case OMPC_sizes:
+      case OMPC_depth:
       case OMPC_default:
       case OMPC_proc_bind:
       case OMPC_private:
@@ -16395,29 +16396,6 @@ StmtResult SemaOpenMP::ActOnOpenMPInterchangeDirective(
                                          buildPreInits(Context, PreInits));
 }
 
-/// Counts the perfectly nested canonical loop depth at \p AStmt. Returns
-/// std::nullopt if a loop-generating transformation prevents a static count.
-static std::optional<unsigned> getCanonicalLoopNestDepth(Stmt *AStmt) {
-  unsigned Depth = 0;
-  Stmt *CurStmt = AStmt ? AStmt->IgnoreContainers() : nullptr;
-  while (CurStmt) {
-    if (isa<OMPLoopTransformationDirective>(CurStmt))
-      return std::nullopt;
-    if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(CurStmt))
-      CurStmt = CanonLoop->getLoopStmt();
-    Stmt *Body = nullptr;
-    if (auto *For = dyn_cast<ForStmt>(CurStmt))
-      Body = For->getBody();
-    else if (auto *RangeFor = dyn_cast<CXXForRangeStmt>(CurStmt))
-      Body = RangeFor->getBody();
-    else
-      break;
-    ++Depth;
-    CurStmt = Body ? Body->IgnoreContainers() : nullptr;
-  }
-  return Depth;
-}
-
 StmtResult
 SemaOpenMP::ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
                                         Stmt *AStmt, SourceLocation StartLoc,
@@ -16432,36 +16410,49 @@ SemaOpenMP::ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
 
   // flatten without 'depth' clause combines two loops; 'depth(k)' selects k.
   unsigned NumLoops = 2;
-  bool DepthIsValueDependent = false;
+  bool DepthIsDependent = false;
   const auto *DepthClause =
       OMPExecutableDirective::getSingleClause<OMPDepthClause>(Clauses);
   if (DepthClause) {
     Expr *DepthExpr = DepthClause->getDepth();
-    if (DepthExpr && DepthExpr->isValueDependent()) {
-      DepthIsValueDependent = true;
+    if (DepthExpr && DepthExpr->isInstantiationDependent()) {
+      DepthIsDependent = true;
     } else if (DepthExpr) {
       Expr::EvalResult EvalResult;
       if (DepthExpr->EvaluateAsInt(EvalResult, Context))
-        NumLoops = EvalResult.Val.getInt().getZExtValue();
+        NumLoops = EvalResult.Val.getInt().getLimitedValue();
     }
   }
 
-  // Report "expected k for loops, but found only n" when depth exceeds the
-  // perfect nest (same form as 'collapse'); skip if depth or nest is unknown.
-  if (DepthClause && !DepthIsValueDependent) {
-    if (std::optional<unsigned> NestDepth = getCanonicalLoopNestDepth(AStmt);
-        NestDepth && NumLoops > *NestDepth) {
+  // Count perfectly nested loops with doForAllLoops. When 'depth' is present,
+  // walk NumLoops iterations to diagnose an insufficient nest. When it is
+  // omitted, walk one extra loop (3 total) so we can warn that default
+  // flatten only combines 2 of a deeper nest.
+  if (!DepthIsDependent) {
+    unsigned WalkLimit = DepthClause ? NumLoops : 3;
+    unsigned Found = 0;
+    bool Enough = OMPLoopBasedDirective::doForAllLoops(
+        AStmt->IgnoreContainers(), /*TryImperfectlyNestedLoops=*/false,
+        WalkLimit, [&](unsigned Cnt, Stmt *S) {
+          if (!isa<ForStmt>(S) && !isa<CXXForRangeStmt>(S))
+            return true;
+          Found = Cnt + 1;
+          return false;
+        });
+    if (DepthClause && !Enough) {
       Diag(AStmt->getBeginLoc(), diag::err_omp_not_for)
           << /*expected N for loops form=*/1
-          << getOpenMPDirectiveName(OMPD_flatten) << NumLoops
-          << (*NestDepth > 0) << *NestDepth;
+          << getOpenMPDirectiveName(OMPD_flatten) << NumLoops << (Found > 0)
+          << Found;
       return StmtError();
     }
+    if (!DepthClause && Found >= 3)
+      Diag(StartLoc, diag::warn_omp_flatten_omitted_depth);
   }
 
-  // Defer when 'depth' is value-dependent (concrete k unknown until
+  // Defer when 'depth' is instantiation-dependent (concrete k unknown until
   // instantiation).
-  if (DepthIsValueDependent)
+  if (DepthIsDependent)
     return OMPFlattenDirective::Create(Context, StartLoc, EndLoc, Clauses,
                                        NumLoops, AStmt, nullptr, nullptr);
 
@@ -16537,20 +16528,30 @@ SemaOpenMP::ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
     AllCountsLessThan32Bits &=
         Context.getTypeSize(LoopHelpers[I].NumIterations->getType()) < 32;
 
-  ExprResult TripCount32 = BuildTripCount(/*Bits=*/32);
-  ExprResult TripCount64 = BuildTripCount(/*Bits=*/64);
-  if (!TripCount32.isUsable() || !TripCount64.isUsable())
+  ExprResult TripCount;
+  if (AllCountsLessThan32Bits || NumLoops == 1) {
+    TripCount = BuildTripCount(/*Bits=*/32);
+  } else {
+    ExprResult TripCount64 = BuildTripCount(/*Bits=*/64);
+    if (!TripCount64.isUsable())
+      return StmtError();
+    if (fitsInto(
+            /*Bits=*/32,
+            TripCount64.get()->getType()->hasSignedIntegerRepresentation(),
+            TripCount64.get(), SemaRef)) {
+      ExprResult TripCount32 = BuildTripCount(/*Bits=*/32);
+      if (TripCount32.isUsable() &&
+          Context.getTypeSize(TripCount32.get()->getType()) == 32)
+        TripCount = TripCount32;
+      else
+        TripCount = TripCount64;
+    } else {
+      TripCount = TripCount64;
+    }
+  }
+  if (!TripCount.isUsable())
     return StmtError();
 
-  ExprResult TripCount = TripCount64;
-  if (Context.getTypeSize(TripCount32.get()->getType()) == 32 &&
-      (AllCountsLessThan32Bits || NumLoops == 1 ||
-       fitsInto(
-           /*Bits=*/32,
-           TripCount32.get()->getType()->hasSignedIntegerRepresentation(),
-           TripCount64.get(), SemaRef)))
-    TripCount = TripCount32;
-
   QualType IVTy = TripCount.get()->getType();
   uint64_t IVWidth = Context.getTypeSize(IVTy);
 
@@ -16680,6 +16681,13 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
   if (!AStmt)
     return StmtError();
 
+  if (const auto *DepthC =
+          OMPExecutableDirective::getSingleClause<OMPDepthClause>(Clauses)) {
+    Diag(DepthC->getBeginLoc(), diag::err_omp_clause_not_supported_yet)
+        << "depth" << getOpenMPDirectiveName(OMPD_fuse);
+    return StmtError();
+  }
+
   // Defer transformation in dependent contexts
   // The NumLoopNests argument is set to a placeholder 1 (even though
   // using looprange fuse could yield up to 3 top level loop nests)
@@ -17911,6 +17919,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPSimpleClause(
   case OMPC_safelen:
   case OMPC_simdlen:
   case OMPC_sizes:
+  case OMPC_depth:
   case OMPC_allocator:
   case OMPC_collapse:
   case OMPC_schedule:
@@ -18532,8 +18541,8 @@ OMPClause *SemaOpenMP::ActOnOpenMPDepthClause(Expr *DepthExpr,
     return nullptr;
   DepthExpr = DepthResult.get();
 
-  return OMPDepthClause::Create(getASTContext(), StartLoc, LParenLoc, EndLoc,
-                                DepthExpr);
+  return new (getASTContext())
+      OMPDepthClause(StartLoc, LParenLoc, EndLoc, DepthExpr);
 }
 
 OMPClause *SemaOpenMP::ActOnOpenMPLoopRangeClause(
@@ -18671,6 +18680,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPSingleExprWithArgClause(
   case OMPC_safelen:
   case OMPC_simdlen:
   case OMPC_sizes:
+  case OMPC_depth:
   case OMPC_allocator:
   case OMPC_collapse:
   case OMPC_proc_bind:
@@ -18959,6 +18969,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPClause(OpenMPClauseKind Kind,
   case OMPC_safelen:
   case OMPC_simdlen:
   case OMPC_sizes:
+  case OMPC_depth:
   case OMPC_allocator:
   case OMPC_collapse:
   case OMPC_schedule:
@@ -19643,6 +19654,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPVarListClause(OpenMPClauseKind Kind,
   case OMPC_safelen:
   case OMPC_simdlen:
   case OMPC_sizes:
+  case OMPC_depth:
   case OMPC_allocator:
   case OMPC_collapse:
   case OMPC_default:


From openmp-commits at lists.llvm.org  Tue Aug 18 04:50:11 2026
From: openmp-commits at lists.llvm.org (Amit Tiwari via Openmp-commits)
Date: Tue, 18 Aug 2026 04:50:11 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [Clang][OpenMP] Add
 `#pragma omp flatten` loop directive + `depth` clause (PR #206977)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206977@github.com>
Message-ID: <6a8446f3.ef5297aa.395e37.697a@mx.google.com>


https://github.com/loopacino updated https://github.com/llvm/llvm-project/pull/206977

>From 0fb5bd42f1e127faea963c359b1e3c69cb969c60 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 1 Jul 2026 09:06:58 -0400
Subject: [PATCH 01/15] add OMPFlattenDirective ast node

---
 clang/include/clang/AST/RecursiveASTVisitor.h |  3 +
 clang/include/clang/AST/StmtOpenMP.h          | 75 ++++++++++++++++++-
 clang/include/clang/Basic/StmtNodes.td        |  1 +
 clang/lib/AST/StmtOpenMP.cpp                  | 20 +++++
 clang/lib/AST/StmtPrinter.cpp                 |  5 ++
 clang/lib/AST/StmtProfile.cpp                 |  4 +
 clang/lib/Basic/OpenMPKinds.cpp               |  2 +-
 7 files changed, 108 insertions(+), 2 deletions(-)

diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index b000a34043696..e425f7a31f59c 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -3227,6 +3227,9 @@ DEF_TRAVERSE_STMT(OMPFuseDirective,
 DEF_TRAVERSE_STMT(OMPInterchangeDirective,
                   { TRY_TO(TraverseOMPExecutableDirective(S)); })
 
+DEF_TRAVERSE_STMT(OMPFlattenDirective,
+                  { TRY_TO(TraverseOMPExecutableDirective(S)); })
+
 DEF_TRAVERSE_STMT(OMPSplitDirective,
                   { TRY_TO(TraverseOMPExecutableDirective(S)); })
 
diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index dbc76e7df8ecd..1420331f37c1f 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -1041,7 +1041,7 @@ class OMPCanonicalLoopNestTransformationDirective
     Stmt::StmtClass C = T->getStmtClass();
     return C == OMPTileDirectiveClass || C == OMPUnrollDirectiveClass ||
            C == OMPReverseDirectiveClass || C == OMPInterchangeDirectiveClass ||
-           C == OMPStripeDirectiveClass;
+           C == OMPStripeDirectiveClass || C == OMPFlattenDirectiveClass;
   }
 };
 
@@ -5959,6 +5959,79 @@ class OMPInterchangeDirective final
   }
 };
 
+/// Represents the '#pragma omp flatten' loop transformation directive.
+///
+/// \code{c}
+///   #pragma omp flatten
+///   for (int i = 0; i < m; ++i)
+///     for (int j = 0; j < n; ++j)
+///       ..
+/// \endcode
+class OMPFlattenDirective final
+    : public OMPCanonicalLoopNestTransformationDirective {
+  friend class ASTStmtReader;
+  friend class OMPExecutableDirective;
+
+  /// Offsets of child members.
+  enum {
+    PreInitsOffset = 0,
+    TransformedStmtOffset,
+  };
+
+  explicit OMPFlattenDirective(SourceLocation StartLoc, SourceLocation EndLoc,
+                               unsigned NumLoops)
+      : OMPCanonicalLoopNestTransformationDirective(
+            OMPFlattenDirectiveClass, llvm::omp::OMPD_flatten, StartLoc, EndLoc,
+            NumLoops) {}
+
+  void setPreInits(Stmt *PreInits) {
+    Data->getChildren()[PreInitsOffset] = PreInits;
+  }
+
+  void setTransformedStmt(Stmt *S) {
+    Data->getChildren()[TransformedStmtOffset] = S;
+  }
+
+public:
+  /// Create a new AST node representation for '#pragma omp flatten'.
+  ///
+  /// \param C         Context of the AST.
+  /// \param StartLoc  Location of the introducer (e.g. the 'omp' token).
+  /// \param EndLoc    Location of the directive's end (e.g. the tok::eod).
+  /// \param Clauses   The directive's clauses.
+  /// \param NumLoops  Number of affected loops (the flatten depth, currently
+  ///                  always 2).
+  /// \param AssociatedStmt  The outermost associated loop.
+  /// \param TransformedStmt The flattened loop, or nullptr in dependent
+  ///                        contexts.
+  /// \param PreInits  Helper preinits statements for the loop nest.
+  static OMPFlattenDirective *
+  Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation EndLoc,
+         ArrayRef<OMPClause *> Clauses, unsigned NumLoops, Stmt *AssociatedStmt,
+         Stmt *TransformedStmt, Stmt *PreInits);
+
+  /// Build an empty '#pragma omp flatten' AST node for deserialization.
+  ///
+  /// \param C          Context of the AST.
+  /// \param NumClauses Number of clauses to allocate.
+  /// \param NumLoops   Number of associated loops to allocate.
+  static OMPFlattenDirective *
+  CreateEmpty(const ASTContext &C, unsigned NumClauses, unsigned NumLoops);
+
+  /// Gets the flattened loop after the transformation. This is the de-sugared
+  /// replacement or nullptr in dependent contexts.
+  Stmt *getTransformedStmt() const {
+    return Data->getChildren()[TransformedStmtOffset];
+  }
+
+  /// Return preinits statement.
+  Stmt *getPreInits() const { return Data->getChildren()[PreInitsOffset]; }
+
+  static bool classof(const Stmt *T) {
+    return T->getStmtClass() == OMPFlattenDirectiveClass;
+  }
+};
+
 /// The base class for all transformation directives of canonical loop
 /// sequences (currently only 'fuse')
 class OMPCanonicalLoopSequenceTransformationDirective
diff --git a/clang/include/clang/Basic/StmtNodes.td b/clang/include/clang/Basic/StmtNodes.td
index e166894ea024b..0d7a0c1490a1e 100644
--- a/clang/include/clang/Basic/StmtNodes.td
+++ b/clang/include/clang/Basic/StmtNodes.td
@@ -247,6 +247,7 @@ def OMPReverseDirective : StmtNode<OMPCanonicalLoopNestTransformationDirective>;
 def OMPSplitDirective : StmtNode<OMPCanonicalLoopNestTransformationDirective>;
 def OMPInterchangeDirective
     : StmtNode<OMPCanonicalLoopNestTransformationDirective>;
+def OMPFlattenDirective : StmtNode<OMPCanonicalLoopNestTransformationDirective>;
 def OMPCanonicalLoopSequenceTransformationDirective
     : StmtNode<OMPExecutableDirective, 1>;
 def OMPFuseDirective
diff --git a/clang/lib/AST/StmtOpenMP.cpp b/clang/lib/AST/StmtOpenMP.cpp
index 9d6b315effb41..477604ef3dafb 100644
--- a/clang/lib/AST/StmtOpenMP.cpp
+++ b/clang/lib/AST/StmtOpenMP.cpp
@@ -552,6 +552,26 @@ OMPInterchangeDirective::CreateEmpty(const ASTContext &C, unsigned NumClauses,
       SourceLocation(), SourceLocation(), NumLoops);
 }
 
+OMPFlattenDirective *OMPFlattenDirective::Create(
+    const ASTContext &C, SourceLocation StartLoc, SourceLocation EndLoc,
+    ArrayRef<OMPClause *> Clauses, unsigned NumLoops, Stmt *AssociatedStmt,
+    Stmt *TransformedStmt, Stmt *PreInits) {
+  OMPFlattenDirective *Dir = createDirective<OMPFlattenDirective>(
+      C, Clauses, AssociatedStmt, TransformedStmtOffset + 1, StartLoc, EndLoc,
+      NumLoops);
+  Dir->setTransformedStmt(TransformedStmt);
+  Dir->setPreInits(PreInits);
+  return Dir;
+}
+
+OMPFlattenDirective *OMPFlattenDirective::CreateEmpty(const ASTContext &C,
+                                                      unsigned NumClauses,
+                                                      unsigned NumLoops) {
+  return createEmptyDirective<OMPFlattenDirective>(
+      C, NumClauses, /*HasAssociatedStmt=*/true, TransformedStmtOffset + 1,
+      SourceLocation(), SourceLocation(), NumLoops);
+}
+
 OMPSplitDirective *
 OMPSplitDirective::Create(const ASTContext &C, SourceLocation StartLoc,
                           SourceLocation EndLoc, ArrayRef<OMPClause *> Clauses,
diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp
index 6c3294573e9d4..0bacb79ed86e3 100644
--- a/clang/lib/AST/StmtPrinter.cpp
+++ b/clang/lib/AST/StmtPrinter.cpp
@@ -800,6 +800,11 @@ void StmtPrinter::VisitOMPInterchangeDirective(OMPInterchangeDirective *Node) {
   PrintOMPExecutableDirective(Node);
 }
 
+void StmtPrinter::VisitOMPFlattenDirective(OMPFlattenDirective *Node) {
+  Indent() << "#pragma omp flatten";
+  PrintOMPExecutableDirective(Node);
+}
+
 void StmtPrinter::VisitOMPSplitDirective(OMPSplitDirective *Node) {
   Indent() << "#pragma omp split";
   PrintOMPExecutableDirective(Node);
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index 90eab530e0c2e..de7ed322d7424 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -1071,6 +1071,10 @@ void StmtProfiler::VisitOMPInterchangeDirective(
   VisitOMPCanonicalLoopNestTransformationDirective(S);
 }
 
+void StmtProfiler::VisitOMPFlattenDirective(const OMPFlattenDirective *S) {
+  VisitOMPCanonicalLoopNestTransformationDirective(S);
+}
+
 void StmtProfiler::VisitOMPSplitDirective(const OMPSplitDirective *S) {
   VisitOMPCanonicalLoopNestTransformationDirective(S);
 }
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index 675d86349c933..57e0347f8ea77 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -818,7 +818,7 @@ bool clang::isOpenMPCanonicalLoopNestTransformationDirective(
     OpenMPDirectiveKind DKind) {
   return DKind == OMPD_tile || DKind == OMPD_unroll || DKind == OMPD_reverse ||
          DKind == OMPD_split || DKind == OMPD_interchange ||
-         DKind == OMPD_stripe;
+         DKind == OMPD_stripe || DKind == OMPD_flatten;
 }
 
 bool clang::isOpenMPCanonicalLoopSequenceTransformationDirective(

>From 933007b6333584fb358adc4f89a61db1ae15d4cb Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:02:27 -0400
Subject: [PATCH 02/15] add OMPDepthClause ast node

---
 clang/include/clang/AST/OpenMPClause.h        | 68 +++++++++++++++++++
 clang/include/clang/AST/RecursiveASTVisitor.h |  6 ++
 clang/lib/AST/OpenMPClause.cpp                | 25 +++++++
 clang/lib/AST/StmtProfile.cpp                 |  5 ++
 clang/lib/Parse/ParseOpenMP.cpp               |  1 +
 llvm/include/llvm/Frontend/OpenMP/OMP.td      |  1 +
 6 files changed, 106 insertions(+)

diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 8ceafc4669297..50d1364a9726d 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -1380,6 +1380,74 @@ class OMPPartialClause final : public OMPClause {
   }
 };
 
+/// This represents the 'depth' clause on the '#pragma omp flatten' (and
+/// '#pragma omp fuse') loop-transformation directives.
+///
+/// \code
+/// #pragma omp flatten depth(3)
+/// \endcode
+/// In this example the 'flatten' directive has a 'depth' clause whose argument
+/// '3' specifies how many perfectly nested loops are combined into one.
+/// The argument must be a positive integer constant expression that evaluates
+/// to at most the loop nest depth of the associated loop nest.
+class OMPDepthClause final : public OMPClause {
+  friend class OMPClauseReader;
+
+  /// Location of '('.
+  SourceLocation LParenLoc;
+
+  /// The depth expression (number of loops to flatten).
+  Stmt *Depth = nullptr;
+
+  /// Build an empty clause.
+  explicit OMPDepthClause() : OMPClause(llvm::omp::OMPC_depth, {}, {}) {}
+
+  /// Set the depth expression.
+  void setDepth(Expr *E) { Depth = E; }
+
+  /// Sets the location of '('.
+  void setLParenLoc(SourceLocation Loc) { LParenLoc = Loc; }
+
+public:
+  /// Build an AST node for a 'depth' clause.
+  ///
+  /// \param C         Context of the AST.
+  /// \param StartLoc  Location of the 'depth' identifier.
+  /// \param LParenLoc Location of '('.
+  /// \param EndLoc    Location of ')'.
+  /// \param Depth     The depth expression.
+  static OMPDepthClause *Create(const ASTContext &C, SourceLocation StartLoc,
+                                SourceLocation LParenLoc, SourceLocation EndLoc,
+                                Expr *Depth);
+
+  /// Build an empty 'depth' AST node for deserialization.
+  ///
+  /// \param C     Context of the AST.
+  static OMPDepthClause *CreateEmpty(const ASTContext &C);
+
+  /// Returns the location of '('.
+  SourceLocation getLParenLoc() const { return LParenLoc; }
+
+  /// Returns the depth expression or nullptr if not set.
+  Expr *getDepth() const { return cast_or_null<Expr>(Depth); }
+
+  child_range children() { return child_range(&Depth, &Depth + 1); }
+  const_child_range children() const {
+    return const_child_range(&Depth, &Depth + 1);
+  }
+
+  child_range used_children() {
+    return child_range(child_iterator(), child_iterator());
+  }
+  const_child_range used_children() const {
+    return const_child_range(const_child_iterator(), const_child_iterator());
+  }
+
+  static bool classof(const OMPClause *T) {
+    return T->getClauseKind() == llvm::omp::OMPC_depth;
+  }
+};
+
 /// This represents 'collapse' clause in the '#pragma omp ...'
 /// directive.
 ///
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index e425f7a31f59c..da55e97fdfcb5 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -3568,6 +3568,12 @@ bool RecursiveASTVisitor<Derived>::VisitOMPPartialClause(OMPPartialClause *C) {
   return true;
 }
 
+template <typename Derived>
+bool RecursiveASTVisitor<Derived>::VisitOMPDepthClause(OMPDepthClause *C) {
+  TRY_TO(TraverseStmt(C->getDepth()));
+  return true;
+}
+
 template <typename Derived>
 bool
 RecursiveASTVisitor<Derived>::VisitOMPCollapseClause(OMPCollapseClause *C) {
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index ed00e80144c25..7b2f67a31cad9 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -1057,6 +1057,22 @@ OMPPartialClause *OMPPartialClause::CreateEmpty(const ASTContext &C) {
   return new (C) OMPPartialClause();
 }
 
+OMPDepthClause *OMPDepthClause::Create(const ASTContext &C,
+                                       SourceLocation StartLoc,
+                                       SourceLocation LParenLoc,
+                                       SourceLocation EndLoc, Expr *Depth) {
+  OMPDepthClause *Clause = CreateEmpty(C);
+  Clause->setLocStart(StartLoc);
+  Clause->setLParenLoc(LParenLoc);
+  Clause->setLocEnd(EndLoc);
+  Clause->setDepth(Depth);
+  return Clause;
+}
+
+OMPDepthClause *OMPDepthClause::CreateEmpty(const ASTContext &C) {
+  return new (C) OMPDepthClause();
+}
+
 OMPLoopRangeClause *
 OMPLoopRangeClause::Create(const ASTContext &C, SourceLocation StartLoc,
                            SourceLocation LParenLoc, SourceLocation FirstLoc,
@@ -2069,6 +2085,15 @@ void OMPClausePrinter::VisitOMPPermutationClause(OMPPermutationClause *Node) {
 
 void OMPClausePrinter::VisitOMPFullClause(OMPFullClause *Node) { OS << "full"; }
 
+void OMPClausePrinter::VisitOMPDepthClause(OMPDepthClause *Node) {
+  OS << "depth";
+  if (Expr *Depth = Node->getDepth()) {
+    OS << '(';
+    Depth->printPretty(OS, nullptr, Policy, 0);
+    OS << ')';
+  }
+}
+
 void OMPClausePrinter::VisitOMPPartialClause(OMPPartialClause *Node) {
   OS << "partial";
 
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index de7ed322d7424..838d6d02f7446 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -521,6 +521,11 @@ void OMPClauseProfiler::VisitOMPPartialClause(const OMPPartialClause *C) {
     Profiler->VisitExpr(Factor);
 }
 
+void OMPClauseProfiler::VisitOMPDepthClause(const OMPDepthClause *C) {
+  if (const Expr *Depth = C->getDepth())
+    Profiler->VisitExpr(Depth);
+}
+
 void OMPClauseProfiler::VisitOMPLoopRangeClause(const OMPLoopRangeClause *C) {
   if (const Expr *First = C->getFirst())
     Profiler->VisitExpr(First);
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index af52295df2d8b..952a517205381 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -3247,6 +3247,7 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
   case OMPC_nocontext:
   case OMPC_filter:
   case OMPC_partial:
+  case OMPC_depth:
   case OMPC_align:
   case OMPC_message:
   case OMPC_ompx_dyn_cgroup_mem:
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index 679a944fc4358..d49c5ee1a0db3 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -161,6 +161,7 @@ def OMPC_Depobj : Clause<[Spelling<"depobj">]> {
   let isImplicit = true;
 }
 def OMPC_Depth : Clause<[Spelling<"depth">]> {
+  let clangClass = "OMPDepthClause";
   let flangClass = "ScalarIntConstantExpr";
 }
 def OMPC_Destroy : Clause<[Spelling<"destroy">]> {

>From b63024e1e342cd168ef128a3f6a24909df2a82db Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:04:44 -0400
Subject: [PATCH 03/15] template-transform support for flatten directive &
 depth clause

---
 clang/lib/Sema/TreeTransform.h | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 3b99ff4bb9e23..865622ad50f1c 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -1804,6 +1804,14 @@ class TreeTransform {
                                                        LParenLoc, EndLoc);
   }
 
+  /// Build a new OpenMP 'depth' clause.
+  OMPClause *RebuildOMPDepthClause(Expr *Depth, SourceLocation StartLoc,
+                                   SourceLocation LParenLoc,
+                                   SourceLocation EndLoc) {
+    return getSema().OpenMP().ActOnOpenMPDepthClause(Depth, StartLoc, LParenLoc,
+                                                     EndLoc);
+  }
+
   OMPClause *
   RebuildOMPLoopRangeClause(Expr *First, Expr *Count, SourceLocation StartLoc,
                             SourceLocation LParenLoc, SourceLocation FirstLoc,
@@ -9791,6 +9799,17 @@ TreeTransform<Derived>::TransformOMPSplitDirective(OMPSplitDirective *D) {
   return Res;
 }
 
+template <typename Derived>
+StmtResult
+TreeTransform<Derived>::TransformOMPFlattenDirective(OMPFlattenDirective *D) {
+  DeclarationNameInfo DirName;
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      D->getDirectiveKind(), DirName, nullptr, D->getBeginLoc());
+  StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
+  return Res;
+}
+
 template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPFuseDirective(OMPFuseDirective *D) {
@@ -10717,6 +10736,20 @@ TreeTransform<Derived>::TransformOMPPartialClause(OMPPartialClause *C) {
                                  C->getEndLoc());
 }
 
+template <typename Derived>
+OMPClause *TreeTransform<Derived>::TransformOMPDepthClause(OMPDepthClause *C) {
+  ExprResult T = getDerived().TransformExpr(C->getDepth());
+  if (T.isInvalid())
+    return nullptr;
+  Expr *Depth = T.get();
+  bool Changed = Depth != C->getDepth();
+
+  if (!Changed && !getDerived().AlwaysRebuild())
+    return C;
+  return RebuildOMPDepthClause(Depth, C->getBeginLoc(), C->getLParenLoc(),
+                               C->getEndLoc());
+}
+
 template <typename Derived>
 OMPClause *
 TreeTransform<Derived>::TransformOMPLoopRangeClause(OMPLoopRangeClause *C) {

>From deafb222a7d251069c68a44fa267e58e750f1064 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:08:12 -0400
Subject: [PATCH 04/15] serialize flatten

---
 clang/include/clang/Serialization/ASTBitCodes.h |  1 +
 clang/lib/Serialization/ASTReaderStmt.cpp       | 11 +++++++++++
 clang/lib/Serialization/ASTWriterStmt.cpp       |  5 +++++
 3 files changed, 17 insertions(+)

diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h
index 279380de2f7fe..93a0e0a454b68 100644
--- a/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/clang/include/clang/Serialization/ASTBitCodes.h
@@ -1970,6 +1970,7 @@ enum StmtCode {
   STMT_OMP_REVERSE_DIRECTIVE,
   STMT_OMP_SPLIT_DIRECTIVE,
   STMT_OMP_INTERCHANGE_DIRECTIVE,
+  STMT_OMP_FLATTEN_DIRECTIVE,
   STMT_OMP_FUSE_DIRECTIVE,
   STMT_OMP_FOR_DIRECTIVE,
   STMT_OMP_FOR_SIMD_DIRECTIVE,
diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp
index 02ccf8d4d41c2..8c2debd6cdf32 100644
--- a/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -2537,6 +2537,10 @@ void ASTStmtReader::VisitOMPInterchangeDirective(OMPInterchangeDirective *D) {
   VisitOMPCanonicalLoopNestTransformationDirective(D);
 }
 
+void ASTStmtReader::VisitOMPFlattenDirective(OMPFlattenDirective *D) {
+  VisitOMPCanonicalLoopNestTransformationDirective(D);
+}
+
 void ASTStmtReader::VisitOMPSplitDirective(OMPSplitDirective *D) {
   VisitOMPCanonicalLoopNestTransformationDirective(D);
 }
@@ -3719,6 +3723,13 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) {
       break;
     }
 
+    case STMT_OMP_FLATTEN_DIRECTIVE: {
+      unsigned NumLoops = Record[ASTStmtReader::NumStmtFields];
+      unsigned NumClauses = Record[ASTStmtReader::NumStmtFields + 1];
+      S = OMPFlattenDirective::CreateEmpty(Context, NumClauses, NumLoops);
+      break;
+    }
+
     case STMT_OMP_FOR_DIRECTIVE: {
       unsigned CollapsedNum = Record[ASTStmtReader::NumStmtFields];
       unsigned NumClauses = Record[ASTStmtReader::NumStmtFields + 1];
diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp
index 4e9eadd730a56..dfe28a9b915e3 100644
--- a/clang/lib/Serialization/ASTWriterStmt.cpp
+++ b/clang/lib/Serialization/ASTWriterStmt.cpp
@@ -2547,6 +2547,11 @@ void ASTStmtWriter::VisitOMPInterchangeDirective(OMPInterchangeDirective *D) {
   Code = serialization::STMT_OMP_INTERCHANGE_DIRECTIVE;
 }
 
+void ASTStmtWriter::VisitOMPFlattenDirective(OMPFlattenDirective *D) {
+  VisitOMPCanonicalLoopNestTransformationDirective(D);
+  Code = serialization::STMT_OMP_FLATTEN_DIRECTIVE;
+}
+
 void ASTStmtWriter::VisitOMPSplitDirective(OMPSplitDirective *D) {
   VisitOMPCanonicalLoopNestTransformationDirective(D);
   Code = serialization::STMT_OMP_SPLIT_DIRECTIVE;

>From 3e2dfeb99230057b7cb9c484182ae33befbe1e41 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:09:03 -0400
Subject: [PATCH 05/15] serialize depth

---
 clang/lib/Serialization/ASTReader.cpp | 8 ++++++++
 clang/lib/Serialization/ASTWriter.cpp | 5 +++++
 2 files changed, 13 insertions(+)

diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index f8a6a38bb9b5c..4f51d79a52105 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -11495,6 +11495,9 @@ OMPClause *OMPClauseReader::readClause() {
   case llvm::omp::OMPC_partial:
     C = OMPPartialClause::CreateEmpty(Context);
     break;
+  case llvm::omp::OMPC_depth:
+    C = OMPDepthClause::CreateEmpty(Context);
+    break;
   case llvm::omp::OMPC_looprange:
     C = OMPLoopRangeClause::CreateEmpty(Context);
     break;
@@ -11924,6 +11927,11 @@ void OMPClauseReader::VisitOMPPartialClause(OMPPartialClause *C) {
   C->setLParenLoc(Record.readSourceLocation());
 }
 
+void OMPClauseReader::VisitOMPDepthClause(OMPDepthClause *C) {
+  C->setDepth(Record.readSubExpr());
+  C->setLParenLoc(Record.readSourceLocation());
+}
+
 void OMPClauseReader::VisitOMPLoopRangeClause(OMPLoopRangeClause *C) {
   C->setFirst(Record.readSubExpr());
   C->setCount(Record.readSubExpr());
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 357a7f7e95fa0..cbfeaa9e6638a 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -8106,6 +8106,11 @@ void OMPClauseWriter::VisitOMPPartialClause(OMPPartialClause *C) {
   Record.AddSourceLocation(C->getLParenLoc());
 }
 
+void OMPClauseWriter::VisitOMPDepthClause(OMPDepthClause *C) {
+  Record.AddStmt(C->getDepth());
+  Record.AddSourceLocation(C->getLParenLoc());
+}
+
 void OMPClauseWriter::VisitOMPLoopRangeClause(OMPLoopRangeClause *C) {
   Record.AddStmt(C->getFirst());
   Record.AddStmt(C->getCount());

>From 4ae746ca1eeaf11db10e859572a4500029d01cce Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:17:29 -0400
Subject: [PATCH 06/15] add analysis traversal support for flatten directive

---
 clang/lib/Sema/SemaExceptionSpec.cpp         | 1 +
 clang/lib/StaticAnalyzer/Core/ExprEngine.cpp | 1 +
 2 files changed, 2 insertions(+)

diff --git a/clang/lib/Sema/SemaExceptionSpec.cpp b/clang/lib/Sema/SemaExceptionSpec.cpp
index 40d530a1f3925..995b5b6010978 100644
--- a/clang/lib/Sema/SemaExceptionSpec.cpp
+++ b/clang/lib/Sema/SemaExceptionSpec.cpp
@@ -1508,6 +1508,7 @@ CanThrowResult Sema::canThrow(const Stmt *S) {
   case Stmt::OMPUnrollDirectiveClass:
   case Stmt::OMPReverseDirectiveClass:
   case Stmt::OMPInterchangeDirectiveClass:
+  case Stmt::OMPFlattenDirectiveClass:
   case Stmt::OMPSplitDirectiveClass:
   case Stmt::OMPFuseDirectiveClass:
   case Stmt::OMPSingleDirectiveClass:
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
index cfb294736ee02..3f666be00cc24 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
@@ -1783,6 +1783,7 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
     case Stmt::OMPStripeDirectiveClass:
     case Stmt::OMPTileDirectiveClass:
     case Stmt::OMPInterchangeDirectiveClass:
+    case Stmt::OMPFlattenDirectiveClass:
     case Stmt::OMPSplitDirectiveClass:
     case Stmt::OMPFuseDirectiveClass:
     case Stmt::OMPInteropDirectiveClass:

>From 415eba88ad0af63c552b4c5c2eabd660bcb29c02 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:21:25 -0400
Subject: [PATCH 07/15] codegen flatten directive

---
 clang/lib/CodeGen/CGStmt.cpp        | 3 +++
 clang/lib/CodeGen/CGStmtOpenMP.cpp  | 8 ++++++++
 clang/lib/CodeGen/CodeGenFunction.h | 1 +
 3 files changed, 12 insertions(+)

diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index 232094777f233..27f48cb63b553 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -236,6 +236,9 @@ void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {
   case Stmt::OMPInterchangeDirectiveClass:
     EmitOMPInterchangeDirective(cast<OMPInterchangeDirective>(*S));
     break;
+  case Stmt::OMPFlattenDirectiveClass:
+    EmitOMPFlattenDirective(cast<OMPFlattenDirective>(*S));
+    break;
   case Stmt::OMPFuseDirectiveClass:
     EmitOMPFuseDirective(cast<OMPFuseDirective>(*S));
     break;
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 95fd6694437fe..25da3430a23fe 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -218,6 +218,8 @@ class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
     } else if (const auto *Interchange =
                    dyn_cast<OMPInterchangeDirective>(&S)) {
       PreInits = Interchange->getPreInits();
+    } else if (const auto *Flatten = dyn_cast<OMPFlattenDirective>(&S)) {
+      PreInits = Flatten->getPreInits();
     } else {
       llvm_unreachable("Unknown loop-based directive kind.");
     }
@@ -3248,6 +3250,12 @@ void CodeGenFunction::EmitOMPInterchangeDirective(
   EmitStmt(S.getTransformedStmt());
 }
 
+void CodeGenFunction::EmitOMPFlattenDirective(const OMPFlattenDirective &S) {
+  // Emit the de-sugared statement.
+  OMPTransformDirectiveScopeRAII FlattenScope(*this, &S);
+  EmitStmt(S.getTransformedStmt());
+}
+
 void CodeGenFunction::EmitOMPFuseDirective(const OMPFuseDirective &S) {
   // Emit the de-sugared statement
   OMPTransformDirectiveScopeRAII FuseScope(*this, &S);
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 6d0718c243812..eea0163422cd3 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -3934,6 +3934,7 @@ class CodeGenFunction : public CodeGenTypeCache {
   void EmitOMPReverseDirective(const OMPReverseDirective &S);
   void EmitOMPSplitDirective(const OMPSplitDirective &S);
   void EmitOMPInterchangeDirective(const OMPInterchangeDirective &S);
+  void EmitOMPFlattenDirective(const OMPFlattenDirective &S);
   void EmitOMPFuseDirective(const OMPFuseDirective &S);
   void EmitOMPForDirective(const OMPForDirective &S);
   void EmitOMPForSimdDirective(const OMPForSimdDirective &S);

>From 814e2e9fe9bf4e28e4557999fbc16beec144cc39 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:24:42 -0400
Subject: [PATCH 08/15] add NYI CIR stub

---
 clang/lib/CIR/CodeGen/CIRGenFunction.h     | 1 +
 clang/lib/CIR/CodeGen/CIRGenStmt.cpp       | 2 ++
 clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp | 5 +++++
 3 files changed, 8 insertions(+)

diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.h b/clang/lib/CIR/CodeGen/CIRGenFunction.h
index 322355fde3957..8374583ffd4d0 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.h
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.h
@@ -2559,6 +2559,7 @@ class CIRGenFunction : public CIRGenTypeCache {
   mlir::LogicalResult emitOMPSplitDirective(const OMPSplitDirective &s);
   mlir::LogicalResult
   emitOMPInterchangeDirective(const OMPInterchangeDirective &s);
+  mlir::LogicalResult emitOMPFlattenDirective(const OMPFlattenDirective &s);
   mlir::LogicalResult emitOMPAssumeDirective(const OMPAssumeDirective &s);
   mlir::LogicalResult emitOMPMaskedDirective(const OMPMaskedDirective &s);
   mlir::LogicalResult emitOMPStripeDirective(const OMPStripeDirective &s);
diff --git a/clang/lib/CIR/CodeGen/CIRGenStmt.cpp b/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
index d3acac5801e74..33d425b22d61a 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
@@ -408,6 +408,8 @@ mlir::LogicalResult CIRGenFunction::emitStmt(const Stmt *s,
     return emitOMPSplitDirective(cast<OMPSplitDirective>(*s));
   case Stmt::OMPInterchangeDirectiveClass:
     return emitOMPInterchangeDirective(cast<OMPInterchangeDirective>(*s));
+  case Stmt::OMPFlattenDirectiveClass:
+    return emitOMPFlattenDirective(cast<OMPFlattenDirective>(*s));
   case Stmt::OMPAssumeDirectiveClass:
     return emitOMPAssumeDirective(cast<OMPAssumeDirective>(*s));
   case Stmt::OMPMaskedDirectiveClass:
diff --git a/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp b/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp
index 17a1fb8090f5c..4fffd17b25a16 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp
@@ -587,6 +587,11 @@ CIRGenFunction::emitOMPInterchangeDirective(const OMPInterchangeDirective &s) {
   return mlir::failure();
 }
 mlir::LogicalResult
+CIRGenFunction::emitOMPFlattenDirective(const OMPFlattenDirective &s) {
+  getCIRGenModule().errorNYI(s.getSourceRange(), "OpenMP OMPFlattenDirective");
+  return mlir::failure();
+}
+mlir::LogicalResult
 CIRGenFunction::emitOMPAssumeDirective(const OMPAssumeDirective &s) {
   getCIRGenModule().errorNYI(s.getSourceRange(), "OpenMP OMPAssumeDirective");
   return mlir::failure();

>From 0027ec6ab982a83766bd65f3e1bc45d9a671fc62 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:26:31 -0400
Subject: [PATCH 09/15] add libclang traversal support for depth clause

---
 clang/tools/libclang/CIndex.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index ac2fad38a1348..f8ab63e3e0dae 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2378,6 +2378,10 @@ void OMPClauseEnqueue::VisitOMPLoopRangeClause(const OMPLoopRangeClause *C) {
   Visitor->AddStmt(C->getCount());
 }
 
+void OMPClauseEnqueue::VisitOMPDepthClause(const OMPDepthClause *C) {
+  Visitor->AddStmt(C->getDepth());
+}
+
 void OMPClauseEnqueue::VisitOMPAllocatorClause(const OMPAllocatorClause *C) {
   Visitor->AddStmt(C->getAllocator());
 }

>From f5ac5f740c414310c8482f652f1bed936ae7cda3 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Fri, 10 Jul 2026 06:54:16 -0400
Subject: [PATCH 10/15] sema

---
 clang/include/clang/Sema/SemaOpenMP.h |  10 +
 clang/lib/Sema/SemaOpenMP.cpp         | 295 ++++++++++++++++++++++++++
 2 files changed, 305 insertions(+)

diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index 3621ce96b8724..7b086388416a0 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -470,6 +470,12 @@ class SemaOpenMP : public SemaBase {
                                              SourceLocation StartLoc,
                                              SourceLocation EndLoc);
 
+  /// Called on well-formed '#pragma omp flatten' after parsing of its
+  /// clauses and the associated statement.
+  StmtResult ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
+                                         Stmt *AStmt, SourceLocation StartLoc,
+                                         SourceLocation EndLoc);
+
   /// Called on well-formed '#pragma omp fuse' after parsing of its
   /// clauses and the associated statement.
   StmtResult ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
@@ -935,6 +941,10 @@ class SemaOpenMP : public SemaBase {
   OMPClause *ActOnOpenMPPartialClause(Expr *FactorExpr, SourceLocation StartLoc,
                                       SourceLocation LParenLoc,
                                       SourceLocation EndLoc);
+  /// Called on well-formed 'depth' clause.
+  OMPClause *ActOnOpenMPDepthClause(Expr *DepthExpr, SourceLocation StartLoc,
+                                    SourceLocation LParenLoc,
+                                    SourceLocation EndLoc);
   /// Called on well-formed 'collapse' clause.
   OMPClause *ActOnOpenMPCollapseClause(Expr *NumForLoops,
                                        SourceLocation StartLoc,
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 247a4a6ad9271..fcc256dec3c88 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -4629,6 +4629,7 @@ void SemaOpenMP::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind,
   case OMPD_reverse:
   case OMPD_split:
   case OMPD_interchange:
+  case OMPD_flatten:
   case OMPD_fuse:
   case OMPD_assume:
     break;
@@ -6475,6 +6476,10 @@ StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
     Res = ActOnOpenMPInterchangeDirective(ClausesWithImplicit, AStmt, StartLoc,
                                           EndLoc);
     break;
+  case OMPD_flatten:
+    Res = ActOnOpenMPFlattenDirective(ClausesWithImplicit, AStmt, StartLoc,
+                                      EndLoc);
+    break;
   case OMPD_fuse:
     Res =
         ActOnOpenMPFuseDirective(ClausesWithImplicit, AStmt, StartLoc, EndLoc);
@@ -16390,6 +16395,277 @@ StmtResult SemaOpenMP::ActOnOpenMPInterchangeDirective(
                                          buildPreInits(Context, PreInits));
 }
 
+/// Counts the perfectly nested canonical loop depth at \p AStmt. Returns
+/// std::nullopt if a loop-generating transformation prevents a static count.
+static std::optional<unsigned> getCanonicalLoopNestDepth(Stmt *AStmt) {
+  unsigned Depth = 0;
+  Stmt *CurStmt = AStmt ? AStmt->IgnoreContainers() : nullptr;
+  while (CurStmt) {
+    if (isa<OMPLoopTransformationDirective>(CurStmt))
+      return std::nullopt;
+    if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(CurStmt))
+      CurStmt = CanonLoop->getLoopStmt();
+    Stmt *Body = nullptr;
+    if (auto *For = dyn_cast<ForStmt>(CurStmt))
+      Body = For->getBody();
+    else if (auto *RangeFor = dyn_cast<CXXForRangeStmt>(CurStmt))
+      Body = RangeFor->getBody();
+    else
+      break;
+    ++Depth;
+    CurStmt = Body ? Body->IgnoreContainers() : nullptr;
+  }
+  return Depth;
+}
+
+StmtResult
+SemaOpenMP::ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
+                                        Stmt *AStmt, SourceLocation StartLoc,
+                                        SourceLocation EndLoc) {
+  ASTContext &Context = getASTContext();
+  DeclContext *CurContext = SemaRef.CurContext;
+  Scope *CurScope = SemaRef.getCurScope();
+
+  // Empty statement should only be possible if there already was an error.
+  if (!AStmt)
+    return StmtError();
+
+  // flatten without 'depth' clause combines two loops; 'depth(k)' selects k.
+  unsigned NumLoops = 2;
+  bool DepthIsValueDependent = false;
+  const auto *DepthClause =
+      OMPExecutableDirective::getSingleClause<OMPDepthClause>(Clauses);
+  if (DepthClause) {
+    Expr *DepthExpr = DepthClause->getDepth();
+    if (DepthExpr && DepthExpr->isValueDependent()) {
+      DepthIsValueDependent = true;
+    } else if (DepthExpr) {
+      Expr::EvalResult EvalResult;
+      if (DepthExpr->EvaluateAsInt(EvalResult, Context))
+        NumLoops = EvalResult.Val.getInt().getZExtValue();
+    }
+  }
+
+  // Report "expected k for loops, but found only n" when depth exceeds the
+  // perfect nest (same form as 'collapse'); skip if depth or nest is unknown.
+  if (DepthClause && !DepthIsValueDependent) {
+    if (std::optional<unsigned> NestDepth = getCanonicalLoopNestDepth(AStmt);
+        NestDepth && NumLoops > *NestDepth) {
+      Diag(AStmt->getBeginLoc(), diag::err_omp_not_for)
+          << /*expected N for loops form=*/1
+          << getOpenMPDirectiveName(OMPD_flatten) << NumLoops
+          << (*NestDepth > 0) << *NestDepth;
+      return StmtError();
+    }
+  }
+
+  // Defer when 'depth' is value-dependent (concrete k unknown until
+  // instantiation).
+  if (DepthIsValueDependent)
+    return OMPFlattenDirective::Create(Context, StartLoc, EndLoc, Clauses,
+                                       NumLoops, AStmt, nullptr, nullptr);
+
+  // Verify and diagnose loop nest.
+  SmallVector<OMPLoopBasedDirective::HelperExprs, 4> LoopHelpers(NumLoops);
+  Stmt *Body = nullptr;
+  SmallVector<SmallVector<Stmt *>, 4> OriginalInits;
+  if (!checkTransformableLoopNest(OMPD_flatten, AStmt, NumLoops, LoopHelpers,
+                                  Body, OriginalInits))
+    return StmtError();
+
+  // Delay flattening to when template is completely instantiated.
+  if (CurContext->isDependentContext())
+    return OMPFlattenDirective::Create(Context, StartLoc, EndLoc, Clauses,
+                                       NumLoops, AStmt, nullptr, nullptr);
+
+  assert(LoopHelpers.size() == NumLoops &&
+         "Expecting loop iteration space dimensionality to match number of "
+         "affected loops");
+  assert(OriginalInits.size() == NumLoops &&
+         "Expecting loop iteration space dimensionality to match number of "
+         "affected loops");
+
+  // Find the affected loops.
+  SmallVector<Stmt *> LoopStmts(NumLoops, nullptr);
+  collectLoopStmts(AStmt, LoopStmts);
+
+  // Collect pre-init statements in outer-to-inner order.
+  SmallVector<Stmt *> PreInits;
+  for (auto I : llvm::seq<int>(NumLoops)) {
+    OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
+    assert(LoopHelper.Counters.size() == 1 &&
+           "Single-dimensional loop iteration space expected");
+    addLoopPreInits(Context, LoopHelper, LoopStmts[I], OriginalInits[I],
+                    PreInits);
+  }
+
+  CaptureVars CopyTransformer(SemaRef);
+  auto MakeNumIterations = [&CopyTransformer,
+                            &LoopHelpers](unsigned I) -> Expr * {
+    return AssertSuccess(
+        CopyTransformer.TransformExpr(LoopHelpers[I].NumIterations));
+  };
+
+  OMPLoopBasedDirective::HelperExprs &OutermostHelper = LoopHelpers[0];
+  auto *OutermostCntVar = cast<DeclRefExpr>(OutermostHelper.Counters.front());
+  SourceLocation OrigVarLoc = OutermostCntVar->getExprLoc();
+  SourceLocation OrigVarLocBegin = OutermostCntVar->getBeginLoc();
+  SourceLocation OrigVarLocEnd = OutermostCntVar->getEndLoc();
+  SourceLocation CondLoc = OutermostHelper.Cond->getExprLoc();
+
+  // Product of trip counts; mirror 'collapse' IV-width selection to avoid
+  // overflow when several counts are multiplied.
+  auto BuildTripCount = [&](unsigned Bits) -> ExprResult {
+    ExprResult Product;
+    for (unsigned I = 0; I < NumLoops; ++I) {
+      ExprResult N = widenIterationCount(Bits, MakeNumIterations(I), SemaRef);
+      if (!N.isUsable())
+        return ExprError();
+      if (I == 0)
+        Product = N;
+      else
+        Product = SemaRef.BuildBinOp(CurScope, CondLoc, BO_Mul, Product.get(),
+                                     N.get());
+      if (!Product.isUsable())
+        return ExprError();
+    }
+    return Product;
+  };
+
+  bool AllCountsLessThan32Bits = true;
+  for (unsigned I = 0; I < NumLoops; ++I)
+    AllCountsLessThan32Bits &=
+        Context.getTypeSize(LoopHelpers[I].NumIterations->getType()) < 32;
+
+  ExprResult TripCount32 = BuildTripCount(/*Bits=*/32);
+  ExprResult TripCount64 = BuildTripCount(/*Bits=*/64);
+  if (!TripCount32.isUsable() || !TripCount64.isUsable())
+    return StmtError();
+
+  ExprResult TripCount = TripCount64;
+  if (Context.getTypeSize(TripCount32.get()->getType()) == 32 &&
+      (AllCountsLessThan32Bits || NumLoops == 1 ||
+       fitsInto(
+           /*Bits=*/32,
+           TripCount32.get()->getType()->hasSignedIntegerRepresentation(),
+           TripCount64.get(), SemaRef)))
+    TripCount = TripCount32;
+
+  QualType IVTy = TripCount.get()->getType();
+  uint64_t IVWidth = Context.getTypeSize(IVTy);
+
+  auto MakeNumIterationsInIVTy = [&](unsigned I) -> Expr * {
+    return AssertSuccess(SemaRef.PerformImplicitConversion(
+        MakeNumIterations(I), IVTy, AssignmentAction::Converting,
+        /*AllowExplicit=*/true));
+  };
+
+  // \code{.cpp}
+  //   for (auto .flatten.iv = 0; .flatten.iv < n0 * n1 * ...; ++.flatten.iv) {
+  //     .flatten.iv.0 = .flatten.iv / (n1 * ...);
+  //     i0 = ...;                                        // Updates[0]
+  //     .flatten.iv.1 = (.flatten.iv / ...) % n1;
+  //     i1 = ...;                                        // Updates[1]
+  //     ...
+  //     body(i0, i1, ...);
+  //   }
+  // \endcode
+  SmallString<64> FlattenedIVName(".flatten.iv");
+  VarDecl *FlattenedIVDecl = buildVarDecl(SemaRef, {}, IVTy, FlattenedIVName,
+                                          nullptr, OutermostCntVar);
+  auto MakeFlattenedRef = [&SemaRef = this->SemaRef, FlattenedIVDecl, IVTy,
+                           OrigVarLoc]() {
+    return buildDeclRefExpr(SemaRef, FlattenedIVDecl, IVTy, OrigVarLoc);
+  };
+
+  // For init-statement:
+  // \code{.cpp}
+  //   auto .flatten.iv = 0;
+  // \endcode
+  auto *Zero = IntegerLiteral::Create(Context, llvm::APInt::getZero(IVWidth),
+                                      IVTy, OrigVarLoc);
+  SemaRef.AddInitializerToDecl(FlattenedIVDecl, Zero, /*DirectInit=*/false);
+  StmtResult Init = new (Context)
+      DeclStmt(DeclGroupRef(FlattenedIVDecl), OrigVarLocBegin, OrigVarLocEnd);
+  if (!Init.isUsable())
+    return StmtError();
+
+  // For cond-expression:
+  // \code{.cpp}
+  //   .flatten.iv < n0 * n1 * ... * n(k-1)
+  // \endcode
+  ExprResult Cond = SemaRef.BuildBinOp(CurScope, CondLoc, BO_LT,
+                                       MakeFlattenedRef(), TripCount.get());
+  if (!Cond.isUsable())
+    return StmtError();
+
+  // For incr-statement:
+  // \code{.cpp}
+  //   ++.flatten.iv
+  // \endcode
+  ExprResult Incr =
+      SemaRef.BuildUnaryOp(CurScope, OutermostHelper.Inc->getExprLoc(),
+                           UO_PreInc, MakeFlattenedRef());
+  if (!Incr.isUsable())
+    return StmtError();
+
+  // Recover each logical iteration counter via mixed-radix div/mod; reuse the
+  // iteration variables from checkOpenMPLoop so Updates compute user counters.
+  SmallVector<Stmt *, 8> BodyStmts;
+  for (unsigned I = 0; I < NumLoops; ++I) {
+    OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
+    auto *IVRef = cast<DeclRefExpr>(LoopHelper.IterationVarRef);
+    auto *IVDecl = cast<VarDecl>(IVRef->getDecl());
+    std::string IVName = (".flatten.iv." + llvm::Twine(I)).str();
+    IVDecl->setDeclName(&SemaRef.PP.getIdentifierTable().get(IVName));
+
+    ExprResult Value = MakeFlattenedRef();
+    if (I + 1 < NumLoops) {
+      ExprResult Divisor = MakeNumIterationsInIVTy(I + 1);
+      for (unsigned J = I + 2; J < NumLoops; ++J) {
+        Divisor = SemaRef.BuildBinOp(CurScope, OrigVarLoc, BO_Mul,
+                                     Divisor.get(), MakeNumIterationsInIVTy(J));
+        if (!Divisor.isUsable())
+          return StmtError();
+      }
+      Value = SemaRef.BuildBinOp(CurScope, OrigVarLoc, BO_Div, Value.get(),
+                                 Divisor.get());
+      if (!Value.isUsable())
+        return StmtError();
+    }
+    if (I > 0) {
+      Value = SemaRef.BuildBinOp(CurScope, OrigVarLoc, BO_Rem, Value.get(),
+                                 MakeNumIterationsInIVTy(I));
+      if (!Value.isUsable())
+        return StmtError();
+    }
+
+    SemaRef.AddInitializerToDecl(IVDecl, Value.get(), /*DirectInit=*/false);
+    StmtResult IVStmt = new (Context)
+        DeclStmt(DeclGroupRef(IVDecl), OrigVarLocBegin, OrigVarLocEnd);
+    if (!IVStmt.isUsable())
+      return StmtError();
+
+    BodyStmts.push_back(IVStmt.get());
+    llvm::append_range(BodyStmts, LoopHelper.Updates);
+    if (auto *CXXFor = dyn_cast<CXXForRangeStmt>(LoopStmts[I]))
+      BodyStmts.push_back(CXXFor->getLoopVarStmt());
+  }
+  BodyStmts.push_back(Body);
+  auto *FlattenedBody =
+      CompoundStmt::Create(Context, BodyStmts, FPOptionsOverride(),
+                           Body->getBeginLoc(), Body->getEndLoc());
+
+  auto *FlattenedFor = new (Context) ForStmt(
+      Context, Init.get(), Cond.get(), nullptr, Incr.get(), FlattenedBody,
+      OutermostHelper.Init->getBeginLoc(), OutermostHelper.Init->getBeginLoc(),
+      OutermostHelper.Inc->getEndLoc());
+
+  return OMPFlattenDirective::Create(Context, StartLoc, EndLoc, Clauses,
+                                     NumLoops, AStmt, FlattenedFor,
+                                     buildPreInits(Context, PreInits));
+}
+
 StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
                                                 Stmt *AStmt,
                                                 SourceLocation StartLoc,
@@ -16920,6 +17196,9 @@ OMPClause *SemaOpenMP::ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind,
   case OMPC_partial:
     Res = ActOnOpenMPPartialClause(Expr, StartLoc, LParenLoc, EndLoc);
     break;
+  case OMPC_depth:
+    Res = ActOnOpenMPDepthClause(Expr, StartLoc, LParenLoc, EndLoc);
+    break;
   case OMPC_message:
     Res = ActOnOpenMPMessageClause(Expr, StartLoc, LParenLoc, EndLoc);
     break;
@@ -18241,6 +18520,22 @@ OMPClause *SemaOpenMP::ActOnOpenMPPartialClause(Expr *FactorExpr,
                                   FactorExpr);
 }
 
+OMPClause *SemaOpenMP::ActOnOpenMPDepthClause(Expr *DepthExpr,
+                                              SourceLocation StartLoc,
+                                              SourceLocation LParenLoc,
+                                              SourceLocation EndLoc) {
+  // The depth-expr must be a positive integer constant expression and
+  // not greater than the number of loops in the associated loop nest.
+  ExprResult DepthResult = VerifyPositiveIntegerConstantInClause(
+      DepthExpr, OMPC_depth, /*StrictlyPositive=*/true);
+  if (DepthResult.isInvalid())
+    return nullptr;
+  DepthExpr = DepthResult.get();
+
+  return OMPDepthClause::Create(getASTContext(), StartLoc, LParenLoc, EndLoc,
+                                DepthExpr);
+}
+
 OMPClause *SemaOpenMP::ActOnOpenMPLoopRangeClause(
     Expr *First, Expr *Count, SourceLocation StartLoc, SourceLocation LParenLoc,
     SourceLocation FirstLoc, SourceLocation CountLoc, SourceLocation EndLoc) {

>From 54be7347979638725cd4b6c421d11ec1cd618c43 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Mon, 27 Jul 2026 07:03:06 -0400
Subject: [PATCH 11/15] add libclang traversal support for flatten directive

---
 clang/tools/libclang/CIndex.cpp   | 7 +++++++
 clang/tools/libclang/CXCursor.cpp | 3 +++
 2 files changed, 10 insertions(+)

diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index f8ab63e3e0dae..9eb0ec8c21158 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2160,6 +2160,7 @@ class EnqueueVisitor : public ConstStmtVisitor<EnqueueVisitor, void>,
   void VisitOMPUnrollDirective(const OMPUnrollDirective *D);
   void VisitOMPReverseDirective(const OMPReverseDirective *D);
   void VisitOMPInterchangeDirective(const OMPInterchangeDirective *D);
+  void VisitOMPFlattenDirective(const OMPFlattenDirective *D);
   void VisitOMPCanonicalLoopSequenceTransformationDirective(
       const OMPCanonicalLoopSequenceTransformationDirective *D);
   void VisitOMPFuseDirective(const OMPFuseDirective *D);
@@ -3358,6 +3359,10 @@ void EnqueueVisitor::VisitOMPInterchangeDirective(
   VisitOMPCanonicalLoopNestTransformationDirective(D);
 }
 
+void EnqueueVisitor::VisitOMPFlattenDirective(const OMPFlattenDirective *D) {
+  VisitOMPCanonicalLoopNestTransformationDirective(D);
+}
+
 void EnqueueVisitor::VisitOMPCanonicalLoopSequenceTransformationDirective(
     const OMPCanonicalLoopSequenceTransformationDirective *D) {
   VisitOMPExecutableDirective(D);
@@ -6329,6 +6334,8 @@ CXString clang_getCursorKindSpelling(enum CXCursorKind Kind) {
     return cxstring::createRef("OMPReverseDirective");
   case CXCursor_OMPInterchangeDirective:
     return cxstring::createRef("OMPInterchangeDirective");
+  case CXCursor_OMPFlattenDirective:
+    return cxstring::createRef("OMPFlattenDirective");
   case CXCursor_OMPFuseDirective:
     return cxstring::createRef("OMPFuseDirective");
   case CXCursor_OMPSplitDirective:
diff --git a/clang/tools/libclang/CXCursor.cpp b/clang/tools/libclang/CXCursor.cpp
index b2feed578fc0f..97bbfd36558a9 100644
--- a/clang/tools/libclang/CXCursor.cpp
+++ b/clang/tools/libclang/CXCursor.cpp
@@ -703,6 +703,9 @@ CXCursor cxcursor::MakeCXCursor(const Stmt *S, const Decl *Parent,
   case Stmt::OMPInterchangeDirectiveClass:
     K = CXCursor_OMPInterchangeDirective;
     break;
+  case Stmt::OMPFlattenDirectiveClass:
+    K = CXCursor_OMPFlattenDirective;
+    break;
   case Stmt::OMPFuseDirectiveClass:
     K = CXCursor_OMPFuseDirective;
     break;

>From cdedb1045baf373937b0da858624a6ace14a3d21 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Mon, 27 Jul 2026 07:06:36 -0400
Subject: [PATCH 12/15] bindings

---
 clang/bindings/python/clang/cindex.py | 3 +++
 clang/include/clang-c/Index.h         | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/clang/bindings/python/clang/cindex.py b/clang/bindings/python/clang/cindex.py
index 24b737139dba8..ccad140e8c861 100644
--- a/clang/bindings/python/clang/cindex.py
+++ b/clang/bindings/python/clang/cindex.py
@@ -1456,6 +1456,9 @@ def is_unexposed(self):
     # OpenMP split directive.
     OMP_SPLIT_DIRECTIVE = 312
 
+    # OpenMP flatten directive.
+    OMP_FLATTEN_DIRECTIVE = 313
+
     # OpenACC Compute Construct.
     OPEN_ACC_COMPUTE_DIRECTIVE = 320
 
diff --git a/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h
index 8427236e0b444..f561fb5724897 100644
--- a/clang/include/clang-c/Index.h
+++ b/clang/include/clang-c/Index.h
@@ -2170,6 +2170,10 @@ enum CXCursorKind {
    */
   CXCursor_OMPSplitDirective = 312,
 
+  /** OpenMP flatten directive.
+   */
+  CXCursor_OMPFlattenDirective = 313,
+
   /** OpenACC Compute Construct.
    */
   CXCursor_OpenACCComputeConstruct = 320,

>From 94dee8fe1351f74760df0fb2d9903a291a5ccf66 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 5 Aug 2026 04:46:14 -0400
Subject: [PATCH 13/15] core tests-flatten and depth

---
 clang/test/OpenMP/flatten_codegen.cpp         |  65 ++++++++
 clang/test/OpenMP/flatten_depth_codegen.cpp   | 154 ++++++++++++++++++
 .../test/transform/flatten/depth-intfor.c     |  40 +++++
 .../runtime/test/transform/flatten/intfor.c   |  33 ++++
 .../test/transform/flatten/lit.local.cfg      |   5 +
 5 files changed, 297 insertions(+)
 create mode 100644 clang/test/OpenMP/flatten_codegen.cpp
 create mode 100644 clang/test/OpenMP/flatten_depth_codegen.cpp
 create mode 100644 openmp/runtime/test/transform/flatten/depth-intfor.c
 create mode 100644 openmp/runtime/test/transform/flatten/intfor.c
 create mode 100644 openmp/runtime/test/transform/flatten/lit.local.cfg

diff --git a/clang/test/OpenMP/flatten_codegen.cpp b/clang/test/OpenMP/flatten_codegen.cpp
new file mode 100644
index 0000000000000..f77b2fb00ce9b
--- /dev/null
+++ b/clang/test/OpenMP/flatten_codegen.cpp
@@ -0,0 +1,65 @@
+// Check code generation
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=60 -emit-llvm %s -o - | FileCheck %s
+
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+extern "C" void body(int, int);
+
+// CHECK-LABEL: define {{.*}}void @foo(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK:   store i64 0, ptr %.flatten.iv,
+// CHECK:   %[[COND_IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[MUL:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   icmp slt i64 %[[COND_IV]], %[[MUL]]
+// CHECK:   %[[DIV_IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[DIV:.+]] = sdiv i64 %[[DIV_IV]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   %[[REM_IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[REM:.+]] = srem i64 %[[REM_IV]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+// CHECK:   %[[I_VAL:.+]] = load i32, ptr %i,
+// CHECK:   %[[J_VAL:.+]] = load i32, ptr %j,
+// CHECK:   call void @body(i32{{.*}} %[[I_VAL]], i32{{.*}} %[[J_VAL]])
+// CHECK: for.inc:
+// CHECK:   %[[INC_IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[NEXT:.+]] = add nsw i64 %[[INC_IV]], 1
+// CHECK:   store i64 %[[NEXT]], ptr %.flatten.iv,
+// CHECK:   br label %for.cond
+extern "C" void foo(int n, int m) {
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+
+// CHECK-LABEL: define {{.*}}void @bar(
+// CHECK:   icmp slt i32 %{{.+}}, 63
+// CHECK:   sdiv i32 %{{.+}}, 9
+// CHECK:   srem i32 %{{.+}}, 9
+// CHECK:   call void @body(
+extern "C" void bar() {
+#pragma omp flatten
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+}
+
+// CHECK-LABEL: define {{.*}}void @baz(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   icmp slt i64 %{{.+}}, 10000000000
+// CHECK:   sdiv i64 %{{.+}}, 100000
+// CHECK:   srem i64 %{{.+}}, 100000
+// CHECK:   call void @body(
+extern "C" void baz() {
+#pragma omp flatten
+  for (int i = 0; i < 100000; ++i)
+    for (int j = 0; j < 100000; ++j)
+      body(i, j);
+}
+
+#endif
diff --git a/clang/test/OpenMP/flatten_depth_codegen.cpp b/clang/test/OpenMP/flatten_depth_codegen.cpp
new file mode 100644
index 0000000000000..77408fbcbdcf1
--- /dev/null
+++ b/clang/test/OpenMP/flatten_depth_codegen.cpp
@@ -0,0 +1,154 @@
+// Check code generation for the 'depth' clause on '#pragma omp flatten'.
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=61 -emit-llvm %s -o - | FileCheck %s
+
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+extern "C" void body(int, int, int);
+
+// CHECK-LABEL: define {{.*}}void @foo3(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK:   %.flatten.iv.2 = alloca i32
+// CHECK:   %[[CIV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[T01:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   %[[TALL:.+]] = mul nsw i64 %[[T01]], %{{.+}}
+// CHECK:   icmp slt i64 %[[CIV]], %[[TALL]]
+// CHECK:   %[[D0IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[D0M:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   %[[D0:.+]] = sdiv i64 %[[D0IV]], %[[D0M]]
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   %[[D1IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[D1:.+]] = sdiv i64 %[[D1IV]], %{{.+}}
+// CHECK:   %[[R1:.+]] = srem i64 %[[D1]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+// CHECK:   %[[R2IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[R2:.+]] = srem i64 %[[R2IV]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.2
+// CHECK:   call void @body(
+extern "C" void foo3(int n, int m, int p) {
+#pragma omp flatten depth(3)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        body(i, j, k);
+}
+
+// CHECK-LABEL: define {{.*}}void @foo2(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK:   %[[CIV2:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[M:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   icmp slt i64 %[[CIV2]], %[[M]]
+// CHECK:   sdiv i64 %{{.+}}, %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   srem i64 %{{.+}}, %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+extern "C" void foo2(int n, int m) {
+#pragma omp flatten depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j, 0);
+}
+
+// CHECK-LABEL: define {{.*}}void @foo1(
+// CHECK:   %.flatten.iv = alloca i32
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK: for.body:
+// CHECK:   %[[FV:.+]] = load i32, ptr %.flatten.iv,
+// CHECK:   store i32 %[[FV]], ptr %.flatten.iv.0,
+// CHECK-NOT: sdiv
+// CHECK-NOT: srem
+// CHECK:   call void @body(
+extern "C" void foo1(int n) {
+#pragma omp flatten depth(1)
+  for (int i = 0; i < n; ++i)
+    body(i, 0, 0);
+}
+
+// CHECK-LABEL: define {{.*}}void @foo2partial(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK-NOT: %.flatten.iv.2 = alloca
+// CHECK:   %[[CIV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[M:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   icmp slt i64 %[[CIV]], %[[M]]
+// CHECK:   sdiv i64 %{{.+}}, %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   srem i64 %{{.+}}, %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+// CHECK:   br label %for.cond{{.*}}
+// CHECK: for.cond{{.*}}:
+// CHECK:   icmp slt i32 %{{.+}}, 5
+extern "C" void foo2partial(int n, int m) {
+#pragma omp flatten depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < 5; ++k)
+        body(i, j, k);
+}
+
+// A value-dependent 'depth' (the argument is a non-type template parameter) is
+// deferred until instantiation, where the concrete depth drives the flattening.
+
+// CHECK-LABEL: define {{.*}}void @_Z11tmpl_depth3ILi3EEviii(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK:   %.flatten.iv.2 = alloca i32
+// CHECK:   %[[CIV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[T01:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   %[[TALL:.+]] = mul nsw i64 %[[T01]], %{{.+}}
+// CHECK:   icmp slt i64 %[[CIV]], %[[TALL]]
+// CHECK:   %[[D0IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[D0M:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   %[[D0:.+]] = sdiv i64 %[[D0IV]], %[[D0M]]
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   %[[D1IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[D1:.+]] = sdiv i64 %[[D1IV]], %{{.+}}
+// CHECK:   %[[R1:.+]] = srem i64 %[[D1]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+// CHECK:   %[[R2IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[R2:.+]] = srem i64 %[[R2IV]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.2
+// CHECK:   call void @body(
+template <int D>
+void tmpl_depth3(int n, int m, int p) {
+#pragma omp flatten depth(D)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        body(i, j, k);
+}
+
+// A value-dependent 'depth' that instantiates to depth(1) on a single loop is
+// well-formed: the shallower-than-default nest is accepted because validation is
+// deferred to instantiation. The single loop needs no div/mod recovery.
+
+// CHECK-LABEL: define {{.*}}void @_Z11tmpl_depth1ILi1EEvi(
+// CHECK:   %.flatten.iv = alloca i32
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK: for.body:
+// CHECK:   %[[FV:.+]] = load i32, ptr %.flatten.iv,
+// CHECK:   store i32 %[[FV]], ptr %.flatten.iv.0,
+// CHECK-NOT: sdiv
+// CHECK-NOT: srem
+// CHECK:   call void @body(
+template <int D>
+void tmpl_depth1(int n) {
+#pragma omp flatten depth(D)
+  for (int i = 0; i < n; ++i)
+    body(i, 0, 0);
+}
+
+void inst() {
+  tmpl_depth3<3>(1, 1, 1);
+  tmpl_depth1<1>(1);
+}
+
+#endif
diff --git a/openmp/runtime/test/transform/flatten/depth-intfor.c b/openmp/runtime/test/transform/flatten/depth-intfor.c
new file mode 100644
index 0000000000000..7211e02e25313
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/depth-intfor.c
@@ -0,0 +1,40 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+// 'depth(3)' fully flattens a three-deep perfectly nested loop into one loop.
+// This verifies that the flattened loop preserves the exact row-major
+// visitation order of the original nest, including non-unit steps and non-zero
+// start values.
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+int main() {
+  printf("do\n");
+#pragma omp flatten depth(3)
+  for (int i = 7; i < 11; i += 2)
+    for (int j = 1; j < 4; j += 1)
+      for (int k = 0; k < 2; ++k)
+        printf("i=%d j=%d k=%d\n", i, j, k);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: i=7 j=1 k=0
+// CHECK-NEXT: i=7 j=1 k=1
+// CHECK-NEXT: i=7 j=2 k=0
+// CHECK-NEXT: i=7 j=2 k=1
+// CHECK-NEXT: i=7 j=3 k=0
+// CHECK-NEXT: i=7 j=3 k=1
+// CHECK-NEXT: i=9 j=1 k=0
+// CHECK-NEXT: i=9 j=1 k=1
+// CHECK-NEXT: i=9 j=2 k=0
+// CHECK-NEXT: i=9 j=2 k=1
+// CHECK-NEXT: i=9 j=3 k=0
+// CHECK-NEXT: i=9 j=3 k=1
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/flatten/intfor.c b/openmp/runtime/test/transform/flatten/intfor.c
new file mode 100644
index 0000000000000..f937659fbebe1
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/intfor.c
@@ -0,0 +1,33 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+int main() {
+  printf("do\n");
+#pragma omp flatten
+  for (int i = 5; i < 11; i += 2)
+    for (int j = 1; j < 7; j += 2)
+      printf("i=%d j=%d\n", i, j);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// The flattened loop visits the original iteration space in the original
+// (row-major) order: for each outer iteration the inner loop runs fully.
+// CHECK:      do
+// CHECK-NEXT: i=5 j=1
+// CHECK-NEXT: i=5 j=3
+// CHECK-NEXT: i=5 j=5
+// CHECK-NEXT: i=7 j=1
+// CHECK-NEXT: i=7 j=3
+// CHECK-NEXT: i=7 j=5
+// CHECK-NEXT: i=9 j=1
+// CHECK-NEXT: i=9 j=3
+// CHECK-NEXT: i=9 j=5
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/flatten/lit.local.cfg b/openmp/runtime/test/transform/flatten/lit.local.cfg
new file mode 100644
index 0000000000000..e05bd627df1e9
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/lit.local.cfg
@@ -0,0 +1,5 @@
+# The flatten directive's depth clause requires OpenMP 6.1.
+for i, (pattern, replacement) in enumerate(config.substitutions):
+    if pattern == "%openmp_flags":
+        config.substitutions[i] = (pattern, replacement + " -fopenmp-version=61")
+        break

>From 667fe837a39dbc711dcc5288b736aa52fbe893b6 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 5 Aug 2026 04:48:45 -0400
Subject: [PATCH 14/15] TBD

---
 clang/docs/OpenMPSupport.rst                  |   2 +-
 clang/include/clang/ASTMatchers/ASTMatchers.h |  14 ++
 clang/lib/ASTMatchers/ASTMatchersInternal.cpp |   2 +
 clang/lib/ASTMatchers/Dynamic/Registry.cpp    |   1 +
 clang/test/Index/openmp-flatten.c             |  12 ++
 clang/test/OpenMP/flatten_ast_print.cpp       | 124 ++++++++++++++++++
 clang/test/OpenMP/flatten_codegen_for.cpp     |  67 ++++++++++
 clang/test/OpenMP/flatten_depth_ast_print.cpp |  74 +++++++++++
 clang/test/OpenMP/flatten_depth_messages.cpp  |  48 +++++++
 clang/test/OpenMP/flatten_messages.cpp        |  89 +++++++++++++
 .../test/OpenMP/flatten_serialize_module.cpp  |  25 ++++
 .../ASTMatchers/ASTMatchersNodeTest.cpp       |  22 ++++
 flatten-verify-tmp/01_default_depth2.cpp      |   8 ++
 flatten-verify-tmp/02_depth3_full.cpp         |   9 ++
 flatten-verify-tmp/03_depth2_partial.cpp      |  10 ++
 flatten-verify-tmp/04_depth1_identity.cpp     |   8 ++
 flatten-verify-tmp/05_rangefor_template.cpp   |  20 +++
 flatten-verify-tmp/README.md                  |  39 ++++++
 flatten-verify-tmp/end_result_check.cpp       |  82 ++++++++++++
 .../test/transform/flatten/depth-partial.c    |  45 +++++++
 openmp/runtime/test/transform/flatten/empty.c |  42 ++++++
 .../test/transform/flatten/foreach.cpp        |  31 +++++
 .../flatten/parallel-wsloop-foreach.cpp       |  32 +++++
 23 files changed, 805 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/Index/openmp-flatten.c
 create mode 100644 clang/test/OpenMP/flatten_ast_print.cpp
 create mode 100644 clang/test/OpenMP/flatten_codegen_for.cpp
 create mode 100644 clang/test/OpenMP/flatten_depth_ast_print.cpp
 create mode 100644 clang/test/OpenMP/flatten_depth_messages.cpp
 create mode 100644 clang/test/OpenMP/flatten_messages.cpp
 create mode 100644 clang/test/OpenMP/flatten_serialize_module.cpp
 create mode 100644 flatten-verify-tmp/01_default_depth2.cpp
 create mode 100644 flatten-verify-tmp/02_depth3_full.cpp
 create mode 100644 flatten-verify-tmp/03_depth2_partial.cpp
 create mode 100644 flatten-verify-tmp/04_depth1_identity.cpp
 create mode 100644 flatten-verify-tmp/05_rangefor_template.cpp
 create mode 100644 flatten-verify-tmp/README.md
 create mode 100644 flatten-verify-tmp/end_result_check.cpp
 create mode 100644 openmp/runtime/test/transform/flatten/depth-partial.c
 create mode 100644 openmp/runtime/test/transform/flatten/empty.c
 create mode 100644 openmp/runtime/test/transform/flatten/foreach.cpp
 create mode 100644 openmp/runtime/test/transform/flatten/parallel-wsloop-foreach.cpp

diff --git a/clang/docs/OpenMPSupport.rst b/clang/docs/OpenMPSupport.rst
index 0c21aa41ff269..c11a46c3bb6a6 100644
--- a/clang/docs/OpenMPSupport.rst
+++ b/clang/docs/OpenMPSupport.rst
@@ -666,7 +666,7 @@ implementation.
 +=============================================================+===========================+===========================+==========================================================================+
 | dyn_groupprivate clause                                     | :part:`partial`           | :part:`In Progress`       | C/C++: Host device support missing                                       |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
-| loop flatten transformation                                 | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
+| loop flatten transformation                                 | :part:`partial`           | :none:`unclaimed`         | Clang: depth clause supported (OpenMP 6.1); apply clause not yet supported|
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | loop grid/tile modifiers for sizes clause                   | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
diff --git a/clang/include/clang/ASTMatchers/ASTMatchers.h b/clang/include/clang/ASTMatchers/ASTMatchers.h
index bc0f35898a2c9..602bab9aa8cb8 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchers.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchers.h
@@ -8842,6 +8842,20 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt,
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, OMPSplitDirective>
     ompSplitDirective;
 
+/// Matches OpenMP ``flatten`` directive.
+///
+/// Given
+///
+/// \code
+///   #pragma omp flatten
+///   for (int i = 0; i < m; ++i)
+///     for (int j = 0; j < n; ++j) {}
+/// \endcode
+///
+/// ``ompFlattenDirective()`` matches the flatten directive.
+extern const internal::VariadicDynCastAllOfMatcher<Stmt, OMPFlattenDirective>
+    ompFlattenDirective;
+
 /// Matches OpenMP ``counts`` clause used by ``#pragma omp split``.
 ///
 /// Given ``#pragma omp split counts(1, 2, omp_fill)``, ``ompCountsClause()``
diff --git a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
index 5cbf134620e34..e2c0c25518b70 100644
--- a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
+++ b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
@@ -1141,6 +1141,8 @@ const internal::VariadicDynCastAllOfMatcher<Stmt, OMPTargetUpdateDirective>
     ompTargetUpdateDirective;
 const internal::VariadicDynCastAllOfMatcher<Stmt, OMPSplitDirective>
     ompSplitDirective;
+const internal::VariadicDynCastAllOfMatcher<Stmt, OMPFlattenDirective>
+    ompFlattenDirective;
 const internal::VariadicDynCastAllOfMatcher<OMPClause, OMPCountsClause>
     ompCountsClause;
 const internal::VariadicDynCastAllOfMatcher<OMPClause, OMPDefaultClause>
diff --git a/clang/lib/ASTMatchers/Dynamic/Registry.cpp b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
index a04070971f0eb..a6252363e08ce 100644
--- a/clang/lib/ASTMatchers/Dynamic/Registry.cpp
+++ b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
@@ -534,6 +534,7 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(ompFromClause);
   REGISTER_MATCHER(ompToClause);
   REGISTER_MATCHER(ompExecutableDirective);
+  REGISTER_MATCHER(ompFlattenDirective);
   REGISTER_MATCHER(ompSplitDirective);
   REGISTER_MATCHER(ompTargetUpdateDirective);
   REGISTER_MATCHER(on);
diff --git a/clang/test/Index/openmp-flatten.c b/clang/test/Index/openmp-flatten.c
new file mode 100644
index 0000000000000..74bc3cda3ef1a
--- /dev/null
+++ b/clang/test/Index/openmp-flatten.c
@@ -0,0 +1,12 @@
+// RUN: c-index-test -test-load-source local %s -fopenmp=libomp -fopenmp-version=60 | FileCheck %s
+
+void test(void) {
+#pragma omp flatten
+  for (int i = 0; i < 20; i += 1)
+    for (int j = 0; j < 30; j += 1)
+      ;
+}
+
+// CHECK: openmp-flatten.c:4:1: OMPFlattenDirective= Extent=[4:1 - 4:20]
+// CHECK: openmp-flatten.c:5:3: ForStmt= Extent=[5:3 - 7:8]
+// CHECK: openmp-flatten.c:6:5: ForStmt= Extent=[6:5 - 7:8]
diff --git a/clang/test/OpenMP/flatten_ast_print.cpp b/clang/test/OpenMP/flatten_ast_print.cpp
new file mode 100644
index 0000000000000..e1115bf112c40
--- /dev/null
+++ b/clang/test/OpenMP/flatten_ast_print.cpp
@@ -0,0 +1,124 @@
+// Check no warnings/errors
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -fsyntax-only -verify %s
+// expected-no-diagnostics
+
+// Check AST and unparsing
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -ast-dump  %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -ast-print %s | FileCheck %s --check-prefix=PRINT
+
+// Check same results after serialization round-trip
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -emit-pch -o %t %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -include-pch %t -ast-dump-all %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -include-pch %t -ast-print    %s | FileCheck %s --check-prefix=PRINT
+
+#ifndef HEADER
+#define HEADER
+
+// placeholder for loop body code.
+extern "C" void body(...);
+
+// PRINT-LABEL: void foo1(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo1
+void foo1() {
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (int i = 0; i < 7; i += 1)
+  // DUMP-NEXT: ForStmt
+  for (int i = 0; i < 7; i += 1)
+    // PRINT: for (int j = 0; j < 9; j += 1)
+    // DUMP:  ForStmt
+    for (int j = 0; j < 9; j += 1)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// PRINT-LABEL: void foo2(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo2
+void foo2(int start1, int end1, int step1, int start2, int end2, int step2) {
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (int i = start1; i < end1; i += step1)
+  // DUMP-NEXT: ForStmt
+  for (int i = start1; i < end1; i += step1)
+    // PRINT: for (int j = start2; j < end2; j += step2)
+    // DUMP:  ForStmt
+    for (int j = start2; j < end2; j += step2)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// PRINT-LABEL: void foo3(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo3
+void foo3() {
+  // Verify that a flattened loop can itself be the operand of another loop
+  // transformation. The outer reverse consumes the loop generated by flatten.
+  // PRINT:     #pragma omp reverse
+  // DUMP:      OMPReverseDirective
+  #pragma omp reverse
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (int i = 0; i < 13; i += 1)
+  // DUMP-NEXT: ForStmt
+  for (int i = 0; i < 13; i += 1)
+    // PRINT: for (int j = 0; j < 17; j += 1)
+    // DUMP:  ForStmt
+    for (int j = 0; j < 17; j += 1)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// Range-based for loops are supported as affected loops.
+// PRINT-LABEL: void foo4(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo4
+struct Range {
+  int *begin();
+  int *end();
+};
+void foo4(Range r1, Range r2) {
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (int i : r1)
+  // DUMP-NEXT: CXXForRangeStmt
+  for (int i : r1)
+    // PRINT: for (int j : r2)
+    // DUMP:  CXXForRangeStmt
+    for (int j : r2)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// The directive is instantiated together with its enclosing template.
+// PRINT-LABEL: template <typename T, int LO, int HI> void foo5()
+// DUMP-LABEL:  FunctionTemplateDecl {{.*}} foo5
+template <typename T, int LO, int HI>
+void foo5() {
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (T i = LO; i < HI; ++i)
+  // DUMP-NEXT: ForStmt
+  for (T i = LO; i < HI; ++i)
+    // PRINT: for (T j = LO; j < HI; ++j)
+    // DUMP:  ForStmt
+    for (T j = LO; j < HI; ++j)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// PRINT-LABEL: template<> void foo5<int, 0, 8>()
+// DUMP-LABEL:  FunctionDecl {{.*}} foo5 'void ()' implicit_instantiation
+// DUMP:        OMPFlattenDirective
+void inst() {
+  foo5<int, 0, 8>();
+}
+
+#endif
diff --git a/clang/test/OpenMP/flatten_codegen_for.cpp b/clang/test/OpenMP/flatten_codegen_for.cpp
new file mode 100644
index 0000000000000..d64128bd25d78
--- /dev/null
+++ b/clang/test/OpenMP/flatten_codegen_for.cpp
@@ -0,0 +1,67 @@
+// Check that the loop generated by 'flatten' can be consumed by a loop
+// associated directive ('omp for' / 'omp parallel for'), and that signed and
+// unsigned induction variables generate signed/unsigned division.
+
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=60 -emit-llvm %s -o - | FileCheck %s
+
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+extern "C" void body(int, int);
+extern "C" void ubody(unsigned, unsigned);
+
+// 'omp for' worksharing over the flattened iteration space: the flattened loop
+// uses a 64-bit induction variable for the variable-bound product trip count,
+// so the worksharing loop is distributed with the 64-bit static-init call and
+// the body recovers the logical counters via signed division/remainder.
+//
+// CHECK-LABEL: define {{.*}}void @wsfor(
+// CHECK:   mul nsw i64
+// CHECK:   call void @__kmpc_for_static_init_8(
+// CHECK:   sdiv i64
+// CHECK:   srem i64
+// CHECK:   call void @body(
+// CHECK:   call void @__kmpc_for_static_fini(
+extern "C" void wsfor(int n, int m) {
+#pragma omp for
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+
+// 'omp parallel for' outlines the flattened loop and worksharing-distributes
+// it.
+//
+// CHECK-LABEL: define {{.*}}void @pfor(
+// CHECK:   call {{.*}}@__kmpc_fork_call(
+// CHECK-LABEL: define internal void @pfor.omp_outlined(
+// CHECK:   call void @__kmpc_for_static_init_8(
+// CHECK:   sdiv i64
+// CHECK:   srem i64
+// CHECK:   call void @body(
+extern "C" void pfor(int n, int m) {
+#pragma omp parallel for
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+
+// Unsigned induction variables produce unsigned division/remainder.
+//
+// CHECK-LABEL: define {{.*}}void @ufor(
+// CHECK:   icmp ult i32 %{{.+}}, 63
+// CHECK:   udiv i32 %{{.+}}, 9
+// CHECK:   urem i32 %{{.+}}, 9
+// CHECK:   call void @ubody(
+extern "C" void ufor() {
+#pragma omp flatten
+  for (unsigned i = 0; i < 7u; ++i)
+    for (unsigned j = 0; j < 9u; ++j)
+      ubody(i, j);
+}
+
+#endif
diff --git a/clang/test/OpenMP/flatten_depth_ast_print.cpp b/clang/test/OpenMP/flatten_depth_ast_print.cpp
new file mode 100644
index 0000000000000..ceaaa68e1f79a
--- /dev/null
+++ b/clang/test/OpenMP/flatten_depth_ast_print.cpp
@@ -0,0 +1,74 @@
+// Check no warnings/errors
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -fsyntax-only -verify %s
+// expected-no-diagnostics
+
+// Check AST and unparsing
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-dump  %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-print %s | FileCheck %s --check-prefix=PRINT
+
+// Check same results after serialization round-trip
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -emit-pch -o %t %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -include-pch %t -ast-dump-all %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -include-pch %t -ast-print    %s | FileCheck %s --check-prefix=PRINT
+
+#ifndef HEADER
+#define HEADER
+
+// placeholder for loop body code.
+extern "C" void body(...);
+
+// The depth clause round-trips through -ast-print and PCH serialization.
+// PRINT-LABEL: void foo_depth3(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo_depth3
+void foo_depth3(int n, int m, int p) {
+  // PRINT:     #pragma omp flatten depth(3)
+  // DUMP:      OMPFlattenDirective
+  // DUMP-NEXT: OMPDepthClause
+  #pragma omp flatten depth(3)
+  // PRINT: for (int i = 0; i < n; ++i)
+  // DUMP:  ForStmt
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        // PRINT: body(i, j, k);
+        // DUMP:  CallExpr
+        body(i, j, k);
+}
+
+// depth(2) is accepted and printed like any other depth argument.
+// PRINT-LABEL: void foo_depth2(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo_depth2
+void foo_depth2(int n, int m) {
+  // PRINT:     #pragma omp flatten depth(2)
+  // DUMP:      OMPFlattenDirective
+  // DUMP-NEXT: OMPDepthClause
+  #pragma omp flatten depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+
+// The depth clause is instantiated together with its enclosing template.
+// PRINT-LABEL: template <typename T> void foo_tmpl()
+// DUMP-LABEL:  FunctionTemplateDecl {{.*}} foo_tmpl
+template <typename T>
+void foo_tmpl() {
+  // PRINT:     #pragma omp flatten depth(3)
+  // DUMP:      OMPFlattenDirective
+  // DUMP-NEXT: OMPDepthClause
+  #pragma omp flatten depth(3)
+  for (T i = 0; i < 8; ++i)
+    for (T j = 0; j < 8; ++j)
+      for (T k = 0; k < 8; ++k)
+        body(i, j, k);
+}
+
+// PRINT-LABEL: template<> void foo_tmpl<int>()
+// DUMP-LABEL:  FunctionDecl {{.*}} foo_tmpl 'void ()' implicit_instantiation
+// DUMP:        OMPFlattenDirective
+// DUMP-NEXT:   OMPDepthClause
+void inst() {
+  foo_tmpl<int>();
+}
+
+#endif
diff --git a/clang/test/OpenMP/flatten_depth_messages.cpp b/clang/test/OpenMP/flatten_depth_messages.cpp
new file mode 100644
index 0000000000000..16f60318d0d06
--- /dev/null
+++ b/clang/test/OpenMP/flatten_depth_messages.cpp
@@ -0,0 +1,48 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=61 -fsyntax-only -Wuninitialized -verify %s
+
+extern "C" void body(...);
+
+// expected-note at +1 {{declared here}}
+void func(int n) {
+
+  // The depth argument must be a positive integer constant expression.
+  // expected-error at +1 {{argument to 'depth' clause must be a strictly positive integer value}}
+  #pragma omp flatten depth(0)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // expected-error at +1 {{argument to 'depth' clause must be a strictly positive integer value}}
+  #pragma omp flatten depth(-1)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // A non-constant argument is rejected.
+  // expected-error at +2 {{expression is not an integral constant expression}}
+  // expected-note at +1 {{function parameter 'n' with unknown value cannot be used in a constant expression}}
+  #pragma omp flatten depth(n)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // At most one 'depth' clause may appear on the directive.
+  // expected-error at +1 {{directive '#pragma omp flatten' cannot contain more than one 'depth' clause}}
+  #pragma omp flatten depth(2) depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // The depth must be at most the loop nest depth: depth(3) requires three
+  // perfectly nested loops, but only two are present here.
+  #pragma omp flatten depth(3)
+  // expected-error at +1 {{expected 3 for loops after '#pragma omp flatten', but found only 2}}
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // depth(1) flattens a single loop and is well-formed (no diagnostic).
+  #pragma omp flatten depth(1)
+  for (int i = 0; i < n; ++i)
+    body(i);
+}
diff --git a/clang/test/OpenMP/flatten_messages.cpp b/clang/test/OpenMP/flatten_messages.cpp
new file mode 100644
index 0000000000000..a022e0e8999d2
--- /dev/null
+++ b/clang/test/OpenMP/flatten_messages.cpp
@@ -0,0 +1,89 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=60 -fsyntax-only -Wuninitialized -verify %s
+
+extern "C" void body(...);
+
+void func(int n) {
+
+  // The associated statement must be a for loop.
+  // expected-error at +2 {{statement after '#pragma omp flatten' must be a for loop}}
+  #pragma omp flatten
+  ;
+
+  // A non-loop statement is rejected as well.
+  // expected-error at +2 {{statement after '#pragma omp flatten' must be a for loop}}
+  #pragma omp flatten
+  int b = 0;
+
+  // A single loop is not enough: flatten combines two perfectly nested loops,
+  // so the body of the outer loop must itself be a for loop.
+  #pragma omp flatten
+  for (int i = 0; i < 7; ++i)
+    // expected-error at +1 {{statement after '#pragma omp flatten' must be a for loop}}
+    ;
+
+  // The associated statement of a directive is not a for loop.
+  // expected-error at +2 {{statement after '#pragma omp flatten' must be a for loop}}
+  #pragma omp flatten
+  #pragma omp for
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 7; ++j)
+      body(i, j);
+
+  {
+    // expected-error at +2 {{expected statement}}
+    #pragma omp flatten
+  }
+
+  // The loops must be perfectly nested: no code is allowed between them.
+  #pragma omp flatten
+  for (int i = 0; i < n; ++i) {
+    int x = 0;
+    // expected-error at -2 {{statement after '#pragma omp flatten' must be a for loop}}
+    for (int j = 0; j < n; ++j)
+      body(i, j, x);
+  }
+
+  // Each affected loop must be in OpenMP canonical form.
+  #pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    // expected-error at +1 {{condition of OpenMP for loop must be a relational comparison ('<', '<=', '>', '>=', or '!=') of loop variable 'j'}}
+    for (int j = 0; j / 3 < n; ++j)
+      body(i, j);
+
+  // The affected loops must be rectangular: an inner bound may not depend on an
+  // outer loop counter.
+  #pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    // expected-error at +1 {{expected loop invariant expression}}
+    for (int j = i; j < n; ++j)
+      body(i, j);
+
+  // The 'sizes' clause is not allowed on 'flatten'.
+  // expected-error at +1 {{unexpected OpenMP clause 'sizes' in directive '#pragma omp flatten'}}
+  #pragma omp flatten sizes(2)
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+
+  // The 'permutation' clause is not allowed on 'flatten'.
+  // expected-error at +1 {{unexpected OpenMP clause 'permutation' in directive '#pragma omp flatten'}}
+  #pragma omp flatten permutation(2, 1)
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+
+  // Tokens after the directive name are ignored with a warning.
+  // expected-warning at +1 {{extra tokens at the end of '#pragma omp flatten' are ignored}}
+  #pragma omp flatten foo
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+
+  // The 'depth' clause is only available from OpenMP 6.1; it is rejected here
+  // under -fopenmp-version=60.
+  // expected-error at +1 {{unexpected OpenMP clause 'depth' in directive '#pragma omp flatten'}}
+  #pragma omp flatten depth(2)
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+}
diff --git a/clang/test/OpenMP/flatten_serialize_module.cpp b/clang/test/OpenMP/flatten_serialize_module.cpp
new file mode 100644
index 0000000000000..5d3420500b282
--- /dev/null
+++ b/clang/test/OpenMP/flatten_serialize_module.cpp
@@ -0,0 +1,25 @@
+// C++20 module interface with `#pragma omp flatten` — emit BMI + import; AST retains directive.
+//
+// RUN: rm -rf %t && split-file %s %t && cd %t
+// RUN: %clang_cc1 -std=c++20 -fopenmp -fopenmp-version=60 -triple x86_64-unknown-linux-gnu %t/FlattenMod.cppm -emit-module-interface -o %t/FlattenMod.pcm
+// RUN: %clang_cc1 -std=c++20 -fopenmp -fopenmp-version=60 -triple x86_64-unknown-linux-gnu %t/UseFlattenMod.cpp -fmodule-file=FlattenMod=%t/FlattenMod.pcm -ast-dump-all | FileCheck %t/FlattenMod.cppm
+
+// expected-no-diagnostics
+
+//--- FlattenMod.cppm
+module;
+export module FlattenMod;
+
+export void flattenfoo(int n, int m) {
+// CHECK: OMPFlattenDirective
+// CHECK: ForStmt
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j) {
+    }
+}
+
+//--- UseFlattenMod.cpp
+import FlattenMod;
+
+void g(void) { flattenfoo(10, 20); }
diff --git a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
index 4190d4703e37d..34c0fd27821d1 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
@@ -3126,6 +3126,28 @@ void f() {
   EXPECT_TRUE(notMatchesWithOpenMP60(ParallelOnly, Matcher));
 }
 
+// OpenMP 6 flatten directive
+TEST(ASTMatchersTestOpenMP, OMPFlattenDirective) {
+  auto Matcher = stmt(ompFlattenDirective());
+
+  StringRef FlattenOk = R"(
+void f() {
+#pragma omp flatten
+  for (int i = 0; i < 10; ++i)
+    for (int j = 0; j < 10; ++j) {}
+}
+)";
+  EXPECT_TRUE(matchesWithOpenMP60(FlattenOk, Matcher));
+
+  StringRef ParallelOnly = R"(
+void f() {
+#pragma omp parallel
+  ;
+}
+)";
+  EXPECT_TRUE(notMatchesWithOpenMP60(ParallelOnly, Matcher));
+}
+
 TEST(ASTMatchersTestOpenMP, OMPSplitDirective_HasCountsClause) {
   auto Matcher = stmt(ompSplitDirective(hasAnyClause(ompCountsClause())));
 
diff --git a/flatten-verify-tmp/01_default_depth2.cpp b/flatten-verify-tmp/01_default_depth2.cpp
new file mode 100644
index 0000000000000..900d0263622cd
--- /dev/null
+++ b/flatten-verify-tmp/01_default_depth2.cpp
@@ -0,0 +1,8 @@
+// Default flatten: combine 2 perfectly nested loops (depth defaults to 2).
+extern "C" void body(int, int);
+void t(int n, int m) {
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
diff --git a/flatten-verify-tmp/02_depth3_full.cpp b/flatten-verify-tmp/02_depth3_full.cpp
new file mode 100644
index 0000000000000..ba9f0b7ed8808
--- /dev/null
+++ b/flatten-verify-tmp/02_depth3_full.cpp
@@ -0,0 +1,9 @@
+// depth(3): fully flatten a three-deep perfect nest (OpenMP 6.1).
+extern "C" void body(int, int, int);
+void t(int n, int m, int p) {
+#pragma omp flatten depth(3)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        body(i, j, k);
+}
diff --git a/flatten-verify-tmp/03_depth2_partial.cpp b/flatten-verify-tmp/03_depth2_partial.cpp
new file mode 100644
index 0000000000000..35b9dc83834fd
--- /dev/null
+++ b/flatten-verify-tmp/03_depth2_partial.cpp
@@ -0,0 +1,10 @@
+// depth(2) on a three-deep nest: only the outer two loops are combined; the
+// innermost loop must remain as ordinary body of the flattened loop.
+extern "C" void body(int, int, int);
+void t(int n, int m, int p) {
+#pragma omp flatten depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        body(i, j, k);
+}
diff --git a/flatten-verify-tmp/04_depth1_identity.cpp b/flatten-verify-tmp/04_depth1_identity.cpp
new file mode 100644
index 0000000000000..ae62a11312fe8
--- /dev/null
+++ b/flatten-verify-tmp/04_depth1_identity.cpp
@@ -0,0 +1,8 @@
+// depth(1): identity transform. A single loop is "combined"; no mixed-radix
+// decomposition (no srem), the flattened counter maps straight to the loop var.
+extern "C" void body(int);
+void t(int n) {
+#pragma omp flatten depth(1)
+  for (int i = 0; i < n; ++i)
+    body(i);
+}
diff --git a/flatten-verify-tmp/05_rangefor_template.cpp b/flatten-verify-tmp/05_rangefor_template.cpp
new file mode 100644
index 0000000000000..89928c34a8a15
--- /dev/null
+++ b/flatten-verify-tmp/05_rangefor_template.cpp
@@ -0,0 +1,20 @@
+// Range-for (CXXForRangeStmt) nest + a function template instantiated with a
+// dependent depth(K). Exercises the __begin/__end pre-inits and the deferred
+// (dependent-context) desugaring path.
+#include <stdlibc++.h>
+extern "C" void body(int, int);
+
+void ranges(std::vector<int> &a, std::vector<int> &b) {
+#pragma omp flatten
+  for (int i : a)
+    for (int j : b)
+      body(i, j);
+}
+
+template <int K> void tmpl(int n, int m) {
+#pragma omp flatten depth(K)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+template void tmpl<2>(int, int);
diff --git a/flatten-verify-tmp/README.md b/flatten-verify-tmp/README.md
new file mode 100644
index 0000000000000..f48f4d06cfec0
--- /dev/null
+++ b/flatten-verify-tmp/README.md
@@ -0,0 +1,39 @@
+# Temporary flatten + depth verification cases
+
+Throwaway sandbox for eyeballing the flatten desugaring (AST / IR / runtime).
+Not part of any PR — delete freely. The committed regression tests live under
+`clang/test/OpenMP/flatten*` and `openmp/runtime/test/transform/flatten/*`.
+
+`CLANG=build/bin/clang`, `CLANGXX=build/bin/clang++`, run from the repo root.
+
+| File | Case | What to look for |
+|------|------|------------------|
+| `01_default_depth2.cpp` | default (no clause) = depth 2 | i64 `.flatten.iv`, `mul nsw`, `sdiv`+`srem` |
+| `02_depth3_full.cpp` | `depth(3)` full flatten | 3 `.flatten.iv.N`, two-level mixed radix |
+| `03_depth2_partial.cpp` | `depth(2)` on 3-deep nest | 2 flatten IVs, **inner `%k` loop stays** |
+| `04_depth1_identity.cpp` | `depth(1)` identity | i32 IV, **no `srem`**, direct store |
+| `05_rangefor_template.cpp` | range-for + template `depth(K)` | ast-print round-trips; instantiation |
+| `end_result_check.cpp` | runtime semantic check | prints `OK`, exit 0 (order+set preserved) |
+
+## Commands
+
+AST (round-trip original loops) and the desugared node:
+```
+$CLANG -cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-print FILE
+$CLANG -cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-dump  FILE
+```
+
+IR (the div/mod arithmetic that implements the transform):
+```
+$CLANG -cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -emit-llvm -o - FILE
+```
+
+Range-for / template need the driver (libstdc++ + template instantiation):
+```
+$CLANGXX -fopenmp -fopenmp-version=61 -Xclang -ast-print -fsyntax-only 05_rangefor_template.cpp
+```
+
+Runtime end-result (must print OK, exit 0):
+```
+$CLANGXX -fopenmp -fopenmp-version=61 -O0 end_result_check.cpp -o /tmp/erc && /tmp/erc
+```
diff --git a/flatten-verify-tmp/end_result_check.cpp b/flatten-verify-tmp/end_result_check.cpp
new file mode 100644
index 0000000000000..3aa8850829058
--- /dev/null
+++ b/flatten-verify-tmp/end_result_check.cpp
@@ -0,0 +1,82 @@
+// End-result (semantic) verification: flatten must preserve BOTH the set of
+// iterations and their row-major visitation ORDER. We record the sequence of
+// visited tuples from a flattened nest and compare it against the sequence from
+// an identical non-flattened reference nest. Exit code 0 == identical.
+//
+// flatten is a pure frontend AST desugaring to an ordinary loop, so this runs
+// as a normal program (build with: clang++ -fopenmp -fopenmp-version=61).
+#include <cassert>
+#include <cstdio>
+#include <vector>
+
+using Seq = std::vector<long>;
+
+static void rec(Seq &s, int i, int j, int k = -1) {
+  s.push_back(i);
+  s.push_back(j);
+  if (k >= 0)
+    s.push_back(k);
+}
+
+int main() {
+  const int n = 3, m = 4, p = 2;
+
+  // --- depth(2) default over 2 loops ---
+  {
+    Seq flat, ref;
+#pragma omp flatten
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        rec(flat, i, j);
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        rec(ref, i, j);
+    assert(flat == ref && "depth(2) order/set mismatch");
+  }
+
+  // --- depth(3) full flatten over 3 loops ---
+  {
+    Seq flat, ref;
+#pragma omp flatten depth(3)
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        for (int k = 0; k < p; ++k)
+          rec(flat, i, j, k);
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        for (int k = 0; k < p; ++k)
+          rec(ref, i, j, k);
+    assert(flat == ref && "depth(3) order/set mismatch");
+  }
+
+  // --- depth(2) partial: inner k-loop stays intact ---
+  {
+    Seq flat, ref;
+#pragma omp flatten depth(2)
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        for (int k = 0; k < p; ++k)
+          rec(flat, i, j, k);
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        for (int k = 0; k < p; ++k)
+          rec(ref, i, j, k);
+    assert(flat == ref && "depth(2)-partial order/set mismatch");
+  }
+
+  // --- non-unit step / non-zero start must survive ---
+  {
+    Seq flat, ref;
+#pragma omp flatten
+    for (int i = 5; i < 12; i += 2)
+      for (int j = 1; j < 7; j += 3)
+        rec(flat, i, j);
+    for (int i = 5; i < 12; i += 2)
+      for (int j = 1; j < 7; j += 3)
+        rec(ref, i, j);
+    assert(flat == ref && "stride/offset mismatch");
+  }
+
+  std::puts("flatten end-result preserved: OK");
+  return 0;
+}
diff --git a/openmp/runtime/test/transform/flatten/depth-partial.c b/openmp/runtime/test/transform/flatten/depth-partial.c
new file mode 100644
index 0000000000000..2fda3a04cbd84
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/depth-partial.c
@@ -0,0 +1,45 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+// A depth(2) clause on a three-deep loop nest flattens only the outermost two
+// loops; the innermost loop is left untouched and executes normally inside the
+// flattened loop body.
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+int main() {
+  printf("do\n");
+#pragma omp flatten depth(2)
+  for (int i = 5; i < 11; i += 2)
+    for (int j = 1; j < 7; j += 2)
+      for (int k = 0; k < 2; ++k)
+        printf("i=%d j=%d k=%d\n", i, j, k);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: i=5 j=1 k=0
+// CHECK-NEXT: i=5 j=1 k=1
+// CHECK-NEXT: i=5 j=3 k=0
+// CHECK-NEXT: i=5 j=3 k=1
+// CHECK-NEXT: i=5 j=5 k=0
+// CHECK-NEXT: i=5 j=5 k=1
+// CHECK-NEXT: i=7 j=1 k=0
+// CHECK-NEXT: i=7 j=1 k=1
+// CHECK-NEXT: i=7 j=3 k=0
+// CHECK-NEXT: i=7 j=3 k=1
+// CHECK-NEXT: i=7 j=5 k=0
+// CHECK-NEXT: i=7 j=5 k=1
+// CHECK-NEXT: i=9 j=1 k=0
+// CHECK-NEXT: i=9 j=1 k=1
+// CHECK-NEXT: i=9 j=3 k=0
+// CHECK-NEXT: i=9 j=3 k=1
+// CHECK-NEXT: i=9 j=5 k=0
+// CHECK-NEXT: i=9 j=5 k=1
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/flatten/empty.c b/openmp/runtime/test/transform/flatten/empty.c
new file mode 100644
index 0000000000000..ecf1e8ef622f2
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/empty.c
@@ -0,0 +1,42 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+// The bounds are runtime values so that an empty iteration space does not
+// trigger a compile-time division-by-zero diagnostic for the (never executed)
+// induction-variable recovery.
+static void flatten(int n, int m) {
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      printf("i=%d j=%d\n", i, j);
+}
+
+int main() {
+  printf("empty-inner-begin\n");
+  flatten(3, 0);
+  printf("empty-inner-end\n");
+
+  printf("empty-outer-begin\n");
+  flatten(0, 3);
+  printf("empty-outer-end\n");
+
+  printf("single-begin\n");
+  flatten(1, 1);
+  printf("single-end\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      empty-inner-begin
+// CHECK-NEXT: empty-inner-end
+// CHECK-NEXT: empty-outer-begin
+// CHECK-NEXT: empty-outer-end
+// CHECK-NEXT: single-begin
+// CHECK-NEXT: i=0 j=0
+// CHECK-NEXT: single-end
diff --git a/openmp/runtime/test/transform/flatten/foreach.cpp b/openmp/runtime/test/transform/flatten/foreach.cpp
new file mode 100644
index 0000000000000..14997f7c4051d
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/foreach.cpp
@@ -0,0 +1,31 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdio>
+#include <vector>
+
+int main() {
+  printf("do\n");
+  std::vector<int> outer{10, 20};
+  std::vector<int> inner{1, 2, 3};
+#pragma omp flatten
+  for (int a : outer)
+    for (int b : inner)
+      printf("a=%d b=%d\n", a, b);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: a=10 b=1
+// CHECK-NEXT: a=10 b=2
+// CHECK-NEXT: a=10 b=3
+// CHECK-NEXT: a=20 b=1
+// CHECK-NEXT: a=20 b=2
+// CHECK-NEXT: a=20 b=3
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/flatten/parallel-wsloop-foreach.cpp b/openmp/runtime/test/transform/flatten/parallel-wsloop-foreach.cpp
new file mode 100644
index 0000000000000..7ddcde2fd622c
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/parallel-wsloop-foreach.cpp
@@ -0,0 +1,32 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdio>
+
+int main() {
+  printf("do\n");
+  // The loop generated by 'flatten' is the canonical loop associated with the
+  // worksharing 'omp for'. With a single thread the iteration order is the
+  // original row-major order.
+#pragma omp parallel for num_threads(1)
+#pragma omp flatten
+  for (int i = 0; i < 2; ++i)
+    for (int j = 0; j < 3; ++j)
+      printf("i=%d j=%d\n", i, j);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: i=0 j=0
+// CHECK-NEXT: i=0 j=1
+// CHECK-NEXT: i=0 j=2
+// CHECK-NEXT: i=1 j=0
+// CHECK-NEXT: i=1 j=1
+// CHECK-NEXT: i=1 j=2
+// CHECK-NEXT: done

>From 152ee3bed82f0aa84079151c559d46564d9304df Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Tue, 18 Aug 2026 07:49:02 -0400
Subject: [PATCH 15/15] Remove flatten-verify-tmp and restore
 OpenMPSupport.rst.

Co-authored-by: Cursor <cursoragent at cursor.com>
---
 clang/docs/OpenMPSupport.rst                |  2 +-
 flatten-verify-tmp/01_default_depth2.cpp    |  8 --
 flatten-verify-tmp/02_depth3_full.cpp       |  9 ---
 flatten-verify-tmp/03_depth2_partial.cpp    | 10 ---
 flatten-verify-tmp/04_depth1_identity.cpp   |  8 --
 flatten-verify-tmp/05_rangefor_template.cpp | 20 -----
 flatten-verify-tmp/README.md                | 39 ----------
 flatten-verify-tmp/end_result_check.cpp     | 82 ---------------------
 8 files changed, 1 insertion(+), 177 deletions(-)
 delete mode 100644 flatten-verify-tmp/01_default_depth2.cpp
 delete mode 100644 flatten-verify-tmp/02_depth3_full.cpp
 delete mode 100644 flatten-verify-tmp/03_depth2_partial.cpp
 delete mode 100644 flatten-verify-tmp/04_depth1_identity.cpp
 delete mode 100644 flatten-verify-tmp/05_rangefor_template.cpp
 delete mode 100644 flatten-verify-tmp/README.md
 delete mode 100644 flatten-verify-tmp/end_result_check.cpp

diff --git a/clang/docs/OpenMPSupport.rst b/clang/docs/OpenMPSupport.rst
index c11a46c3bb6a6..0c21aa41ff269 100644
--- a/clang/docs/OpenMPSupport.rst
+++ b/clang/docs/OpenMPSupport.rst
@@ -666,7 +666,7 @@ implementation.
 +=============================================================+===========================+===========================+==========================================================================+
 | dyn_groupprivate clause                                     | :part:`partial`           | :part:`In Progress`       | C/C++: Host device support missing                                       |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
-| loop flatten transformation                                 | :part:`partial`           | :none:`unclaimed`         | Clang: depth clause supported (OpenMP 6.1); apply clause not yet supported|
+| loop flatten transformation                                 | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | loop grid/tile modifiers for sizes clause                   | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
diff --git a/flatten-verify-tmp/01_default_depth2.cpp b/flatten-verify-tmp/01_default_depth2.cpp
deleted file mode 100644
index 900d0263622cd..0000000000000
--- a/flatten-verify-tmp/01_default_depth2.cpp
+++ /dev/null
@@ -1,8 +0,0 @@
-// Default flatten: combine 2 perfectly nested loops (depth defaults to 2).
-extern "C" void body(int, int);
-void t(int n, int m) {
-#pragma omp flatten
-  for (int i = 0; i < n; ++i)
-    for (int j = 0; j < m; ++j)
-      body(i, j);
-}
diff --git a/flatten-verify-tmp/02_depth3_full.cpp b/flatten-verify-tmp/02_depth3_full.cpp
deleted file mode 100644
index ba9f0b7ed8808..0000000000000
--- a/flatten-verify-tmp/02_depth3_full.cpp
+++ /dev/null
@@ -1,9 +0,0 @@
-// depth(3): fully flatten a three-deep perfect nest (OpenMP 6.1).
-extern "C" void body(int, int, int);
-void t(int n, int m, int p) {
-#pragma omp flatten depth(3)
-  for (int i = 0; i < n; ++i)
-    for (int j = 0; j < m; ++j)
-      for (int k = 0; k < p; ++k)
-        body(i, j, k);
-}
diff --git a/flatten-verify-tmp/03_depth2_partial.cpp b/flatten-verify-tmp/03_depth2_partial.cpp
deleted file mode 100644
index 35b9dc83834fd..0000000000000
--- a/flatten-verify-tmp/03_depth2_partial.cpp
+++ /dev/null
@@ -1,10 +0,0 @@
-// depth(2) on a three-deep nest: only the outer two loops are combined; the
-// innermost loop must remain as ordinary body of the flattened loop.
-extern "C" void body(int, int, int);
-void t(int n, int m, int p) {
-#pragma omp flatten depth(2)
-  for (int i = 0; i < n; ++i)
-    for (int j = 0; j < m; ++j)
-      for (int k = 0; k < p; ++k)
-        body(i, j, k);
-}
diff --git a/flatten-verify-tmp/04_depth1_identity.cpp b/flatten-verify-tmp/04_depth1_identity.cpp
deleted file mode 100644
index ae62a11312fe8..0000000000000
--- a/flatten-verify-tmp/04_depth1_identity.cpp
+++ /dev/null
@@ -1,8 +0,0 @@
-// depth(1): identity transform. A single loop is "combined"; no mixed-radix
-// decomposition (no srem), the flattened counter maps straight to the loop var.
-extern "C" void body(int);
-void t(int n) {
-#pragma omp flatten depth(1)
-  for (int i = 0; i < n; ++i)
-    body(i);
-}
diff --git a/flatten-verify-tmp/05_rangefor_template.cpp b/flatten-verify-tmp/05_rangefor_template.cpp
deleted file mode 100644
index 89928c34a8a15..0000000000000
--- a/flatten-verify-tmp/05_rangefor_template.cpp
+++ /dev/null
@@ -1,20 +0,0 @@
-// Range-for (CXXForRangeStmt) nest + a function template instantiated with a
-// dependent depth(K). Exercises the __begin/__end pre-inits and the deferred
-// (dependent-context) desugaring path.
-#include <stdlibc++.h>
-extern "C" void body(int, int);
-
-void ranges(std::vector<int> &a, std::vector<int> &b) {
-#pragma omp flatten
-  for (int i : a)
-    for (int j : b)
-      body(i, j);
-}
-
-template <int K> void tmpl(int n, int m) {
-#pragma omp flatten depth(K)
-  for (int i = 0; i < n; ++i)
-    for (int j = 0; j < m; ++j)
-      body(i, j);
-}
-template void tmpl<2>(int, int);
diff --git a/flatten-verify-tmp/README.md b/flatten-verify-tmp/README.md
deleted file mode 100644
index f48f4d06cfec0..0000000000000
--- a/flatten-verify-tmp/README.md
+++ /dev/null
@@ -1,39 +0,0 @@
-# Temporary flatten + depth verification cases
-
-Throwaway sandbox for eyeballing the flatten desugaring (AST / IR / runtime).
-Not part of any PR — delete freely. The committed regression tests live under
-`clang/test/OpenMP/flatten*` and `openmp/runtime/test/transform/flatten/*`.
-
-`CLANG=build/bin/clang`, `CLANGXX=build/bin/clang++`, run from the repo root.
-
-| File | Case | What to look for |
-|------|------|------------------|
-| `01_default_depth2.cpp` | default (no clause) = depth 2 | i64 `.flatten.iv`, `mul nsw`, `sdiv`+`srem` |
-| `02_depth3_full.cpp` | `depth(3)` full flatten | 3 `.flatten.iv.N`, two-level mixed radix |
-| `03_depth2_partial.cpp` | `depth(2)` on 3-deep nest | 2 flatten IVs, **inner `%k` loop stays** |
-| `04_depth1_identity.cpp` | `depth(1)` identity | i32 IV, **no `srem`**, direct store |
-| `05_rangefor_template.cpp` | range-for + template `depth(K)` | ast-print round-trips; instantiation |
-| `end_result_check.cpp` | runtime semantic check | prints `OK`, exit 0 (order+set preserved) |
-
-## Commands
-
-AST (round-trip original loops) and the desugared node:
-```
-$CLANG -cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-print FILE
-$CLANG -cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-dump  FILE
-```
-
-IR (the div/mod arithmetic that implements the transform):
-```
-$CLANG -cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -emit-llvm -o - FILE
-```
-
-Range-for / template need the driver (libstdc++ + template instantiation):
-```
-$CLANGXX -fopenmp -fopenmp-version=61 -Xclang -ast-print -fsyntax-only 05_rangefor_template.cpp
-```
-
-Runtime end-result (must print OK, exit 0):
-```
-$CLANGXX -fopenmp -fopenmp-version=61 -O0 end_result_check.cpp -o /tmp/erc && /tmp/erc
-```
diff --git a/flatten-verify-tmp/end_result_check.cpp b/flatten-verify-tmp/end_result_check.cpp
deleted file mode 100644
index 3aa8850829058..0000000000000
--- a/flatten-verify-tmp/end_result_check.cpp
+++ /dev/null
@@ -1,82 +0,0 @@
-// End-result (semantic) verification: flatten must preserve BOTH the set of
-// iterations and their row-major visitation ORDER. We record the sequence of
-// visited tuples from a flattened nest and compare it against the sequence from
-// an identical non-flattened reference nest. Exit code 0 == identical.
-//
-// flatten is a pure frontend AST desugaring to an ordinary loop, so this runs
-// as a normal program (build with: clang++ -fopenmp -fopenmp-version=61).
-#include <cassert>
-#include <cstdio>
-#include <vector>
-
-using Seq = std::vector<long>;
-
-static void rec(Seq &s, int i, int j, int k = -1) {
-  s.push_back(i);
-  s.push_back(j);
-  if (k >= 0)
-    s.push_back(k);
-}
-
-int main() {
-  const int n = 3, m = 4, p = 2;
-
-  // --- depth(2) default over 2 loops ---
-  {
-    Seq flat, ref;
-#pragma omp flatten
-    for (int i = 0; i < n; ++i)
-      for (int j = 0; j < m; ++j)
-        rec(flat, i, j);
-    for (int i = 0; i < n; ++i)
-      for (int j = 0; j < m; ++j)
-        rec(ref, i, j);
-    assert(flat == ref && "depth(2) order/set mismatch");
-  }
-
-  // --- depth(3) full flatten over 3 loops ---
-  {
-    Seq flat, ref;
-#pragma omp flatten depth(3)
-    for (int i = 0; i < n; ++i)
-      for (int j = 0; j < m; ++j)
-        for (int k = 0; k < p; ++k)
-          rec(flat, i, j, k);
-    for (int i = 0; i < n; ++i)
-      for (int j = 0; j < m; ++j)
-        for (int k = 0; k < p; ++k)
-          rec(ref, i, j, k);
-    assert(flat == ref && "depth(3) order/set mismatch");
-  }
-
-  // --- depth(2) partial: inner k-loop stays intact ---
-  {
-    Seq flat, ref;
-#pragma omp flatten depth(2)
-    for (int i = 0; i < n; ++i)
-      for (int j = 0; j < m; ++j)
-        for (int k = 0; k < p; ++k)
-          rec(flat, i, j, k);
-    for (int i = 0; i < n; ++i)
-      for (int j = 0; j < m; ++j)
-        for (int k = 0; k < p; ++k)
-          rec(ref, i, j, k);
-    assert(flat == ref && "depth(2)-partial order/set mismatch");
-  }
-
-  // --- non-unit step / non-zero start must survive ---
-  {
-    Seq flat, ref;
-#pragma omp flatten
-    for (int i = 5; i < 12; i += 2)
-      for (int j = 1; j < 7; j += 3)
-        rec(flat, i, j);
-    for (int i = 5; i < 12; i += 2)
-      for (int j = 1; j < 7; j += 3)
-        rec(ref, i, j);
-    assert(flat == ref && "stride/offset mismatch");
-  }
-
-  std::puts("flatten end-result preserved: OK");
-  return 0;
-}


From openmp-commits at lists.llvm.org  Tue Aug 18 04:55:27 2026
From: openmp-commits at lists.llvm.org (Amit Tiwari via Openmp-commits)
Date: Tue, 18 Aug 2026 04:55:27 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [Clang][OpenMP] Add
 `#pragma omp flatten` loop directive + `depth` clause (PR #206977)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206977@github.com>
Message-ID: <6a84482f.ee10049a.50084.03fc@mx.google.com>


https://github.com/loopacino updated https://github.com/llvm/llvm-project/pull/206977

>From 0fb5bd42f1e127faea963c359b1e3c69cb969c60 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 1 Jul 2026 09:06:58 -0400
Subject: [PATCH 01/17] add OMPFlattenDirective ast node

---
 clang/include/clang/AST/RecursiveASTVisitor.h |  3 +
 clang/include/clang/AST/StmtOpenMP.h          | 75 ++++++++++++++++++-
 clang/include/clang/Basic/StmtNodes.td        |  1 +
 clang/lib/AST/StmtOpenMP.cpp                  | 20 +++++
 clang/lib/AST/StmtPrinter.cpp                 |  5 ++
 clang/lib/AST/StmtProfile.cpp                 |  4 +
 clang/lib/Basic/OpenMPKinds.cpp               |  2 +-
 7 files changed, 108 insertions(+), 2 deletions(-)

diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index b000a34043696..e425f7a31f59c 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -3227,6 +3227,9 @@ DEF_TRAVERSE_STMT(OMPFuseDirective,
 DEF_TRAVERSE_STMT(OMPInterchangeDirective,
                   { TRY_TO(TraverseOMPExecutableDirective(S)); })
 
+DEF_TRAVERSE_STMT(OMPFlattenDirective,
+                  { TRY_TO(TraverseOMPExecutableDirective(S)); })
+
 DEF_TRAVERSE_STMT(OMPSplitDirective,
                   { TRY_TO(TraverseOMPExecutableDirective(S)); })
 
diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index dbc76e7df8ecd..1420331f37c1f 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -1041,7 +1041,7 @@ class OMPCanonicalLoopNestTransformationDirective
     Stmt::StmtClass C = T->getStmtClass();
     return C == OMPTileDirectiveClass || C == OMPUnrollDirectiveClass ||
            C == OMPReverseDirectiveClass || C == OMPInterchangeDirectiveClass ||
-           C == OMPStripeDirectiveClass;
+           C == OMPStripeDirectiveClass || C == OMPFlattenDirectiveClass;
   }
 };
 
@@ -5959,6 +5959,79 @@ class OMPInterchangeDirective final
   }
 };
 
+/// Represents the '#pragma omp flatten' loop transformation directive.
+///
+/// \code{c}
+///   #pragma omp flatten
+///   for (int i = 0; i < m; ++i)
+///     for (int j = 0; j < n; ++j)
+///       ..
+/// \endcode
+class OMPFlattenDirective final
+    : public OMPCanonicalLoopNestTransformationDirective {
+  friend class ASTStmtReader;
+  friend class OMPExecutableDirective;
+
+  /// Offsets of child members.
+  enum {
+    PreInitsOffset = 0,
+    TransformedStmtOffset,
+  };
+
+  explicit OMPFlattenDirective(SourceLocation StartLoc, SourceLocation EndLoc,
+                               unsigned NumLoops)
+      : OMPCanonicalLoopNestTransformationDirective(
+            OMPFlattenDirectiveClass, llvm::omp::OMPD_flatten, StartLoc, EndLoc,
+            NumLoops) {}
+
+  void setPreInits(Stmt *PreInits) {
+    Data->getChildren()[PreInitsOffset] = PreInits;
+  }
+
+  void setTransformedStmt(Stmt *S) {
+    Data->getChildren()[TransformedStmtOffset] = S;
+  }
+
+public:
+  /// Create a new AST node representation for '#pragma omp flatten'.
+  ///
+  /// \param C         Context of the AST.
+  /// \param StartLoc  Location of the introducer (e.g. the 'omp' token).
+  /// \param EndLoc    Location of the directive's end (e.g. the tok::eod).
+  /// \param Clauses   The directive's clauses.
+  /// \param NumLoops  Number of affected loops (the flatten depth, currently
+  ///                  always 2).
+  /// \param AssociatedStmt  The outermost associated loop.
+  /// \param TransformedStmt The flattened loop, or nullptr in dependent
+  ///                        contexts.
+  /// \param PreInits  Helper preinits statements for the loop nest.
+  static OMPFlattenDirective *
+  Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation EndLoc,
+         ArrayRef<OMPClause *> Clauses, unsigned NumLoops, Stmt *AssociatedStmt,
+         Stmt *TransformedStmt, Stmt *PreInits);
+
+  /// Build an empty '#pragma omp flatten' AST node for deserialization.
+  ///
+  /// \param C          Context of the AST.
+  /// \param NumClauses Number of clauses to allocate.
+  /// \param NumLoops   Number of associated loops to allocate.
+  static OMPFlattenDirective *
+  CreateEmpty(const ASTContext &C, unsigned NumClauses, unsigned NumLoops);
+
+  /// Gets the flattened loop after the transformation. This is the de-sugared
+  /// replacement or nullptr in dependent contexts.
+  Stmt *getTransformedStmt() const {
+    return Data->getChildren()[TransformedStmtOffset];
+  }
+
+  /// Return preinits statement.
+  Stmt *getPreInits() const { return Data->getChildren()[PreInitsOffset]; }
+
+  static bool classof(const Stmt *T) {
+    return T->getStmtClass() == OMPFlattenDirectiveClass;
+  }
+};
+
 /// The base class for all transformation directives of canonical loop
 /// sequences (currently only 'fuse')
 class OMPCanonicalLoopSequenceTransformationDirective
diff --git a/clang/include/clang/Basic/StmtNodes.td b/clang/include/clang/Basic/StmtNodes.td
index e166894ea024b..0d7a0c1490a1e 100644
--- a/clang/include/clang/Basic/StmtNodes.td
+++ b/clang/include/clang/Basic/StmtNodes.td
@@ -247,6 +247,7 @@ def OMPReverseDirective : StmtNode<OMPCanonicalLoopNestTransformationDirective>;
 def OMPSplitDirective : StmtNode<OMPCanonicalLoopNestTransformationDirective>;
 def OMPInterchangeDirective
     : StmtNode<OMPCanonicalLoopNestTransformationDirective>;
+def OMPFlattenDirective : StmtNode<OMPCanonicalLoopNestTransformationDirective>;
 def OMPCanonicalLoopSequenceTransformationDirective
     : StmtNode<OMPExecutableDirective, 1>;
 def OMPFuseDirective
diff --git a/clang/lib/AST/StmtOpenMP.cpp b/clang/lib/AST/StmtOpenMP.cpp
index 9d6b315effb41..477604ef3dafb 100644
--- a/clang/lib/AST/StmtOpenMP.cpp
+++ b/clang/lib/AST/StmtOpenMP.cpp
@@ -552,6 +552,26 @@ OMPInterchangeDirective::CreateEmpty(const ASTContext &C, unsigned NumClauses,
       SourceLocation(), SourceLocation(), NumLoops);
 }
 
+OMPFlattenDirective *OMPFlattenDirective::Create(
+    const ASTContext &C, SourceLocation StartLoc, SourceLocation EndLoc,
+    ArrayRef<OMPClause *> Clauses, unsigned NumLoops, Stmt *AssociatedStmt,
+    Stmt *TransformedStmt, Stmt *PreInits) {
+  OMPFlattenDirective *Dir = createDirective<OMPFlattenDirective>(
+      C, Clauses, AssociatedStmt, TransformedStmtOffset + 1, StartLoc, EndLoc,
+      NumLoops);
+  Dir->setTransformedStmt(TransformedStmt);
+  Dir->setPreInits(PreInits);
+  return Dir;
+}
+
+OMPFlattenDirective *OMPFlattenDirective::CreateEmpty(const ASTContext &C,
+                                                      unsigned NumClauses,
+                                                      unsigned NumLoops) {
+  return createEmptyDirective<OMPFlattenDirective>(
+      C, NumClauses, /*HasAssociatedStmt=*/true, TransformedStmtOffset + 1,
+      SourceLocation(), SourceLocation(), NumLoops);
+}
+
 OMPSplitDirective *
 OMPSplitDirective::Create(const ASTContext &C, SourceLocation StartLoc,
                           SourceLocation EndLoc, ArrayRef<OMPClause *> Clauses,
diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp
index 6c3294573e9d4..0bacb79ed86e3 100644
--- a/clang/lib/AST/StmtPrinter.cpp
+++ b/clang/lib/AST/StmtPrinter.cpp
@@ -800,6 +800,11 @@ void StmtPrinter::VisitOMPInterchangeDirective(OMPInterchangeDirective *Node) {
   PrintOMPExecutableDirective(Node);
 }
 
+void StmtPrinter::VisitOMPFlattenDirective(OMPFlattenDirective *Node) {
+  Indent() << "#pragma omp flatten";
+  PrintOMPExecutableDirective(Node);
+}
+
 void StmtPrinter::VisitOMPSplitDirective(OMPSplitDirective *Node) {
   Indent() << "#pragma omp split";
   PrintOMPExecutableDirective(Node);
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index 90eab530e0c2e..de7ed322d7424 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -1071,6 +1071,10 @@ void StmtProfiler::VisitOMPInterchangeDirective(
   VisitOMPCanonicalLoopNestTransformationDirective(S);
 }
 
+void StmtProfiler::VisitOMPFlattenDirective(const OMPFlattenDirective *S) {
+  VisitOMPCanonicalLoopNestTransformationDirective(S);
+}
+
 void StmtProfiler::VisitOMPSplitDirective(const OMPSplitDirective *S) {
   VisitOMPCanonicalLoopNestTransformationDirective(S);
 }
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index 675d86349c933..57e0347f8ea77 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -818,7 +818,7 @@ bool clang::isOpenMPCanonicalLoopNestTransformationDirective(
     OpenMPDirectiveKind DKind) {
   return DKind == OMPD_tile || DKind == OMPD_unroll || DKind == OMPD_reverse ||
          DKind == OMPD_split || DKind == OMPD_interchange ||
-         DKind == OMPD_stripe;
+         DKind == OMPD_stripe || DKind == OMPD_flatten;
 }
 
 bool clang::isOpenMPCanonicalLoopSequenceTransformationDirective(

>From 933007b6333584fb358adc4f89a61db1ae15d4cb Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:02:27 -0400
Subject: [PATCH 02/17] add OMPDepthClause ast node

---
 clang/include/clang/AST/OpenMPClause.h        | 68 +++++++++++++++++++
 clang/include/clang/AST/RecursiveASTVisitor.h |  6 ++
 clang/lib/AST/OpenMPClause.cpp                | 25 +++++++
 clang/lib/AST/StmtProfile.cpp                 |  5 ++
 clang/lib/Parse/ParseOpenMP.cpp               |  1 +
 llvm/include/llvm/Frontend/OpenMP/OMP.td      |  1 +
 6 files changed, 106 insertions(+)

diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 8ceafc4669297..50d1364a9726d 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -1380,6 +1380,74 @@ class OMPPartialClause final : public OMPClause {
   }
 };
 
+/// This represents the 'depth' clause on the '#pragma omp flatten' (and
+/// '#pragma omp fuse') loop-transformation directives.
+///
+/// \code
+/// #pragma omp flatten depth(3)
+/// \endcode
+/// In this example the 'flatten' directive has a 'depth' clause whose argument
+/// '3' specifies how many perfectly nested loops are combined into one.
+/// The argument must be a positive integer constant expression that evaluates
+/// to at most the loop nest depth of the associated loop nest.
+class OMPDepthClause final : public OMPClause {
+  friend class OMPClauseReader;
+
+  /// Location of '('.
+  SourceLocation LParenLoc;
+
+  /// The depth expression (number of loops to flatten).
+  Stmt *Depth = nullptr;
+
+  /// Build an empty clause.
+  explicit OMPDepthClause() : OMPClause(llvm::omp::OMPC_depth, {}, {}) {}
+
+  /// Set the depth expression.
+  void setDepth(Expr *E) { Depth = E; }
+
+  /// Sets the location of '('.
+  void setLParenLoc(SourceLocation Loc) { LParenLoc = Loc; }
+
+public:
+  /// Build an AST node for a 'depth' clause.
+  ///
+  /// \param C         Context of the AST.
+  /// \param StartLoc  Location of the 'depth' identifier.
+  /// \param LParenLoc Location of '('.
+  /// \param EndLoc    Location of ')'.
+  /// \param Depth     The depth expression.
+  static OMPDepthClause *Create(const ASTContext &C, SourceLocation StartLoc,
+                                SourceLocation LParenLoc, SourceLocation EndLoc,
+                                Expr *Depth);
+
+  /// Build an empty 'depth' AST node for deserialization.
+  ///
+  /// \param C     Context of the AST.
+  static OMPDepthClause *CreateEmpty(const ASTContext &C);
+
+  /// Returns the location of '('.
+  SourceLocation getLParenLoc() const { return LParenLoc; }
+
+  /// Returns the depth expression or nullptr if not set.
+  Expr *getDepth() const { return cast_or_null<Expr>(Depth); }
+
+  child_range children() { return child_range(&Depth, &Depth + 1); }
+  const_child_range children() const {
+    return const_child_range(&Depth, &Depth + 1);
+  }
+
+  child_range used_children() {
+    return child_range(child_iterator(), child_iterator());
+  }
+  const_child_range used_children() const {
+    return const_child_range(const_child_iterator(), const_child_iterator());
+  }
+
+  static bool classof(const OMPClause *T) {
+    return T->getClauseKind() == llvm::omp::OMPC_depth;
+  }
+};
+
 /// This represents 'collapse' clause in the '#pragma omp ...'
 /// directive.
 ///
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index e425f7a31f59c..da55e97fdfcb5 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -3568,6 +3568,12 @@ bool RecursiveASTVisitor<Derived>::VisitOMPPartialClause(OMPPartialClause *C) {
   return true;
 }
 
+template <typename Derived>
+bool RecursiveASTVisitor<Derived>::VisitOMPDepthClause(OMPDepthClause *C) {
+  TRY_TO(TraverseStmt(C->getDepth()));
+  return true;
+}
+
 template <typename Derived>
 bool
 RecursiveASTVisitor<Derived>::VisitOMPCollapseClause(OMPCollapseClause *C) {
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index ed00e80144c25..7b2f67a31cad9 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -1057,6 +1057,22 @@ OMPPartialClause *OMPPartialClause::CreateEmpty(const ASTContext &C) {
   return new (C) OMPPartialClause();
 }
 
+OMPDepthClause *OMPDepthClause::Create(const ASTContext &C,
+                                       SourceLocation StartLoc,
+                                       SourceLocation LParenLoc,
+                                       SourceLocation EndLoc, Expr *Depth) {
+  OMPDepthClause *Clause = CreateEmpty(C);
+  Clause->setLocStart(StartLoc);
+  Clause->setLParenLoc(LParenLoc);
+  Clause->setLocEnd(EndLoc);
+  Clause->setDepth(Depth);
+  return Clause;
+}
+
+OMPDepthClause *OMPDepthClause::CreateEmpty(const ASTContext &C) {
+  return new (C) OMPDepthClause();
+}
+
 OMPLoopRangeClause *
 OMPLoopRangeClause::Create(const ASTContext &C, SourceLocation StartLoc,
                            SourceLocation LParenLoc, SourceLocation FirstLoc,
@@ -2069,6 +2085,15 @@ void OMPClausePrinter::VisitOMPPermutationClause(OMPPermutationClause *Node) {
 
 void OMPClausePrinter::VisitOMPFullClause(OMPFullClause *Node) { OS << "full"; }
 
+void OMPClausePrinter::VisitOMPDepthClause(OMPDepthClause *Node) {
+  OS << "depth";
+  if (Expr *Depth = Node->getDepth()) {
+    OS << '(';
+    Depth->printPretty(OS, nullptr, Policy, 0);
+    OS << ')';
+  }
+}
+
 void OMPClausePrinter::VisitOMPPartialClause(OMPPartialClause *Node) {
   OS << "partial";
 
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index de7ed322d7424..838d6d02f7446 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -521,6 +521,11 @@ void OMPClauseProfiler::VisitOMPPartialClause(const OMPPartialClause *C) {
     Profiler->VisitExpr(Factor);
 }
 
+void OMPClauseProfiler::VisitOMPDepthClause(const OMPDepthClause *C) {
+  if (const Expr *Depth = C->getDepth())
+    Profiler->VisitExpr(Depth);
+}
+
 void OMPClauseProfiler::VisitOMPLoopRangeClause(const OMPLoopRangeClause *C) {
   if (const Expr *First = C->getFirst())
     Profiler->VisitExpr(First);
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index af52295df2d8b..952a517205381 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -3247,6 +3247,7 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
   case OMPC_nocontext:
   case OMPC_filter:
   case OMPC_partial:
+  case OMPC_depth:
   case OMPC_align:
   case OMPC_message:
   case OMPC_ompx_dyn_cgroup_mem:
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index 679a944fc4358..d49c5ee1a0db3 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -161,6 +161,7 @@ def OMPC_Depobj : Clause<[Spelling<"depobj">]> {
   let isImplicit = true;
 }
 def OMPC_Depth : Clause<[Spelling<"depth">]> {
+  let clangClass = "OMPDepthClause";
   let flangClass = "ScalarIntConstantExpr";
 }
 def OMPC_Destroy : Clause<[Spelling<"destroy">]> {

>From b63024e1e342cd168ef128a3f6a24909df2a82db Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:04:44 -0400
Subject: [PATCH 03/17] template-transform support for flatten directive &
 depth clause

---
 clang/lib/Sema/TreeTransform.h | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 3b99ff4bb9e23..865622ad50f1c 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -1804,6 +1804,14 @@ class TreeTransform {
                                                        LParenLoc, EndLoc);
   }
 
+  /// Build a new OpenMP 'depth' clause.
+  OMPClause *RebuildOMPDepthClause(Expr *Depth, SourceLocation StartLoc,
+                                   SourceLocation LParenLoc,
+                                   SourceLocation EndLoc) {
+    return getSema().OpenMP().ActOnOpenMPDepthClause(Depth, StartLoc, LParenLoc,
+                                                     EndLoc);
+  }
+
   OMPClause *
   RebuildOMPLoopRangeClause(Expr *First, Expr *Count, SourceLocation StartLoc,
                             SourceLocation LParenLoc, SourceLocation FirstLoc,
@@ -9791,6 +9799,17 @@ TreeTransform<Derived>::TransformOMPSplitDirective(OMPSplitDirective *D) {
   return Res;
 }
 
+template <typename Derived>
+StmtResult
+TreeTransform<Derived>::TransformOMPFlattenDirective(OMPFlattenDirective *D) {
+  DeclarationNameInfo DirName;
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      D->getDirectiveKind(), DirName, nullptr, D->getBeginLoc());
+  StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
+  return Res;
+}
+
 template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPFuseDirective(OMPFuseDirective *D) {
@@ -10717,6 +10736,20 @@ TreeTransform<Derived>::TransformOMPPartialClause(OMPPartialClause *C) {
                                  C->getEndLoc());
 }
 
+template <typename Derived>
+OMPClause *TreeTransform<Derived>::TransformOMPDepthClause(OMPDepthClause *C) {
+  ExprResult T = getDerived().TransformExpr(C->getDepth());
+  if (T.isInvalid())
+    return nullptr;
+  Expr *Depth = T.get();
+  bool Changed = Depth != C->getDepth();
+
+  if (!Changed && !getDerived().AlwaysRebuild())
+    return C;
+  return RebuildOMPDepthClause(Depth, C->getBeginLoc(), C->getLParenLoc(),
+                               C->getEndLoc());
+}
+
 template <typename Derived>
 OMPClause *
 TreeTransform<Derived>::TransformOMPLoopRangeClause(OMPLoopRangeClause *C) {

>From deafb222a7d251069c68a44fa267e58e750f1064 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:08:12 -0400
Subject: [PATCH 04/17] serialize flatten

---
 clang/include/clang/Serialization/ASTBitCodes.h |  1 +
 clang/lib/Serialization/ASTReaderStmt.cpp       | 11 +++++++++++
 clang/lib/Serialization/ASTWriterStmt.cpp       |  5 +++++
 3 files changed, 17 insertions(+)

diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h
index 279380de2f7fe..93a0e0a454b68 100644
--- a/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/clang/include/clang/Serialization/ASTBitCodes.h
@@ -1970,6 +1970,7 @@ enum StmtCode {
   STMT_OMP_REVERSE_DIRECTIVE,
   STMT_OMP_SPLIT_DIRECTIVE,
   STMT_OMP_INTERCHANGE_DIRECTIVE,
+  STMT_OMP_FLATTEN_DIRECTIVE,
   STMT_OMP_FUSE_DIRECTIVE,
   STMT_OMP_FOR_DIRECTIVE,
   STMT_OMP_FOR_SIMD_DIRECTIVE,
diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp
index 02ccf8d4d41c2..8c2debd6cdf32 100644
--- a/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -2537,6 +2537,10 @@ void ASTStmtReader::VisitOMPInterchangeDirective(OMPInterchangeDirective *D) {
   VisitOMPCanonicalLoopNestTransformationDirective(D);
 }
 
+void ASTStmtReader::VisitOMPFlattenDirective(OMPFlattenDirective *D) {
+  VisitOMPCanonicalLoopNestTransformationDirective(D);
+}
+
 void ASTStmtReader::VisitOMPSplitDirective(OMPSplitDirective *D) {
   VisitOMPCanonicalLoopNestTransformationDirective(D);
 }
@@ -3719,6 +3723,13 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) {
       break;
     }
 
+    case STMT_OMP_FLATTEN_DIRECTIVE: {
+      unsigned NumLoops = Record[ASTStmtReader::NumStmtFields];
+      unsigned NumClauses = Record[ASTStmtReader::NumStmtFields + 1];
+      S = OMPFlattenDirective::CreateEmpty(Context, NumClauses, NumLoops);
+      break;
+    }
+
     case STMT_OMP_FOR_DIRECTIVE: {
       unsigned CollapsedNum = Record[ASTStmtReader::NumStmtFields];
       unsigned NumClauses = Record[ASTStmtReader::NumStmtFields + 1];
diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp
index 4e9eadd730a56..dfe28a9b915e3 100644
--- a/clang/lib/Serialization/ASTWriterStmt.cpp
+++ b/clang/lib/Serialization/ASTWriterStmt.cpp
@@ -2547,6 +2547,11 @@ void ASTStmtWriter::VisitOMPInterchangeDirective(OMPInterchangeDirective *D) {
   Code = serialization::STMT_OMP_INTERCHANGE_DIRECTIVE;
 }
 
+void ASTStmtWriter::VisitOMPFlattenDirective(OMPFlattenDirective *D) {
+  VisitOMPCanonicalLoopNestTransformationDirective(D);
+  Code = serialization::STMT_OMP_FLATTEN_DIRECTIVE;
+}
+
 void ASTStmtWriter::VisitOMPSplitDirective(OMPSplitDirective *D) {
   VisitOMPCanonicalLoopNestTransformationDirective(D);
   Code = serialization::STMT_OMP_SPLIT_DIRECTIVE;

>From 3e2dfeb99230057b7cb9c484182ae33befbe1e41 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:09:03 -0400
Subject: [PATCH 05/17] serialize depth

---
 clang/lib/Serialization/ASTReader.cpp | 8 ++++++++
 clang/lib/Serialization/ASTWriter.cpp | 5 +++++
 2 files changed, 13 insertions(+)

diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index f8a6a38bb9b5c..4f51d79a52105 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -11495,6 +11495,9 @@ OMPClause *OMPClauseReader::readClause() {
   case llvm::omp::OMPC_partial:
     C = OMPPartialClause::CreateEmpty(Context);
     break;
+  case llvm::omp::OMPC_depth:
+    C = OMPDepthClause::CreateEmpty(Context);
+    break;
   case llvm::omp::OMPC_looprange:
     C = OMPLoopRangeClause::CreateEmpty(Context);
     break;
@@ -11924,6 +11927,11 @@ void OMPClauseReader::VisitOMPPartialClause(OMPPartialClause *C) {
   C->setLParenLoc(Record.readSourceLocation());
 }
 
+void OMPClauseReader::VisitOMPDepthClause(OMPDepthClause *C) {
+  C->setDepth(Record.readSubExpr());
+  C->setLParenLoc(Record.readSourceLocation());
+}
+
 void OMPClauseReader::VisitOMPLoopRangeClause(OMPLoopRangeClause *C) {
   C->setFirst(Record.readSubExpr());
   C->setCount(Record.readSubExpr());
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 357a7f7e95fa0..cbfeaa9e6638a 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -8106,6 +8106,11 @@ void OMPClauseWriter::VisitOMPPartialClause(OMPPartialClause *C) {
   Record.AddSourceLocation(C->getLParenLoc());
 }
 
+void OMPClauseWriter::VisitOMPDepthClause(OMPDepthClause *C) {
+  Record.AddStmt(C->getDepth());
+  Record.AddSourceLocation(C->getLParenLoc());
+}
+
 void OMPClauseWriter::VisitOMPLoopRangeClause(OMPLoopRangeClause *C) {
   Record.AddStmt(C->getFirst());
   Record.AddStmt(C->getCount());

>From 4ae746ca1eeaf11db10e859572a4500029d01cce Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:17:29 -0400
Subject: [PATCH 06/17] add analysis traversal support for flatten directive

---
 clang/lib/Sema/SemaExceptionSpec.cpp         | 1 +
 clang/lib/StaticAnalyzer/Core/ExprEngine.cpp | 1 +
 2 files changed, 2 insertions(+)

diff --git a/clang/lib/Sema/SemaExceptionSpec.cpp b/clang/lib/Sema/SemaExceptionSpec.cpp
index 40d530a1f3925..995b5b6010978 100644
--- a/clang/lib/Sema/SemaExceptionSpec.cpp
+++ b/clang/lib/Sema/SemaExceptionSpec.cpp
@@ -1508,6 +1508,7 @@ CanThrowResult Sema::canThrow(const Stmt *S) {
   case Stmt::OMPUnrollDirectiveClass:
   case Stmt::OMPReverseDirectiveClass:
   case Stmt::OMPInterchangeDirectiveClass:
+  case Stmt::OMPFlattenDirectiveClass:
   case Stmt::OMPSplitDirectiveClass:
   case Stmt::OMPFuseDirectiveClass:
   case Stmt::OMPSingleDirectiveClass:
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
index cfb294736ee02..3f666be00cc24 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
@@ -1783,6 +1783,7 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
     case Stmt::OMPStripeDirectiveClass:
     case Stmt::OMPTileDirectiveClass:
     case Stmt::OMPInterchangeDirectiveClass:
+    case Stmt::OMPFlattenDirectiveClass:
     case Stmt::OMPSplitDirectiveClass:
     case Stmt::OMPFuseDirectiveClass:
     case Stmt::OMPInteropDirectiveClass:

>From 415eba88ad0af63c552b4c5c2eabd660bcb29c02 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:21:25 -0400
Subject: [PATCH 07/17] codegen flatten directive

---
 clang/lib/CodeGen/CGStmt.cpp        | 3 +++
 clang/lib/CodeGen/CGStmtOpenMP.cpp  | 8 ++++++++
 clang/lib/CodeGen/CodeGenFunction.h | 1 +
 3 files changed, 12 insertions(+)

diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index 232094777f233..27f48cb63b553 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -236,6 +236,9 @@ void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {
   case Stmt::OMPInterchangeDirectiveClass:
     EmitOMPInterchangeDirective(cast<OMPInterchangeDirective>(*S));
     break;
+  case Stmt::OMPFlattenDirectiveClass:
+    EmitOMPFlattenDirective(cast<OMPFlattenDirective>(*S));
+    break;
   case Stmt::OMPFuseDirectiveClass:
     EmitOMPFuseDirective(cast<OMPFuseDirective>(*S));
     break;
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 95fd6694437fe..25da3430a23fe 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -218,6 +218,8 @@ class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
     } else if (const auto *Interchange =
                    dyn_cast<OMPInterchangeDirective>(&S)) {
       PreInits = Interchange->getPreInits();
+    } else if (const auto *Flatten = dyn_cast<OMPFlattenDirective>(&S)) {
+      PreInits = Flatten->getPreInits();
     } else {
       llvm_unreachable("Unknown loop-based directive kind.");
     }
@@ -3248,6 +3250,12 @@ void CodeGenFunction::EmitOMPInterchangeDirective(
   EmitStmt(S.getTransformedStmt());
 }
 
+void CodeGenFunction::EmitOMPFlattenDirective(const OMPFlattenDirective &S) {
+  // Emit the de-sugared statement.
+  OMPTransformDirectiveScopeRAII FlattenScope(*this, &S);
+  EmitStmt(S.getTransformedStmt());
+}
+
 void CodeGenFunction::EmitOMPFuseDirective(const OMPFuseDirective &S) {
   // Emit the de-sugared statement
   OMPTransformDirectiveScopeRAII FuseScope(*this, &S);
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 6d0718c243812..eea0163422cd3 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -3934,6 +3934,7 @@ class CodeGenFunction : public CodeGenTypeCache {
   void EmitOMPReverseDirective(const OMPReverseDirective &S);
   void EmitOMPSplitDirective(const OMPSplitDirective &S);
   void EmitOMPInterchangeDirective(const OMPInterchangeDirective &S);
+  void EmitOMPFlattenDirective(const OMPFlattenDirective &S);
   void EmitOMPFuseDirective(const OMPFuseDirective &S);
   void EmitOMPForDirective(const OMPForDirective &S);
   void EmitOMPForSimdDirective(const OMPForSimdDirective &S);

>From 814e2e9fe9bf4e28e4557999fbc16beec144cc39 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:24:42 -0400
Subject: [PATCH 08/17] add NYI CIR stub

---
 clang/lib/CIR/CodeGen/CIRGenFunction.h     | 1 +
 clang/lib/CIR/CodeGen/CIRGenStmt.cpp       | 2 ++
 clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp | 5 +++++
 3 files changed, 8 insertions(+)

diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.h b/clang/lib/CIR/CodeGen/CIRGenFunction.h
index 322355fde3957..8374583ffd4d0 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.h
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.h
@@ -2559,6 +2559,7 @@ class CIRGenFunction : public CIRGenTypeCache {
   mlir::LogicalResult emitOMPSplitDirective(const OMPSplitDirective &s);
   mlir::LogicalResult
   emitOMPInterchangeDirective(const OMPInterchangeDirective &s);
+  mlir::LogicalResult emitOMPFlattenDirective(const OMPFlattenDirective &s);
   mlir::LogicalResult emitOMPAssumeDirective(const OMPAssumeDirective &s);
   mlir::LogicalResult emitOMPMaskedDirective(const OMPMaskedDirective &s);
   mlir::LogicalResult emitOMPStripeDirective(const OMPStripeDirective &s);
diff --git a/clang/lib/CIR/CodeGen/CIRGenStmt.cpp b/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
index d3acac5801e74..33d425b22d61a 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
@@ -408,6 +408,8 @@ mlir::LogicalResult CIRGenFunction::emitStmt(const Stmt *s,
     return emitOMPSplitDirective(cast<OMPSplitDirective>(*s));
   case Stmt::OMPInterchangeDirectiveClass:
     return emitOMPInterchangeDirective(cast<OMPInterchangeDirective>(*s));
+  case Stmt::OMPFlattenDirectiveClass:
+    return emitOMPFlattenDirective(cast<OMPFlattenDirective>(*s));
   case Stmt::OMPAssumeDirectiveClass:
     return emitOMPAssumeDirective(cast<OMPAssumeDirective>(*s));
   case Stmt::OMPMaskedDirectiveClass:
diff --git a/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp b/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp
index 17a1fb8090f5c..4fffd17b25a16 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp
@@ -587,6 +587,11 @@ CIRGenFunction::emitOMPInterchangeDirective(const OMPInterchangeDirective &s) {
   return mlir::failure();
 }
 mlir::LogicalResult
+CIRGenFunction::emitOMPFlattenDirective(const OMPFlattenDirective &s) {
+  getCIRGenModule().errorNYI(s.getSourceRange(), "OpenMP OMPFlattenDirective");
+  return mlir::failure();
+}
+mlir::LogicalResult
 CIRGenFunction::emitOMPAssumeDirective(const OMPAssumeDirective &s) {
   getCIRGenModule().errorNYI(s.getSourceRange(), "OpenMP OMPAssumeDirective");
   return mlir::failure();

>From 0027ec6ab982a83766bd65f3e1bc45d9a671fc62 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:26:31 -0400
Subject: [PATCH 09/17] add libclang traversal support for depth clause

---
 clang/tools/libclang/CIndex.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index ac2fad38a1348..f8ab63e3e0dae 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2378,6 +2378,10 @@ void OMPClauseEnqueue::VisitOMPLoopRangeClause(const OMPLoopRangeClause *C) {
   Visitor->AddStmt(C->getCount());
 }
 
+void OMPClauseEnqueue::VisitOMPDepthClause(const OMPDepthClause *C) {
+  Visitor->AddStmt(C->getDepth());
+}
+
 void OMPClauseEnqueue::VisitOMPAllocatorClause(const OMPAllocatorClause *C) {
   Visitor->AddStmt(C->getAllocator());
 }

>From f5ac5f740c414310c8482f652f1bed936ae7cda3 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Fri, 10 Jul 2026 06:54:16 -0400
Subject: [PATCH 10/17] sema

---
 clang/include/clang/Sema/SemaOpenMP.h |  10 +
 clang/lib/Sema/SemaOpenMP.cpp         | 295 ++++++++++++++++++++++++++
 2 files changed, 305 insertions(+)

diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index 3621ce96b8724..7b086388416a0 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -470,6 +470,12 @@ class SemaOpenMP : public SemaBase {
                                              SourceLocation StartLoc,
                                              SourceLocation EndLoc);
 
+  /// Called on well-formed '#pragma omp flatten' after parsing of its
+  /// clauses and the associated statement.
+  StmtResult ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
+                                         Stmt *AStmt, SourceLocation StartLoc,
+                                         SourceLocation EndLoc);
+
   /// Called on well-formed '#pragma omp fuse' after parsing of its
   /// clauses and the associated statement.
   StmtResult ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
@@ -935,6 +941,10 @@ class SemaOpenMP : public SemaBase {
   OMPClause *ActOnOpenMPPartialClause(Expr *FactorExpr, SourceLocation StartLoc,
                                       SourceLocation LParenLoc,
                                       SourceLocation EndLoc);
+  /// Called on well-formed 'depth' clause.
+  OMPClause *ActOnOpenMPDepthClause(Expr *DepthExpr, SourceLocation StartLoc,
+                                    SourceLocation LParenLoc,
+                                    SourceLocation EndLoc);
   /// Called on well-formed 'collapse' clause.
   OMPClause *ActOnOpenMPCollapseClause(Expr *NumForLoops,
                                        SourceLocation StartLoc,
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 247a4a6ad9271..fcc256dec3c88 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -4629,6 +4629,7 @@ void SemaOpenMP::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind,
   case OMPD_reverse:
   case OMPD_split:
   case OMPD_interchange:
+  case OMPD_flatten:
   case OMPD_fuse:
   case OMPD_assume:
     break;
@@ -6475,6 +6476,10 @@ StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
     Res = ActOnOpenMPInterchangeDirective(ClausesWithImplicit, AStmt, StartLoc,
                                           EndLoc);
     break;
+  case OMPD_flatten:
+    Res = ActOnOpenMPFlattenDirective(ClausesWithImplicit, AStmt, StartLoc,
+                                      EndLoc);
+    break;
   case OMPD_fuse:
     Res =
         ActOnOpenMPFuseDirective(ClausesWithImplicit, AStmt, StartLoc, EndLoc);
@@ -16390,6 +16395,277 @@ StmtResult SemaOpenMP::ActOnOpenMPInterchangeDirective(
                                          buildPreInits(Context, PreInits));
 }
 
+/// Counts the perfectly nested canonical loop depth at \p AStmt. Returns
+/// std::nullopt if a loop-generating transformation prevents a static count.
+static std::optional<unsigned> getCanonicalLoopNestDepth(Stmt *AStmt) {
+  unsigned Depth = 0;
+  Stmt *CurStmt = AStmt ? AStmt->IgnoreContainers() : nullptr;
+  while (CurStmt) {
+    if (isa<OMPLoopTransformationDirective>(CurStmt))
+      return std::nullopt;
+    if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(CurStmt))
+      CurStmt = CanonLoop->getLoopStmt();
+    Stmt *Body = nullptr;
+    if (auto *For = dyn_cast<ForStmt>(CurStmt))
+      Body = For->getBody();
+    else if (auto *RangeFor = dyn_cast<CXXForRangeStmt>(CurStmt))
+      Body = RangeFor->getBody();
+    else
+      break;
+    ++Depth;
+    CurStmt = Body ? Body->IgnoreContainers() : nullptr;
+  }
+  return Depth;
+}
+
+StmtResult
+SemaOpenMP::ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
+                                        Stmt *AStmt, SourceLocation StartLoc,
+                                        SourceLocation EndLoc) {
+  ASTContext &Context = getASTContext();
+  DeclContext *CurContext = SemaRef.CurContext;
+  Scope *CurScope = SemaRef.getCurScope();
+
+  // Empty statement should only be possible if there already was an error.
+  if (!AStmt)
+    return StmtError();
+
+  // flatten without 'depth' clause combines two loops; 'depth(k)' selects k.
+  unsigned NumLoops = 2;
+  bool DepthIsValueDependent = false;
+  const auto *DepthClause =
+      OMPExecutableDirective::getSingleClause<OMPDepthClause>(Clauses);
+  if (DepthClause) {
+    Expr *DepthExpr = DepthClause->getDepth();
+    if (DepthExpr && DepthExpr->isValueDependent()) {
+      DepthIsValueDependent = true;
+    } else if (DepthExpr) {
+      Expr::EvalResult EvalResult;
+      if (DepthExpr->EvaluateAsInt(EvalResult, Context))
+        NumLoops = EvalResult.Val.getInt().getZExtValue();
+    }
+  }
+
+  // Report "expected k for loops, but found only n" when depth exceeds the
+  // perfect nest (same form as 'collapse'); skip if depth or nest is unknown.
+  if (DepthClause && !DepthIsValueDependent) {
+    if (std::optional<unsigned> NestDepth = getCanonicalLoopNestDepth(AStmt);
+        NestDepth && NumLoops > *NestDepth) {
+      Diag(AStmt->getBeginLoc(), diag::err_omp_not_for)
+          << /*expected N for loops form=*/1
+          << getOpenMPDirectiveName(OMPD_flatten) << NumLoops
+          << (*NestDepth > 0) << *NestDepth;
+      return StmtError();
+    }
+  }
+
+  // Defer when 'depth' is value-dependent (concrete k unknown until
+  // instantiation).
+  if (DepthIsValueDependent)
+    return OMPFlattenDirective::Create(Context, StartLoc, EndLoc, Clauses,
+                                       NumLoops, AStmt, nullptr, nullptr);
+
+  // Verify and diagnose loop nest.
+  SmallVector<OMPLoopBasedDirective::HelperExprs, 4> LoopHelpers(NumLoops);
+  Stmt *Body = nullptr;
+  SmallVector<SmallVector<Stmt *>, 4> OriginalInits;
+  if (!checkTransformableLoopNest(OMPD_flatten, AStmt, NumLoops, LoopHelpers,
+                                  Body, OriginalInits))
+    return StmtError();
+
+  // Delay flattening to when template is completely instantiated.
+  if (CurContext->isDependentContext())
+    return OMPFlattenDirective::Create(Context, StartLoc, EndLoc, Clauses,
+                                       NumLoops, AStmt, nullptr, nullptr);
+
+  assert(LoopHelpers.size() == NumLoops &&
+         "Expecting loop iteration space dimensionality to match number of "
+         "affected loops");
+  assert(OriginalInits.size() == NumLoops &&
+         "Expecting loop iteration space dimensionality to match number of "
+         "affected loops");
+
+  // Find the affected loops.
+  SmallVector<Stmt *> LoopStmts(NumLoops, nullptr);
+  collectLoopStmts(AStmt, LoopStmts);
+
+  // Collect pre-init statements in outer-to-inner order.
+  SmallVector<Stmt *> PreInits;
+  for (auto I : llvm::seq<int>(NumLoops)) {
+    OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
+    assert(LoopHelper.Counters.size() == 1 &&
+           "Single-dimensional loop iteration space expected");
+    addLoopPreInits(Context, LoopHelper, LoopStmts[I], OriginalInits[I],
+                    PreInits);
+  }
+
+  CaptureVars CopyTransformer(SemaRef);
+  auto MakeNumIterations = [&CopyTransformer,
+                            &LoopHelpers](unsigned I) -> Expr * {
+    return AssertSuccess(
+        CopyTransformer.TransformExpr(LoopHelpers[I].NumIterations));
+  };
+
+  OMPLoopBasedDirective::HelperExprs &OutermostHelper = LoopHelpers[0];
+  auto *OutermostCntVar = cast<DeclRefExpr>(OutermostHelper.Counters.front());
+  SourceLocation OrigVarLoc = OutermostCntVar->getExprLoc();
+  SourceLocation OrigVarLocBegin = OutermostCntVar->getBeginLoc();
+  SourceLocation OrigVarLocEnd = OutermostCntVar->getEndLoc();
+  SourceLocation CondLoc = OutermostHelper.Cond->getExprLoc();
+
+  // Product of trip counts; mirror 'collapse' IV-width selection to avoid
+  // overflow when several counts are multiplied.
+  auto BuildTripCount = [&](unsigned Bits) -> ExprResult {
+    ExprResult Product;
+    for (unsigned I = 0; I < NumLoops; ++I) {
+      ExprResult N = widenIterationCount(Bits, MakeNumIterations(I), SemaRef);
+      if (!N.isUsable())
+        return ExprError();
+      if (I == 0)
+        Product = N;
+      else
+        Product = SemaRef.BuildBinOp(CurScope, CondLoc, BO_Mul, Product.get(),
+                                     N.get());
+      if (!Product.isUsable())
+        return ExprError();
+    }
+    return Product;
+  };
+
+  bool AllCountsLessThan32Bits = true;
+  for (unsigned I = 0; I < NumLoops; ++I)
+    AllCountsLessThan32Bits &=
+        Context.getTypeSize(LoopHelpers[I].NumIterations->getType()) < 32;
+
+  ExprResult TripCount32 = BuildTripCount(/*Bits=*/32);
+  ExprResult TripCount64 = BuildTripCount(/*Bits=*/64);
+  if (!TripCount32.isUsable() || !TripCount64.isUsable())
+    return StmtError();
+
+  ExprResult TripCount = TripCount64;
+  if (Context.getTypeSize(TripCount32.get()->getType()) == 32 &&
+      (AllCountsLessThan32Bits || NumLoops == 1 ||
+       fitsInto(
+           /*Bits=*/32,
+           TripCount32.get()->getType()->hasSignedIntegerRepresentation(),
+           TripCount64.get(), SemaRef)))
+    TripCount = TripCount32;
+
+  QualType IVTy = TripCount.get()->getType();
+  uint64_t IVWidth = Context.getTypeSize(IVTy);
+
+  auto MakeNumIterationsInIVTy = [&](unsigned I) -> Expr * {
+    return AssertSuccess(SemaRef.PerformImplicitConversion(
+        MakeNumIterations(I), IVTy, AssignmentAction::Converting,
+        /*AllowExplicit=*/true));
+  };
+
+  // \code{.cpp}
+  //   for (auto .flatten.iv = 0; .flatten.iv < n0 * n1 * ...; ++.flatten.iv) {
+  //     .flatten.iv.0 = .flatten.iv / (n1 * ...);
+  //     i0 = ...;                                        // Updates[0]
+  //     .flatten.iv.1 = (.flatten.iv / ...) % n1;
+  //     i1 = ...;                                        // Updates[1]
+  //     ...
+  //     body(i0, i1, ...);
+  //   }
+  // \endcode
+  SmallString<64> FlattenedIVName(".flatten.iv");
+  VarDecl *FlattenedIVDecl = buildVarDecl(SemaRef, {}, IVTy, FlattenedIVName,
+                                          nullptr, OutermostCntVar);
+  auto MakeFlattenedRef = [&SemaRef = this->SemaRef, FlattenedIVDecl, IVTy,
+                           OrigVarLoc]() {
+    return buildDeclRefExpr(SemaRef, FlattenedIVDecl, IVTy, OrigVarLoc);
+  };
+
+  // For init-statement:
+  // \code{.cpp}
+  //   auto .flatten.iv = 0;
+  // \endcode
+  auto *Zero = IntegerLiteral::Create(Context, llvm::APInt::getZero(IVWidth),
+                                      IVTy, OrigVarLoc);
+  SemaRef.AddInitializerToDecl(FlattenedIVDecl, Zero, /*DirectInit=*/false);
+  StmtResult Init = new (Context)
+      DeclStmt(DeclGroupRef(FlattenedIVDecl), OrigVarLocBegin, OrigVarLocEnd);
+  if (!Init.isUsable())
+    return StmtError();
+
+  // For cond-expression:
+  // \code{.cpp}
+  //   .flatten.iv < n0 * n1 * ... * n(k-1)
+  // \endcode
+  ExprResult Cond = SemaRef.BuildBinOp(CurScope, CondLoc, BO_LT,
+                                       MakeFlattenedRef(), TripCount.get());
+  if (!Cond.isUsable())
+    return StmtError();
+
+  // For incr-statement:
+  // \code{.cpp}
+  //   ++.flatten.iv
+  // \endcode
+  ExprResult Incr =
+      SemaRef.BuildUnaryOp(CurScope, OutermostHelper.Inc->getExprLoc(),
+                           UO_PreInc, MakeFlattenedRef());
+  if (!Incr.isUsable())
+    return StmtError();
+
+  // Recover each logical iteration counter via mixed-radix div/mod; reuse the
+  // iteration variables from checkOpenMPLoop so Updates compute user counters.
+  SmallVector<Stmt *, 8> BodyStmts;
+  for (unsigned I = 0; I < NumLoops; ++I) {
+    OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
+    auto *IVRef = cast<DeclRefExpr>(LoopHelper.IterationVarRef);
+    auto *IVDecl = cast<VarDecl>(IVRef->getDecl());
+    std::string IVName = (".flatten.iv." + llvm::Twine(I)).str();
+    IVDecl->setDeclName(&SemaRef.PP.getIdentifierTable().get(IVName));
+
+    ExprResult Value = MakeFlattenedRef();
+    if (I + 1 < NumLoops) {
+      ExprResult Divisor = MakeNumIterationsInIVTy(I + 1);
+      for (unsigned J = I + 2; J < NumLoops; ++J) {
+        Divisor = SemaRef.BuildBinOp(CurScope, OrigVarLoc, BO_Mul,
+                                     Divisor.get(), MakeNumIterationsInIVTy(J));
+        if (!Divisor.isUsable())
+          return StmtError();
+      }
+      Value = SemaRef.BuildBinOp(CurScope, OrigVarLoc, BO_Div, Value.get(),
+                                 Divisor.get());
+      if (!Value.isUsable())
+        return StmtError();
+    }
+    if (I > 0) {
+      Value = SemaRef.BuildBinOp(CurScope, OrigVarLoc, BO_Rem, Value.get(),
+                                 MakeNumIterationsInIVTy(I));
+      if (!Value.isUsable())
+        return StmtError();
+    }
+
+    SemaRef.AddInitializerToDecl(IVDecl, Value.get(), /*DirectInit=*/false);
+    StmtResult IVStmt = new (Context)
+        DeclStmt(DeclGroupRef(IVDecl), OrigVarLocBegin, OrigVarLocEnd);
+    if (!IVStmt.isUsable())
+      return StmtError();
+
+    BodyStmts.push_back(IVStmt.get());
+    llvm::append_range(BodyStmts, LoopHelper.Updates);
+    if (auto *CXXFor = dyn_cast<CXXForRangeStmt>(LoopStmts[I]))
+      BodyStmts.push_back(CXXFor->getLoopVarStmt());
+  }
+  BodyStmts.push_back(Body);
+  auto *FlattenedBody =
+      CompoundStmt::Create(Context, BodyStmts, FPOptionsOverride(),
+                           Body->getBeginLoc(), Body->getEndLoc());
+
+  auto *FlattenedFor = new (Context) ForStmt(
+      Context, Init.get(), Cond.get(), nullptr, Incr.get(), FlattenedBody,
+      OutermostHelper.Init->getBeginLoc(), OutermostHelper.Init->getBeginLoc(),
+      OutermostHelper.Inc->getEndLoc());
+
+  return OMPFlattenDirective::Create(Context, StartLoc, EndLoc, Clauses,
+                                     NumLoops, AStmt, FlattenedFor,
+                                     buildPreInits(Context, PreInits));
+}
+
 StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
                                                 Stmt *AStmt,
                                                 SourceLocation StartLoc,
@@ -16920,6 +17196,9 @@ OMPClause *SemaOpenMP::ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind,
   case OMPC_partial:
     Res = ActOnOpenMPPartialClause(Expr, StartLoc, LParenLoc, EndLoc);
     break;
+  case OMPC_depth:
+    Res = ActOnOpenMPDepthClause(Expr, StartLoc, LParenLoc, EndLoc);
+    break;
   case OMPC_message:
     Res = ActOnOpenMPMessageClause(Expr, StartLoc, LParenLoc, EndLoc);
     break;
@@ -18241,6 +18520,22 @@ OMPClause *SemaOpenMP::ActOnOpenMPPartialClause(Expr *FactorExpr,
                                   FactorExpr);
 }
 
+OMPClause *SemaOpenMP::ActOnOpenMPDepthClause(Expr *DepthExpr,
+                                              SourceLocation StartLoc,
+                                              SourceLocation LParenLoc,
+                                              SourceLocation EndLoc) {
+  // The depth-expr must be a positive integer constant expression and
+  // not greater than the number of loops in the associated loop nest.
+  ExprResult DepthResult = VerifyPositiveIntegerConstantInClause(
+      DepthExpr, OMPC_depth, /*StrictlyPositive=*/true);
+  if (DepthResult.isInvalid())
+    return nullptr;
+  DepthExpr = DepthResult.get();
+
+  return OMPDepthClause::Create(getASTContext(), StartLoc, LParenLoc, EndLoc,
+                                DepthExpr);
+}
+
 OMPClause *SemaOpenMP::ActOnOpenMPLoopRangeClause(
     Expr *First, Expr *Count, SourceLocation StartLoc, SourceLocation LParenLoc,
     SourceLocation FirstLoc, SourceLocation CountLoc, SourceLocation EndLoc) {

>From 54be7347979638725cd4b6c421d11ec1cd618c43 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Mon, 27 Jul 2026 07:03:06 -0400
Subject: [PATCH 11/17] add libclang traversal support for flatten directive

---
 clang/tools/libclang/CIndex.cpp   | 7 +++++++
 clang/tools/libclang/CXCursor.cpp | 3 +++
 2 files changed, 10 insertions(+)

diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index f8ab63e3e0dae..9eb0ec8c21158 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2160,6 +2160,7 @@ class EnqueueVisitor : public ConstStmtVisitor<EnqueueVisitor, void>,
   void VisitOMPUnrollDirective(const OMPUnrollDirective *D);
   void VisitOMPReverseDirective(const OMPReverseDirective *D);
   void VisitOMPInterchangeDirective(const OMPInterchangeDirective *D);
+  void VisitOMPFlattenDirective(const OMPFlattenDirective *D);
   void VisitOMPCanonicalLoopSequenceTransformationDirective(
       const OMPCanonicalLoopSequenceTransformationDirective *D);
   void VisitOMPFuseDirective(const OMPFuseDirective *D);
@@ -3358,6 +3359,10 @@ void EnqueueVisitor::VisitOMPInterchangeDirective(
   VisitOMPCanonicalLoopNestTransformationDirective(D);
 }
 
+void EnqueueVisitor::VisitOMPFlattenDirective(const OMPFlattenDirective *D) {
+  VisitOMPCanonicalLoopNestTransformationDirective(D);
+}
+
 void EnqueueVisitor::VisitOMPCanonicalLoopSequenceTransformationDirective(
     const OMPCanonicalLoopSequenceTransformationDirective *D) {
   VisitOMPExecutableDirective(D);
@@ -6329,6 +6334,8 @@ CXString clang_getCursorKindSpelling(enum CXCursorKind Kind) {
     return cxstring::createRef("OMPReverseDirective");
   case CXCursor_OMPInterchangeDirective:
     return cxstring::createRef("OMPInterchangeDirective");
+  case CXCursor_OMPFlattenDirective:
+    return cxstring::createRef("OMPFlattenDirective");
   case CXCursor_OMPFuseDirective:
     return cxstring::createRef("OMPFuseDirective");
   case CXCursor_OMPSplitDirective:
diff --git a/clang/tools/libclang/CXCursor.cpp b/clang/tools/libclang/CXCursor.cpp
index b2feed578fc0f..97bbfd36558a9 100644
--- a/clang/tools/libclang/CXCursor.cpp
+++ b/clang/tools/libclang/CXCursor.cpp
@@ -703,6 +703,9 @@ CXCursor cxcursor::MakeCXCursor(const Stmt *S, const Decl *Parent,
   case Stmt::OMPInterchangeDirectiveClass:
     K = CXCursor_OMPInterchangeDirective;
     break;
+  case Stmt::OMPFlattenDirectiveClass:
+    K = CXCursor_OMPFlattenDirective;
+    break;
   case Stmt::OMPFuseDirectiveClass:
     K = CXCursor_OMPFuseDirective;
     break;

>From cdedb1045baf373937b0da858624a6ace14a3d21 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Mon, 27 Jul 2026 07:06:36 -0400
Subject: [PATCH 12/17] bindings

---
 clang/bindings/python/clang/cindex.py | 3 +++
 clang/include/clang-c/Index.h         | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/clang/bindings/python/clang/cindex.py b/clang/bindings/python/clang/cindex.py
index 24b737139dba8..ccad140e8c861 100644
--- a/clang/bindings/python/clang/cindex.py
+++ b/clang/bindings/python/clang/cindex.py
@@ -1456,6 +1456,9 @@ def is_unexposed(self):
     # OpenMP split directive.
     OMP_SPLIT_DIRECTIVE = 312
 
+    # OpenMP flatten directive.
+    OMP_FLATTEN_DIRECTIVE = 313
+
     # OpenACC Compute Construct.
     OPEN_ACC_COMPUTE_DIRECTIVE = 320
 
diff --git a/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h
index 8427236e0b444..f561fb5724897 100644
--- a/clang/include/clang-c/Index.h
+++ b/clang/include/clang-c/Index.h
@@ -2170,6 +2170,10 @@ enum CXCursorKind {
    */
   CXCursor_OMPSplitDirective = 312,
 
+  /** OpenMP flatten directive.
+   */
+  CXCursor_OMPFlattenDirective = 313,
+
   /** OpenACC Compute Construct.
    */
   CXCursor_OpenACCComputeConstruct = 320,

>From 94dee8fe1351f74760df0fb2d9903a291a5ccf66 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 5 Aug 2026 04:46:14 -0400
Subject: [PATCH 13/17] core tests-flatten and depth

---
 clang/test/OpenMP/flatten_codegen.cpp         |  65 ++++++++
 clang/test/OpenMP/flatten_depth_codegen.cpp   | 154 ++++++++++++++++++
 .../test/transform/flatten/depth-intfor.c     |  40 +++++
 .../runtime/test/transform/flatten/intfor.c   |  33 ++++
 .../test/transform/flatten/lit.local.cfg      |   5 +
 5 files changed, 297 insertions(+)
 create mode 100644 clang/test/OpenMP/flatten_codegen.cpp
 create mode 100644 clang/test/OpenMP/flatten_depth_codegen.cpp
 create mode 100644 openmp/runtime/test/transform/flatten/depth-intfor.c
 create mode 100644 openmp/runtime/test/transform/flatten/intfor.c
 create mode 100644 openmp/runtime/test/transform/flatten/lit.local.cfg

diff --git a/clang/test/OpenMP/flatten_codegen.cpp b/clang/test/OpenMP/flatten_codegen.cpp
new file mode 100644
index 0000000000000..f77b2fb00ce9b
--- /dev/null
+++ b/clang/test/OpenMP/flatten_codegen.cpp
@@ -0,0 +1,65 @@
+// Check code generation
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=60 -emit-llvm %s -o - | FileCheck %s
+
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+extern "C" void body(int, int);
+
+// CHECK-LABEL: define {{.*}}void @foo(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK:   store i64 0, ptr %.flatten.iv,
+// CHECK:   %[[COND_IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[MUL:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   icmp slt i64 %[[COND_IV]], %[[MUL]]
+// CHECK:   %[[DIV_IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[DIV:.+]] = sdiv i64 %[[DIV_IV]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   %[[REM_IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[REM:.+]] = srem i64 %[[REM_IV]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+// CHECK:   %[[I_VAL:.+]] = load i32, ptr %i,
+// CHECK:   %[[J_VAL:.+]] = load i32, ptr %j,
+// CHECK:   call void @body(i32{{.*}} %[[I_VAL]], i32{{.*}} %[[J_VAL]])
+// CHECK: for.inc:
+// CHECK:   %[[INC_IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[NEXT:.+]] = add nsw i64 %[[INC_IV]], 1
+// CHECK:   store i64 %[[NEXT]], ptr %.flatten.iv,
+// CHECK:   br label %for.cond
+extern "C" void foo(int n, int m) {
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+
+// CHECK-LABEL: define {{.*}}void @bar(
+// CHECK:   icmp slt i32 %{{.+}}, 63
+// CHECK:   sdiv i32 %{{.+}}, 9
+// CHECK:   srem i32 %{{.+}}, 9
+// CHECK:   call void @body(
+extern "C" void bar() {
+#pragma omp flatten
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+}
+
+// CHECK-LABEL: define {{.*}}void @baz(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   icmp slt i64 %{{.+}}, 10000000000
+// CHECK:   sdiv i64 %{{.+}}, 100000
+// CHECK:   srem i64 %{{.+}}, 100000
+// CHECK:   call void @body(
+extern "C" void baz() {
+#pragma omp flatten
+  for (int i = 0; i < 100000; ++i)
+    for (int j = 0; j < 100000; ++j)
+      body(i, j);
+}
+
+#endif
diff --git a/clang/test/OpenMP/flatten_depth_codegen.cpp b/clang/test/OpenMP/flatten_depth_codegen.cpp
new file mode 100644
index 0000000000000..77408fbcbdcf1
--- /dev/null
+++ b/clang/test/OpenMP/flatten_depth_codegen.cpp
@@ -0,0 +1,154 @@
+// Check code generation for the 'depth' clause on '#pragma omp flatten'.
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=61 -emit-llvm %s -o - | FileCheck %s
+
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+extern "C" void body(int, int, int);
+
+// CHECK-LABEL: define {{.*}}void @foo3(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK:   %.flatten.iv.2 = alloca i32
+// CHECK:   %[[CIV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[T01:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   %[[TALL:.+]] = mul nsw i64 %[[T01]], %{{.+}}
+// CHECK:   icmp slt i64 %[[CIV]], %[[TALL]]
+// CHECK:   %[[D0IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[D0M:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   %[[D0:.+]] = sdiv i64 %[[D0IV]], %[[D0M]]
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   %[[D1IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[D1:.+]] = sdiv i64 %[[D1IV]], %{{.+}}
+// CHECK:   %[[R1:.+]] = srem i64 %[[D1]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+// CHECK:   %[[R2IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[R2:.+]] = srem i64 %[[R2IV]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.2
+// CHECK:   call void @body(
+extern "C" void foo3(int n, int m, int p) {
+#pragma omp flatten depth(3)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        body(i, j, k);
+}
+
+// CHECK-LABEL: define {{.*}}void @foo2(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK:   %[[CIV2:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[M:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   icmp slt i64 %[[CIV2]], %[[M]]
+// CHECK:   sdiv i64 %{{.+}}, %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   srem i64 %{{.+}}, %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+extern "C" void foo2(int n, int m) {
+#pragma omp flatten depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j, 0);
+}
+
+// CHECK-LABEL: define {{.*}}void @foo1(
+// CHECK:   %.flatten.iv = alloca i32
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK: for.body:
+// CHECK:   %[[FV:.+]] = load i32, ptr %.flatten.iv,
+// CHECK:   store i32 %[[FV]], ptr %.flatten.iv.0,
+// CHECK-NOT: sdiv
+// CHECK-NOT: srem
+// CHECK:   call void @body(
+extern "C" void foo1(int n) {
+#pragma omp flatten depth(1)
+  for (int i = 0; i < n; ++i)
+    body(i, 0, 0);
+}
+
+// CHECK-LABEL: define {{.*}}void @foo2partial(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK-NOT: %.flatten.iv.2 = alloca
+// CHECK:   %[[CIV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[M:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   icmp slt i64 %[[CIV]], %[[M]]
+// CHECK:   sdiv i64 %{{.+}}, %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   srem i64 %{{.+}}, %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+// CHECK:   br label %for.cond{{.*}}
+// CHECK: for.cond{{.*}}:
+// CHECK:   icmp slt i32 %{{.+}}, 5
+extern "C" void foo2partial(int n, int m) {
+#pragma omp flatten depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < 5; ++k)
+        body(i, j, k);
+}
+
+// A value-dependent 'depth' (the argument is a non-type template parameter) is
+// deferred until instantiation, where the concrete depth drives the flattening.
+
+// CHECK-LABEL: define {{.*}}void @_Z11tmpl_depth3ILi3EEviii(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK:   %.flatten.iv.2 = alloca i32
+// CHECK:   %[[CIV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[T01:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   %[[TALL:.+]] = mul nsw i64 %[[T01]], %{{.+}}
+// CHECK:   icmp slt i64 %[[CIV]], %[[TALL]]
+// CHECK:   %[[D0IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[D0M:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   %[[D0:.+]] = sdiv i64 %[[D0IV]], %[[D0M]]
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   %[[D1IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[D1:.+]] = sdiv i64 %[[D1IV]], %{{.+}}
+// CHECK:   %[[R1:.+]] = srem i64 %[[D1]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+// CHECK:   %[[R2IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[R2:.+]] = srem i64 %[[R2IV]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.2
+// CHECK:   call void @body(
+template <int D>
+void tmpl_depth3(int n, int m, int p) {
+#pragma omp flatten depth(D)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        body(i, j, k);
+}
+
+// A value-dependent 'depth' that instantiates to depth(1) on a single loop is
+// well-formed: the shallower-than-default nest is accepted because validation is
+// deferred to instantiation. The single loop needs no div/mod recovery.
+
+// CHECK-LABEL: define {{.*}}void @_Z11tmpl_depth1ILi1EEvi(
+// CHECK:   %.flatten.iv = alloca i32
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK: for.body:
+// CHECK:   %[[FV:.+]] = load i32, ptr %.flatten.iv,
+// CHECK:   store i32 %[[FV]], ptr %.flatten.iv.0,
+// CHECK-NOT: sdiv
+// CHECK-NOT: srem
+// CHECK:   call void @body(
+template <int D>
+void tmpl_depth1(int n) {
+#pragma omp flatten depth(D)
+  for (int i = 0; i < n; ++i)
+    body(i, 0, 0);
+}
+
+void inst() {
+  tmpl_depth3<3>(1, 1, 1);
+  tmpl_depth1<1>(1);
+}
+
+#endif
diff --git a/openmp/runtime/test/transform/flatten/depth-intfor.c b/openmp/runtime/test/transform/flatten/depth-intfor.c
new file mode 100644
index 0000000000000..7211e02e25313
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/depth-intfor.c
@@ -0,0 +1,40 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+// 'depth(3)' fully flattens a three-deep perfectly nested loop into one loop.
+// This verifies that the flattened loop preserves the exact row-major
+// visitation order of the original nest, including non-unit steps and non-zero
+// start values.
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+int main() {
+  printf("do\n");
+#pragma omp flatten depth(3)
+  for (int i = 7; i < 11; i += 2)
+    for (int j = 1; j < 4; j += 1)
+      for (int k = 0; k < 2; ++k)
+        printf("i=%d j=%d k=%d\n", i, j, k);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: i=7 j=1 k=0
+// CHECK-NEXT: i=7 j=1 k=1
+// CHECK-NEXT: i=7 j=2 k=0
+// CHECK-NEXT: i=7 j=2 k=1
+// CHECK-NEXT: i=7 j=3 k=0
+// CHECK-NEXT: i=7 j=3 k=1
+// CHECK-NEXT: i=9 j=1 k=0
+// CHECK-NEXT: i=9 j=1 k=1
+// CHECK-NEXT: i=9 j=2 k=0
+// CHECK-NEXT: i=9 j=2 k=1
+// CHECK-NEXT: i=9 j=3 k=0
+// CHECK-NEXT: i=9 j=3 k=1
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/flatten/intfor.c b/openmp/runtime/test/transform/flatten/intfor.c
new file mode 100644
index 0000000000000..f937659fbebe1
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/intfor.c
@@ -0,0 +1,33 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+int main() {
+  printf("do\n");
+#pragma omp flatten
+  for (int i = 5; i < 11; i += 2)
+    for (int j = 1; j < 7; j += 2)
+      printf("i=%d j=%d\n", i, j);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// The flattened loop visits the original iteration space in the original
+// (row-major) order: for each outer iteration the inner loop runs fully.
+// CHECK:      do
+// CHECK-NEXT: i=5 j=1
+// CHECK-NEXT: i=5 j=3
+// CHECK-NEXT: i=5 j=5
+// CHECK-NEXT: i=7 j=1
+// CHECK-NEXT: i=7 j=3
+// CHECK-NEXT: i=7 j=5
+// CHECK-NEXT: i=9 j=1
+// CHECK-NEXT: i=9 j=3
+// CHECK-NEXT: i=9 j=5
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/flatten/lit.local.cfg b/openmp/runtime/test/transform/flatten/lit.local.cfg
new file mode 100644
index 0000000000000..e05bd627df1e9
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/lit.local.cfg
@@ -0,0 +1,5 @@
+# The flatten directive's depth clause requires OpenMP 6.1.
+for i, (pattern, replacement) in enumerate(config.substitutions):
+    if pattern == "%openmp_flags":
+        config.substitutions[i] = (pattern, replacement + " -fopenmp-version=61")
+        break

>From 667fe837a39dbc711dcc5288b736aa52fbe893b6 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 5 Aug 2026 04:48:45 -0400
Subject: [PATCH 14/17] TBD

---
 clang/docs/OpenMPSupport.rst                  |   2 +-
 clang/include/clang/ASTMatchers/ASTMatchers.h |  14 ++
 clang/lib/ASTMatchers/ASTMatchersInternal.cpp |   2 +
 clang/lib/ASTMatchers/Dynamic/Registry.cpp    |   1 +
 clang/test/Index/openmp-flatten.c             |  12 ++
 clang/test/OpenMP/flatten_ast_print.cpp       | 124 ++++++++++++++++++
 clang/test/OpenMP/flatten_codegen_for.cpp     |  67 ++++++++++
 clang/test/OpenMP/flatten_depth_ast_print.cpp |  74 +++++++++++
 clang/test/OpenMP/flatten_depth_messages.cpp  |  48 +++++++
 clang/test/OpenMP/flatten_messages.cpp        |  89 +++++++++++++
 .../test/OpenMP/flatten_serialize_module.cpp  |  25 ++++
 .../ASTMatchers/ASTMatchersNodeTest.cpp       |  22 ++++
 flatten-verify-tmp/01_default_depth2.cpp      |   8 ++
 flatten-verify-tmp/02_depth3_full.cpp         |   9 ++
 flatten-verify-tmp/03_depth2_partial.cpp      |  10 ++
 flatten-verify-tmp/04_depth1_identity.cpp     |   8 ++
 flatten-verify-tmp/05_rangefor_template.cpp   |  20 +++
 flatten-verify-tmp/README.md                  |  39 ++++++
 flatten-verify-tmp/end_result_check.cpp       |  82 ++++++++++++
 .../test/transform/flatten/depth-partial.c    |  45 +++++++
 openmp/runtime/test/transform/flatten/empty.c |  42 ++++++
 .../test/transform/flatten/foreach.cpp        |  31 +++++
 .../flatten/parallel-wsloop-foreach.cpp       |  32 +++++
 23 files changed, 805 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/Index/openmp-flatten.c
 create mode 100644 clang/test/OpenMP/flatten_ast_print.cpp
 create mode 100644 clang/test/OpenMP/flatten_codegen_for.cpp
 create mode 100644 clang/test/OpenMP/flatten_depth_ast_print.cpp
 create mode 100644 clang/test/OpenMP/flatten_depth_messages.cpp
 create mode 100644 clang/test/OpenMP/flatten_messages.cpp
 create mode 100644 clang/test/OpenMP/flatten_serialize_module.cpp
 create mode 100644 flatten-verify-tmp/01_default_depth2.cpp
 create mode 100644 flatten-verify-tmp/02_depth3_full.cpp
 create mode 100644 flatten-verify-tmp/03_depth2_partial.cpp
 create mode 100644 flatten-verify-tmp/04_depth1_identity.cpp
 create mode 100644 flatten-verify-tmp/05_rangefor_template.cpp
 create mode 100644 flatten-verify-tmp/README.md
 create mode 100644 flatten-verify-tmp/end_result_check.cpp
 create mode 100644 openmp/runtime/test/transform/flatten/depth-partial.c
 create mode 100644 openmp/runtime/test/transform/flatten/empty.c
 create mode 100644 openmp/runtime/test/transform/flatten/foreach.cpp
 create mode 100644 openmp/runtime/test/transform/flatten/parallel-wsloop-foreach.cpp

diff --git a/clang/docs/OpenMPSupport.rst b/clang/docs/OpenMPSupport.rst
index 0c21aa41ff269..c11a46c3bb6a6 100644
--- a/clang/docs/OpenMPSupport.rst
+++ b/clang/docs/OpenMPSupport.rst
@@ -666,7 +666,7 @@ implementation.
 +=============================================================+===========================+===========================+==========================================================================+
 | dyn_groupprivate clause                                     | :part:`partial`           | :part:`In Progress`       | C/C++: Host device support missing                                       |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
-| loop flatten transformation                                 | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
+| loop flatten transformation                                 | :part:`partial`           | :none:`unclaimed`         | Clang: depth clause supported (OpenMP 6.1); apply clause not yet supported|
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | loop grid/tile modifiers for sizes clause                   | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
diff --git a/clang/include/clang/ASTMatchers/ASTMatchers.h b/clang/include/clang/ASTMatchers/ASTMatchers.h
index bc0f35898a2c9..602bab9aa8cb8 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchers.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchers.h
@@ -8842,6 +8842,20 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt,
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, OMPSplitDirective>
     ompSplitDirective;
 
+/// Matches OpenMP ``flatten`` directive.
+///
+/// Given
+///
+/// \code
+///   #pragma omp flatten
+///   for (int i = 0; i < m; ++i)
+///     for (int j = 0; j < n; ++j) {}
+/// \endcode
+///
+/// ``ompFlattenDirective()`` matches the flatten directive.
+extern const internal::VariadicDynCastAllOfMatcher<Stmt, OMPFlattenDirective>
+    ompFlattenDirective;
+
 /// Matches OpenMP ``counts`` clause used by ``#pragma omp split``.
 ///
 /// Given ``#pragma omp split counts(1, 2, omp_fill)``, ``ompCountsClause()``
diff --git a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
index 5cbf134620e34..e2c0c25518b70 100644
--- a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
+++ b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
@@ -1141,6 +1141,8 @@ const internal::VariadicDynCastAllOfMatcher<Stmt, OMPTargetUpdateDirective>
     ompTargetUpdateDirective;
 const internal::VariadicDynCastAllOfMatcher<Stmt, OMPSplitDirective>
     ompSplitDirective;
+const internal::VariadicDynCastAllOfMatcher<Stmt, OMPFlattenDirective>
+    ompFlattenDirective;
 const internal::VariadicDynCastAllOfMatcher<OMPClause, OMPCountsClause>
     ompCountsClause;
 const internal::VariadicDynCastAllOfMatcher<OMPClause, OMPDefaultClause>
diff --git a/clang/lib/ASTMatchers/Dynamic/Registry.cpp b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
index a04070971f0eb..a6252363e08ce 100644
--- a/clang/lib/ASTMatchers/Dynamic/Registry.cpp
+++ b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
@@ -534,6 +534,7 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(ompFromClause);
   REGISTER_MATCHER(ompToClause);
   REGISTER_MATCHER(ompExecutableDirective);
+  REGISTER_MATCHER(ompFlattenDirective);
   REGISTER_MATCHER(ompSplitDirective);
   REGISTER_MATCHER(ompTargetUpdateDirective);
   REGISTER_MATCHER(on);
diff --git a/clang/test/Index/openmp-flatten.c b/clang/test/Index/openmp-flatten.c
new file mode 100644
index 0000000000000..74bc3cda3ef1a
--- /dev/null
+++ b/clang/test/Index/openmp-flatten.c
@@ -0,0 +1,12 @@
+// RUN: c-index-test -test-load-source local %s -fopenmp=libomp -fopenmp-version=60 | FileCheck %s
+
+void test(void) {
+#pragma omp flatten
+  for (int i = 0; i < 20; i += 1)
+    for (int j = 0; j < 30; j += 1)
+      ;
+}
+
+// CHECK: openmp-flatten.c:4:1: OMPFlattenDirective= Extent=[4:1 - 4:20]
+// CHECK: openmp-flatten.c:5:3: ForStmt= Extent=[5:3 - 7:8]
+// CHECK: openmp-flatten.c:6:5: ForStmt= Extent=[6:5 - 7:8]
diff --git a/clang/test/OpenMP/flatten_ast_print.cpp b/clang/test/OpenMP/flatten_ast_print.cpp
new file mode 100644
index 0000000000000..e1115bf112c40
--- /dev/null
+++ b/clang/test/OpenMP/flatten_ast_print.cpp
@@ -0,0 +1,124 @@
+// Check no warnings/errors
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -fsyntax-only -verify %s
+// expected-no-diagnostics
+
+// Check AST and unparsing
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -ast-dump  %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -ast-print %s | FileCheck %s --check-prefix=PRINT
+
+// Check same results after serialization round-trip
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -emit-pch -o %t %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -include-pch %t -ast-dump-all %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -include-pch %t -ast-print    %s | FileCheck %s --check-prefix=PRINT
+
+#ifndef HEADER
+#define HEADER
+
+// placeholder for loop body code.
+extern "C" void body(...);
+
+// PRINT-LABEL: void foo1(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo1
+void foo1() {
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (int i = 0; i < 7; i += 1)
+  // DUMP-NEXT: ForStmt
+  for (int i = 0; i < 7; i += 1)
+    // PRINT: for (int j = 0; j < 9; j += 1)
+    // DUMP:  ForStmt
+    for (int j = 0; j < 9; j += 1)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// PRINT-LABEL: void foo2(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo2
+void foo2(int start1, int end1, int step1, int start2, int end2, int step2) {
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (int i = start1; i < end1; i += step1)
+  // DUMP-NEXT: ForStmt
+  for (int i = start1; i < end1; i += step1)
+    // PRINT: for (int j = start2; j < end2; j += step2)
+    // DUMP:  ForStmt
+    for (int j = start2; j < end2; j += step2)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// PRINT-LABEL: void foo3(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo3
+void foo3() {
+  // Verify that a flattened loop can itself be the operand of another loop
+  // transformation. The outer reverse consumes the loop generated by flatten.
+  // PRINT:     #pragma omp reverse
+  // DUMP:      OMPReverseDirective
+  #pragma omp reverse
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (int i = 0; i < 13; i += 1)
+  // DUMP-NEXT: ForStmt
+  for (int i = 0; i < 13; i += 1)
+    // PRINT: for (int j = 0; j < 17; j += 1)
+    // DUMP:  ForStmt
+    for (int j = 0; j < 17; j += 1)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// Range-based for loops are supported as affected loops.
+// PRINT-LABEL: void foo4(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo4
+struct Range {
+  int *begin();
+  int *end();
+};
+void foo4(Range r1, Range r2) {
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (int i : r1)
+  // DUMP-NEXT: CXXForRangeStmt
+  for (int i : r1)
+    // PRINT: for (int j : r2)
+    // DUMP:  CXXForRangeStmt
+    for (int j : r2)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// The directive is instantiated together with its enclosing template.
+// PRINT-LABEL: template <typename T, int LO, int HI> void foo5()
+// DUMP-LABEL:  FunctionTemplateDecl {{.*}} foo5
+template <typename T, int LO, int HI>
+void foo5() {
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (T i = LO; i < HI; ++i)
+  // DUMP-NEXT: ForStmt
+  for (T i = LO; i < HI; ++i)
+    // PRINT: for (T j = LO; j < HI; ++j)
+    // DUMP:  ForStmt
+    for (T j = LO; j < HI; ++j)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// PRINT-LABEL: template<> void foo5<int, 0, 8>()
+// DUMP-LABEL:  FunctionDecl {{.*}} foo5 'void ()' implicit_instantiation
+// DUMP:        OMPFlattenDirective
+void inst() {
+  foo5<int, 0, 8>();
+}
+
+#endif
diff --git a/clang/test/OpenMP/flatten_codegen_for.cpp b/clang/test/OpenMP/flatten_codegen_for.cpp
new file mode 100644
index 0000000000000..d64128bd25d78
--- /dev/null
+++ b/clang/test/OpenMP/flatten_codegen_for.cpp
@@ -0,0 +1,67 @@
+// Check that the loop generated by 'flatten' can be consumed by a loop
+// associated directive ('omp for' / 'omp parallel for'), and that signed and
+// unsigned induction variables generate signed/unsigned division.
+
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=60 -emit-llvm %s -o - | FileCheck %s
+
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+extern "C" void body(int, int);
+extern "C" void ubody(unsigned, unsigned);
+
+// 'omp for' worksharing over the flattened iteration space: the flattened loop
+// uses a 64-bit induction variable for the variable-bound product trip count,
+// so the worksharing loop is distributed with the 64-bit static-init call and
+// the body recovers the logical counters via signed division/remainder.
+//
+// CHECK-LABEL: define {{.*}}void @wsfor(
+// CHECK:   mul nsw i64
+// CHECK:   call void @__kmpc_for_static_init_8(
+// CHECK:   sdiv i64
+// CHECK:   srem i64
+// CHECK:   call void @body(
+// CHECK:   call void @__kmpc_for_static_fini(
+extern "C" void wsfor(int n, int m) {
+#pragma omp for
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+
+// 'omp parallel for' outlines the flattened loop and worksharing-distributes
+// it.
+//
+// CHECK-LABEL: define {{.*}}void @pfor(
+// CHECK:   call {{.*}}@__kmpc_fork_call(
+// CHECK-LABEL: define internal void @pfor.omp_outlined(
+// CHECK:   call void @__kmpc_for_static_init_8(
+// CHECK:   sdiv i64
+// CHECK:   srem i64
+// CHECK:   call void @body(
+extern "C" void pfor(int n, int m) {
+#pragma omp parallel for
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+
+// Unsigned induction variables produce unsigned division/remainder.
+//
+// CHECK-LABEL: define {{.*}}void @ufor(
+// CHECK:   icmp ult i32 %{{.+}}, 63
+// CHECK:   udiv i32 %{{.+}}, 9
+// CHECK:   urem i32 %{{.+}}, 9
+// CHECK:   call void @ubody(
+extern "C" void ufor() {
+#pragma omp flatten
+  for (unsigned i = 0; i < 7u; ++i)
+    for (unsigned j = 0; j < 9u; ++j)
+      ubody(i, j);
+}
+
+#endif
diff --git a/clang/test/OpenMP/flatten_depth_ast_print.cpp b/clang/test/OpenMP/flatten_depth_ast_print.cpp
new file mode 100644
index 0000000000000..ceaaa68e1f79a
--- /dev/null
+++ b/clang/test/OpenMP/flatten_depth_ast_print.cpp
@@ -0,0 +1,74 @@
+// Check no warnings/errors
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -fsyntax-only -verify %s
+// expected-no-diagnostics
+
+// Check AST and unparsing
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-dump  %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-print %s | FileCheck %s --check-prefix=PRINT
+
+// Check same results after serialization round-trip
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -emit-pch -o %t %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -include-pch %t -ast-dump-all %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -include-pch %t -ast-print    %s | FileCheck %s --check-prefix=PRINT
+
+#ifndef HEADER
+#define HEADER
+
+// placeholder for loop body code.
+extern "C" void body(...);
+
+// The depth clause round-trips through -ast-print and PCH serialization.
+// PRINT-LABEL: void foo_depth3(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo_depth3
+void foo_depth3(int n, int m, int p) {
+  // PRINT:     #pragma omp flatten depth(3)
+  // DUMP:      OMPFlattenDirective
+  // DUMP-NEXT: OMPDepthClause
+  #pragma omp flatten depth(3)
+  // PRINT: for (int i = 0; i < n; ++i)
+  // DUMP:  ForStmt
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        // PRINT: body(i, j, k);
+        // DUMP:  CallExpr
+        body(i, j, k);
+}
+
+// depth(2) is accepted and printed like any other depth argument.
+// PRINT-LABEL: void foo_depth2(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo_depth2
+void foo_depth2(int n, int m) {
+  // PRINT:     #pragma omp flatten depth(2)
+  // DUMP:      OMPFlattenDirective
+  // DUMP-NEXT: OMPDepthClause
+  #pragma omp flatten depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+
+// The depth clause is instantiated together with its enclosing template.
+// PRINT-LABEL: template <typename T> void foo_tmpl()
+// DUMP-LABEL:  FunctionTemplateDecl {{.*}} foo_tmpl
+template <typename T>
+void foo_tmpl() {
+  // PRINT:     #pragma omp flatten depth(3)
+  // DUMP:      OMPFlattenDirective
+  // DUMP-NEXT: OMPDepthClause
+  #pragma omp flatten depth(3)
+  for (T i = 0; i < 8; ++i)
+    for (T j = 0; j < 8; ++j)
+      for (T k = 0; k < 8; ++k)
+        body(i, j, k);
+}
+
+// PRINT-LABEL: template<> void foo_tmpl<int>()
+// DUMP-LABEL:  FunctionDecl {{.*}} foo_tmpl 'void ()' implicit_instantiation
+// DUMP:        OMPFlattenDirective
+// DUMP-NEXT:   OMPDepthClause
+void inst() {
+  foo_tmpl<int>();
+}
+
+#endif
diff --git a/clang/test/OpenMP/flatten_depth_messages.cpp b/clang/test/OpenMP/flatten_depth_messages.cpp
new file mode 100644
index 0000000000000..16f60318d0d06
--- /dev/null
+++ b/clang/test/OpenMP/flatten_depth_messages.cpp
@@ -0,0 +1,48 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=61 -fsyntax-only -Wuninitialized -verify %s
+
+extern "C" void body(...);
+
+// expected-note at +1 {{declared here}}
+void func(int n) {
+
+  // The depth argument must be a positive integer constant expression.
+  // expected-error at +1 {{argument to 'depth' clause must be a strictly positive integer value}}
+  #pragma omp flatten depth(0)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // expected-error at +1 {{argument to 'depth' clause must be a strictly positive integer value}}
+  #pragma omp flatten depth(-1)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // A non-constant argument is rejected.
+  // expected-error at +2 {{expression is not an integral constant expression}}
+  // expected-note at +1 {{function parameter 'n' with unknown value cannot be used in a constant expression}}
+  #pragma omp flatten depth(n)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // At most one 'depth' clause may appear on the directive.
+  // expected-error at +1 {{directive '#pragma omp flatten' cannot contain more than one 'depth' clause}}
+  #pragma omp flatten depth(2) depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // The depth must be at most the loop nest depth: depth(3) requires three
+  // perfectly nested loops, but only two are present here.
+  #pragma omp flatten depth(3)
+  // expected-error at +1 {{expected 3 for loops after '#pragma omp flatten', but found only 2}}
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // depth(1) flattens a single loop and is well-formed (no diagnostic).
+  #pragma omp flatten depth(1)
+  for (int i = 0; i < n; ++i)
+    body(i);
+}
diff --git a/clang/test/OpenMP/flatten_messages.cpp b/clang/test/OpenMP/flatten_messages.cpp
new file mode 100644
index 0000000000000..a022e0e8999d2
--- /dev/null
+++ b/clang/test/OpenMP/flatten_messages.cpp
@@ -0,0 +1,89 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=60 -fsyntax-only -Wuninitialized -verify %s
+
+extern "C" void body(...);
+
+void func(int n) {
+
+  // The associated statement must be a for loop.
+  // expected-error at +2 {{statement after '#pragma omp flatten' must be a for loop}}
+  #pragma omp flatten
+  ;
+
+  // A non-loop statement is rejected as well.
+  // expected-error at +2 {{statement after '#pragma omp flatten' must be a for loop}}
+  #pragma omp flatten
+  int b = 0;
+
+  // A single loop is not enough: flatten combines two perfectly nested loops,
+  // so the body of the outer loop must itself be a for loop.
+  #pragma omp flatten
+  for (int i = 0; i < 7; ++i)
+    // expected-error at +1 {{statement after '#pragma omp flatten' must be a for loop}}
+    ;
+
+  // The associated statement of a directive is not a for loop.
+  // expected-error at +2 {{statement after '#pragma omp flatten' must be a for loop}}
+  #pragma omp flatten
+  #pragma omp for
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 7; ++j)
+      body(i, j);
+
+  {
+    // expected-error at +2 {{expected statement}}
+    #pragma omp flatten
+  }
+
+  // The loops must be perfectly nested: no code is allowed between them.
+  #pragma omp flatten
+  for (int i = 0; i < n; ++i) {
+    int x = 0;
+    // expected-error at -2 {{statement after '#pragma omp flatten' must be a for loop}}
+    for (int j = 0; j < n; ++j)
+      body(i, j, x);
+  }
+
+  // Each affected loop must be in OpenMP canonical form.
+  #pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    // expected-error at +1 {{condition of OpenMP for loop must be a relational comparison ('<', '<=', '>', '>=', or '!=') of loop variable 'j'}}
+    for (int j = 0; j / 3 < n; ++j)
+      body(i, j);
+
+  // The affected loops must be rectangular: an inner bound may not depend on an
+  // outer loop counter.
+  #pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    // expected-error at +1 {{expected loop invariant expression}}
+    for (int j = i; j < n; ++j)
+      body(i, j);
+
+  // The 'sizes' clause is not allowed on 'flatten'.
+  // expected-error at +1 {{unexpected OpenMP clause 'sizes' in directive '#pragma omp flatten'}}
+  #pragma omp flatten sizes(2)
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+
+  // The 'permutation' clause is not allowed on 'flatten'.
+  // expected-error at +1 {{unexpected OpenMP clause 'permutation' in directive '#pragma omp flatten'}}
+  #pragma omp flatten permutation(2, 1)
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+
+  // Tokens after the directive name are ignored with a warning.
+  // expected-warning at +1 {{extra tokens at the end of '#pragma omp flatten' are ignored}}
+  #pragma omp flatten foo
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+
+  // The 'depth' clause is only available from OpenMP 6.1; it is rejected here
+  // under -fopenmp-version=60.
+  // expected-error at +1 {{unexpected OpenMP clause 'depth' in directive '#pragma omp flatten'}}
+  #pragma omp flatten depth(2)
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+}
diff --git a/clang/test/OpenMP/flatten_serialize_module.cpp b/clang/test/OpenMP/flatten_serialize_module.cpp
new file mode 100644
index 0000000000000..5d3420500b282
--- /dev/null
+++ b/clang/test/OpenMP/flatten_serialize_module.cpp
@@ -0,0 +1,25 @@
+// C++20 module interface with `#pragma omp flatten` — emit BMI + import; AST retains directive.
+//
+// RUN: rm -rf %t && split-file %s %t && cd %t
+// RUN: %clang_cc1 -std=c++20 -fopenmp -fopenmp-version=60 -triple x86_64-unknown-linux-gnu %t/FlattenMod.cppm -emit-module-interface -o %t/FlattenMod.pcm
+// RUN: %clang_cc1 -std=c++20 -fopenmp -fopenmp-version=60 -triple x86_64-unknown-linux-gnu %t/UseFlattenMod.cpp -fmodule-file=FlattenMod=%t/FlattenMod.pcm -ast-dump-all | FileCheck %t/FlattenMod.cppm
+
+// expected-no-diagnostics
+
+//--- FlattenMod.cppm
+module;
+export module FlattenMod;
+
+export void flattenfoo(int n, int m) {
+// CHECK: OMPFlattenDirective
+// CHECK: ForStmt
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j) {
+    }
+}
+
+//--- UseFlattenMod.cpp
+import FlattenMod;
+
+void g(void) { flattenfoo(10, 20); }
diff --git a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
index 4190d4703e37d..34c0fd27821d1 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
@@ -3126,6 +3126,28 @@ void f() {
   EXPECT_TRUE(notMatchesWithOpenMP60(ParallelOnly, Matcher));
 }
 
+// OpenMP 6 flatten directive
+TEST(ASTMatchersTestOpenMP, OMPFlattenDirective) {
+  auto Matcher = stmt(ompFlattenDirective());
+
+  StringRef FlattenOk = R"(
+void f() {
+#pragma omp flatten
+  for (int i = 0; i < 10; ++i)
+    for (int j = 0; j < 10; ++j) {}
+}
+)";
+  EXPECT_TRUE(matchesWithOpenMP60(FlattenOk, Matcher));
+
+  StringRef ParallelOnly = R"(
+void f() {
+#pragma omp parallel
+  ;
+}
+)";
+  EXPECT_TRUE(notMatchesWithOpenMP60(ParallelOnly, Matcher));
+}
+
 TEST(ASTMatchersTestOpenMP, OMPSplitDirective_HasCountsClause) {
   auto Matcher = stmt(ompSplitDirective(hasAnyClause(ompCountsClause())));
 
diff --git a/flatten-verify-tmp/01_default_depth2.cpp b/flatten-verify-tmp/01_default_depth2.cpp
new file mode 100644
index 0000000000000..900d0263622cd
--- /dev/null
+++ b/flatten-verify-tmp/01_default_depth2.cpp
@@ -0,0 +1,8 @@
+// Default flatten: combine 2 perfectly nested loops (depth defaults to 2).
+extern "C" void body(int, int);
+void t(int n, int m) {
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
diff --git a/flatten-verify-tmp/02_depth3_full.cpp b/flatten-verify-tmp/02_depth3_full.cpp
new file mode 100644
index 0000000000000..ba9f0b7ed8808
--- /dev/null
+++ b/flatten-verify-tmp/02_depth3_full.cpp
@@ -0,0 +1,9 @@
+// depth(3): fully flatten a three-deep perfect nest (OpenMP 6.1).
+extern "C" void body(int, int, int);
+void t(int n, int m, int p) {
+#pragma omp flatten depth(3)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        body(i, j, k);
+}
diff --git a/flatten-verify-tmp/03_depth2_partial.cpp b/flatten-verify-tmp/03_depth2_partial.cpp
new file mode 100644
index 0000000000000..35b9dc83834fd
--- /dev/null
+++ b/flatten-verify-tmp/03_depth2_partial.cpp
@@ -0,0 +1,10 @@
+// depth(2) on a three-deep nest: only the outer two loops are combined; the
+// innermost loop must remain as ordinary body of the flattened loop.
+extern "C" void body(int, int, int);
+void t(int n, int m, int p) {
+#pragma omp flatten depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        body(i, j, k);
+}
diff --git a/flatten-verify-tmp/04_depth1_identity.cpp b/flatten-verify-tmp/04_depth1_identity.cpp
new file mode 100644
index 0000000000000..ae62a11312fe8
--- /dev/null
+++ b/flatten-verify-tmp/04_depth1_identity.cpp
@@ -0,0 +1,8 @@
+// depth(1): identity transform. A single loop is "combined"; no mixed-radix
+// decomposition (no srem), the flattened counter maps straight to the loop var.
+extern "C" void body(int);
+void t(int n) {
+#pragma omp flatten depth(1)
+  for (int i = 0; i < n; ++i)
+    body(i);
+}
diff --git a/flatten-verify-tmp/05_rangefor_template.cpp b/flatten-verify-tmp/05_rangefor_template.cpp
new file mode 100644
index 0000000000000..89928c34a8a15
--- /dev/null
+++ b/flatten-verify-tmp/05_rangefor_template.cpp
@@ -0,0 +1,20 @@
+// Range-for (CXXForRangeStmt) nest + a function template instantiated with a
+// dependent depth(K). Exercises the __begin/__end pre-inits and the deferred
+// (dependent-context) desugaring path.
+#include <stdlibc++.h>
+extern "C" void body(int, int);
+
+void ranges(std::vector<int> &a, std::vector<int> &b) {
+#pragma omp flatten
+  for (int i : a)
+    for (int j : b)
+      body(i, j);
+}
+
+template <int K> void tmpl(int n, int m) {
+#pragma omp flatten depth(K)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+template void tmpl<2>(int, int);
diff --git a/flatten-verify-tmp/README.md b/flatten-verify-tmp/README.md
new file mode 100644
index 0000000000000..f48f4d06cfec0
--- /dev/null
+++ b/flatten-verify-tmp/README.md
@@ -0,0 +1,39 @@
+# Temporary flatten + depth verification cases
+
+Throwaway sandbox for eyeballing the flatten desugaring (AST / IR / runtime).
+Not part of any PR — delete freely. The committed regression tests live under
+`clang/test/OpenMP/flatten*` and `openmp/runtime/test/transform/flatten/*`.
+
+`CLANG=build/bin/clang`, `CLANGXX=build/bin/clang++`, run from the repo root.
+
+| File | Case | What to look for |
+|------|------|------------------|
+| `01_default_depth2.cpp` | default (no clause) = depth 2 | i64 `.flatten.iv`, `mul nsw`, `sdiv`+`srem` |
+| `02_depth3_full.cpp` | `depth(3)` full flatten | 3 `.flatten.iv.N`, two-level mixed radix |
+| `03_depth2_partial.cpp` | `depth(2)` on 3-deep nest | 2 flatten IVs, **inner `%k` loop stays** |
+| `04_depth1_identity.cpp` | `depth(1)` identity | i32 IV, **no `srem`**, direct store |
+| `05_rangefor_template.cpp` | range-for + template `depth(K)` | ast-print round-trips; instantiation |
+| `end_result_check.cpp` | runtime semantic check | prints `OK`, exit 0 (order+set preserved) |
+
+## Commands
+
+AST (round-trip original loops) and the desugared node:
+```
+$CLANG -cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-print FILE
+$CLANG -cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-dump  FILE
+```
+
+IR (the div/mod arithmetic that implements the transform):
+```
+$CLANG -cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -emit-llvm -o - FILE
+```
+
+Range-for / template need the driver (libstdc++ + template instantiation):
+```
+$CLANGXX -fopenmp -fopenmp-version=61 -Xclang -ast-print -fsyntax-only 05_rangefor_template.cpp
+```
+
+Runtime end-result (must print OK, exit 0):
+```
+$CLANGXX -fopenmp -fopenmp-version=61 -O0 end_result_check.cpp -o /tmp/erc && /tmp/erc
+```
diff --git a/flatten-verify-tmp/end_result_check.cpp b/flatten-verify-tmp/end_result_check.cpp
new file mode 100644
index 0000000000000..3aa8850829058
--- /dev/null
+++ b/flatten-verify-tmp/end_result_check.cpp
@@ -0,0 +1,82 @@
+// End-result (semantic) verification: flatten must preserve BOTH the set of
+// iterations and their row-major visitation ORDER. We record the sequence of
+// visited tuples from a flattened nest and compare it against the sequence from
+// an identical non-flattened reference nest. Exit code 0 == identical.
+//
+// flatten is a pure frontend AST desugaring to an ordinary loop, so this runs
+// as a normal program (build with: clang++ -fopenmp -fopenmp-version=61).
+#include <cassert>
+#include <cstdio>
+#include <vector>
+
+using Seq = std::vector<long>;
+
+static void rec(Seq &s, int i, int j, int k = -1) {
+  s.push_back(i);
+  s.push_back(j);
+  if (k >= 0)
+    s.push_back(k);
+}
+
+int main() {
+  const int n = 3, m = 4, p = 2;
+
+  // --- depth(2) default over 2 loops ---
+  {
+    Seq flat, ref;
+#pragma omp flatten
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        rec(flat, i, j);
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        rec(ref, i, j);
+    assert(flat == ref && "depth(2) order/set mismatch");
+  }
+
+  // --- depth(3) full flatten over 3 loops ---
+  {
+    Seq flat, ref;
+#pragma omp flatten depth(3)
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        for (int k = 0; k < p; ++k)
+          rec(flat, i, j, k);
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        for (int k = 0; k < p; ++k)
+          rec(ref, i, j, k);
+    assert(flat == ref && "depth(3) order/set mismatch");
+  }
+
+  // --- depth(2) partial: inner k-loop stays intact ---
+  {
+    Seq flat, ref;
+#pragma omp flatten depth(2)
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        for (int k = 0; k < p; ++k)
+          rec(flat, i, j, k);
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        for (int k = 0; k < p; ++k)
+          rec(ref, i, j, k);
+    assert(flat == ref && "depth(2)-partial order/set mismatch");
+  }
+
+  // --- non-unit step / non-zero start must survive ---
+  {
+    Seq flat, ref;
+#pragma omp flatten
+    for (int i = 5; i < 12; i += 2)
+      for (int j = 1; j < 7; j += 3)
+        rec(flat, i, j);
+    for (int i = 5; i < 12; i += 2)
+      for (int j = 1; j < 7; j += 3)
+        rec(ref, i, j);
+    assert(flat == ref && "stride/offset mismatch");
+  }
+
+  std::puts("flatten end-result preserved: OK");
+  return 0;
+}
diff --git a/openmp/runtime/test/transform/flatten/depth-partial.c b/openmp/runtime/test/transform/flatten/depth-partial.c
new file mode 100644
index 0000000000000..2fda3a04cbd84
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/depth-partial.c
@@ -0,0 +1,45 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+// A depth(2) clause on a three-deep loop nest flattens only the outermost two
+// loops; the innermost loop is left untouched and executes normally inside the
+// flattened loop body.
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+int main() {
+  printf("do\n");
+#pragma omp flatten depth(2)
+  for (int i = 5; i < 11; i += 2)
+    for (int j = 1; j < 7; j += 2)
+      for (int k = 0; k < 2; ++k)
+        printf("i=%d j=%d k=%d\n", i, j, k);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: i=5 j=1 k=0
+// CHECK-NEXT: i=5 j=1 k=1
+// CHECK-NEXT: i=5 j=3 k=0
+// CHECK-NEXT: i=5 j=3 k=1
+// CHECK-NEXT: i=5 j=5 k=0
+// CHECK-NEXT: i=5 j=5 k=1
+// CHECK-NEXT: i=7 j=1 k=0
+// CHECK-NEXT: i=7 j=1 k=1
+// CHECK-NEXT: i=7 j=3 k=0
+// CHECK-NEXT: i=7 j=3 k=1
+// CHECK-NEXT: i=7 j=5 k=0
+// CHECK-NEXT: i=7 j=5 k=1
+// CHECK-NEXT: i=9 j=1 k=0
+// CHECK-NEXT: i=9 j=1 k=1
+// CHECK-NEXT: i=9 j=3 k=0
+// CHECK-NEXT: i=9 j=3 k=1
+// CHECK-NEXT: i=9 j=5 k=0
+// CHECK-NEXT: i=9 j=5 k=1
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/flatten/empty.c b/openmp/runtime/test/transform/flatten/empty.c
new file mode 100644
index 0000000000000..ecf1e8ef622f2
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/empty.c
@@ -0,0 +1,42 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+// The bounds are runtime values so that an empty iteration space does not
+// trigger a compile-time division-by-zero diagnostic for the (never executed)
+// induction-variable recovery.
+static void flatten(int n, int m) {
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      printf("i=%d j=%d\n", i, j);
+}
+
+int main() {
+  printf("empty-inner-begin\n");
+  flatten(3, 0);
+  printf("empty-inner-end\n");
+
+  printf("empty-outer-begin\n");
+  flatten(0, 3);
+  printf("empty-outer-end\n");
+
+  printf("single-begin\n");
+  flatten(1, 1);
+  printf("single-end\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      empty-inner-begin
+// CHECK-NEXT: empty-inner-end
+// CHECK-NEXT: empty-outer-begin
+// CHECK-NEXT: empty-outer-end
+// CHECK-NEXT: single-begin
+// CHECK-NEXT: i=0 j=0
+// CHECK-NEXT: single-end
diff --git a/openmp/runtime/test/transform/flatten/foreach.cpp b/openmp/runtime/test/transform/flatten/foreach.cpp
new file mode 100644
index 0000000000000..14997f7c4051d
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/foreach.cpp
@@ -0,0 +1,31 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdio>
+#include <vector>
+
+int main() {
+  printf("do\n");
+  std::vector<int> outer{10, 20};
+  std::vector<int> inner{1, 2, 3};
+#pragma omp flatten
+  for (int a : outer)
+    for (int b : inner)
+      printf("a=%d b=%d\n", a, b);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: a=10 b=1
+// CHECK-NEXT: a=10 b=2
+// CHECK-NEXT: a=10 b=3
+// CHECK-NEXT: a=20 b=1
+// CHECK-NEXT: a=20 b=2
+// CHECK-NEXT: a=20 b=3
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/flatten/parallel-wsloop-foreach.cpp b/openmp/runtime/test/transform/flatten/parallel-wsloop-foreach.cpp
new file mode 100644
index 0000000000000..7ddcde2fd622c
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/parallel-wsloop-foreach.cpp
@@ -0,0 +1,32 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdio>
+
+int main() {
+  printf("do\n");
+  // The loop generated by 'flatten' is the canonical loop associated with the
+  // worksharing 'omp for'. With a single thread the iteration order is the
+  // original row-major order.
+#pragma omp parallel for num_threads(1)
+#pragma omp flatten
+  for (int i = 0; i < 2; ++i)
+    for (int j = 0; j < 3; ++j)
+      printf("i=%d j=%d\n", i, j);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: i=0 j=0
+// CHECK-NEXT: i=0 j=1
+// CHECK-NEXT: i=0 j=2
+// CHECK-NEXT: i=1 j=0
+// CHECK-NEXT: i=1 j=1
+// CHECK-NEXT: i=1 j=2
+// CHECK-NEXT: done

>From 152ee3bed82f0aa84079151c559d46564d9304df Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Tue, 18 Aug 2026 07:49:02 -0400
Subject: [PATCH 15/17] Remove flatten-verify-tmp and restore
 OpenMPSupport.rst.

Co-authored-by: Cursor <cursoragent at cursor.com>
---
 clang/docs/OpenMPSupport.rst                |  2 +-
 flatten-verify-tmp/01_default_depth2.cpp    |  8 --
 flatten-verify-tmp/02_depth3_full.cpp       |  9 ---
 flatten-verify-tmp/03_depth2_partial.cpp    | 10 ---
 flatten-verify-tmp/04_depth1_identity.cpp   |  8 --
 flatten-verify-tmp/05_rangefor_template.cpp | 20 -----
 flatten-verify-tmp/README.md                | 39 ----------
 flatten-verify-tmp/end_result_check.cpp     | 82 ---------------------
 8 files changed, 1 insertion(+), 177 deletions(-)
 delete mode 100644 flatten-verify-tmp/01_default_depth2.cpp
 delete mode 100644 flatten-verify-tmp/02_depth3_full.cpp
 delete mode 100644 flatten-verify-tmp/03_depth2_partial.cpp
 delete mode 100644 flatten-verify-tmp/04_depth1_identity.cpp
 delete mode 100644 flatten-verify-tmp/05_rangefor_template.cpp
 delete mode 100644 flatten-verify-tmp/README.md
 delete mode 100644 flatten-verify-tmp/end_result_check.cpp

diff --git a/clang/docs/OpenMPSupport.rst b/clang/docs/OpenMPSupport.rst
index c11a46c3bb6a6..0c21aa41ff269 100644
--- a/clang/docs/OpenMPSupport.rst
+++ b/clang/docs/OpenMPSupport.rst
@@ -666,7 +666,7 @@ implementation.
 +=============================================================+===========================+===========================+==========================================================================+
 | dyn_groupprivate clause                                     | :part:`partial`           | :part:`In Progress`       | C/C++: Host device support missing                                       |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
-| loop flatten transformation                                 | :part:`partial`           | :none:`unclaimed`         | Clang: depth clause supported (OpenMP 6.1); apply clause not yet supported|
+| loop flatten transformation                                 | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | loop grid/tile modifiers for sizes clause                   | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
diff --git a/flatten-verify-tmp/01_default_depth2.cpp b/flatten-verify-tmp/01_default_depth2.cpp
deleted file mode 100644
index 900d0263622cd..0000000000000
--- a/flatten-verify-tmp/01_default_depth2.cpp
+++ /dev/null
@@ -1,8 +0,0 @@
-// Default flatten: combine 2 perfectly nested loops (depth defaults to 2).
-extern "C" void body(int, int);
-void t(int n, int m) {
-#pragma omp flatten
-  for (int i = 0; i < n; ++i)
-    for (int j = 0; j < m; ++j)
-      body(i, j);
-}
diff --git a/flatten-verify-tmp/02_depth3_full.cpp b/flatten-verify-tmp/02_depth3_full.cpp
deleted file mode 100644
index ba9f0b7ed8808..0000000000000
--- a/flatten-verify-tmp/02_depth3_full.cpp
+++ /dev/null
@@ -1,9 +0,0 @@
-// depth(3): fully flatten a three-deep perfect nest (OpenMP 6.1).
-extern "C" void body(int, int, int);
-void t(int n, int m, int p) {
-#pragma omp flatten depth(3)
-  for (int i = 0; i < n; ++i)
-    for (int j = 0; j < m; ++j)
-      for (int k = 0; k < p; ++k)
-        body(i, j, k);
-}
diff --git a/flatten-verify-tmp/03_depth2_partial.cpp b/flatten-verify-tmp/03_depth2_partial.cpp
deleted file mode 100644
index 35b9dc83834fd..0000000000000
--- a/flatten-verify-tmp/03_depth2_partial.cpp
+++ /dev/null
@@ -1,10 +0,0 @@
-// depth(2) on a three-deep nest: only the outer two loops are combined; the
-// innermost loop must remain as ordinary body of the flattened loop.
-extern "C" void body(int, int, int);
-void t(int n, int m, int p) {
-#pragma omp flatten depth(2)
-  for (int i = 0; i < n; ++i)
-    for (int j = 0; j < m; ++j)
-      for (int k = 0; k < p; ++k)
-        body(i, j, k);
-}
diff --git a/flatten-verify-tmp/04_depth1_identity.cpp b/flatten-verify-tmp/04_depth1_identity.cpp
deleted file mode 100644
index ae62a11312fe8..0000000000000
--- a/flatten-verify-tmp/04_depth1_identity.cpp
+++ /dev/null
@@ -1,8 +0,0 @@
-// depth(1): identity transform. A single loop is "combined"; no mixed-radix
-// decomposition (no srem), the flattened counter maps straight to the loop var.
-extern "C" void body(int);
-void t(int n) {
-#pragma omp flatten depth(1)
-  for (int i = 0; i < n; ++i)
-    body(i);
-}
diff --git a/flatten-verify-tmp/05_rangefor_template.cpp b/flatten-verify-tmp/05_rangefor_template.cpp
deleted file mode 100644
index 89928c34a8a15..0000000000000
--- a/flatten-verify-tmp/05_rangefor_template.cpp
+++ /dev/null
@@ -1,20 +0,0 @@
-// Range-for (CXXForRangeStmt) nest + a function template instantiated with a
-// dependent depth(K). Exercises the __begin/__end pre-inits and the deferred
-// (dependent-context) desugaring path.
-#include <stdlibc++.h>
-extern "C" void body(int, int);
-
-void ranges(std::vector<int> &a, std::vector<int> &b) {
-#pragma omp flatten
-  for (int i : a)
-    for (int j : b)
-      body(i, j);
-}
-
-template <int K> void tmpl(int n, int m) {
-#pragma omp flatten depth(K)
-  for (int i = 0; i < n; ++i)
-    for (int j = 0; j < m; ++j)
-      body(i, j);
-}
-template void tmpl<2>(int, int);
diff --git a/flatten-verify-tmp/README.md b/flatten-verify-tmp/README.md
deleted file mode 100644
index f48f4d06cfec0..0000000000000
--- a/flatten-verify-tmp/README.md
+++ /dev/null
@@ -1,39 +0,0 @@
-# Temporary flatten + depth verification cases
-
-Throwaway sandbox for eyeballing the flatten desugaring (AST / IR / runtime).
-Not part of any PR — delete freely. The committed regression tests live under
-`clang/test/OpenMP/flatten*` and `openmp/runtime/test/transform/flatten/*`.
-
-`CLANG=build/bin/clang`, `CLANGXX=build/bin/clang++`, run from the repo root.
-
-| File | Case | What to look for |
-|------|------|------------------|
-| `01_default_depth2.cpp` | default (no clause) = depth 2 | i64 `.flatten.iv`, `mul nsw`, `sdiv`+`srem` |
-| `02_depth3_full.cpp` | `depth(3)` full flatten | 3 `.flatten.iv.N`, two-level mixed radix |
-| `03_depth2_partial.cpp` | `depth(2)` on 3-deep nest | 2 flatten IVs, **inner `%k` loop stays** |
-| `04_depth1_identity.cpp` | `depth(1)` identity | i32 IV, **no `srem`**, direct store |
-| `05_rangefor_template.cpp` | range-for + template `depth(K)` | ast-print round-trips; instantiation |
-| `end_result_check.cpp` | runtime semantic check | prints `OK`, exit 0 (order+set preserved) |
-
-## Commands
-
-AST (round-trip original loops) and the desugared node:
-```
-$CLANG -cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-print FILE
-$CLANG -cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-dump  FILE
-```
-
-IR (the div/mod arithmetic that implements the transform):
-```
-$CLANG -cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -emit-llvm -o - FILE
-```
-
-Range-for / template need the driver (libstdc++ + template instantiation):
-```
-$CLANGXX -fopenmp -fopenmp-version=61 -Xclang -ast-print -fsyntax-only 05_rangefor_template.cpp
-```
-
-Runtime end-result (must print OK, exit 0):
-```
-$CLANGXX -fopenmp -fopenmp-version=61 -O0 end_result_check.cpp -o /tmp/erc && /tmp/erc
-```
diff --git a/flatten-verify-tmp/end_result_check.cpp b/flatten-verify-tmp/end_result_check.cpp
deleted file mode 100644
index 3aa8850829058..0000000000000
--- a/flatten-verify-tmp/end_result_check.cpp
+++ /dev/null
@@ -1,82 +0,0 @@
-// End-result (semantic) verification: flatten must preserve BOTH the set of
-// iterations and their row-major visitation ORDER. We record the sequence of
-// visited tuples from a flattened nest and compare it against the sequence from
-// an identical non-flattened reference nest. Exit code 0 == identical.
-//
-// flatten is a pure frontend AST desugaring to an ordinary loop, so this runs
-// as a normal program (build with: clang++ -fopenmp -fopenmp-version=61).
-#include <cassert>
-#include <cstdio>
-#include <vector>
-
-using Seq = std::vector<long>;
-
-static void rec(Seq &s, int i, int j, int k = -1) {
-  s.push_back(i);
-  s.push_back(j);
-  if (k >= 0)
-    s.push_back(k);
-}
-
-int main() {
-  const int n = 3, m = 4, p = 2;
-
-  // --- depth(2) default over 2 loops ---
-  {
-    Seq flat, ref;
-#pragma omp flatten
-    for (int i = 0; i < n; ++i)
-      for (int j = 0; j < m; ++j)
-        rec(flat, i, j);
-    for (int i = 0; i < n; ++i)
-      for (int j = 0; j < m; ++j)
-        rec(ref, i, j);
-    assert(flat == ref && "depth(2) order/set mismatch");
-  }
-
-  // --- depth(3) full flatten over 3 loops ---
-  {
-    Seq flat, ref;
-#pragma omp flatten depth(3)
-    for (int i = 0; i < n; ++i)
-      for (int j = 0; j < m; ++j)
-        for (int k = 0; k < p; ++k)
-          rec(flat, i, j, k);
-    for (int i = 0; i < n; ++i)
-      for (int j = 0; j < m; ++j)
-        for (int k = 0; k < p; ++k)
-          rec(ref, i, j, k);
-    assert(flat == ref && "depth(3) order/set mismatch");
-  }
-
-  // --- depth(2) partial: inner k-loop stays intact ---
-  {
-    Seq flat, ref;
-#pragma omp flatten depth(2)
-    for (int i = 0; i < n; ++i)
-      for (int j = 0; j < m; ++j)
-        for (int k = 0; k < p; ++k)
-          rec(flat, i, j, k);
-    for (int i = 0; i < n; ++i)
-      for (int j = 0; j < m; ++j)
-        for (int k = 0; k < p; ++k)
-          rec(ref, i, j, k);
-    assert(flat == ref && "depth(2)-partial order/set mismatch");
-  }
-
-  // --- non-unit step / non-zero start must survive ---
-  {
-    Seq flat, ref;
-#pragma omp flatten
-    for (int i = 5; i < 12; i += 2)
-      for (int j = 1; j < 7; j += 3)
-        rec(flat, i, j);
-    for (int i = 5; i < 12; i += 2)
-      for (int j = 1; j < 7; j += 3)
-        rec(ref, i, j);
-    assert(flat == ref && "stride/offset mismatch");
-  }
-
-  std::puts("flatten end-result preserved: OK");
-  return 0;
-}

>From a169b5202ef4f4a5d9f8fbb47c4f8f9132ad0edb Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Tue, 18 Aug 2026 07:15:20 -0400
Subject: [PATCH 16/17] conflict-resolve

---
 clang/bindings/python/clang/cindex.py | 5 ++++-
 clang/include/clang-c/Index.h         | 6 +++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/clang/bindings/python/clang/cindex.py b/clang/bindings/python/clang/cindex.py
index ccad140e8c861..56e20fdfad781 100644
--- a/clang/bindings/python/clang/cindex.py
+++ b/clang/bindings/python/clang/cindex.py
@@ -1456,8 +1456,11 @@ def is_unexposed(self):
     # OpenMP split directive.
     OMP_SPLIT_DIRECTIVE = 312
 
+    # OpenMP ordered-blockassoc directive.
+    OMP_ORDERED_BLOCK_ASSOC_DIRECTIVE = 313
+
     # OpenMP flatten directive.
-    OMP_FLATTEN_DIRECTIVE = 313
+    OMP_FLATTEN_DIRECTIVE = 314
 
     # OpenACC Compute Construct.
     OPEN_ACC_COMPUTE_DIRECTIVE = 320
diff --git a/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h
index f561fb5724897..4c4cf392f172e 100644
--- a/clang/include/clang-c/Index.h
+++ b/clang/include/clang-c/Index.h
@@ -2170,9 +2170,13 @@ enum CXCursorKind {
    */
   CXCursor_OMPSplitDirective = 312,
 
+  /** OpenMP ordered-blockassoc directive.
+   */
+  CXCursor_OMPOrderedBlockAssocDirective = 313,
+
   /** OpenMP flatten directive.
    */
-  CXCursor_OMPFlattenDirective = 313,
+  CXCursor_OMPFlattenDirective = 314,
 
   /** OpenACC Compute Construct.
    */

>From 147063ca2af28aecf2a0135373cabb9c179c0999 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Tue, 18 Aug 2026 07:17:36 -0400
Subject: [PATCH 17/17] revised

---
 clang/include/clang/AST/OpenMPClause.h        |   8 +-
 clang/include/clang/AST/StmtOpenMP.h          |   4 +-
 .../clang/Basic/DiagnosticSemaKinds.td        |   6 +
 clang/lib/AST/OpenMPClause.cpp                |  12 --
 clang/lib/Sema/SemaOpenMP.cpp                 | 112 ++++++++++--------
 5 files changed, 74 insertions(+), 68 deletions(-)

diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 50d1364a9726d..05ccf4db5752b 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -1411,14 +1411,14 @@ class OMPDepthClause final : public OMPClause {
 public:
   /// Build an AST node for a 'depth' clause.
   ///
-  /// \param C         Context of the AST.
   /// \param StartLoc  Location of the 'depth' identifier.
   /// \param LParenLoc Location of '('.
   /// \param EndLoc    Location of ')'.
   /// \param Depth     The depth expression.
-  static OMPDepthClause *Create(const ASTContext &C, SourceLocation StartLoc,
-                                SourceLocation LParenLoc, SourceLocation EndLoc,
-                                Expr *Depth);
+  OMPDepthClause(SourceLocation StartLoc, SourceLocation LParenLoc,
+                 SourceLocation EndLoc, Expr *Depth)
+      : OMPClause(llvm::omp::OMPC_depth, StartLoc, EndLoc),
+        LParenLoc(LParenLoc), Depth(Depth) {}
 
   /// Build an empty 'depth' AST node for deserialization.
   ///
diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index 1420331f37c1f..4ae6f364dced3 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -5999,8 +5999,8 @@ class OMPFlattenDirective final
   /// \param StartLoc  Location of the introducer (e.g. the 'omp' token).
   /// \param EndLoc    Location of the directive's end (e.g. the tok::eod).
   /// \param Clauses   The directive's clauses.
-  /// \param NumLoops  Number of affected loops (the flatten depth, currently
-  ///                  always 2).
+  /// \param NumLoops  Number of affected loops (the flatten depth: the
+  ///                  argument of the 'depth' clause, or 2 if omitted).
   /// \param AssociatedStmt  The outermost associated loop.
   /// \param TransformedStmt The flattened loop, or nullptr in dependent
   ///                        contexts.
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 7e20630708312..fc44b8d15239e 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12172,6 +12172,12 @@ def err_omp_invalid_looprange
 def warn_omp_redundant_fusion : Warning<"looprange clause selects a single "
                                         "loop, resulting in redundant fusion">,
                                 InGroup<OpenMPClauses>;
+def warn_omp_flatten_omitted_depth : Warning<
+  "'flatten' without a 'depth' clause only combines 2 loops, but 3 or more "
+  "loops are perfectly nested">,
+  InGroup<OpenMPClauses>;
+def err_omp_clause_not_supported_yet : Error<
+  "'%0' clause is not yet supported on '#pragma omp %1' directive">;
 def err_omp_not_for : Error<
   "%select{statement after '#pragma omp %1' must be a for loop|"
   "expected %2 for loops after '#pragma omp %1'%select{|, but found only %4}3}0">;
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index 7b2f67a31cad9..7311e404b66c3 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -1057,18 +1057,6 @@ OMPPartialClause *OMPPartialClause::CreateEmpty(const ASTContext &C) {
   return new (C) OMPPartialClause();
 }
 
-OMPDepthClause *OMPDepthClause::Create(const ASTContext &C,
-                                       SourceLocation StartLoc,
-                                       SourceLocation LParenLoc,
-                                       SourceLocation EndLoc, Expr *Depth) {
-  OMPDepthClause *Clause = CreateEmpty(C);
-  Clause->setLocStart(StartLoc);
-  Clause->setLParenLoc(LParenLoc);
-  Clause->setLocEnd(EndLoc);
-  Clause->setDepth(Depth);
-  return Clause;
-}
-
 OMPDepthClause *OMPDepthClause::CreateEmpty(const ASTContext &C) {
   return new (C) OMPDepthClause();
 }
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index fcc256dec3c88..caaba5a3be9fa 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -6839,6 +6839,7 @@ StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
       case OMPC_safelen:
       case OMPC_simdlen:
       case OMPC_sizes:
+      case OMPC_depth:
       case OMPC_default:
       case OMPC_proc_bind:
       case OMPC_private:
@@ -16395,29 +16396,6 @@ StmtResult SemaOpenMP::ActOnOpenMPInterchangeDirective(
                                          buildPreInits(Context, PreInits));
 }
 
-/// Counts the perfectly nested canonical loop depth at \p AStmt. Returns
-/// std::nullopt if a loop-generating transformation prevents a static count.
-static std::optional<unsigned> getCanonicalLoopNestDepth(Stmt *AStmt) {
-  unsigned Depth = 0;
-  Stmt *CurStmt = AStmt ? AStmt->IgnoreContainers() : nullptr;
-  while (CurStmt) {
-    if (isa<OMPLoopTransformationDirective>(CurStmt))
-      return std::nullopt;
-    if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(CurStmt))
-      CurStmt = CanonLoop->getLoopStmt();
-    Stmt *Body = nullptr;
-    if (auto *For = dyn_cast<ForStmt>(CurStmt))
-      Body = For->getBody();
-    else if (auto *RangeFor = dyn_cast<CXXForRangeStmt>(CurStmt))
-      Body = RangeFor->getBody();
-    else
-      break;
-    ++Depth;
-    CurStmt = Body ? Body->IgnoreContainers() : nullptr;
-  }
-  return Depth;
-}
-
 StmtResult
 SemaOpenMP::ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
                                         Stmt *AStmt, SourceLocation StartLoc,
@@ -16432,36 +16410,49 @@ SemaOpenMP::ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
 
   // flatten without 'depth' clause combines two loops; 'depth(k)' selects k.
   unsigned NumLoops = 2;
-  bool DepthIsValueDependent = false;
+  bool DepthIsDependent = false;
   const auto *DepthClause =
       OMPExecutableDirective::getSingleClause<OMPDepthClause>(Clauses);
   if (DepthClause) {
     Expr *DepthExpr = DepthClause->getDepth();
-    if (DepthExpr && DepthExpr->isValueDependent()) {
-      DepthIsValueDependent = true;
+    if (DepthExpr && DepthExpr->isInstantiationDependent()) {
+      DepthIsDependent = true;
     } else if (DepthExpr) {
       Expr::EvalResult EvalResult;
       if (DepthExpr->EvaluateAsInt(EvalResult, Context))
-        NumLoops = EvalResult.Val.getInt().getZExtValue();
+        NumLoops = EvalResult.Val.getInt().getLimitedValue();
     }
   }
 
-  // Report "expected k for loops, but found only n" when depth exceeds the
-  // perfect nest (same form as 'collapse'); skip if depth or nest is unknown.
-  if (DepthClause && !DepthIsValueDependent) {
-    if (std::optional<unsigned> NestDepth = getCanonicalLoopNestDepth(AStmt);
-        NestDepth && NumLoops > *NestDepth) {
+  // Count perfectly nested loops with doForAllLoops. When 'depth' is present,
+  // walk NumLoops iterations to diagnose an insufficient nest. When it is
+  // omitted, walk one extra loop (3 total) so we can warn that default
+  // flatten only combines 2 of a deeper nest.
+  if (!DepthIsDependent) {
+    unsigned WalkLimit = DepthClause ? NumLoops : 3;
+    unsigned Found = 0;
+    bool Enough = OMPLoopBasedDirective::doForAllLoops(
+        AStmt->IgnoreContainers(), /*TryImperfectlyNestedLoops=*/false,
+        WalkLimit, [&](unsigned Cnt, Stmt *S) {
+          if (!isa<ForStmt>(S) && !isa<CXXForRangeStmt>(S))
+            return true;
+          Found = Cnt + 1;
+          return false;
+        });
+    if (DepthClause && !Enough) {
       Diag(AStmt->getBeginLoc(), diag::err_omp_not_for)
           << /*expected N for loops form=*/1
-          << getOpenMPDirectiveName(OMPD_flatten) << NumLoops
-          << (*NestDepth > 0) << *NestDepth;
+          << getOpenMPDirectiveName(OMPD_flatten) << NumLoops << (Found > 0)
+          << Found;
       return StmtError();
     }
+    if (!DepthClause && Found >= 3)
+      Diag(StartLoc, diag::warn_omp_flatten_omitted_depth);
   }
 
-  // Defer when 'depth' is value-dependent (concrete k unknown until
+  // Defer when 'depth' is instantiation-dependent (concrete k unknown until
   // instantiation).
-  if (DepthIsValueDependent)
+  if (DepthIsDependent)
     return OMPFlattenDirective::Create(Context, StartLoc, EndLoc, Clauses,
                                        NumLoops, AStmt, nullptr, nullptr);
 
@@ -16537,20 +16528,30 @@ SemaOpenMP::ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
     AllCountsLessThan32Bits &=
         Context.getTypeSize(LoopHelpers[I].NumIterations->getType()) < 32;
 
-  ExprResult TripCount32 = BuildTripCount(/*Bits=*/32);
-  ExprResult TripCount64 = BuildTripCount(/*Bits=*/64);
-  if (!TripCount32.isUsable() || !TripCount64.isUsable())
+  ExprResult TripCount;
+  if (AllCountsLessThan32Bits || NumLoops == 1) {
+    TripCount = BuildTripCount(/*Bits=*/32);
+  } else {
+    ExprResult TripCount64 = BuildTripCount(/*Bits=*/64);
+    if (!TripCount64.isUsable())
+      return StmtError();
+    if (fitsInto(
+            /*Bits=*/32,
+            TripCount64.get()->getType()->hasSignedIntegerRepresentation(),
+            TripCount64.get(), SemaRef)) {
+      ExprResult TripCount32 = BuildTripCount(/*Bits=*/32);
+      if (TripCount32.isUsable() &&
+          Context.getTypeSize(TripCount32.get()->getType()) == 32)
+        TripCount = TripCount32;
+      else
+        TripCount = TripCount64;
+    } else {
+      TripCount = TripCount64;
+    }
+  }
+  if (!TripCount.isUsable())
     return StmtError();
 
-  ExprResult TripCount = TripCount64;
-  if (Context.getTypeSize(TripCount32.get()->getType()) == 32 &&
-      (AllCountsLessThan32Bits || NumLoops == 1 ||
-       fitsInto(
-           /*Bits=*/32,
-           TripCount32.get()->getType()->hasSignedIntegerRepresentation(),
-           TripCount64.get(), SemaRef)))
-    TripCount = TripCount32;
-
   QualType IVTy = TripCount.get()->getType();
   uint64_t IVWidth = Context.getTypeSize(IVTy);
 
@@ -16680,6 +16681,13 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
   if (!AStmt)
     return StmtError();
 
+  if (const auto *DepthC =
+          OMPExecutableDirective::getSingleClause<OMPDepthClause>(Clauses)) {
+    Diag(DepthC->getBeginLoc(), diag::err_omp_clause_not_supported_yet)
+        << "depth" << getOpenMPDirectiveName(OMPD_fuse);
+    return StmtError();
+  }
+
   // Defer transformation in dependent contexts
   // The NumLoopNests argument is set to a placeholder 1 (even though
   // using looprange fuse could yield up to 3 top level loop nests)
@@ -17911,6 +17919,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPSimpleClause(
   case OMPC_safelen:
   case OMPC_simdlen:
   case OMPC_sizes:
+  case OMPC_depth:
   case OMPC_allocator:
   case OMPC_collapse:
   case OMPC_schedule:
@@ -18532,8 +18541,8 @@ OMPClause *SemaOpenMP::ActOnOpenMPDepthClause(Expr *DepthExpr,
     return nullptr;
   DepthExpr = DepthResult.get();
 
-  return OMPDepthClause::Create(getASTContext(), StartLoc, LParenLoc, EndLoc,
-                                DepthExpr);
+  return new (getASTContext())
+      OMPDepthClause(StartLoc, LParenLoc, EndLoc, DepthExpr);
 }
 
 OMPClause *SemaOpenMP::ActOnOpenMPLoopRangeClause(
@@ -18671,6 +18680,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPSingleExprWithArgClause(
   case OMPC_safelen:
   case OMPC_simdlen:
   case OMPC_sizes:
+  case OMPC_depth:
   case OMPC_allocator:
   case OMPC_collapse:
   case OMPC_proc_bind:
@@ -18959,6 +18969,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPClause(OpenMPClauseKind Kind,
   case OMPC_safelen:
   case OMPC_simdlen:
   case OMPC_sizes:
+  case OMPC_depth:
   case OMPC_allocator:
   case OMPC_collapse:
   case OMPC_schedule:
@@ -19643,6 +19654,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPVarListClause(OpenMPClauseKind Kind,
   case OMPC_safelen:
   case OMPC_simdlen:
   case OMPC_sizes:
+  case OMPC_depth:
   case OMPC_allocator:
   case OMPC_collapse:
   case OMPC_default:


From openmp-commits at lists.llvm.org  Tue Aug 18 05:46:08 2026
From: openmp-commits at lists.llvm.org (Amit Tiwari via Openmp-commits)
Date: Tue, 18 Aug 2026 05:46:08 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [Clang][OpenMP] Add
 `#pragma omp flatten` loop directive + `depth` clause (PR #206977)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206977@github.com>
Message-ID: <6a845410.d681cab9.3ab686.09d3@mx.google.com>


loopacino wrote:

> I think we need more tests:
> 
> Diagnostics output
> 
> * Diagnostics when applied to a non-perfectly nested loop

clang/test/OpenMP/flatten_messages.cpp

> * depth argument diagnostics: `0`, `-1`, insufficient number of loops
> * depth argument is not a constant

clang/test/OpenMP/flatten_depth_messages.cpp

> 
> Codegen/execution
> 
> * When applied to a non-rectangular loop

It would be a Sema error. There is no IR or run test, because the code is not allowed to compile. Kept diag test only at clang/test/OpenMP/flatten_messages.cpp

> * Applied to the generated loops of other constructs: fuse, reverse, unroll, etc

Flatten around reverse/unroll/another flatten does not compile (flatten_messages.cpp). reverse on flatten AST only: flatten_ast_print.cpp.

> * Other loop-associated constructs applied to it, e.g. flatten+for, flatten+unroll, etc.

 unroll/tile codegen (flatten_codegen.cpp), runtime: unroll-on-flatten.c/tile-on-flatten.c/reverse-on-flatten.c, interchange is an error

> * Applied onto itself: flatten + flatten ≈ flatten depth(3)

openmp/runtime/test/transform/flatten/depth-intfor.c

> * depth argument is a template parameter

clang/test/OpenMP/flatten_depth_codegen.cpp

> * Degenerate loop: `for (int i = 0; i < -1; ++i)`

openmp/runtime/test/transform/flatten/empty.c, 

> 
> Optional: A warning diagnostic if `depth` is omitted (so the default of 2 applies), but there are 3 or more loops that could be flattened

Done in SemaOpenMP.cpp



https://github.com/llvm/llvm-project/pull/206977

From openmp-commits at lists.llvm.org  Tue Aug 18 05:52:25 2026
From: openmp-commits at lists.llvm.org (Amit Tiwari via Openmp-commits)
Date: Tue, 18 Aug 2026 05:52:25 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [Clang][OpenMP] Add
 `#pragma omp flatten` loop directive + `depth` clause (PR #206977)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206977@github.com>
Message-ID: <6a845589.ee10049a.50084.0b74@mx.google.com>


https://github.com/loopacino updated https://github.com/llvm/llvm-project/pull/206977

>From 0fb5bd42f1e127faea963c359b1e3c69cb969c60 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 1 Jul 2026 09:06:58 -0400
Subject: [PATCH 01/18] add OMPFlattenDirective ast node

---
 clang/include/clang/AST/RecursiveASTVisitor.h |  3 +
 clang/include/clang/AST/StmtOpenMP.h          | 75 ++++++++++++++++++-
 clang/include/clang/Basic/StmtNodes.td        |  1 +
 clang/lib/AST/StmtOpenMP.cpp                  | 20 +++++
 clang/lib/AST/StmtPrinter.cpp                 |  5 ++
 clang/lib/AST/StmtProfile.cpp                 |  4 +
 clang/lib/Basic/OpenMPKinds.cpp               |  2 +-
 7 files changed, 108 insertions(+), 2 deletions(-)

diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index b000a34043696..e425f7a31f59c 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -3227,6 +3227,9 @@ DEF_TRAVERSE_STMT(OMPFuseDirective,
 DEF_TRAVERSE_STMT(OMPInterchangeDirective,
                   { TRY_TO(TraverseOMPExecutableDirective(S)); })
 
+DEF_TRAVERSE_STMT(OMPFlattenDirective,
+                  { TRY_TO(TraverseOMPExecutableDirective(S)); })
+
 DEF_TRAVERSE_STMT(OMPSplitDirective,
                   { TRY_TO(TraverseOMPExecutableDirective(S)); })
 
diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index dbc76e7df8ecd..1420331f37c1f 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -1041,7 +1041,7 @@ class OMPCanonicalLoopNestTransformationDirective
     Stmt::StmtClass C = T->getStmtClass();
     return C == OMPTileDirectiveClass || C == OMPUnrollDirectiveClass ||
            C == OMPReverseDirectiveClass || C == OMPInterchangeDirectiveClass ||
-           C == OMPStripeDirectiveClass;
+           C == OMPStripeDirectiveClass || C == OMPFlattenDirectiveClass;
   }
 };
 
@@ -5959,6 +5959,79 @@ class OMPInterchangeDirective final
   }
 };
 
+/// Represents the '#pragma omp flatten' loop transformation directive.
+///
+/// \code{c}
+///   #pragma omp flatten
+///   for (int i = 0; i < m; ++i)
+///     for (int j = 0; j < n; ++j)
+///       ..
+/// \endcode
+class OMPFlattenDirective final
+    : public OMPCanonicalLoopNestTransformationDirective {
+  friend class ASTStmtReader;
+  friend class OMPExecutableDirective;
+
+  /// Offsets of child members.
+  enum {
+    PreInitsOffset = 0,
+    TransformedStmtOffset,
+  };
+
+  explicit OMPFlattenDirective(SourceLocation StartLoc, SourceLocation EndLoc,
+                               unsigned NumLoops)
+      : OMPCanonicalLoopNestTransformationDirective(
+            OMPFlattenDirectiveClass, llvm::omp::OMPD_flatten, StartLoc, EndLoc,
+            NumLoops) {}
+
+  void setPreInits(Stmt *PreInits) {
+    Data->getChildren()[PreInitsOffset] = PreInits;
+  }
+
+  void setTransformedStmt(Stmt *S) {
+    Data->getChildren()[TransformedStmtOffset] = S;
+  }
+
+public:
+  /// Create a new AST node representation for '#pragma omp flatten'.
+  ///
+  /// \param C         Context of the AST.
+  /// \param StartLoc  Location of the introducer (e.g. the 'omp' token).
+  /// \param EndLoc    Location of the directive's end (e.g. the tok::eod).
+  /// \param Clauses   The directive's clauses.
+  /// \param NumLoops  Number of affected loops (the flatten depth, currently
+  ///                  always 2).
+  /// \param AssociatedStmt  The outermost associated loop.
+  /// \param TransformedStmt The flattened loop, or nullptr in dependent
+  ///                        contexts.
+  /// \param PreInits  Helper preinits statements for the loop nest.
+  static OMPFlattenDirective *
+  Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation EndLoc,
+         ArrayRef<OMPClause *> Clauses, unsigned NumLoops, Stmt *AssociatedStmt,
+         Stmt *TransformedStmt, Stmt *PreInits);
+
+  /// Build an empty '#pragma omp flatten' AST node for deserialization.
+  ///
+  /// \param C          Context of the AST.
+  /// \param NumClauses Number of clauses to allocate.
+  /// \param NumLoops   Number of associated loops to allocate.
+  static OMPFlattenDirective *
+  CreateEmpty(const ASTContext &C, unsigned NumClauses, unsigned NumLoops);
+
+  /// Gets the flattened loop after the transformation. This is the de-sugared
+  /// replacement or nullptr in dependent contexts.
+  Stmt *getTransformedStmt() const {
+    return Data->getChildren()[TransformedStmtOffset];
+  }
+
+  /// Return preinits statement.
+  Stmt *getPreInits() const { return Data->getChildren()[PreInitsOffset]; }
+
+  static bool classof(const Stmt *T) {
+    return T->getStmtClass() == OMPFlattenDirectiveClass;
+  }
+};
+
 /// The base class for all transformation directives of canonical loop
 /// sequences (currently only 'fuse')
 class OMPCanonicalLoopSequenceTransformationDirective
diff --git a/clang/include/clang/Basic/StmtNodes.td b/clang/include/clang/Basic/StmtNodes.td
index e166894ea024b..0d7a0c1490a1e 100644
--- a/clang/include/clang/Basic/StmtNodes.td
+++ b/clang/include/clang/Basic/StmtNodes.td
@@ -247,6 +247,7 @@ def OMPReverseDirective : StmtNode<OMPCanonicalLoopNestTransformationDirective>;
 def OMPSplitDirective : StmtNode<OMPCanonicalLoopNestTransformationDirective>;
 def OMPInterchangeDirective
     : StmtNode<OMPCanonicalLoopNestTransformationDirective>;
+def OMPFlattenDirective : StmtNode<OMPCanonicalLoopNestTransformationDirective>;
 def OMPCanonicalLoopSequenceTransformationDirective
     : StmtNode<OMPExecutableDirective, 1>;
 def OMPFuseDirective
diff --git a/clang/lib/AST/StmtOpenMP.cpp b/clang/lib/AST/StmtOpenMP.cpp
index 9d6b315effb41..477604ef3dafb 100644
--- a/clang/lib/AST/StmtOpenMP.cpp
+++ b/clang/lib/AST/StmtOpenMP.cpp
@@ -552,6 +552,26 @@ OMPInterchangeDirective::CreateEmpty(const ASTContext &C, unsigned NumClauses,
       SourceLocation(), SourceLocation(), NumLoops);
 }
 
+OMPFlattenDirective *OMPFlattenDirective::Create(
+    const ASTContext &C, SourceLocation StartLoc, SourceLocation EndLoc,
+    ArrayRef<OMPClause *> Clauses, unsigned NumLoops, Stmt *AssociatedStmt,
+    Stmt *TransformedStmt, Stmt *PreInits) {
+  OMPFlattenDirective *Dir = createDirective<OMPFlattenDirective>(
+      C, Clauses, AssociatedStmt, TransformedStmtOffset + 1, StartLoc, EndLoc,
+      NumLoops);
+  Dir->setTransformedStmt(TransformedStmt);
+  Dir->setPreInits(PreInits);
+  return Dir;
+}
+
+OMPFlattenDirective *OMPFlattenDirective::CreateEmpty(const ASTContext &C,
+                                                      unsigned NumClauses,
+                                                      unsigned NumLoops) {
+  return createEmptyDirective<OMPFlattenDirective>(
+      C, NumClauses, /*HasAssociatedStmt=*/true, TransformedStmtOffset + 1,
+      SourceLocation(), SourceLocation(), NumLoops);
+}
+
 OMPSplitDirective *
 OMPSplitDirective::Create(const ASTContext &C, SourceLocation StartLoc,
                           SourceLocation EndLoc, ArrayRef<OMPClause *> Clauses,
diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp
index 6c3294573e9d4..0bacb79ed86e3 100644
--- a/clang/lib/AST/StmtPrinter.cpp
+++ b/clang/lib/AST/StmtPrinter.cpp
@@ -800,6 +800,11 @@ void StmtPrinter::VisitOMPInterchangeDirective(OMPInterchangeDirective *Node) {
   PrintOMPExecutableDirective(Node);
 }
 
+void StmtPrinter::VisitOMPFlattenDirective(OMPFlattenDirective *Node) {
+  Indent() << "#pragma omp flatten";
+  PrintOMPExecutableDirective(Node);
+}
+
 void StmtPrinter::VisitOMPSplitDirective(OMPSplitDirective *Node) {
   Indent() << "#pragma omp split";
   PrintOMPExecutableDirective(Node);
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index 90eab530e0c2e..de7ed322d7424 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -1071,6 +1071,10 @@ void StmtProfiler::VisitOMPInterchangeDirective(
   VisitOMPCanonicalLoopNestTransformationDirective(S);
 }
 
+void StmtProfiler::VisitOMPFlattenDirective(const OMPFlattenDirective *S) {
+  VisitOMPCanonicalLoopNestTransformationDirective(S);
+}
+
 void StmtProfiler::VisitOMPSplitDirective(const OMPSplitDirective *S) {
   VisitOMPCanonicalLoopNestTransformationDirective(S);
 }
diff --git a/clang/lib/Basic/OpenMPKinds.cpp b/clang/lib/Basic/OpenMPKinds.cpp
index 675d86349c933..57e0347f8ea77 100644
--- a/clang/lib/Basic/OpenMPKinds.cpp
+++ b/clang/lib/Basic/OpenMPKinds.cpp
@@ -818,7 +818,7 @@ bool clang::isOpenMPCanonicalLoopNestTransformationDirective(
     OpenMPDirectiveKind DKind) {
   return DKind == OMPD_tile || DKind == OMPD_unroll || DKind == OMPD_reverse ||
          DKind == OMPD_split || DKind == OMPD_interchange ||
-         DKind == OMPD_stripe;
+         DKind == OMPD_stripe || DKind == OMPD_flatten;
 }
 
 bool clang::isOpenMPCanonicalLoopSequenceTransformationDirective(

>From 933007b6333584fb358adc4f89a61db1ae15d4cb Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:02:27 -0400
Subject: [PATCH 02/18] add OMPDepthClause ast node

---
 clang/include/clang/AST/OpenMPClause.h        | 68 +++++++++++++++++++
 clang/include/clang/AST/RecursiveASTVisitor.h |  6 ++
 clang/lib/AST/OpenMPClause.cpp                | 25 +++++++
 clang/lib/AST/StmtProfile.cpp                 |  5 ++
 clang/lib/Parse/ParseOpenMP.cpp               |  1 +
 llvm/include/llvm/Frontend/OpenMP/OMP.td      |  1 +
 6 files changed, 106 insertions(+)

diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 8ceafc4669297..50d1364a9726d 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -1380,6 +1380,74 @@ class OMPPartialClause final : public OMPClause {
   }
 };
 
+/// This represents the 'depth' clause on the '#pragma omp flatten' (and
+/// '#pragma omp fuse') loop-transformation directives.
+///
+/// \code
+/// #pragma omp flatten depth(3)
+/// \endcode
+/// In this example the 'flatten' directive has a 'depth' clause whose argument
+/// '3' specifies how many perfectly nested loops are combined into one.
+/// The argument must be a positive integer constant expression that evaluates
+/// to at most the loop nest depth of the associated loop nest.
+class OMPDepthClause final : public OMPClause {
+  friend class OMPClauseReader;
+
+  /// Location of '('.
+  SourceLocation LParenLoc;
+
+  /// The depth expression (number of loops to flatten).
+  Stmt *Depth = nullptr;
+
+  /// Build an empty clause.
+  explicit OMPDepthClause() : OMPClause(llvm::omp::OMPC_depth, {}, {}) {}
+
+  /// Set the depth expression.
+  void setDepth(Expr *E) { Depth = E; }
+
+  /// Sets the location of '('.
+  void setLParenLoc(SourceLocation Loc) { LParenLoc = Loc; }
+
+public:
+  /// Build an AST node for a 'depth' clause.
+  ///
+  /// \param C         Context of the AST.
+  /// \param StartLoc  Location of the 'depth' identifier.
+  /// \param LParenLoc Location of '('.
+  /// \param EndLoc    Location of ')'.
+  /// \param Depth     The depth expression.
+  static OMPDepthClause *Create(const ASTContext &C, SourceLocation StartLoc,
+                                SourceLocation LParenLoc, SourceLocation EndLoc,
+                                Expr *Depth);
+
+  /// Build an empty 'depth' AST node for deserialization.
+  ///
+  /// \param C     Context of the AST.
+  static OMPDepthClause *CreateEmpty(const ASTContext &C);
+
+  /// Returns the location of '('.
+  SourceLocation getLParenLoc() const { return LParenLoc; }
+
+  /// Returns the depth expression or nullptr if not set.
+  Expr *getDepth() const { return cast_or_null<Expr>(Depth); }
+
+  child_range children() { return child_range(&Depth, &Depth + 1); }
+  const_child_range children() const {
+    return const_child_range(&Depth, &Depth + 1);
+  }
+
+  child_range used_children() {
+    return child_range(child_iterator(), child_iterator());
+  }
+  const_child_range used_children() const {
+    return const_child_range(const_child_iterator(), const_child_iterator());
+  }
+
+  static bool classof(const OMPClause *T) {
+    return T->getClauseKind() == llvm::omp::OMPC_depth;
+  }
+};
+
 /// This represents 'collapse' clause in the '#pragma omp ...'
 /// directive.
 ///
diff --git a/clang/include/clang/AST/RecursiveASTVisitor.h b/clang/include/clang/AST/RecursiveASTVisitor.h
index e425f7a31f59c..da55e97fdfcb5 100644
--- a/clang/include/clang/AST/RecursiveASTVisitor.h
+++ b/clang/include/clang/AST/RecursiveASTVisitor.h
@@ -3568,6 +3568,12 @@ bool RecursiveASTVisitor<Derived>::VisitOMPPartialClause(OMPPartialClause *C) {
   return true;
 }
 
+template <typename Derived>
+bool RecursiveASTVisitor<Derived>::VisitOMPDepthClause(OMPDepthClause *C) {
+  TRY_TO(TraverseStmt(C->getDepth()));
+  return true;
+}
+
 template <typename Derived>
 bool
 RecursiveASTVisitor<Derived>::VisitOMPCollapseClause(OMPCollapseClause *C) {
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index ed00e80144c25..7b2f67a31cad9 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -1057,6 +1057,22 @@ OMPPartialClause *OMPPartialClause::CreateEmpty(const ASTContext &C) {
   return new (C) OMPPartialClause();
 }
 
+OMPDepthClause *OMPDepthClause::Create(const ASTContext &C,
+                                       SourceLocation StartLoc,
+                                       SourceLocation LParenLoc,
+                                       SourceLocation EndLoc, Expr *Depth) {
+  OMPDepthClause *Clause = CreateEmpty(C);
+  Clause->setLocStart(StartLoc);
+  Clause->setLParenLoc(LParenLoc);
+  Clause->setLocEnd(EndLoc);
+  Clause->setDepth(Depth);
+  return Clause;
+}
+
+OMPDepthClause *OMPDepthClause::CreateEmpty(const ASTContext &C) {
+  return new (C) OMPDepthClause();
+}
+
 OMPLoopRangeClause *
 OMPLoopRangeClause::Create(const ASTContext &C, SourceLocation StartLoc,
                            SourceLocation LParenLoc, SourceLocation FirstLoc,
@@ -2069,6 +2085,15 @@ void OMPClausePrinter::VisitOMPPermutationClause(OMPPermutationClause *Node) {
 
 void OMPClausePrinter::VisitOMPFullClause(OMPFullClause *Node) { OS << "full"; }
 
+void OMPClausePrinter::VisitOMPDepthClause(OMPDepthClause *Node) {
+  OS << "depth";
+  if (Expr *Depth = Node->getDepth()) {
+    OS << '(';
+    Depth->printPretty(OS, nullptr, Policy, 0);
+    OS << ')';
+  }
+}
+
 void OMPClausePrinter::VisitOMPPartialClause(OMPPartialClause *Node) {
   OS << "partial";
 
diff --git a/clang/lib/AST/StmtProfile.cpp b/clang/lib/AST/StmtProfile.cpp
index de7ed322d7424..838d6d02f7446 100644
--- a/clang/lib/AST/StmtProfile.cpp
+++ b/clang/lib/AST/StmtProfile.cpp
@@ -521,6 +521,11 @@ void OMPClauseProfiler::VisitOMPPartialClause(const OMPPartialClause *C) {
     Profiler->VisitExpr(Factor);
 }
 
+void OMPClauseProfiler::VisitOMPDepthClause(const OMPDepthClause *C) {
+  if (const Expr *Depth = C->getDepth())
+    Profiler->VisitExpr(Depth);
+}
+
 void OMPClauseProfiler::VisitOMPLoopRangeClause(const OMPLoopRangeClause *C) {
   if (const Expr *First = C->getFirst())
     Profiler->VisitExpr(First);
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index af52295df2d8b..952a517205381 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -3247,6 +3247,7 @@ OMPClause *Parser::ParseOpenMPClause(OpenMPDirectiveKind DKind,
   case OMPC_nocontext:
   case OMPC_filter:
   case OMPC_partial:
+  case OMPC_depth:
   case OMPC_align:
   case OMPC_message:
   case OMPC_ompx_dyn_cgroup_mem:
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMP.td b/llvm/include/llvm/Frontend/OpenMP/OMP.td
index 679a944fc4358..d49c5ee1a0db3 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMP.td
+++ b/llvm/include/llvm/Frontend/OpenMP/OMP.td
@@ -161,6 +161,7 @@ def OMPC_Depobj : Clause<[Spelling<"depobj">]> {
   let isImplicit = true;
 }
 def OMPC_Depth : Clause<[Spelling<"depth">]> {
+  let clangClass = "OMPDepthClause";
   let flangClass = "ScalarIntConstantExpr";
 }
 def OMPC_Destroy : Clause<[Spelling<"destroy">]> {

>From b63024e1e342cd168ef128a3f6a24909df2a82db Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:04:44 -0400
Subject: [PATCH 03/18] template-transform support for flatten directive &
 depth clause

---
 clang/lib/Sema/TreeTransform.h | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h
index 3b99ff4bb9e23..865622ad50f1c 100644
--- a/clang/lib/Sema/TreeTransform.h
+++ b/clang/lib/Sema/TreeTransform.h
@@ -1804,6 +1804,14 @@ class TreeTransform {
                                                        LParenLoc, EndLoc);
   }
 
+  /// Build a new OpenMP 'depth' clause.
+  OMPClause *RebuildOMPDepthClause(Expr *Depth, SourceLocation StartLoc,
+                                   SourceLocation LParenLoc,
+                                   SourceLocation EndLoc) {
+    return getSema().OpenMP().ActOnOpenMPDepthClause(Depth, StartLoc, LParenLoc,
+                                                     EndLoc);
+  }
+
   OMPClause *
   RebuildOMPLoopRangeClause(Expr *First, Expr *Count, SourceLocation StartLoc,
                             SourceLocation LParenLoc, SourceLocation FirstLoc,
@@ -9791,6 +9799,17 @@ TreeTransform<Derived>::TransformOMPSplitDirective(OMPSplitDirective *D) {
   return Res;
 }
 
+template <typename Derived>
+StmtResult
+TreeTransform<Derived>::TransformOMPFlattenDirective(OMPFlattenDirective *D) {
+  DeclarationNameInfo DirName;
+  getDerived().getSema().OpenMP().StartOpenMPDSABlock(
+      D->getDirectiveKind(), DirName, nullptr, D->getBeginLoc());
+  StmtResult Res = getDerived().TransformOMPExecutableDirective(D);
+  getDerived().getSema().OpenMP().EndOpenMPDSABlock(Res.get());
+  return Res;
+}
+
 template <typename Derived>
 StmtResult
 TreeTransform<Derived>::TransformOMPFuseDirective(OMPFuseDirective *D) {
@@ -10717,6 +10736,20 @@ TreeTransform<Derived>::TransformOMPPartialClause(OMPPartialClause *C) {
                                  C->getEndLoc());
 }
 
+template <typename Derived>
+OMPClause *TreeTransform<Derived>::TransformOMPDepthClause(OMPDepthClause *C) {
+  ExprResult T = getDerived().TransformExpr(C->getDepth());
+  if (T.isInvalid())
+    return nullptr;
+  Expr *Depth = T.get();
+  bool Changed = Depth != C->getDepth();
+
+  if (!Changed && !getDerived().AlwaysRebuild())
+    return C;
+  return RebuildOMPDepthClause(Depth, C->getBeginLoc(), C->getLParenLoc(),
+                               C->getEndLoc());
+}
+
 template <typename Derived>
 OMPClause *
 TreeTransform<Derived>::TransformOMPLoopRangeClause(OMPLoopRangeClause *C) {

>From deafb222a7d251069c68a44fa267e58e750f1064 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:08:12 -0400
Subject: [PATCH 04/18] serialize flatten

---
 clang/include/clang/Serialization/ASTBitCodes.h |  1 +
 clang/lib/Serialization/ASTReaderStmt.cpp       | 11 +++++++++++
 clang/lib/Serialization/ASTWriterStmt.cpp       |  5 +++++
 3 files changed, 17 insertions(+)

diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h
index 279380de2f7fe..93a0e0a454b68 100644
--- a/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/clang/include/clang/Serialization/ASTBitCodes.h
@@ -1970,6 +1970,7 @@ enum StmtCode {
   STMT_OMP_REVERSE_DIRECTIVE,
   STMT_OMP_SPLIT_DIRECTIVE,
   STMT_OMP_INTERCHANGE_DIRECTIVE,
+  STMT_OMP_FLATTEN_DIRECTIVE,
   STMT_OMP_FUSE_DIRECTIVE,
   STMT_OMP_FOR_DIRECTIVE,
   STMT_OMP_FOR_SIMD_DIRECTIVE,
diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp
index 02ccf8d4d41c2..8c2debd6cdf32 100644
--- a/clang/lib/Serialization/ASTReaderStmt.cpp
+++ b/clang/lib/Serialization/ASTReaderStmt.cpp
@@ -2537,6 +2537,10 @@ void ASTStmtReader::VisitOMPInterchangeDirective(OMPInterchangeDirective *D) {
   VisitOMPCanonicalLoopNestTransformationDirective(D);
 }
 
+void ASTStmtReader::VisitOMPFlattenDirective(OMPFlattenDirective *D) {
+  VisitOMPCanonicalLoopNestTransformationDirective(D);
+}
+
 void ASTStmtReader::VisitOMPSplitDirective(OMPSplitDirective *D) {
   VisitOMPCanonicalLoopNestTransformationDirective(D);
 }
@@ -3719,6 +3723,13 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) {
       break;
     }
 
+    case STMT_OMP_FLATTEN_DIRECTIVE: {
+      unsigned NumLoops = Record[ASTStmtReader::NumStmtFields];
+      unsigned NumClauses = Record[ASTStmtReader::NumStmtFields + 1];
+      S = OMPFlattenDirective::CreateEmpty(Context, NumClauses, NumLoops);
+      break;
+    }
+
     case STMT_OMP_FOR_DIRECTIVE: {
       unsigned CollapsedNum = Record[ASTStmtReader::NumStmtFields];
       unsigned NumClauses = Record[ASTStmtReader::NumStmtFields + 1];
diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp
index 4e9eadd730a56..dfe28a9b915e3 100644
--- a/clang/lib/Serialization/ASTWriterStmt.cpp
+++ b/clang/lib/Serialization/ASTWriterStmt.cpp
@@ -2547,6 +2547,11 @@ void ASTStmtWriter::VisitOMPInterchangeDirective(OMPInterchangeDirective *D) {
   Code = serialization::STMT_OMP_INTERCHANGE_DIRECTIVE;
 }
 
+void ASTStmtWriter::VisitOMPFlattenDirective(OMPFlattenDirective *D) {
+  VisitOMPCanonicalLoopNestTransformationDirective(D);
+  Code = serialization::STMT_OMP_FLATTEN_DIRECTIVE;
+}
+
 void ASTStmtWriter::VisitOMPSplitDirective(OMPSplitDirective *D) {
   VisitOMPCanonicalLoopNestTransformationDirective(D);
   Code = serialization::STMT_OMP_SPLIT_DIRECTIVE;

>From 3e2dfeb99230057b7cb9c484182ae33befbe1e41 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:09:03 -0400
Subject: [PATCH 05/18] serialize depth

---
 clang/lib/Serialization/ASTReader.cpp | 8 ++++++++
 clang/lib/Serialization/ASTWriter.cpp | 5 +++++
 2 files changed, 13 insertions(+)

diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index f8a6a38bb9b5c..4f51d79a52105 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -11495,6 +11495,9 @@ OMPClause *OMPClauseReader::readClause() {
   case llvm::omp::OMPC_partial:
     C = OMPPartialClause::CreateEmpty(Context);
     break;
+  case llvm::omp::OMPC_depth:
+    C = OMPDepthClause::CreateEmpty(Context);
+    break;
   case llvm::omp::OMPC_looprange:
     C = OMPLoopRangeClause::CreateEmpty(Context);
     break;
@@ -11924,6 +11927,11 @@ void OMPClauseReader::VisitOMPPartialClause(OMPPartialClause *C) {
   C->setLParenLoc(Record.readSourceLocation());
 }
 
+void OMPClauseReader::VisitOMPDepthClause(OMPDepthClause *C) {
+  C->setDepth(Record.readSubExpr());
+  C->setLParenLoc(Record.readSourceLocation());
+}
+
 void OMPClauseReader::VisitOMPLoopRangeClause(OMPLoopRangeClause *C) {
   C->setFirst(Record.readSubExpr());
   C->setCount(Record.readSubExpr());
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index 357a7f7e95fa0..cbfeaa9e6638a 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -8106,6 +8106,11 @@ void OMPClauseWriter::VisitOMPPartialClause(OMPPartialClause *C) {
   Record.AddSourceLocation(C->getLParenLoc());
 }
 
+void OMPClauseWriter::VisitOMPDepthClause(OMPDepthClause *C) {
+  Record.AddStmt(C->getDepth());
+  Record.AddSourceLocation(C->getLParenLoc());
+}
+
 void OMPClauseWriter::VisitOMPLoopRangeClause(OMPLoopRangeClause *C) {
   Record.AddStmt(C->getFirst());
   Record.AddStmt(C->getCount());

>From 4ae746ca1eeaf11db10e859572a4500029d01cce Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:17:29 -0400
Subject: [PATCH 06/18] add analysis traversal support for flatten directive

---
 clang/lib/Sema/SemaExceptionSpec.cpp         | 1 +
 clang/lib/StaticAnalyzer/Core/ExprEngine.cpp | 1 +
 2 files changed, 2 insertions(+)

diff --git a/clang/lib/Sema/SemaExceptionSpec.cpp b/clang/lib/Sema/SemaExceptionSpec.cpp
index 40d530a1f3925..995b5b6010978 100644
--- a/clang/lib/Sema/SemaExceptionSpec.cpp
+++ b/clang/lib/Sema/SemaExceptionSpec.cpp
@@ -1508,6 +1508,7 @@ CanThrowResult Sema::canThrow(const Stmt *S) {
   case Stmt::OMPUnrollDirectiveClass:
   case Stmt::OMPReverseDirectiveClass:
   case Stmt::OMPInterchangeDirectiveClass:
+  case Stmt::OMPFlattenDirectiveClass:
   case Stmt::OMPSplitDirectiveClass:
   case Stmt::OMPFuseDirectiveClass:
   case Stmt::OMPSingleDirectiveClass:
diff --git a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
index cfb294736ee02..3f666be00cc24 100644
--- a/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
+++ b/clang/lib/StaticAnalyzer/Core/ExprEngine.cpp
@@ -1783,6 +1783,7 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
     case Stmt::OMPStripeDirectiveClass:
     case Stmt::OMPTileDirectiveClass:
     case Stmt::OMPInterchangeDirectiveClass:
+    case Stmt::OMPFlattenDirectiveClass:
     case Stmt::OMPSplitDirectiveClass:
     case Stmt::OMPFuseDirectiveClass:
     case Stmt::OMPInteropDirectiveClass:

>From 415eba88ad0af63c552b4c5c2eabd660bcb29c02 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:21:25 -0400
Subject: [PATCH 07/18] codegen flatten directive

---
 clang/lib/CodeGen/CGStmt.cpp        | 3 +++
 clang/lib/CodeGen/CGStmtOpenMP.cpp  | 8 ++++++++
 clang/lib/CodeGen/CodeGenFunction.h | 1 +
 3 files changed, 12 insertions(+)

diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index 232094777f233..27f48cb63b553 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -236,6 +236,9 @@ void CodeGenFunction::EmitStmt(const Stmt *S, ArrayRef<const Attr *> Attrs) {
   case Stmt::OMPInterchangeDirectiveClass:
     EmitOMPInterchangeDirective(cast<OMPInterchangeDirective>(*S));
     break;
+  case Stmt::OMPFlattenDirectiveClass:
+    EmitOMPFlattenDirective(cast<OMPFlattenDirective>(*S));
+    break;
   case Stmt::OMPFuseDirectiveClass:
     EmitOMPFuseDirective(cast<OMPFuseDirective>(*S));
     break;
diff --git a/clang/lib/CodeGen/CGStmtOpenMP.cpp b/clang/lib/CodeGen/CGStmtOpenMP.cpp
index 95fd6694437fe..25da3430a23fe 100644
--- a/clang/lib/CodeGen/CGStmtOpenMP.cpp
+++ b/clang/lib/CodeGen/CGStmtOpenMP.cpp
@@ -218,6 +218,8 @@ class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
     } else if (const auto *Interchange =
                    dyn_cast<OMPInterchangeDirective>(&S)) {
       PreInits = Interchange->getPreInits();
+    } else if (const auto *Flatten = dyn_cast<OMPFlattenDirective>(&S)) {
+      PreInits = Flatten->getPreInits();
     } else {
       llvm_unreachable("Unknown loop-based directive kind.");
     }
@@ -3248,6 +3250,12 @@ void CodeGenFunction::EmitOMPInterchangeDirective(
   EmitStmt(S.getTransformedStmt());
 }
 
+void CodeGenFunction::EmitOMPFlattenDirective(const OMPFlattenDirective &S) {
+  // Emit the de-sugared statement.
+  OMPTransformDirectiveScopeRAII FlattenScope(*this, &S);
+  EmitStmt(S.getTransformedStmt());
+}
+
 void CodeGenFunction::EmitOMPFuseDirective(const OMPFuseDirective &S) {
   // Emit the de-sugared statement
   OMPTransformDirectiveScopeRAII FuseScope(*this, &S);
diff --git a/clang/lib/CodeGen/CodeGenFunction.h b/clang/lib/CodeGen/CodeGenFunction.h
index 6d0718c243812..eea0163422cd3 100644
--- a/clang/lib/CodeGen/CodeGenFunction.h
+++ b/clang/lib/CodeGen/CodeGenFunction.h
@@ -3934,6 +3934,7 @@ class CodeGenFunction : public CodeGenTypeCache {
   void EmitOMPReverseDirective(const OMPReverseDirective &S);
   void EmitOMPSplitDirective(const OMPSplitDirective &S);
   void EmitOMPInterchangeDirective(const OMPInterchangeDirective &S);
+  void EmitOMPFlattenDirective(const OMPFlattenDirective &S);
   void EmitOMPFuseDirective(const OMPFuseDirective &S);
   void EmitOMPForDirective(const OMPForDirective &S);
   void EmitOMPForSimdDirective(const OMPForSimdDirective &S);

>From 814e2e9fe9bf4e28e4557999fbc16beec144cc39 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:24:42 -0400
Subject: [PATCH 08/18] add NYI CIR stub

---
 clang/lib/CIR/CodeGen/CIRGenFunction.h     | 1 +
 clang/lib/CIR/CodeGen/CIRGenStmt.cpp       | 2 ++
 clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp | 5 +++++
 3 files changed, 8 insertions(+)

diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.h b/clang/lib/CIR/CodeGen/CIRGenFunction.h
index 322355fde3957..8374583ffd4d0 100644
--- a/clang/lib/CIR/CodeGen/CIRGenFunction.h
+++ b/clang/lib/CIR/CodeGen/CIRGenFunction.h
@@ -2559,6 +2559,7 @@ class CIRGenFunction : public CIRGenTypeCache {
   mlir::LogicalResult emitOMPSplitDirective(const OMPSplitDirective &s);
   mlir::LogicalResult
   emitOMPInterchangeDirective(const OMPInterchangeDirective &s);
+  mlir::LogicalResult emitOMPFlattenDirective(const OMPFlattenDirective &s);
   mlir::LogicalResult emitOMPAssumeDirective(const OMPAssumeDirective &s);
   mlir::LogicalResult emitOMPMaskedDirective(const OMPMaskedDirective &s);
   mlir::LogicalResult emitOMPStripeDirective(const OMPStripeDirective &s);
diff --git a/clang/lib/CIR/CodeGen/CIRGenStmt.cpp b/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
index d3acac5801e74..33d425b22d61a 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmt.cpp
@@ -408,6 +408,8 @@ mlir::LogicalResult CIRGenFunction::emitStmt(const Stmt *s,
     return emitOMPSplitDirective(cast<OMPSplitDirective>(*s));
   case Stmt::OMPInterchangeDirectiveClass:
     return emitOMPInterchangeDirective(cast<OMPInterchangeDirective>(*s));
+  case Stmt::OMPFlattenDirectiveClass:
+    return emitOMPFlattenDirective(cast<OMPFlattenDirective>(*s));
   case Stmt::OMPAssumeDirectiveClass:
     return emitOMPAssumeDirective(cast<OMPAssumeDirective>(*s));
   case Stmt::OMPMaskedDirectiveClass:
diff --git a/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp b/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp
index 17a1fb8090f5c..4fffd17b25a16 100644
--- a/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp
+++ b/clang/lib/CIR/CodeGen/CIRGenStmtOpenMP.cpp
@@ -587,6 +587,11 @@ CIRGenFunction::emitOMPInterchangeDirective(const OMPInterchangeDirective &s) {
   return mlir::failure();
 }
 mlir::LogicalResult
+CIRGenFunction::emitOMPFlattenDirective(const OMPFlattenDirective &s) {
+  getCIRGenModule().errorNYI(s.getSourceRange(), "OpenMP OMPFlattenDirective");
+  return mlir::failure();
+}
+mlir::LogicalResult
 CIRGenFunction::emitOMPAssumeDirective(const OMPAssumeDirective &s) {
   getCIRGenModule().errorNYI(s.getSourceRange(), "OpenMP OMPAssumeDirective");
   return mlir::failure();

>From 0027ec6ab982a83766bd65f3e1bc45d9a671fc62 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 8 Jul 2026 16:26:31 -0400
Subject: [PATCH 09/18] add libclang traversal support for depth clause

---
 clang/tools/libclang/CIndex.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index ac2fad38a1348..f8ab63e3e0dae 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2378,6 +2378,10 @@ void OMPClauseEnqueue::VisitOMPLoopRangeClause(const OMPLoopRangeClause *C) {
   Visitor->AddStmt(C->getCount());
 }
 
+void OMPClauseEnqueue::VisitOMPDepthClause(const OMPDepthClause *C) {
+  Visitor->AddStmt(C->getDepth());
+}
+
 void OMPClauseEnqueue::VisitOMPAllocatorClause(const OMPAllocatorClause *C) {
   Visitor->AddStmt(C->getAllocator());
 }

>From f5ac5f740c414310c8482f652f1bed936ae7cda3 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Fri, 10 Jul 2026 06:54:16 -0400
Subject: [PATCH 10/18] sema

---
 clang/include/clang/Sema/SemaOpenMP.h |  10 +
 clang/lib/Sema/SemaOpenMP.cpp         | 295 ++++++++++++++++++++++++++
 2 files changed, 305 insertions(+)

diff --git a/clang/include/clang/Sema/SemaOpenMP.h b/clang/include/clang/Sema/SemaOpenMP.h
index 3621ce96b8724..7b086388416a0 100644
--- a/clang/include/clang/Sema/SemaOpenMP.h
+++ b/clang/include/clang/Sema/SemaOpenMP.h
@@ -470,6 +470,12 @@ class SemaOpenMP : public SemaBase {
                                              SourceLocation StartLoc,
                                              SourceLocation EndLoc);
 
+  /// Called on well-formed '#pragma omp flatten' after parsing of its
+  /// clauses and the associated statement.
+  StmtResult ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
+                                         Stmt *AStmt, SourceLocation StartLoc,
+                                         SourceLocation EndLoc);
+
   /// Called on well-formed '#pragma omp fuse' after parsing of its
   /// clauses and the associated statement.
   StmtResult ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
@@ -935,6 +941,10 @@ class SemaOpenMP : public SemaBase {
   OMPClause *ActOnOpenMPPartialClause(Expr *FactorExpr, SourceLocation StartLoc,
                                       SourceLocation LParenLoc,
                                       SourceLocation EndLoc);
+  /// Called on well-formed 'depth' clause.
+  OMPClause *ActOnOpenMPDepthClause(Expr *DepthExpr, SourceLocation StartLoc,
+                                    SourceLocation LParenLoc,
+                                    SourceLocation EndLoc);
   /// Called on well-formed 'collapse' clause.
   OMPClause *ActOnOpenMPCollapseClause(Expr *NumForLoops,
                                        SourceLocation StartLoc,
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index 247a4a6ad9271..fcc256dec3c88 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -4629,6 +4629,7 @@ void SemaOpenMP::ActOnOpenMPRegionStart(OpenMPDirectiveKind DKind,
   case OMPD_reverse:
   case OMPD_split:
   case OMPD_interchange:
+  case OMPD_flatten:
   case OMPD_fuse:
   case OMPD_assume:
     break;
@@ -6475,6 +6476,10 @@ StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
     Res = ActOnOpenMPInterchangeDirective(ClausesWithImplicit, AStmt, StartLoc,
                                           EndLoc);
     break;
+  case OMPD_flatten:
+    Res = ActOnOpenMPFlattenDirective(ClausesWithImplicit, AStmt, StartLoc,
+                                      EndLoc);
+    break;
   case OMPD_fuse:
     Res =
         ActOnOpenMPFuseDirective(ClausesWithImplicit, AStmt, StartLoc, EndLoc);
@@ -16390,6 +16395,277 @@ StmtResult SemaOpenMP::ActOnOpenMPInterchangeDirective(
                                          buildPreInits(Context, PreInits));
 }
 
+/// Counts the perfectly nested canonical loop depth at \p AStmt. Returns
+/// std::nullopt if a loop-generating transformation prevents a static count.
+static std::optional<unsigned> getCanonicalLoopNestDepth(Stmt *AStmt) {
+  unsigned Depth = 0;
+  Stmt *CurStmt = AStmt ? AStmt->IgnoreContainers() : nullptr;
+  while (CurStmt) {
+    if (isa<OMPLoopTransformationDirective>(CurStmt))
+      return std::nullopt;
+    if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(CurStmt))
+      CurStmt = CanonLoop->getLoopStmt();
+    Stmt *Body = nullptr;
+    if (auto *For = dyn_cast<ForStmt>(CurStmt))
+      Body = For->getBody();
+    else if (auto *RangeFor = dyn_cast<CXXForRangeStmt>(CurStmt))
+      Body = RangeFor->getBody();
+    else
+      break;
+    ++Depth;
+    CurStmt = Body ? Body->IgnoreContainers() : nullptr;
+  }
+  return Depth;
+}
+
+StmtResult
+SemaOpenMP::ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
+                                        Stmt *AStmt, SourceLocation StartLoc,
+                                        SourceLocation EndLoc) {
+  ASTContext &Context = getASTContext();
+  DeclContext *CurContext = SemaRef.CurContext;
+  Scope *CurScope = SemaRef.getCurScope();
+
+  // Empty statement should only be possible if there already was an error.
+  if (!AStmt)
+    return StmtError();
+
+  // flatten without 'depth' clause combines two loops; 'depth(k)' selects k.
+  unsigned NumLoops = 2;
+  bool DepthIsValueDependent = false;
+  const auto *DepthClause =
+      OMPExecutableDirective::getSingleClause<OMPDepthClause>(Clauses);
+  if (DepthClause) {
+    Expr *DepthExpr = DepthClause->getDepth();
+    if (DepthExpr && DepthExpr->isValueDependent()) {
+      DepthIsValueDependent = true;
+    } else if (DepthExpr) {
+      Expr::EvalResult EvalResult;
+      if (DepthExpr->EvaluateAsInt(EvalResult, Context))
+        NumLoops = EvalResult.Val.getInt().getZExtValue();
+    }
+  }
+
+  // Report "expected k for loops, but found only n" when depth exceeds the
+  // perfect nest (same form as 'collapse'); skip if depth or nest is unknown.
+  if (DepthClause && !DepthIsValueDependent) {
+    if (std::optional<unsigned> NestDepth = getCanonicalLoopNestDepth(AStmt);
+        NestDepth && NumLoops > *NestDepth) {
+      Diag(AStmt->getBeginLoc(), diag::err_omp_not_for)
+          << /*expected N for loops form=*/1
+          << getOpenMPDirectiveName(OMPD_flatten) << NumLoops
+          << (*NestDepth > 0) << *NestDepth;
+      return StmtError();
+    }
+  }
+
+  // Defer when 'depth' is value-dependent (concrete k unknown until
+  // instantiation).
+  if (DepthIsValueDependent)
+    return OMPFlattenDirective::Create(Context, StartLoc, EndLoc, Clauses,
+                                       NumLoops, AStmt, nullptr, nullptr);
+
+  // Verify and diagnose loop nest.
+  SmallVector<OMPLoopBasedDirective::HelperExprs, 4> LoopHelpers(NumLoops);
+  Stmt *Body = nullptr;
+  SmallVector<SmallVector<Stmt *>, 4> OriginalInits;
+  if (!checkTransformableLoopNest(OMPD_flatten, AStmt, NumLoops, LoopHelpers,
+                                  Body, OriginalInits))
+    return StmtError();
+
+  // Delay flattening to when template is completely instantiated.
+  if (CurContext->isDependentContext())
+    return OMPFlattenDirective::Create(Context, StartLoc, EndLoc, Clauses,
+                                       NumLoops, AStmt, nullptr, nullptr);
+
+  assert(LoopHelpers.size() == NumLoops &&
+         "Expecting loop iteration space dimensionality to match number of "
+         "affected loops");
+  assert(OriginalInits.size() == NumLoops &&
+         "Expecting loop iteration space dimensionality to match number of "
+         "affected loops");
+
+  // Find the affected loops.
+  SmallVector<Stmt *> LoopStmts(NumLoops, nullptr);
+  collectLoopStmts(AStmt, LoopStmts);
+
+  // Collect pre-init statements in outer-to-inner order.
+  SmallVector<Stmt *> PreInits;
+  for (auto I : llvm::seq<int>(NumLoops)) {
+    OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
+    assert(LoopHelper.Counters.size() == 1 &&
+           "Single-dimensional loop iteration space expected");
+    addLoopPreInits(Context, LoopHelper, LoopStmts[I], OriginalInits[I],
+                    PreInits);
+  }
+
+  CaptureVars CopyTransformer(SemaRef);
+  auto MakeNumIterations = [&CopyTransformer,
+                            &LoopHelpers](unsigned I) -> Expr * {
+    return AssertSuccess(
+        CopyTransformer.TransformExpr(LoopHelpers[I].NumIterations));
+  };
+
+  OMPLoopBasedDirective::HelperExprs &OutermostHelper = LoopHelpers[0];
+  auto *OutermostCntVar = cast<DeclRefExpr>(OutermostHelper.Counters.front());
+  SourceLocation OrigVarLoc = OutermostCntVar->getExprLoc();
+  SourceLocation OrigVarLocBegin = OutermostCntVar->getBeginLoc();
+  SourceLocation OrigVarLocEnd = OutermostCntVar->getEndLoc();
+  SourceLocation CondLoc = OutermostHelper.Cond->getExprLoc();
+
+  // Product of trip counts; mirror 'collapse' IV-width selection to avoid
+  // overflow when several counts are multiplied.
+  auto BuildTripCount = [&](unsigned Bits) -> ExprResult {
+    ExprResult Product;
+    for (unsigned I = 0; I < NumLoops; ++I) {
+      ExprResult N = widenIterationCount(Bits, MakeNumIterations(I), SemaRef);
+      if (!N.isUsable())
+        return ExprError();
+      if (I == 0)
+        Product = N;
+      else
+        Product = SemaRef.BuildBinOp(CurScope, CondLoc, BO_Mul, Product.get(),
+                                     N.get());
+      if (!Product.isUsable())
+        return ExprError();
+    }
+    return Product;
+  };
+
+  bool AllCountsLessThan32Bits = true;
+  for (unsigned I = 0; I < NumLoops; ++I)
+    AllCountsLessThan32Bits &=
+        Context.getTypeSize(LoopHelpers[I].NumIterations->getType()) < 32;
+
+  ExprResult TripCount32 = BuildTripCount(/*Bits=*/32);
+  ExprResult TripCount64 = BuildTripCount(/*Bits=*/64);
+  if (!TripCount32.isUsable() || !TripCount64.isUsable())
+    return StmtError();
+
+  ExprResult TripCount = TripCount64;
+  if (Context.getTypeSize(TripCount32.get()->getType()) == 32 &&
+      (AllCountsLessThan32Bits || NumLoops == 1 ||
+       fitsInto(
+           /*Bits=*/32,
+           TripCount32.get()->getType()->hasSignedIntegerRepresentation(),
+           TripCount64.get(), SemaRef)))
+    TripCount = TripCount32;
+
+  QualType IVTy = TripCount.get()->getType();
+  uint64_t IVWidth = Context.getTypeSize(IVTy);
+
+  auto MakeNumIterationsInIVTy = [&](unsigned I) -> Expr * {
+    return AssertSuccess(SemaRef.PerformImplicitConversion(
+        MakeNumIterations(I), IVTy, AssignmentAction::Converting,
+        /*AllowExplicit=*/true));
+  };
+
+  // \code{.cpp}
+  //   for (auto .flatten.iv = 0; .flatten.iv < n0 * n1 * ...; ++.flatten.iv) {
+  //     .flatten.iv.0 = .flatten.iv / (n1 * ...);
+  //     i0 = ...;                                        // Updates[0]
+  //     .flatten.iv.1 = (.flatten.iv / ...) % n1;
+  //     i1 = ...;                                        // Updates[1]
+  //     ...
+  //     body(i0, i1, ...);
+  //   }
+  // \endcode
+  SmallString<64> FlattenedIVName(".flatten.iv");
+  VarDecl *FlattenedIVDecl = buildVarDecl(SemaRef, {}, IVTy, FlattenedIVName,
+                                          nullptr, OutermostCntVar);
+  auto MakeFlattenedRef = [&SemaRef = this->SemaRef, FlattenedIVDecl, IVTy,
+                           OrigVarLoc]() {
+    return buildDeclRefExpr(SemaRef, FlattenedIVDecl, IVTy, OrigVarLoc);
+  };
+
+  // For init-statement:
+  // \code{.cpp}
+  //   auto .flatten.iv = 0;
+  // \endcode
+  auto *Zero = IntegerLiteral::Create(Context, llvm::APInt::getZero(IVWidth),
+                                      IVTy, OrigVarLoc);
+  SemaRef.AddInitializerToDecl(FlattenedIVDecl, Zero, /*DirectInit=*/false);
+  StmtResult Init = new (Context)
+      DeclStmt(DeclGroupRef(FlattenedIVDecl), OrigVarLocBegin, OrigVarLocEnd);
+  if (!Init.isUsable())
+    return StmtError();
+
+  // For cond-expression:
+  // \code{.cpp}
+  //   .flatten.iv < n0 * n1 * ... * n(k-1)
+  // \endcode
+  ExprResult Cond = SemaRef.BuildBinOp(CurScope, CondLoc, BO_LT,
+                                       MakeFlattenedRef(), TripCount.get());
+  if (!Cond.isUsable())
+    return StmtError();
+
+  // For incr-statement:
+  // \code{.cpp}
+  //   ++.flatten.iv
+  // \endcode
+  ExprResult Incr =
+      SemaRef.BuildUnaryOp(CurScope, OutermostHelper.Inc->getExprLoc(),
+                           UO_PreInc, MakeFlattenedRef());
+  if (!Incr.isUsable())
+    return StmtError();
+
+  // Recover each logical iteration counter via mixed-radix div/mod; reuse the
+  // iteration variables from checkOpenMPLoop so Updates compute user counters.
+  SmallVector<Stmt *, 8> BodyStmts;
+  for (unsigned I = 0; I < NumLoops; ++I) {
+    OMPLoopBasedDirective::HelperExprs &LoopHelper = LoopHelpers[I];
+    auto *IVRef = cast<DeclRefExpr>(LoopHelper.IterationVarRef);
+    auto *IVDecl = cast<VarDecl>(IVRef->getDecl());
+    std::string IVName = (".flatten.iv." + llvm::Twine(I)).str();
+    IVDecl->setDeclName(&SemaRef.PP.getIdentifierTable().get(IVName));
+
+    ExprResult Value = MakeFlattenedRef();
+    if (I + 1 < NumLoops) {
+      ExprResult Divisor = MakeNumIterationsInIVTy(I + 1);
+      for (unsigned J = I + 2; J < NumLoops; ++J) {
+        Divisor = SemaRef.BuildBinOp(CurScope, OrigVarLoc, BO_Mul,
+                                     Divisor.get(), MakeNumIterationsInIVTy(J));
+        if (!Divisor.isUsable())
+          return StmtError();
+      }
+      Value = SemaRef.BuildBinOp(CurScope, OrigVarLoc, BO_Div, Value.get(),
+                                 Divisor.get());
+      if (!Value.isUsable())
+        return StmtError();
+    }
+    if (I > 0) {
+      Value = SemaRef.BuildBinOp(CurScope, OrigVarLoc, BO_Rem, Value.get(),
+                                 MakeNumIterationsInIVTy(I));
+      if (!Value.isUsable())
+        return StmtError();
+    }
+
+    SemaRef.AddInitializerToDecl(IVDecl, Value.get(), /*DirectInit=*/false);
+    StmtResult IVStmt = new (Context)
+        DeclStmt(DeclGroupRef(IVDecl), OrigVarLocBegin, OrigVarLocEnd);
+    if (!IVStmt.isUsable())
+      return StmtError();
+
+    BodyStmts.push_back(IVStmt.get());
+    llvm::append_range(BodyStmts, LoopHelper.Updates);
+    if (auto *CXXFor = dyn_cast<CXXForRangeStmt>(LoopStmts[I]))
+      BodyStmts.push_back(CXXFor->getLoopVarStmt());
+  }
+  BodyStmts.push_back(Body);
+  auto *FlattenedBody =
+      CompoundStmt::Create(Context, BodyStmts, FPOptionsOverride(),
+                           Body->getBeginLoc(), Body->getEndLoc());
+
+  auto *FlattenedFor = new (Context) ForStmt(
+      Context, Init.get(), Cond.get(), nullptr, Incr.get(), FlattenedBody,
+      OutermostHelper.Init->getBeginLoc(), OutermostHelper.Init->getBeginLoc(),
+      OutermostHelper.Inc->getEndLoc());
+
+  return OMPFlattenDirective::Create(Context, StartLoc, EndLoc, Clauses,
+                                     NumLoops, AStmt, FlattenedFor,
+                                     buildPreInits(Context, PreInits));
+}
+
 StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
                                                 Stmt *AStmt,
                                                 SourceLocation StartLoc,
@@ -16920,6 +17196,9 @@ OMPClause *SemaOpenMP::ActOnOpenMPSingleExprClause(OpenMPClauseKind Kind,
   case OMPC_partial:
     Res = ActOnOpenMPPartialClause(Expr, StartLoc, LParenLoc, EndLoc);
     break;
+  case OMPC_depth:
+    Res = ActOnOpenMPDepthClause(Expr, StartLoc, LParenLoc, EndLoc);
+    break;
   case OMPC_message:
     Res = ActOnOpenMPMessageClause(Expr, StartLoc, LParenLoc, EndLoc);
     break;
@@ -18241,6 +18520,22 @@ OMPClause *SemaOpenMP::ActOnOpenMPPartialClause(Expr *FactorExpr,
                                   FactorExpr);
 }
 
+OMPClause *SemaOpenMP::ActOnOpenMPDepthClause(Expr *DepthExpr,
+                                              SourceLocation StartLoc,
+                                              SourceLocation LParenLoc,
+                                              SourceLocation EndLoc) {
+  // The depth-expr must be a positive integer constant expression and
+  // not greater than the number of loops in the associated loop nest.
+  ExprResult DepthResult = VerifyPositiveIntegerConstantInClause(
+      DepthExpr, OMPC_depth, /*StrictlyPositive=*/true);
+  if (DepthResult.isInvalid())
+    return nullptr;
+  DepthExpr = DepthResult.get();
+
+  return OMPDepthClause::Create(getASTContext(), StartLoc, LParenLoc, EndLoc,
+                                DepthExpr);
+}
+
 OMPClause *SemaOpenMP::ActOnOpenMPLoopRangeClause(
     Expr *First, Expr *Count, SourceLocation StartLoc, SourceLocation LParenLoc,
     SourceLocation FirstLoc, SourceLocation CountLoc, SourceLocation EndLoc) {

>From 54be7347979638725cd4b6c421d11ec1cd618c43 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Mon, 27 Jul 2026 07:03:06 -0400
Subject: [PATCH 11/18] add libclang traversal support for flatten directive

---
 clang/tools/libclang/CIndex.cpp   | 7 +++++++
 clang/tools/libclang/CXCursor.cpp | 3 +++
 2 files changed, 10 insertions(+)

diff --git a/clang/tools/libclang/CIndex.cpp b/clang/tools/libclang/CIndex.cpp
index f8ab63e3e0dae..9eb0ec8c21158 100644
--- a/clang/tools/libclang/CIndex.cpp
+++ b/clang/tools/libclang/CIndex.cpp
@@ -2160,6 +2160,7 @@ class EnqueueVisitor : public ConstStmtVisitor<EnqueueVisitor, void>,
   void VisitOMPUnrollDirective(const OMPUnrollDirective *D);
   void VisitOMPReverseDirective(const OMPReverseDirective *D);
   void VisitOMPInterchangeDirective(const OMPInterchangeDirective *D);
+  void VisitOMPFlattenDirective(const OMPFlattenDirective *D);
   void VisitOMPCanonicalLoopSequenceTransformationDirective(
       const OMPCanonicalLoopSequenceTransformationDirective *D);
   void VisitOMPFuseDirective(const OMPFuseDirective *D);
@@ -3358,6 +3359,10 @@ void EnqueueVisitor::VisitOMPInterchangeDirective(
   VisitOMPCanonicalLoopNestTransformationDirective(D);
 }
 
+void EnqueueVisitor::VisitOMPFlattenDirective(const OMPFlattenDirective *D) {
+  VisitOMPCanonicalLoopNestTransformationDirective(D);
+}
+
 void EnqueueVisitor::VisitOMPCanonicalLoopSequenceTransformationDirective(
     const OMPCanonicalLoopSequenceTransformationDirective *D) {
   VisitOMPExecutableDirective(D);
@@ -6329,6 +6334,8 @@ CXString clang_getCursorKindSpelling(enum CXCursorKind Kind) {
     return cxstring::createRef("OMPReverseDirective");
   case CXCursor_OMPInterchangeDirective:
     return cxstring::createRef("OMPInterchangeDirective");
+  case CXCursor_OMPFlattenDirective:
+    return cxstring::createRef("OMPFlattenDirective");
   case CXCursor_OMPFuseDirective:
     return cxstring::createRef("OMPFuseDirective");
   case CXCursor_OMPSplitDirective:
diff --git a/clang/tools/libclang/CXCursor.cpp b/clang/tools/libclang/CXCursor.cpp
index b2feed578fc0f..97bbfd36558a9 100644
--- a/clang/tools/libclang/CXCursor.cpp
+++ b/clang/tools/libclang/CXCursor.cpp
@@ -703,6 +703,9 @@ CXCursor cxcursor::MakeCXCursor(const Stmt *S, const Decl *Parent,
   case Stmt::OMPInterchangeDirectiveClass:
     K = CXCursor_OMPInterchangeDirective;
     break;
+  case Stmt::OMPFlattenDirectiveClass:
+    K = CXCursor_OMPFlattenDirective;
+    break;
   case Stmt::OMPFuseDirectiveClass:
     K = CXCursor_OMPFuseDirective;
     break;

>From cdedb1045baf373937b0da858624a6ace14a3d21 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Mon, 27 Jul 2026 07:06:36 -0400
Subject: [PATCH 12/18] bindings

---
 clang/bindings/python/clang/cindex.py | 3 +++
 clang/include/clang-c/Index.h         | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/clang/bindings/python/clang/cindex.py b/clang/bindings/python/clang/cindex.py
index 24b737139dba8..ccad140e8c861 100644
--- a/clang/bindings/python/clang/cindex.py
+++ b/clang/bindings/python/clang/cindex.py
@@ -1456,6 +1456,9 @@ def is_unexposed(self):
     # OpenMP split directive.
     OMP_SPLIT_DIRECTIVE = 312
 
+    # OpenMP flatten directive.
+    OMP_FLATTEN_DIRECTIVE = 313
+
     # OpenACC Compute Construct.
     OPEN_ACC_COMPUTE_DIRECTIVE = 320
 
diff --git a/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h
index 8427236e0b444..f561fb5724897 100644
--- a/clang/include/clang-c/Index.h
+++ b/clang/include/clang-c/Index.h
@@ -2170,6 +2170,10 @@ enum CXCursorKind {
    */
   CXCursor_OMPSplitDirective = 312,
 
+  /** OpenMP flatten directive.
+   */
+  CXCursor_OMPFlattenDirective = 313,
+
   /** OpenACC Compute Construct.
    */
   CXCursor_OpenACCComputeConstruct = 320,

>From 94dee8fe1351f74760df0fb2d9903a291a5ccf66 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 5 Aug 2026 04:46:14 -0400
Subject: [PATCH 13/18] core tests-flatten and depth

---
 clang/test/OpenMP/flatten_codegen.cpp         |  65 ++++++++
 clang/test/OpenMP/flatten_depth_codegen.cpp   | 154 ++++++++++++++++++
 .../test/transform/flatten/depth-intfor.c     |  40 +++++
 .../runtime/test/transform/flatten/intfor.c   |  33 ++++
 .../test/transform/flatten/lit.local.cfg      |   5 +
 5 files changed, 297 insertions(+)
 create mode 100644 clang/test/OpenMP/flatten_codegen.cpp
 create mode 100644 clang/test/OpenMP/flatten_depth_codegen.cpp
 create mode 100644 openmp/runtime/test/transform/flatten/depth-intfor.c
 create mode 100644 openmp/runtime/test/transform/flatten/intfor.c
 create mode 100644 openmp/runtime/test/transform/flatten/lit.local.cfg

diff --git a/clang/test/OpenMP/flatten_codegen.cpp b/clang/test/OpenMP/flatten_codegen.cpp
new file mode 100644
index 0000000000000..f77b2fb00ce9b
--- /dev/null
+++ b/clang/test/OpenMP/flatten_codegen.cpp
@@ -0,0 +1,65 @@
+// Check code generation
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=60 -emit-llvm %s -o - | FileCheck %s
+
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+extern "C" void body(int, int);
+
+// CHECK-LABEL: define {{.*}}void @foo(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK:   store i64 0, ptr %.flatten.iv,
+// CHECK:   %[[COND_IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[MUL:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   icmp slt i64 %[[COND_IV]], %[[MUL]]
+// CHECK:   %[[DIV_IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[DIV:.+]] = sdiv i64 %[[DIV_IV]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   %[[REM_IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[REM:.+]] = srem i64 %[[REM_IV]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+// CHECK:   %[[I_VAL:.+]] = load i32, ptr %i,
+// CHECK:   %[[J_VAL:.+]] = load i32, ptr %j,
+// CHECK:   call void @body(i32{{.*}} %[[I_VAL]], i32{{.*}} %[[J_VAL]])
+// CHECK: for.inc:
+// CHECK:   %[[INC_IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[NEXT:.+]] = add nsw i64 %[[INC_IV]], 1
+// CHECK:   store i64 %[[NEXT]], ptr %.flatten.iv,
+// CHECK:   br label %for.cond
+extern "C" void foo(int n, int m) {
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+
+// CHECK-LABEL: define {{.*}}void @bar(
+// CHECK:   icmp slt i32 %{{.+}}, 63
+// CHECK:   sdiv i32 %{{.+}}, 9
+// CHECK:   srem i32 %{{.+}}, 9
+// CHECK:   call void @body(
+extern "C" void bar() {
+#pragma omp flatten
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+}
+
+// CHECK-LABEL: define {{.*}}void @baz(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   icmp slt i64 %{{.+}}, 10000000000
+// CHECK:   sdiv i64 %{{.+}}, 100000
+// CHECK:   srem i64 %{{.+}}, 100000
+// CHECK:   call void @body(
+extern "C" void baz() {
+#pragma omp flatten
+  for (int i = 0; i < 100000; ++i)
+    for (int j = 0; j < 100000; ++j)
+      body(i, j);
+}
+
+#endif
diff --git a/clang/test/OpenMP/flatten_depth_codegen.cpp b/clang/test/OpenMP/flatten_depth_codegen.cpp
new file mode 100644
index 0000000000000..77408fbcbdcf1
--- /dev/null
+++ b/clang/test/OpenMP/flatten_depth_codegen.cpp
@@ -0,0 +1,154 @@
+// Check code generation for the 'depth' clause on '#pragma omp flatten'.
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=61 -emit-llvm %s -o - | FileCheck %s
+
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+extern "C" void body(int, int, int);
+
+// CHECK-LABEL: define {{.*}}void @foo3(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK:   %.flatten.iv.2 = alloca i32
+// CHECK:   %[[CIV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[T01:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   %[[TALL:.+]] = mul nsw i64 %[[T01]], %{{.+}}
+// CHECK:   icmp slt i64 %[[CIV]], %[[TALL]]
+// CHECK:   %[[D0IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[D0M:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   %[[D0:.+]] = sdiv i64 %[[D0IV]], %[[D0M]]
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   %[[D1IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[D1:.+]] = sdiv i64 %[[D1IV]], %{{.+}}
+// CHECK:   %[[R1:.+]] = srem i64 %[[D1]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+// CHECK:   %[[R2IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[R2:.+]] = srem i64 %[[R2IV]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.2
+// CHECK:   call void @body(
+extern "C" void foo3(int n, int m, int p) {
+#pragma omp flatten depth(3)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        body(i, j, k);
+}
+
+// CHECK-LABEL: define {{.*}}void @foo2(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK:   %[[CIV2:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[M:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   icmp slt i64 %[[CIV2]], %[[M]]
+// CHECK:   sdiv i64 %{{.+}}, %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   srem i64 %{{.+}}, %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+extern "C" void foo2(int n, int m) {
+#pragma omp flatten depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j, 0);
+}
+
+// CHECK-LABEL: define {{.*}}void @foo1(
+// CHECK:   %.flatten.iv = alloca i32
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK: for.body:
+// CHECK:   %[[FV:.+]] = load i32, ptr %.flatten.iv,
+// CHECK:   store i32 %[[FV]], ptr %.flatten.iv.0,
+// CHECK-NOT: sdiv
+// CHECK-NOT: srem
+// CHECK:   call void @body(
+extern "C" void foo1(int n) {
+#pragma omp flatten depth(1)
+  for (int i = 0; i < n; ++i)
+    body(i, 0, 0);
+}
+
+// CHECK-LABEL: define {{.*}}void @foo2partial(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK-NOT: %.flatten.iv.2 = alloca
+// CHECK:   %[[CIV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[M:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   icmp slt i64 %[[CIV]], %[[M]]
+// CHECK:   sdiv i64 %{{.+}}, %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   srem i64 %{{.+}}, %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+// CHECK:   br label %for.cond{{.*}}
+// CHECK: for.cond{{.*}}:
+// CHECK:   icmp slt i32 %{{.+}}, 5
+extern "C" void foo2partial(int n, int m) {
+#pragma omp flatten depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < 5; ++k)
+        body(i, j, k);
+}
+
+// A value-dependent 'depth' (the argument is a non-type template parameter) is
+// deferred until instantiation, where the concrete depth drives the flattening.
+
+// CHECK-LABEL: define {{.*}}void @_Z11tmpl_depth3ILi3EEviii(
+// CHECK:   %.flatten.iv = alloca i64
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK:   %.flatten.iv.1 = alloca i32
+// CHECK:   %.flatten.iv.2 = alloca i32
+// CHECK:   %[[CIV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[T01:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   %[[TALL:.+]] = mul nsw i64 %[[T01]], %{{.+}}
+// CHECK:   icmp slt i64 %[[CIV]], %[[TALL]]
+// CHECK:   %[[D0IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[D0M:.+]] = mul nsw i64 %{{.+}}, %{{.+}}
+// CHECK:   %[[D0:.+]] = sdiv i64 %[[D0IV]], %[[D0M]]
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.0
+// CHECK:   %[[D1IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[D1:.+]] = sdiv i64 %[[D1IV]], %{{.+}}
+// CHECK:   %[[R1:.+]] = srem i64 %[[D1]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.1
+// CHECK:   %[[R2IV:.+]] = load i64, ptr %.flatten.iv,
+// CHECK:   %[[R2:.+]] = srem i64 %[[R2IV]], %{{.+}}
+// CHECK:   store i32 %{{.+}}, ptr %.flatten.iv.2
+// CHECK:   call void @body(
+template <int D>
+void tmpl_depth3(int n, int m, int p) {
+#pragma omp flatten depth(D)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        body(i, j, k);
+}
+
+// A value-dependent 'depth' that instantiates to depth(1) on a single loop is
+// well-formed: the shallower-than-default nest is accepted because validation is
+// deferred to instantiation. The single loop needs no div/mod recovery.
+
+// CHECK-LABEL: define {{.*}}void @_Z11tmpl_depth1ILi1EEvi(
+// CHECK:   %.flatten.iv = alloca i32
+// CHECK:   %.flatten.iv.0 = alloca i32
+// CHECK: for.body:
+// CHECK:   %[[FV:.+]] = load i32, ptr %.flatten.iv,
+// CHECK:   store i32 %[[FV]], ptr %.flatten.iv.0,
+// CHECK-NOT: sdiv
+// CHECK-NOT: srem
+// CHECK:   call void @body(
+template <int D>
+void tmpl_depth1(int n) {
+#pragma omp flatten depth(D)
+  for (int i = 0; i < n; ++i)
+    body(i, 0, 0);
+}
+
+void inst() {
+  tmpl_depth3<3>(1, 1, 1);
+  tmpl_depth1<1>(1);
+}
+
+#endif
diff --git a/openmp/runtime/test/transform/flatten/depth-intfor.c b/openmp/runtime/test/transform/flatten/depth-intfor.c
new file mode 100644
index 0000000000000..7211e02e25313
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/depth-intfor.c
@@ -0,0 +1,40 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+// 'depth(3)' fully flattens a three-deep perfectly nested loop into one loop.
+// This verifies that the flattened loop preserves the exact row-major
+// visitation order of the original nest, including non-unit steps and non-zero
+// start values.
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+int main() {
+  printf("do\n");
+#pragma omp flatten depth(3)
+  for (int i = 7; i < 11; i += 2)
+    for (int j = 1; j < 4; j += 1)
+      for (int k = 0; k < 2; ++k)
+        printf("i=%d j=%d k=%d\n", i, j, k);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: i=7 j=1 k=0
+// CHECK-NEXT: i=7 j=1 k=1
+// CHECK-NEXT: i=7 j=2 k=0
+// CHECK-NEXT: i=7 j=2 k=1
+// CHECK-NEXT: i=7 j=3 k=0
+// CHECK-NEXT: i=7 j=3 k=1
+// CHECK-NEXT: i=9 j=1 k=0
+// CHECK-NEXT: i=9 j=1 k=1
+// CHECK-NEXT: i=9 j=2 k=0
+// CHECK-NEXT: i=9 j=2 k=1
+// CHECK-NEXT: i=9 j=3 k=0
+// CHECK-NEXT: i=9 j=3 k=1
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/flatten/intfor.c b/openmp/runtime/test/transform/flatten/intfor.c
new file mode 100644
index 0000000000000..f937659fbebe1
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/intfor.c
@@ -0,0 +1,33 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+int main() {
+  printf("do\n");
+#pragma omp flatten
+  for (int i = 5; i < 11; i += 2)
+    for (int j = 1; j < 7; j += 2)
+      printf("i=%d j=%d\n", i, j);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// The flattened loop visits the original iteration space in the original
+// (row-major) order: for each outer iteration the inner loop runs fully.
+// CHECK:      do
+// CHECK-NEXT: i=5 j=1
+// CHECK-NEXT: i=5 j=3
+// CHECK-NEXT: i=5 j=5
+// CHECK-NEXT: i=7 j=1
+// CHECK-NEXT: i=7 j=3
+// CHECK-NEXT: i=7 j=5
+// CHECK-NEXT: i=9 j=1
+// CHECK-NEXT: i=9 j=3
+// CHECK-NEXT: i=9 j=5
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/flatten/lit.local.cfg b/openmp/runtime/test/transform/flatten/lit.local.cfg
new file mode 100644
index 0000000000000..e05bd627df1e9
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/lit.local.cfg
@@ -0,0 +1,5 @@
+# The flatten directive's depth clause requires OpenMP 6.1.
+for i, (pattern, replacement) in enumerate(config.substitutions):
+    if pattern == "%openmp_flags":
+        config.substitutions[i] = (pattern, replacement + " -fopenmp-version=61")
+        break

>From 667fe837a39dbc711dcc5288b736aa52fbe893b6 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Wed, 5 Aug 2026 04:48:45 -0400
Subject: [PATCH 14/18] TBD

---
 clang/docs/OpenMPSupport.rst                  |   2 +-
 clang/include/clang/ASTMatchers/ASTMatchers.h |  14 ++
 clang/lib/ASTMatchers/ASTMatchersInternal.cpp |   2 +
 clang/lib/ASTMatchers/Dynamic/Registry.cpp    |   1 +
 clang/test/Index/openmp-flatten.c             |  12 ++
 clang/test/OpenMP/flatten_ast_print.cpp       | 124 ++++++++++++++++++
 clang/test/OpenMP/flatten_codegen_for.cpp     |  67 ++++++++++
 clang/test/OpenMP/flatten_depth_ast_print.cpp |  74 +++++++++++
 clang/test/OpenMP/flatten_depth_messages.cpp  |  48 +++++++
 clang/test/OpenMP/flatten_messages.cpp        |  89 +++++++++++++
 .../test/OpenMP/flatten_serialize_module.cpp  |  25 ++++
 .../ASTMatchers/ASTMatchersNodeTest.cpp       |  22 ++++
 flatten-verify-tmp/01_default_depth2.cpp      |   8 ++
 flatten-verify-tmp/02_depth3_full.cpp         |   9 ++
 flatten-verify-tmp/03_depth2_partial.cpp      |  10 ++
 flatten-verify-tmp/04_depth1_identity.cpp     |   8 ++
 flatten-verify-tmp/05_rangefor_template.cpp   |  20 +++
 flatten-verify-tmp/README.md                  |  39 ++++++
 flatten-verify-tmp/end_result_check.cpp       |  82 ++++++++++++
 .../test/transform/flatten/depth-partial.c    |  45 +++++++
 openmp/runtime/test/transform/flatten/empty.c |  42 ++++++
 .../test/transform/flatten/foreach.cpp        |  31 +++++
 .../flatten/parallel-wsloop-foreach.cpp       |  32 +++++
 23 files changed, 805 insertions(+), 1 deletion(-)
 create mode 100644 clang/test/Index/openmp-flatten.c
 create mode 100644 clang/test/OpenMP/flatten_ast_print.cpp
 create mode 100644 clang/test/OpenMP/flatten_codegen_for.cpp
 create mode 100644 clang/test/OpenMP/flatten_depth_ast_print.cpp
 create mode 100644 clang/test/OpenMP/flatten_depth_messages.cpp
 create mode 100644 clang/test/OpenMP/flatten_messages.cpp
 create mode 100644 clang/test/OpenMP/flatten_serialize_module.cpp
 create mode 100644 flatten-verify-tmp/01_default_depth2.cpp
 create mode 100644 flatten-verify-tmp/02_depth3_full.cpp
 create mode 100644 flatten-verify-tmp/03_depth2_partial.cpp
 create mode 100644 flatten-verify-tmp/04_depth1_identity.cpp
 create mode 100644 flatten-verify-tmp/05_rangefor_template.cpp
 create mode 100644 flatten-verify-tmp/README.md
 create mode 100644 flatten-verify-tmp/end_result_check.cpp
 create mode 100644 openmp/runtime/test/transform/flatten/depth-partial.c
 create mode 100644 openmp/runtime/test/transform/flatten/empty.c
 create mode 100644 openmp/runtime/test/transform/flatten/foreach.cpp
 create mode 100644 openmp/runtime/test/transform/flatten/parallel-wsloop-foreach.cpp

diff --git a/clang/docs/OpenMPSupport.rst b/clang/docs/OpenMPSupport.rst
index 0c21aa41ff269..c11a46c3bb6a6 100644
--- a/clang/docs/OpenMPSupport.rst
+++ b/clang/docs/OpenMPSupport.rst
@@ -666,7 +666,7 @@ implementation.
 +=============================================================+===========================+===========================+==========================================================================+
 | dyn_groupprivate clause                                     | :part:`partial`           | :part:`In Progress`       | C/C++: Host device support missing                                       |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
-| loop flatten transformation                                 | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
+| loop flatten transformation                                 | :part:`partial`           | :none:`unclaimed`         | Clang: depth clause supported (OpenMP 6.1); apply clause not yet supported|
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | loop grid/tile modifiers for sizes clause                   | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
diff --git a/clang/include/clang/ASTMatchers/ASTMatchers.h b/clang/include/clang/ASTMatchers/ASTMatchers.h
index bc0f35898a2c9..602bab9aa8cb8 100644
--- a/clang/include/clang/ASTMatchers/ASTMatchers.h
+++ b/clang/include/clang/ASTMatchers/ASTMatchers.h
@@ -8842,6 +8842,20 @@ extern const internal::VariadicDynCastAllOfMatcher<Stmt,
 extern const internal::VariadicDynCastAllOfMatcher<Stmt, OMPSplitDirective>
     ompSplitDirective;
 
+/// Matches OpenMP ``flatten`` directive.
+///
+/// Given
+///
+/// \code
+///   #pragma omp flatten
+///   for (int i = 0; i < m; ++i)
+///     for (int j = 0; j < n; ++j) {}
+/// \endcode
+///
+/// ``ompFlattenDirective()`` matches the flatten directive.
+extern const internal::VariadicDynCastAllOfMatcher<Stmt, OMPFlattenDirective>
+    ompFlattenDirective;
+
 /// Matches OpenMP ``counts`` clause used by ``#pragma omp split``.
 ///
 /// Given ``#pragma omp split counts(1, 2, omp_fill)``, ``ompCountsClause()``
diff --git a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
index 5cbf134620e34..e2c0c25518b70 100644
--- a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
+++ b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
@@ -1141,6 +1141,8 @@ const internal::VariadicDynCastAllOfMatcher<Stmt, OMPTargetUpdateDirective>
     ompTargetUpdateDirective;
 const internal::VariadicDynCastAllOfMatcher<Stmt, OMPSplitDirective>
     ompSplitDirective;
+const internal::VariadicDynCastAllOfMatcher<Stmt, OMPFlattenDirective>
+    ompFlattenDirective;
 const internal::VariadicDynCastAllOfMatcher<OMPClause, OMPCountsClause>
     ompCountsClause;
 const internal::VariadicDynCastAllOfMatcher<OMPClause, OMPDefaultClause>
diff --git a/clang/lib/ASTMatchers/Dynamic/Registry.cpp b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
index a04070971f0eb..a6252363e08ce 100644
--- a/clang/lib/ASTMatchers/Dynamic/Registry.cpp
+++ b/clang/lib/ASTMatchers/Dynamic/Registry.cpp
@@ -534,6 +534,7 @@ RegistryMaps::RegistryMaps() {
   REGISTER_MATCHER(ompFromClause);
   REGISTER_MATCHER(ompToClause);
   REGISTER_MATCHER(ompExecutableDirective);
+  REGISTER_MATCHER(ompFlattenDirective);
   REGISTER_MATCHER(ompSplitDirective);
   REGISTER_MATCHER(ompTargetUpdateDirective);
   REGISTER_MATCHER(on);
diff --git a/clang/test/Index/openmp-flatten.c b/clang/test/Index/openmp-flatten.c
new file mode 100644
index 0000000000000..74bc3cda3ef1a
--- /dev/null
+++ b/clang/test/Index/openmp-flatten.c
@@ -0,0 +1,12 @@
+// RUN: c-index-test -test-load-source local %s -fopenmp=libomp -fopenmp-version=60 | FileCheck %s
+
+void test(void) {
+#pragma omp flatten
+  for (int i = 0; i < 20; i += 1)
+    for (int j = 0; j < 30; j += 1)
+      ;
+}
+
+// CHECK: openmp-flatten.c:4:1: OMPFlattenDirective= Extent=[4:1 - 4:20]
+// CHECK: openmp-flatten.c:5:3: ForStmt= Extent=[5:3 - 7:8]
+// CHECK: openmp-flatten.c:6:5: ForStmt= Extent=[6:5 - 7:8]
diff --git a/clang/test/OpenMP/flatten_ast_print.cpp b/clang/test/OpenMP/flatten_ast_print.cpp
new file mode 100644
index 0000000000000..e1115bf112c40
--- /dev/null
+++ b/clang/test/OpenMP/flatten_ast_print.cpp
@@ -0,0 +1,124 @@
+// Check no warnings/errors
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -fsyntax-only -verify %s
+// expected-no-diagnostics
+
+// Check AST and unparsing
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -ast-dump  %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -ast-print %s | FileCheck %s --check-prefix=PRINT
+
+// Check same results after serialization round-trip
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -emit-pch -o %t %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -include-pch %t -ast-dump-all %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=60 -include-pch %t -ast-print    %s | FileCheck %s --check-prefix=PRINT
+
+#ifndef HEADER
+#define HEADER
+
+// placeholder for loop body code.
+extern "C" void body(...);
+
+// PRINT-LABEL: void foo1(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo1
+void foo1() {
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (int i = 0; i < 7; i += 1)
+  // DUMP-NEXT: ForStmt
+  for (int i = 0; i < 7; i += 1)
+    // PRINT: for (int j = 0; j < 9; j += 1)
+    // DUMP:  ForStmt
+    for (int j = 0; j < 9; j += 1)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// PRINT-LABEL: void foo2(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo2
+void foo2(int start1, int end1, int step1, int start2, int end2, int step2) {
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (int i = start1; i < end1; i += step1)
+  // DUMP-NEXT: ForStmt
+  for (int i = start1; i < end1; i += step1)
+    // PRINT: for (int j = start2; j < end2; j += step2)
+    // DUMP:  ForStmt
+    for (int j = start2; j < end2; j += step2)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// PRINT-LABEL: void foo3(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo3
+void foo3() {
+  // Verify that a flattened loop can itself be the operand of another loop
+  // transformation. The outer reverse consumes the loop generated by flatten.
+  // PRINT:     #pragma omp reverse
+  // DUMP:      OMPReverseDirective
+  #pragma omp reverse
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (int i = 0; i < 13; i += 1)
+  // DUMP-NEXT: ForStmt
+  for (int i = 0; i < 13; i += 1)
+    // PRINT: for (int j = 0; j < 17; j += 1)
+    // DUMP:  ForStmt
+    for (int j = 0; j < 17; j += 1)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// Range-based for loops are supported as affected loops.
+// PRINT-LABEL: void foo4(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo4
+struct Range {
+  int *begin();
+  int *end();
+};
+void foo4(Range r1, Range r2) {
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (int i : r1)
+  // DUMP-NEXT: CXXForRangeStmt
+  for (int i : r1)
+    // PRINT: for (int j : r2)
+    // DUMP:  CXXForRangeStmt
+    for (int j : r2)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// The directive is instantiated together with its enclosing template.
+// PRINT-LABEL: template <typename T, int LO, int HI> void foo5()
+// DUMP-LABEL:  FunctionTemplateDecl {{.*}} foo5
+template <typename T, int LO, int HI>
+void foo5() {
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  // PRINT: for (T i = LO; i < HI; ++i)
+  // DUMP-NEXT: ForStmt
+  for (T i = LO; i < HI; ++i)
+    // PRINT: for (T j = LO; j < HI; ++j)
+    // DUMP:  ForStmt
+    for (T j = LO; j < HI; ++j)
+      // PRINT: body(i, j);
+      // DUMP:  CallExpr
+      body(i, j);
+}
+
+// PRINT-LABEL: template<> void foo5<int, 0, 8>()
+// DUMP-LABEL:  FunctionDecl {{.*}} foo5 'void ()' implicit_instantiation
+// DUMP:        OMPFlattenDirective
+void inst() {
+  foo5<int, 0, 8>();
+}
+
+#endif
diff --git a/clang/test/OpenMP/flatten_codegen_for.cpp b/clang/test/OpenMP/flatten_codegen_for.cpp
new file mode 100644
index 0000000000000..d64128bd25d78
--- /dev/null
+++ b/clang/test/OpenMP/flatten_codegen_for.cpp
@@ -0,0 +1,67 @@
+// Check that the loop generated by 'flatten' can be consumed by a loop
+// associated directive ('omp for' / 'omp parallel for'), and that signed and
+// unsigned induction variables generate signed/unsigned division.
+
+// RUN: %clang_cc1 -verify -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=60 -emit-llvm %s -o - | FileCheck %s
+
+// expected-no-diagnostics
+
+#ifndef HEADER
+#define HEADER
+
+extern "C" void body(int, int);
+extern "C" void ubody(unsigned, unsigned);
+
+// 'omp for' worksharing over the flattened iteration space: the flattened loop
+// uses a 64-bit induction variable for the variable-bound product trip count,
+// so the worksharing loop is distributed with the 64-bit static-init call and
+// the body recovers the logical counters via signed division/remainder.
+//
+// CHECK-LABEL: define {{.*}}void @wsfor(
+// CHECK:   mul nsw i64
+// CHECK:   call void @__kmpc_for_static_init_8(
+// CHECK:   sdiv i64
+// CHECK:   srem i64
+// CHECK:   call void @body(
+// CHECK:   call void @__kmpc_for_static_fini(
+extern "C" void wsfor(int n, int m) {
+#pragma omp for
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+
+// 'omp parallel for' outlines the flattened loop and worksharing-distributes
+// it.
+//
+// CHECK-LABEL: define {{.*}}void @pfor(
+// CHECK:   call {{.*}}@__kmpc_fork_call(
+// CHECK-LABEL: define internal void @pfor.omp_outlined(
+// CHECK:   call void @__kmpc_for_static_init_8(
+// CHECK:   sdiv i64
+// CHECK:   srem i64
+// CHECK:   call void @body(
+extern "C" void pfor(int n, int m) {
+#pragma omp parallel for
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+
+// Unsigned induction variables produce unsigned division/remainder.
+//
+// CHECK-LABEL: define {{.*}}void @ufor(
+// CHECK:   icmp ult i32 %{{.+}}, 63
+// CHECK:   udiv i32 %{{.+}}, 9
+// CHECK:   urem i32 %{{.+}}, 9
+// CHECK:   call void @ubody(
+extern "C" void ufor() {
+#pragma omp flatten
+  for (unsigned i = 0; i < 7u; ++i)
+    for (unsigned j = 0; j < 9u; ++j)
+      ubody(i, j);
+}
+
+#endif
diff --git a/clang/test/OpenMP/flatten_depth_ast_print.cpp b/clang/test/OpenMP/flatten_depth_ast_print.cpp
new file mode 100644
index 0000000000000..ceaaa68e1f79a
--- /dev/null
+++ b/clang/test/OpenMP/flatten_depth_ast_print.cpp
@@ -0,0 +1,74 @@
+// Check no warnings/errors
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -fsyntax-only -verify %s
+// expected-no-diagnostics
+
+// Check AST and unparsing
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-dump  %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-print %s | FileCheck %s --check-prefix=PRINT
+
+// Check same results after serialization round-trip
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -emit-pch -o %t %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -include-pch %t -ast-dump-all %s | FileCheck %s --check-prefix=DUMP
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -include-pch %t -ast-print    %s | FileCheck %s --check-prefix=PRINT
+
+#ifndef HEADER
+#define HEADER
+
+// placeholder for loop body code.
+extern "C" void body(...);
+
+// The depth clause round-trips through -ast-print and PCH serialization.
+// PRINT-LABEL: void foo_depth3(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo_depth3
+void foo_depth3(int n, int m, int p) {
+  // PRINT:     #pragma omp flatten depth(3)
+  // DUMP:      OMPFlattenDirective
+  // DUMP-NEXT: OMPDepthClause
+  #pragma omp flatten depth(3)
+  // PRINT: for (int i = 0; i < n; ++i)
+  // DUMP:  ForStmt
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        // PRINT: body(i, j, k);
+        // DUMP:  CallExpr
+        body(i, j, k);
+}
+
+// depth(2) is accepted and printed like any other depth argument.
+// PRINT-LABEL: void foo_depth2(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo_depth2
+void foo_depth2(int n, int m) {
+  // PRINT:     #pragma omp flatten depth(2)
+  // DUMP:      OMPFlattenDirective
+  // DUMP-NEXT: OMPDepthClause
+  #pragma omp flatten depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+
+// The depth clause is instantiated together with its enclosing template.
+// PRINT-LABEL: template <typename T> void foo_tmpl()
+// DUMP-LABEL:  FunctionTemplateDecl {{.*}} foo_tmpl
+template <typename T>
+void foo_tmpl() {
+  // PRINT:     #pragma omp flatten depth(3)
+  // DUMP:      OMPFlattenDirective
+  // DUMP-NEXT: OMPDepthClause
+  #pragma omp flatten depth(3)
+  for (T i = 0; i < 8; ++i)
+    for (T j = 0; j < 8; ++j)
+      for (T k = 0; k < 8; ++k)
+        body(i, j, k);
+}
+
+// PRINT-LABEL: template<> void foo_tmpl<int>()
+// DUMP-LABEL:  FunctionDecl {{.*}} foo_tmpl 'void ()' implicit_instantiation
+// DUMP:        OMPFlattenDirective
+// DUMP-NEXT:   OMPDepthClause
+void inst() {
+  foo_tmpl<int>();
+}
+
+#endif
diff --git a/clang/test/OpenMP/flatten_depth_messages.cpp b/clang/test/OpenMP/flatten_depth_messages.cpp
new file mode 100644
index 0000000000000..16f60318d0d06
--- /dev/null
+++ b/clang/test/OpenMP/flatten_depth_messages.cpp
@@ -0,0 +1,48 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=61 -fsyntax-only -Wuninitialized -verify %s
+
+extern "C" void body(...);
+
+// expected-note at +1 {{declared here}}
+void func(int n) {
+
+  // The depth argument must be a positive integer constant expression.
+  // expected-error at +1 {{argument to 'depth' clause must be a strictly positive integer value}}
+  #pragma omp flatten depth(0)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // expected-error at +1 {{argument to 'depth' clause must be a strictly positive integer value}}
+  #pragma omp flatten depth(-1)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // A non-constant argument is rejected.
+  // expected-error at +2 {{expression is not an integral constant expression}}
+  // expected-note at +1 {{function parameter 'n' with unknown value cannot be used in a constant expression}}
+  #pragma omp flatten depth(n)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // At most one 'depth' clause may appear on the directive.
+  // expected-error at +1 {{directive '#pragma omp flatten' cannot contain more than one 'depth' clause}}
+  #pragma omp flatten depth(2) depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // The depth must be at most the loop nest depth: depth(3) requires three
+  // perfectly nested loops, but only two are present here.
+  #pragma omp flatten depth(3)
+  // expected-error at +1 {{expected 3 for loops after '#pragma omp flatten', but found only 2}}
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // depth(1) flattens a single loop and is well-formed (no diagnostic).
+  #pragma omp flatten depth(1)
+  for (int i = 0; i < n; ++i)
+    body(i);
+}
diff --git a/clang/test/OpenMP/flatten_messages.cpp b/clang/test/OpenMP/flatten_messages.cpp
new file mode 100644
index 0000000000000..a022e0e8999d2
--- /dev/null
+++ b/clang/test/OpenMP/flatten_messages.cpp
@@ -0,0 +1,89 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=60 -fsyntax-only -Wuninitialized -verify %s
+
+extern "C" void body(...);
+
+void func(int n) {
+
+  // The associated statement must be a for loop.
+  // expected-error at +2 {{statement after '#pragma omp flatten' must be a for loop}}
+  #pragma omp flatten
+  ;
+
+  // A non-loop statement is rejected as well.
+  // expected-error at +2 {{statement after '#pragma omp flatten' must be a for loop}}
+  #pragma omp flatten
+  int b = 0;
+
+  // A single loop is not enough: flatten combines two perfectly nested loops,
+  // so the body of the outer loop must itself be a for loop.
+  #pragma omp flatten
+  for (int i = 0; i < 7; ++i)
+    // expected-error at +1 {{statement after '#pragma omp flatten' must be a for loop}}
+    ;
+
+  // The associated statement of a directive is not a for loop.
+  // expected-error at +2 {{statement after '#pragma omp flatten' must be a for loop}}
+  #pragma omp flatten
+  #pragma omp for
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 7; ++j)
+      body(i, j);
+
+  {
+    // expected-error at +2 {{expected statement}}
+    #pragma omp flatten
+  }
+
+  // The loops must be perfectly nested: no code is allowed between them.
+  #pragma omp flatten
+  for (int i = 0; i < n; ++i) {
+    int x = 0;
+    // expected-error at -2 {{statement after '#pragma omp flatten' must be a for loop}}
+    for (int j = 0; j < n; ++j)
+      body(i, j, x);
+  }
+
+  // Each affected loop must be in OpenMP canonical form.
+  #pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    // expected-error at +1 {{condition of OpenMP for loop must be a relational comparison ('<', '<=', '>', '>=', or '!=') of loop variable 'j'}}
+    for (int j = 0; j / 3 < n; ++j)
+      body(i, j);
+
+  // The affected loops must be rectangular: an inner bound may not depend on an
+  // outer loop counter.
+  #pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    // expected-error at +1 {{expected loop invariant expression}}
+    for (int j = i; j < n; ++j)
+      body(i, j);
+
+  // The 'sizes' clause is not allowed on 'flatten'.
+  // expected-error at +1 {{unexpected OpenMP clause 'sizes' in directive '#pragma omp flatten'}}
+  #pragma omp flatten sizes(2)
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+
+  // The 'permutation' clause is not allowed on 'flatten'.
+  // expected-error at +1 {{unexpected OpenMP clause 'permutation' in directive '#pragma omp flatten'}}
+  #pragma omp flatten permutation(2, 1)
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+
+  // Tokens after the directive name are ignored with a warning.
+  // expected-warning at +1 {{extra tokens at the end of '#pragma omp flatten' are ignored}}
+  #pragma omp flatten foo
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+
+  // The 'depth' clause is only available from OpenMP 6.1; it is rejected here
+  // under -fopenmp-version=60.
+  // expected-error at +1 {{unexpected OpenMP clause 'depth' in directive '#pragma omp flatten'}}
+  #pragma omp flatten depth(2)
+  for (int i = 0; i < 7; ++i)
+    for (int j = 0; j < 9; ++j)
+      body(i, j);
+}
diff --git a/clang/test/OpenMP/flatten_serialize_module.cpp b/clang/test/OpenMP/flatten_serialize_module.cpp
new file mode 100644
index 0000000000000..5d3420500b282
--- /dev/null
+++ b/clang/test/OpenMP/flatten_serialize_module.cpp
@@ -0,0 +1,25 @@
+// C++20 module interface with `#pragma omp flatten` — emit BMI + import; AST retains directive.
+//
+// RUN: rm -rf %t && split-file %s %t && cd %t
+// RUN: %clang_cc1 -std=c++20 -fopenmp -fopenmp-version=60 -triple x86_64-unknown-linux-gnu %t/FlattenMod.cppm -emit-module-interface -o %t/FlattenMod.pcm
+// RUN: %clang_cc1 -std=c++20 -fopenmp -fopenmp-version=60 -triple x86_64-unknown-linux-gnu %t/UseFlattenMod.cpp -fmodule-file=FlattenMod=%t/FlattenMod.pcm -ast-dump-all | FileCheck %t/FlattenMod.cppm
+
+// expected-no-diagnostics
+
+//--- FlattenMod.cppm
+module;
+export module FlattenMod;
+
+export void flattenfoo(int n, int m) {
+// CHECK: OMPFlattenDirective
+// CHECK: ForStmt
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j) {
+    }
+}
+
+//--- UseFlattenMod.cpp
+import FlattenMod;
+
+void g(void) { flattenfoo(10, 20); }
diff --git a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
index 4190d4703e37d..34c0fd27821d1 100644
--- a/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
+++ b/clang/unittests/ASTMatchers/ASTMatchersNodeTest.cpp
@@ -3126,6 +3126,28 @@ void f() {
   EXPECT_TRUE(notMatchesWithOpenMP60(ParallelOnly, Matcher));
 }
 
+// OpenMP 6 flatten directive
+TEST(ASTMatchersTestOpenMP, OMPFlattenDirective) {
+  auto Matcher = stmt(ompFlattenDirective());
+
+  StringRef FlattenOk = R"(
+void f() {
+#pragma omp flatten
+  for (int i = 0; i < 10; ++i)
+    for (int j = 0; j < 10; ++j) {}
+}
+)";
+  EXPECT_TRUE(matchesWithOpenMP60(FlattenOk, Matcher));
+
+  StringRef ParallelOnly = R"(
+void f() {
+#pragma omp parallel
+  ;
+}
+)";
+  EXPECT_TRUE(notMatchesWithOpenMP60(ParallelOnly, Matcher));
+}
+
 TEST(ASTMatchersTestOpenMP, OMPSplitDirective_HasCountsClause) {
   auto Matcher = stmt(ompSplitDirective(hasAnyClause(ompCountsClause())));
 
diff --git a/flatten-verify-tmp/01_default_depth2.cpp b/flatten-verify-tmp/01_default_depth2.cpp
new file mode 100644
index 0000000000000..900d0263622cd
--- /dev/null
+++ b/flatten-verify-tmp/01_default_depth2.cpp
@@ -0,0 +1,8 @@
+// Default flatten: combine 2 perfectly nested loops (depth defaults to 2).
+extern "C" void body(int, int);
+void t(int n, int m) {
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
diff --git a/flatten-verify-tmp/02_depth3_full.cpp b/flatten-verify-tmp/02_depth3_full.cpp
new file mode 100644
index 0000000000000..ba9f0b7ed8808
--- /dev/null
+++ b/flatten-verify-tmp/02_depth3_full.cpp
@@ -0,0 +1,9 @@
+// depth(3): fully flatten a three-deep perfect nest (OpenMP 6.1).
+extern "C" void body(int, int, int);
+void t(int n, int m, int p) {
+#pragma omp flatten depth(3)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        body(i, j, k);
+}
diff --git a/flatten-verify-tmp/03_depth2_partial.cpp b/flatten-verify-tmp/03_depth2_partial.cpp
new file mode 100644
index 0000000000000..35b9dc83834fd
--- /dev/null
+++ b/flatten-verify-tmp/03_depth2_partial.cpp
@@ -0,0 +1,10 @@
+// depth(2) on a three-deep nest: only the outer two loops are combined; the
+// innermost loop must remain as ordinary body of the flattened loop.
+extern "C" void body(int, int, int);
+void t(int n, int m, int p) {
+#pragma omp flatten depth(2)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      for (int k = 0; k < p; ++k)
+        body(i, j, k);
+}
diff --git a/flatten-verify-tmp/04_depth1_identity.cpp b/flatten-verify-tmp/04_depth1_identity.cpp
new file mode 100644
index 0000000000000..ae62a11312fe8
--- /dev/null
+++ b/flatten-verify-tmp/04_depth1_identity.cpp
@@ -0,0 +1,8 @@
+// depth(1): identity transform. A single loop is "combined"; no mixed-radix
+// decomposition (no srem), the flattened counter maps straight to the loop var.
+extern "C" void body(int);
+void t(int n) {
+#pragma omp flatten depth(1)
+  for (int i = 0; i < n; ++i)
+    body(i);
+}
diff --git a/flatten-verify-tmp/05_rangefor_template.cpp b/flatten-verify-tmp/05_rangefor_template.cpp
new file mode 100644
index 0000000000000..89928c34a8a15
--- /dev/null
+++ b/flatten-verify-tmp/05_rangefor_template.cpp
@@ -0,0 +1,20 @@
+// Range-for (CXXForRangeStmt) nest + a function template instantiated with a
+// dependent depth(K). Exercises the __begin/__end pre-inits and the deferred
+// (dependent-context) desugaring path.
+#include <stdlibc++.h>
+extern "C" void body(int, int);
+
+void ranges(std::vector<int> &a, std::vector<int> &b) {
+#pragma omp flatten
+  for (int i : a)
+    for (int j : b)
+      body(i, j);
+}
+
+template <int K> void tmpl(int n, int m) {
+#pragma omp flatten depth(K)
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      body(i, j);
+}
+template void tmpl<2>(int, int);
diff --git a/flatten-verify-tmp/README.md b/flatten-verify-tmp/README.md
new file mode 100644
index 0000000000000..f48f4d06cfec0
--- /dev/null
+++ b/flatten-verify-tmp/README.md
@@ -0,0 +1,39 @@
+# Temporary flatten + depth verification cases
+
+Throwaway sandbox for eyeballing the flatten desugaring (AST / IR / runtime).
+Not part of any PR — delete freely. The committed regression tests live under
+`clang/test/OpenMP/flatten*` and `openmp/runtime/test/transform/flatten/*`.
+
+`CLANG=build/bin/clang`, `CLANGXX=build/bin/clang++`, run from the repo root.
+
+| File | Case | What to look for |
+|------|------|------------------|
+| `01_default_depth2.cpp` | default (no clause) = depth 2 | i64 `.flatten.iv`, `mul nsw`, `sdiv`+`srem` |
+| `02_depth3_full.cpp` | `depth(3)` full flatten | 3 `.flatten.iv.N`, two-level mixed radix |
+| `03_depth2_partial.cpp` | `depth(2)` on 3-deep nest | 2 flatten IVs, **inner `%k` loop stays** |
+| `04_depth1_identity.cpp` | `depth(1)` identity | i32 IV, **no `srem`**, direct store |
+| `05_rangefor_template.cpp` | range-for + template `depth(K)` | ast-print round-trips; instantiation |
+| `end_result_check.cpp` | runtime semantic check | prints `OK`, exit 0 (order+set preserved) |
+
+## Commands
+
+AST (round-trip original loops) and the desugared node:
+```
+$CLANG -cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-print FILE
+$CLANG -cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-dump  FILE
+```
+
+IR (the div/mod arithmetic that implements the transform):
+```
+$CLANG -cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -emit-llvm -o - FILE
+```
+
+Range-for / template need the driver (libstdc++ + template instantiation):
+```
+$CLANGXX -fopenmp -fopenmp-version=61 -Xclang -ast-print -fsyntax-only 05_rangefor_template.cpp
+```
+
+Runtime end-result (must print OK, exit 0):
+```
+$CLANGXX -fopenmp -fopenmp-version=61 -O0 end_result_check.cpp -o /tmp/erc && /tmp/erc
+```
diff --git a/flatten-verify-tmp/end_result_check.cpp b/flatten-verify-tmp/end_result_check.cpp
new file mode 100644
index 0000000000000..3aa8850829058
--- /dev/null
+++ b/flatten-verify-tmp/end_result_check.cpp
@@ -0,0 +1,82 @@
+// End-result (semantic) verification: flatten must preserve BOTH the set of
+// iterations and their row-major visitation ORDER. We record the sequence of
+// visited tuples from a flattened nest and compare it against the sequence from
+// an identical non-flattened reference nest. Exit code 0 == identical.
+//
+// flatten is a pure frontend AST desugaring to an ordinary loop, so this runs
+// as a normal program (build with: clang++ -fopenmp -fopenmp-version=61).
+#include <cassert>
+#include <cstdio>
+#include <vector>
+
+using Seq = std::vector<long>;
+
+static void rec(Seq &s, int i, int j, int k = -1) {
+  s.push_back(i);
+  s.push_back(j);
+  if (k >= 0)
+    s.push_back(k);
+}
+
+int main() {
+  const int n = 3, m = 4, p = 2;
+
+  // --- depth(2) default over 2 loops ---
+  {
+    Seq flat, ref;
+#pragma omp flatten
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        rec(flat, i, j);
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        rec(ref, i, j);
+    assert(flat == ref && "depth(2) order/set mismatch");
+  }
+
+  // --- depth(3) full flatten over 3 loops ---
+  {
+    Seq flat, ref;
+#pragma omp flatten depth(3)
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        for (int k = 0; k < p; ++k)
+          rec(flat, i, j, k);
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        for (int k = 0; k < p; ++k)
+          rec(ref, i, j, k);
+    assert(flat == ref && "depth(3) order/set mismatch");
+  }
+
+  // --- depth(2) partial: inner k-loop stays intact ---
+  {
+    Seq flat, ref;
+#pragma omp flatten depth(2)
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        for (int k = 0; k < p; ++k)
+          rec(flat, i, j, k);
+    for (int i = 0; i < n; ++i)
+      for (int j = 0; j < m; ++j)
+        for (int k = 0; k < p; ++k)
+          rec(ref, i, j, k);
+    assert(flat == ref && "depth(2)-partial order/set mismatch");
+  }
+
+  // --- non-unit step / non-zero start must survive ---
+  {
+    Seq flat, ref;
+#pragma omp flatten
+    for (int i = 5; i < 12; i += 2)
+      for (int j = 1; j < 7; j += 3)
+        rec(flat, i, j);
+    for (int i = 5; i < 12; i += 2)
+      for (int j = 1; j < 7; j += 3)
+        rec(ref, i, j);
+    assert(flat == ref && "stride/offset mismatch");
+  }
+
+  std::puts("flatten end-result preserved: OK");
+  return 0;
+}
diff --git a/openmp/runtime/test/transform/flatten/depth-partial.c b/openmp/runtime/test/transform/flatten/depth-partial.c
new file mode 100644
index 0000000000000..2fda3a04cbd84
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/depth-partial.c
@@ -0,0 +1,45 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+// A depth(2) clause on a three-deep loop nest flattens only the outermost two
+// loops; the innermost loop is left untouched and executes normally inside the
+// flattened loop body.
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+int main() {
+  printf("do\n");
+#pragma omp flatten depth(2)
+  for (int i = 5; i < 11; i += 2)
+    for (int j = 1; j < 7; j += 2)
+      for (int k = 0; k < 2; ++k)
+        printf("i=%d j=%d k=%d\n", i, j, k);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: i=5 j=1 k=0
+// CHECK-NEXT: i=5 j=1 k=1
+// CHECK-NEXT: i=5 j=3 k=0
+// CHECK-NEXT: i=5 j=3 k=1
+// CHECK-NEXT: i=5 j=5 k=0
+// CHECK-NEXT: i=5 j=5 k=1
+// CHECK-NEXT: i=7 j=1 k=0
+// CHECK-NEXT: i=7 j=1 k=1
+// CHECK-NEXT: i=7 j=3 k=0
+// CHECK-NEXT: i=7 j=3 k=1
+// CHECK-NEXT: i=7 j=5 k=0
+// CHECK-NEXT: i=7 j=5 k=1
+// CHECK-NEXT: i=9 j=1 k=0
+// CHECK-NEXT: i=9 j=1 k=1
+// CHECK-NEXT: i=9 j=3 k=0
+// CHECK-NEXT: i=9 j=3 k=1
+// CHECK-NEXT: i=9 j=5 k=0
+// CHECK-NEXT: i=9 j=5 k=1
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/flatten/empty.c b/openmp/runtime/test/transform/flatten/empty.c
new file mode 100644
index 0000000000000..ecf1e8ef622f2
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/empty.c
@@ -0,0 +1,42 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+// The bounds are runtime values so that an empty iteration space does not
+// trigger a compile-time division-by-zero diagnostic for the (never executed)
+// induction-variable recovery.
+static void flatten(int n, int m) {
+#pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < m; ++j)
+      printf("i=%d j=%d\n", i, j);
+}
+
+int main() {
+  printf("empty-inner-begin\n");
+  flatten(3, 0);
+  printf("empty-inner-end\n");
+
+  printf("empty-outer-begin\n");
+  flatten(0, 3);
+  printf("empty-outer-end\n");
+
+  printf("single-begin\n");
+  flatten(1, 1);
+  printf("single-end\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      empty-inner-begin
+// CHECK-NEXT: empty-inner-end
+// CHECK-NEXT: empty-outer-begin
+// CHECK-NEXT: empty-outer-end
+// CHECK-NEXT: single-begin
+// CHECK-NEXT: i=0 j=0
+// CHECK-NEXT: single-end
diff --git a/openmp/runtime/test/transform/flatten/foreach.cpp b/openmp/runtime/test/transform/flatten/foreach.cpp
new file mode 100644
index 0000000000000..14997f7c4051d
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/foreach.cpp
@@ -0,0 +1,31 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdio>
+#include <vector>
+
+int main() {
+  printf("do\n");
+  std::vector<int> outer{10, 20};
+  std::vector<int> inner{1, 2, 3};
+#pragma omp flatten
+  for (int a : outer)
+    for (int b : inner)
+      printf("a=%d b=%d\n", a, b);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: a=10 b=1
+// CHECK-NEXT: a=10 b=2
+// CHECK-NEXT: a=10 b=3
+// CHECK-NEXT: a=20 b=1
+// CHECK-NEXT: a=20 b=2
+// CHECK-NEXT: a=20 b=3
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/flatten/parallel-wsloop-foreach.cpp b/openmp/runtime/test/transform/flatten/parallel-wsloop-foreach.cpp
new file mode 100644
index 0000000000000..7ddcde2fd622c
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/parallel-wsloop-foreach.cpp
@@ -0,0 +1,32 @@
+// RUN: %libomp-cxx-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <cstdlib>
+#include <cstdio>
+
+int main() {
+  printf("do\n");
+  // The loop generated by 'flatten' is the canonical loop associated with the
+  // worksharing 'omp for'. With a single thread the iteration order is the
+  // original row-major order.
+#pragma omp parallel for num_threads(1)
+#pragma omp flatten
+  for (int i = 0; i < 2; ++i)
+    for (int j = 0; j < 3; ++j)
+      printf("i=%d j=%d\n", i, j);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// CHECK:      do
+// CHECK-NEXT: i=0 j=0
+// CHECK-NEXT: i=0 j=1
+// CHECK-NEXT: i=0 j=2
+// CHECK-NEXT: i=1 j=0
+// CHECK-NEXT: i=1 j=1
+// CHECK-NEXT: i=1 j=2
+// CHECK-NEXT: done

>From 152ee3bed82f0aa84079151c559d46564d9304df Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Tue, 18 Aug 2026 07:49:02 -0400
Subject: [PATCH 15/18] Remove flatten-verify-tmp and restore
 OpenMPSupport.rst.

Co-authored-by: Cursor <cursoragent at cursor.com>
---
 clang/docs/OpenMPSupport.rst                |  2 +-
 flatten-verify-tmp/01_default_depth2.cpp    |  8 --
 flatten-verify-tmp/02_depth3_full.cpp       |  9 ---
 flatten-verify-tmp/03_depth2_partial.cpp    | 10 ---
 flatten-verify-tmp/04_depth1_identity.cpp   |  8 --
 flatten-verify-tmp/05_rangefor_template.cpp | 20 -----
 flatten-verify-tmp/README.md                | 39 ----------
 flatten-verify-tmp/end_result_check.cpp     | 82 ---------------------
 8 files changed, 1 insertion(+), 177 deletions(-)
 delete mode 100644 flatten-verify-tmp/01_default_depth2.cpp
 delete mode 100644 flatten-verify-tmp/02_depth3_full.cpp
 delete mode 100644 flatten-verify-tmp/03_depth2_partial.cpp
 delete mode 100644 flatten-verify-tmp/04_depth1_identity.cpp
 delete mode 100644 flatten-verify-tmp/05_rangefor_template.cpp
 delete mode 100644 flatten-verify-tmp/README.md
 delete mode 100644 flatten-verify-tmp/end_result_check.cpp

diff --git a/clang/docs/OpenMPSupport.rst b/clang/docs/OpenMPSupport.rst
index c11a46c3bb6a6..0c21aa41ff269 100644
--- a/clang/docs/OpenMPSupport.rst
+++ b/clang/docs/OpenMPSupport.rst
@@ -666,7 +666,7 @@ implementation.
 +=============================================================+===========================+===========================+==========================================================================+
 | dyn_groupprivate clause                                     | :part:`partial`           | :part:`In Progress`       | C/C++: Host device support missing                                       |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
-| loop flatten transformation                                 | :part:`partial`           | :none:`unclaimed`         | Clang: depth clause supported (OpenMP 6.1); apply clause not yet supported|
+| loop flatten transformation                                 | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
 | loop grid/tile modifiers for sizes clause                   | :none:`unclaimed`         | :none:`unclaimed`         |                                                                          |
 +-------------------------------------------------------------+---------------------------+---------------------------+--------------------------------------------------------------------------+
diff --git a/flatten-verify-tmp/01_default_depth2.cpp b/flatten-verify-tmp/01_default_depth2.cpp
deleted file mode 100644
index 900d0263622cd..0000000000000
--- a/flatten-verify-tmp/01_default_depth2.cpp
+++ /dev/null
@@ -1,8 +0,0 @@
-// Default flatten: combine 2 perfectly nested loops (depth defaults to 2).
-extern "C" void body(int, int);
-void t(int n, int m) {
-#pragma omp flatten
-  for (int i = 0; i < n; ++i)
-    for (int j = 0; j < m; ++j)
-      body(i, j);
-}
diff --git a/flatten-verify-tmp/02_depth3_full.cpp b/flatten-verify-tmp/02_depth3_full.cpp
deleted file mode 100644
index ba9f0b7ed8808..0000000000000
--- a/flatten-verify-tmp/02_depth3_full.cpp
+++ /dev/null
@@ -1,9 +0,0 @@
-// depth(3): fully flatten a three-deep perfect nest (OpenMP 6.1).
-extern "C" void body(int, int, int);
-void t(int n, int m, int p) {
-#pragma omp flatten depth(3)
-  for (int i = 0; i < n; ++i)
-    for (int j = 0; j < m; ++j)
-      for (int k = 0; k < p; ++k)
-        body(i, j, k);
-}
diff --git a/flatten-verify-tmp/03_depth2_partial.cpp b/flatten-verify-tmp/03_depth2_partial.cpp
deleted file mode 100644
index 35b9dc83834fd..0000000000000
--- a/flatten-verify-tmp/03_depth2_partial.cpp
+++ /dev/null
@@ -1,10 +0,0 @@
-// depth(2) on a three-deep nest: only the outer two loops are combined; the
-// innermost loop must remain as ordinary body of the flattened loop.
-extern "C" void body(int, int, int);
-void t(int n, int m, int p) {
-#pragma omp flatten depth(2)
-  for (int i = 0; i < n; ++i)
-    for (int j = 0; j < m; ++j)
-      for (int k = 0; k < p; ++k)
-        body(i, j, k);
-}
diff --git a/flatten-verify-tmp/04_depth1_identity.cpp b/flatten-verify-tmp/04_depth1_identity.cpp
deleted file mode 100644
index ae62a11312fe8..0000000000000
--- a/flatten-verify-tmp/04_depth1_identity.cpp
+++ /dev/null
@@ -1,8 +0,0 @@
-// depth(1): identity transform. A single loop is "combined"; no mixed-radix
-// decomposition (no srem), the flattened counter maps straight to the loop var.
-extern "C" void body(int);
-void t(int n) {
-#pragma omp flatten depth(1)
-  for (int i = 0; i < n; ++i)
-    body(i);
-}
diff --git a/flatten-verify-tmp/05_rangefor_template.cpp b/flatten-verify-tmp/05_rangefor_template.cpp
deleted file mode 100644
index 89928c34a8a15..0000000000000
--- a/flatten-verify-tmp/05_rangefor_template.cpp
+++ /dev/null
@@ -1,20 +0,0 @@
-// Range-for (CXXForRangeStmt) nest + a function template instantiated with a
-// dependent depth(K). Exercises the __begin/__end pre-inits and the deferred
-// (dependent-context) desugaring path.
-#include <stdlibc++.h>
-extern "C" void body(int, int);
-
-void ranges(std::vector<int> &a, std::vector<int> &b) {
-#pragma omp flatten
-  for (int i : a)
-    for (int j : b)
-      body(i, j);
-}
-
-template <int K> void tmpl(int n, int m) {
-#pragma omp flatten depth(K)
-  for (int i = 0; i < n; ++i)
-    for (int j = 0; j < m; ++j)
-      body(i, j);
-}
-template void tmpl<2>(int, int);
diff --git a/flatten-verify-tmp/README.md b/flatten-verify-tmp/README.md
deleted file mode 100644
index f48f4d06cfec0..0000000000000
--- a/flatten-verify-tmp/README.md
+++ /dev/null
@@ -1,39 +0,0 @@
-# Temporary flatten + depth verification cases
-
-Throwaway sandbox for eyeballing the flatten desugaring (AST / IR / runtime).
-Not part of any PR — delete freely. The committed regression tests live under
-`clang/test/OpenMP/flatten*` and `openmp/runtime/test/transform/flatten/*`.
-
-`CLANG=build/bin/clang`, `CLANGXX=build/bin/clang++`, run from the repo root.
-
-| File | Case | What to look for |
-|------|------|------------------|
-| `01_default_depth2.cpp` | default (no clause) = depth 2 | i64 `.flatten.iv`, `mul nsw`, `sdiv`+`srem` |
-| `02_depth3_full.cpp` | `depth(3)` full flatten | 3 `.flatten.iv.N`, two-level mixed radix |
-| `03_depth2_partial.cpp` | `depth(2)` on 3-deep nest | 2 flatten IVs, **inner `%k` loop stays** |
-| `04_depth1_identity.cpp` | `depth(1)` identity | i32 IV, **no `srem`**, direct store |
-| `05_rangefor_template.cpp` | range-for + template `depth(K)` | ast-print round-trips; instantiation |
-| `end_result_check.cpp` | runtime semantic check | prints `OK`, exit 0 (order+set preserved) |
-
-## Commands
-
-AST (round-trip original loops) and the desugared node:
-```
-$CLANG -cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-print FILE
-$CLANG -cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -ast-dump  FILE
-```
-
-IR (the div/mod arithmetic that implements the transform):
-```
-$CLANG -cc1 -triple x86_64-pc-linux-gnu -fopenmp -fopenmp-version=61 -emit-llvm -o - FILE
-```
-
-Range-for / template need the driver (libstdc++ + template instantiation):
-```
-$CLANGXX -fopenmp -fopenmp-version=61 -Xclang -ast-print -fsyntax-only 05_rangefor_template.cpp
-```
-
-Runtime end-result (must print OK, exit 0):
-```
-$CLANGXX -fopenmp -fopenmp-version=61 -O0 end_result_check.cpp -o /tmp/erc && /tmp/erc
-```
diff --git a/flatten-verify-tmp/end_result_check.cpp b/flatten-verify-tmp/end_result_check.cpp
deleted file mode 100644
index 3aa8850829058..0000000000000
--- a/flatten-verify-tmp/end_result_check.cpp
+++ /dev/null
@@ -1,82 +0,0 @@
-// End-result (semantic) verification: flatten must preserve BOTH the set of
-// iterations and their row-major visitation ORDER. We record the sequence of
-// visited tuples from a flattened nest and compare it against the sequence from
-// an identical non-flattened reference nest. Exit code 0 == identical.
-//
-// flatten is a pure frontend AST desugaring to an ordinary loop, so this runs
-// as a normal program (build with: clang++ -fopenmp -fopenmp-version=61).
-#include <cassert>
-#include <cstdio>
-#include <vector>
-
-using Seq = std::vector<long>;
-
-static void rec(Seq &s, int i, int j, int k = -1) {
-  s.push_back(i);
-  s.push_back(j);
-  if (k >= 0)
-    s.push_back(k);
-}
-
-int main() {
-  const int n = 3, m = 4, p = 2;
-
-  // --- depth(2) default over 2 loops ---
-  {
-    Seq flat, ref;
-#pragma omp flatten
-    for (int i = 0; i < n; ++i)
-      for (int j = 0; j < m; ++j)
-        rec(flat, i, j);
-    for (int i = 0; i < n; ++i)
-      for (int j = 0; j < m; ++j)
-        rec(ref, i, j);
-    assert(flat == ref && "depth(2) order/set mismatch");
-  }
-
-  // --- depth(3) full flatten over 3 loops ---
-  {
-    Seq flat, ref;
-#pragma omp flatten depth(3)
-    for (int i = 0; i < n; ++i)
-      for (int j = 0; j < m; ++j)
-        for (int k = 0; k < p; ++k)
-          rec(flat, i, j, k);
-    for (int i = 0; i < n; ++i)
-      for (int j = 0; j < m; ++j)
-        for (int k = 0; k < p; ++k)
-          rec(ref, i, j, k);
-    assert(flat == ref && "depth(3) order/set mismatch");
-  }
-
-  // --- depth(2) partial: inner k-loop stays intact ---
-  {
-    Seq flat, ref;
-#pragma omp flatten depth(2)
-    for (int i = 0; i < n; ++i)
-      for (int j = 0; j < m; ++j)
-        for (int k = 0; k < p; ++k)
-          rec(flat, i, j, k);
-    for (int i = 0; i < n; ++i)
-      for (int j = 0; j < m; ++j)
-        for (int k = 0; k < p; ++k)
-          rec(ref, i, j, k);
-    assert(flat == ref && "depth(2)-partial order/set mismatch");
-  }
-
-  // --- non-unit step / non-zero start must survive ---
-  {
-    Seq flat, ref;
-#pragma omp flatten
-    for (int i = 5; i < 12; i += 2)
-      for (int j = 1; j < 7; j += 3)
-        rec(flat, i, j);
-    for (int i = 5; i < 12; i += 2)
-      for (int j = 1; j < 7; j += 3)
-        rec(ref, i, j);
-    assert(flat == ref && "stride/offset mismatch");
-  }
-
-  std::puts("flatten end-result preserved: OK");
-  return 0;
-}

>From a169b5202ef4f4a5d9f8fbb47c4f8f9132ad0edb Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Tue, 18 Aug 2026 07:15:20 -0400
Subject: [PATCH 16/18] conflict-resolve

---
 clang/bindings/python/clang/cindex.py | 5 ++++-
 clang/include/clang-c/Index.h         | 6 +++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/clang/bindings/python/clang/cindex.py b/clang/bindings/python/clang/cindex.py
index ccad140e8c861..56e20fdfad781 100644
--- a/clang/bindings/python/clang/cindex.py
+++ b/clang/bindings/python/clang/cindex.py
@@ -1456,8 +1456,11 @@ def is_unexposed(self):
     # OpenMP split directive.
     OMP_SPLIT_DIRECTIVE = 312
 
+    # OpenMP ordered-blockassoc directive.
+    OMP_ORDERED_BLOCK_ASSOC_DIRECTIVE = 313
+
     # OpenMP flatten directive.
-    OMP_FLATTEN_DIRECTIVE = 313
+    OMP_FLATTEN_DIRECTIVE = 314
 
     # OpenACC Compute Construct.
     OPEN_ACC_COMPUTE_DIRECTIVE = 320
diff --git a/clang/include/clang-c/Index.h b/clang/include/clang-c/Index.h
index f561fb5724897..4c4cf392f172e 100644
--- a/clang/include/clang-c/Index.h
+++ b/clang/include/clang-c/Index.h
@@ -2170,9 +2170,13 @@ enum CXCursorKind {
    */
   CXCursor_OMPSplitDirective = 312,
 
+  /** OpenMP ordered-blockassoc directive.
+   */
+  CXCursor_OMPOrderedBlockAssocDirective = 313,
+
   /** OpenMP flatten directive.
    */
-  CXCursor_OMPFlattenDirective = 313,
+  CXCursor_OMPFlattenDirective = 314,
 
   /** OpenACC Compute Construct.
    */

>From 147063ca2af28aecf2a0135373cabb9c179c0999 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Tue, 18 Aug 2026 07:17:36 -0400
Subject: [PATCH 17/18] revised

---
 clang/include/clang/AST/OpenMPClause.h        |   8 +-
 clang/include/clang/AST/StmtOpenMP.h          |   4 +-
 .../clang/Basic/DiagnosticSemaKinds.td        |   6 +
 clang/lib/AST/OpenMPClause.cpp                |  12 --
 clang/lib/Sema/SemaOpenMP.cpp                 | 112 ++++++++++--------
 5 files changed, 74 insertions(+), 68 deletions(-)

diff --git a/clang/include/clang/AST/OpenMPClause.h b/clang/include/clang/AST/OpenMPClause.h
index 50d1364a9726d..05ccf4db5752b 100644
--- a/clang/include/clang/AST/OpenMPClause.h
+++ b/clang/include/clang/AST/OpenMPClause.h
@@ -1411,14 +1411,14 @@ class OMPDepthClause final : public OMPClause {
 public:
   /// Build an AST node for a 'depth' clause.
   ///
-  /// \param C         Context of the AST.
   /// \param StartLoc  Location of the 'depth' identifier.
   /// \param LParenLoc Location of '('.
   /// \param EndLoc    Location of ')'.
   /// \param Depth     The depth expression.
-  static OMPDepthClause *Create(const ASTContext &C, SourceLocation StartLoc,
-                                SourceLocation LParenLoc, SourceLocation EndLoc,
-                                Expr *Depth);
+  OMPDepthClause(SourceLocation StartLoc, SourceLocation LParenLoc,
+                 SourceLocation EndLoc, Expr *Depth)
+      : OMPClause(llvm::omp::OMPC_depth, StartLoc, EndLoc),
+        LParenLoc(LParenLoc), Depth(Depth) {}
 
   /// Build an empty 'depth' AST node for deserialization.
   ///
diff --git a/clang/include/clang/AST/StmtOpenMP.h b/clang/include/clang/AST/StmtOpenMP.h
index 1420331f37c1f..4ae6f364dced3 100644
--- a/clang/include/clang/AST/StmtOpenMP.h
+++ b/clang/include/clang/AST/StmtOpenMP.h
@@ -5999,8 +5999,8 @@ class OMPFlattenDirective final
   /// \param StartLoc  Location of the introducer (e.g. the 'omp' token).
   /// \param EndLoc    Location of the directive's end (e.g. the tok::eod).
   /// \param Clauses   The directive's clauses.
-  /// \param NumLoops  Number of affected loops (the flatten depth, currently
-  ///                  always 2).
+  /// \param NumLoops  Number of affected loops (the flatten depth: the
+  ///                  argument of the 'depth' clause, or 2 if omitted).
   /// \param AssociatedStmt  The outermost associated loop.
   /// \param TransformedStmt The flattened loop, or nullptr in dependent
   ///                        contexts.
diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td
index 7e20630708312..fc44b8d15239e 100644
--- a/clang/include/clang/Basic/DiagnosticSemaKinds.td
+++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td
@@ -12172,6 +12172,12 @@ def err_omp_invalid_looprange
 def warn_omp_redundant_fusion : Warning<"looprange clause selects a single "
                                         "loop, resulting in redundant fusion">,
                                 InGroup<OpenMPClauses>;
+def warn_omp_flatten_omitted_depth : Warning<
+  "'flatten' without a 'depth' clause only combines 2 loops, but 3 or more "
+  "loops are perfectly nested">,
+  InGroup<OpenMPClauses>;
+def err_omp_clause_not_supported_yet : Error<
+  "'%0' clause is not yet supported on '#pragma omp %1' directive">;
 def err_omp_not_for : Error<
   "%select{statement after '#pragma omp %1' must be a for loop|"
   "expected %2 for loops after '#pragma omp %1'%select{|, but found only %4}3}0">;
diff --git a/clang/lib/AST/OpenMPClause.cpp b/clang/lib/AST/OpenMPClause.cpp
index 7b2f67a31cad9..7311e404b66c3 100644
--- a/clang/lib/AST/OpenMPClause.cpp
+++ b/clang/lib/AST/OpenMPClause.cpp
@@ -1057,18 +1057,6 @@ OMPPartialClause *OMPPartialClause::CreateEmpty(const ASTContext &C) {
   return new (C) OMPPartialClause();
 }
 
-OMPDepthClause *OMPDepthClause::Create(const ASTContext &C,
-                                       SourceLocation StartLoc,
-                                       SourceLocation LParenLoc,
-                                       SourceLocation EndLoc, Expr *Depth) {
-  OMPDepthClause *Clause = CreateEmpty(C);
-  Clause->setLocStart(StartLoc);
-  Clause->setLParenLoc(LParenLoc);
-  Clause->setLocEnd(EndLoc);
-  Clause->setDepth(Depth);
-  return Clause;
-}
-
 OMPDepthClause *OMPDepthClause::CreateEmpty(const ASTContext &C) {
   return new (C) OMPDepthClause();
 }
diff --git a/clang/lib/Sema/SemaOpenMP.cpp b/clang/lib/Sema/SemaOpenMP.cpp
index fcc256dec3c88..caaba5a3be9fa 100644
--- a/clang/lib/Sema/SemaOpenMP.cpp
+++ b/clang/lib/Sema/SemaOpenMP.cpp
@@ -6839,6 +6839,7 @@ StmtResult SemaOpenMP::ActOnOpenMPExecutableDirective(
       case OMPC_safelen:
       case OMPC_simdlen:
       case OMPC_sizes:
+      case OMPC_depth:
       case OMPC_default:
       case OMPC_proc_bind:
       case OMPC_private:
@@ -16395,29 +16396,6 @@ StmtResult SemaOpenMP::ActOnOpenMPInterchangeDirective(
                                          buildPreInits(Context, PreInits));
 }
 
-/// Counts the perfectly nested canonical loop depth at \p AStmt. Returns
-/// std::nullopt if a loop-generating transformation prevents a static count.
-static std::optional<unsigned> getCanonicalLoopNestDepth(Stmt *AStmt) {
-  unsigned Depth = 0;
-  Stmt *CurStmt = AStmt ? AStmt->IgnoreContainers() : nullptr;
-  while (CurStmt) {
-    if (isa<OMPLoopTransformationDirective>(CurStmt))
-      return std::nullopt;
-    if (auto *CanonLoop = dyn_cast<OMPCanonicalLoop>(CurStmt))
-      CurStmt = CanonLoop->getLoopStmt();
-    Stmt *Body = nullptr;
-    if (auto *For = dyn_cast<ForStmt>(CurStmt))
-      Body = For->getBody();
-    else if (auto *RangeFor = dyn_cast<CXXForRangeStmt>(CurStmt))
-      Body = RangeFor->getBody();
-    else
-      break;
-    ++Depth;
-    CurStmt = Body ? Body->IgnoreContainers() : nullptr;
-  }
-  return Depth;
-}
-
 StmtResult
 SemaOpenMP::ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
                                         Stmt *AStmt, SourceLocation StartLoc,
@@ -16432,36 +16410,49 @@ SemaOpenMP::ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
 
   // flatten without 'depth' clause combines two loops; 'depth(k)' selects k.
   unsigned NumLoops = 2;
-  bool DepthIsValueDependent = false;
+  bool DepthIsDependent = false;
   const auto *DepthClause =
       OMPExecutableDirective::getSingleClause<OMPDepthClause>(Clauses);
   if (DepthClause) {
     Expr *DepthExpr = DepthClause->getDepth();
-    if (DepthExpr && DepthExpr->isValueDependent()) {
-      DepthIsValueDependent = true;
+    if (DepthExpr && DepthExpr->isInstantiationDependent()) {
+      DepthIsDependent = true;
     } else if (DepthExpr) {
       Expr::EvalResult EvalResult;
       if (DepthExpr->EvaluateAsInt(EvalResult, Context))
-        NumLoops = EvalResult.Val.getInt().getZExtValue();
+        NumLoops = EvalResult.Val.getInt().getLimitedValue();
     }
   }
 
-  // Report "expected k for loops, but found only n" when depth exceeds the
-  // perfect nest (same form as 'collapse'); skip if depth or nest is unknown.
-  if (DepthClause && !DepthIsValueDependent) {
-    if (std::optional<unsigned> NestDepth = getCanonicalLoopNestDepth(AStmt);
-        NestDepth && NumLoops > *NestDepth) {
+  // Count perfectly nested loops with doForAllLoops. When 'depth' is present,
+  // walk NumLoops iterations to diagnose an insufficient nest. When it is
+  // omitted, walk one extra loop (3 total) so we can warn that default
+  // flatten only combines 2 of a deeper nest.
+  if (!DepthIsDependent) {
+    unsigned WalkLimit = DepthClause ? NumLoops : 3;
+    unsigned Found = 0;
+    bool Enough = OMPLoopBasedDirective::doForAllLoops(
+        AStmt->IgnoreContainers(), /*TryImperfectlyNestedLoops=*/false,
+        WalkLimit, [&](unsigned Cnt, Stmt *S) {
+          if (!isa<ForStmt>(S) && !isa<CXXForRangeStmt>(S))
+            return true;
+          Found = Cnt + 1;
+          return false;
+        });
+    if (DepthClause && !Enough) {
       Diag(AStmt->getBeginLoc(), diag::err_omp_not_for)
           << /*expected N for loops form=*/1
-          << getOpenMPDirectiveName(OMPD_flatten) << NumLoops
-          << (*NestDepth > 0) << *NestDepth;
+          << getOpenMPDirectiveName(OMPD_flatten) << NumLoops << (Found > 0)
+          << Found;
       return StmtError();
     }
+    if (!DepthClause && Found >= 3)
+      Diag(StartLoc, diag::warn_omp_flatten_omitted_depth);
   }
 
-  // Defer when 'depth' is value-dependent (concrete k unknown until
+  // Defer when 'depth' is instantiation-dependent (concrete k unknown until
   // instantiation).
-  if (DepthIsValueDependent)
+  if (DepthIsDependent)
     return OMPFlattenDirective::Create(Context, StartLoc, EndLoc, Clauses,
                                        NumLoops, AStmt, nullptr, nullptr);
 
@@ -16537,20 +16528,30 @@ SemaOpenMP::ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
     AllCountsLessThan32Bits &=
         Context.getTypeSize(LoopHelpers[I].NumIterations->getType()) < 32;
 
-  ExprResult TripCount32 = BuildTripCount(/*Bits=*/32);
-  ExprResult TripCount64 = BuildTripCount(/*Bits=*/64);
-  if (!TripCount32.isUsable() || !TripCount64.isUsable())
+  ExprResult TripCount;
+  if (AllCountsLessThan32Bits || NumLoops == 1) {
+    TripCount = BuildTripCount(/*Bits=*/32);
+  } else {
+    ExprResult TripCount64 = BuildTripCount(/*Bits=*/64);
+    if (!TripCount64.isUsable())
+      return StmtError();
+    if (fitsInto(
+            /*Bits=*/32,
+            TripCount64.get()->getType()->hasSignedIntegerRepresentation(),
+            TripCount64.get(), SemaRef)) {
+      ExprResult TripCount32 = BuildTripCount(/*Bits=*/32);
+      if (TripCount32.isUsable() &&
+          Context.getTypeSize(TripCount32.get()->getType()) == 32)
+        TripCount = TripCount32;
+      else
+        TripCount = TripCount64;
+    } else {
+      TripCount = TripCount64;
+    }
+  }
+  if (!TripCount.isUsable())
     return StmtError();
 
-  ExprResult TripCount = TripCount64;
-  if (Context.getTypeSize(TripCount32.get()->getType()) == 32 &&
-      (AllCountsLessThan32Bits || NumLoops == 1 ||
-       fitsInto(
-           /*Bits=*/32,
-           TripCount32.get()->getType()->hasSignedIntegerRepresentation(),
-           TripCount64.get(), SemaRef)))
-    TripCount = TripCount32;
-
   QualType IVTy = TripCount.get()->getType();
   uint64_t IVWidth = Context.getTypeSize(IVTy);
 
@@ -16680,6 +16681,13 @@ StmtResult SemaOpenMP::ActOnOpenMPFuseDirective(ArrayRef<OMPClause *> Clauses,
   if (!AStmt)
     return StmtError();
 
+  if (const auto *DepthC =
+          OMPExecutableDirective::getSingleClause<OMPDepthClause>(Clauses)) {
+    Diag(DepthC->getBeginLoc(), diag::err_omp_clause_not_supported_yet)
+        << "depth" << getOpenMPDirectiveName(OMPD_fuse);
+    return StmtError();
+  }
+
   // Defer transformation in dependent contexts
   // The NumLoopNests argument is set to a placeholder 1 (even though
   // using looprange fuse could yield up to 3 top level loop nests)
@@ -17911,6 +17919,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPSimpleClause(
   case OMPC_safelen:
   case OMPC_simdlen:
   case OMPC_sizes:
+  case OMPC_depth:
   case OMPC_allocator:
   case OMPC_collapse:
   case OMPC_schedule:
@@ -18532,8 +18541,8 @@ OMPClause *SemaOpenMP::ActOnOpenMPDepthClause(Expr *DepthExpr,
     return nullptr;
   DepthExpr = DepthResult.get();
 
-  return OMPDepthClause::Create(getASTContext(), StartLoc, LParenLoc, EndLoc,
-                                DepthExpr);
+  return new (getASTContext())
+      OMPDepthClause(StartLoc, LParenLoc, EndLoc, DepthExpr);
 }
 
 OMPClause *SemaOpenMP::ActOnOpenMPLoopRangeClause(
@@ -18671,6 +18680,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPSingleExprWithArgClause(
   case OMPC_safelen:
   case OMPC_simdlen:
   case OMPC_sizes:
+  case OMPC_depth:
   case OMPC_allocator:
   case OMPC_collapse:
   case OMPC_proc_bind:
@@ -18959,6 +18969,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPClause(OpenMPClauseKind Kind,
   case OMPC_safelen:
   case OMPC_simdlen:
   case OMPC_sizes:
+  case OMPC_depth:
   case OMPC_allocator:
   case OMPC_collapse:
   case OMPC_schedule:
@@ -19643,6 +19654,7 @@ OMPClause *SemaOpenMP::ActOnOpenMPVarListClause(OpenMPClauseKind Kind,
   case OMPC_safelen:
   case OMPC_simdlen:
   case OMPC_sizes:
+  case OMPC_depth:
   case OMPC_allocator:
   case OMPC_collapse:
   case OMPC_default:

>From d38388bfaee9a412eea22deb6bb1b10d642a35c7 Mon Sep 17 00:00:00 2001
From: amtiwari <amtiwari at amd.com>
Date: Tue, 18 Aug 2026 08:42:25 -0400
Subject: [PATCH 18/18] added-tests

---
 clang/test/OpenMP/flatten_ast_print.cpp       | 30 +++++++
 clang/test/OpenMP/flatten_codegen.cpp         | 26 ++++++
 clang/test/OpenMP/flatten_depth_ast_print.cpp | 23 ++++++
 clang/test/OpenMP/flatten_messages.cpp        | 39 +++++++++
 clang/test/OpenMP/fuse_depth_messages.cpp     | 12 +++
 .../test/transform/flatten/depth-intfor.c     |  3 +-
 .../test/transform/flatten/depth-partial.c    |  3 +-
 openmp/runtime/test/transform/flatten/empty.c |  9 +++
 .../test/transform/flatten/lit.local.cfg      |  5 --
 .../flatten/parallel-wsloop-intfor.c          | 79 +++++++++++++++++++
 .../transform/flatten/reverse-on-flatten.c    | 29 +++++++
 .../test/transform/flatten/tile-on-flatten.c  | 29 +++++++
 .../transform/flatten/unroll-on-flatten.c     | 29 +++++++
 13 files changed, 309 insertions(+), 7 deletions(-)
 create mode 100644 clang/test/OpenMP/fuse_depth_messages.cpp
 delete mode 100644 openmp/runtime/test/transform/flatten/lit.local.cfg
 create mode 100644 openmp/runtime/test/transform/flatten/parallel-wsloop-intfor.c
 create mode 100644 openmp/runtime/test/transform/flatten/reverse-on-flatten.c
 create mode 100644 openmp/runtime/test/transform/flatten/tile-on-flatten.c
 create mode 100644 openmp/runtime/test/transform/flatten/unroll-on-flatten.c

diff --git a/clang/test/OpenMP/flatten_ast_print.cpp b/clang/test/OpenMP/flatten_ast_print.cpp
index e1115bf112c40..81be842fcd4a6 100644
--- a/clang/test/OpenMP/flatten_ast_print.cpp
+++ b/clang/test/OpenMP/flatten_ast_print.cpp
@@ -121,4 +121,34 @@ void inst() {
   foo5<int, 0, 8>();
 }
 
+// Unroll applied to the loop generated by flatten.
+// PRINT-LABEL: void foo7(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo7
+void foo7() {
+  // PRINT:     #pragma omp unroll partial(2)
+  // DUMP:      OMPUnrollDirective
+  #pragma omp unroll partial(2)
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  for (int i = 0; i < 4; ++i)
+    for (int j = 0; j < 3; ++j)
+      body(i, j);
+}
+
+// Tile applied to the loop generated by flatten.
+// PRINT-LABEL: void foo8(
+// DUMP-LABEL:  FunctionDecl {{.*}} foo8
+void foo8() {
+  // PRINT:     #pragma omp tile sizes(2)
+  // DUMP:      OMPTileDirective
+  #pragma omp tile sizes(2)
+  // PRINT:     #pragma omp flatten
+  // DUMP:      OMPFlattenDirective
+  #pragma omp flatten
+  for (int i = 0; i < 4; ++i)
+    for (int j = 0; j < 3; ++j)
+      body(i, j);
+}
+
 #endif
diff --git a/clang/test/OpenMP/flatten_codegen.cpp b/clang/test/OpenMP/flatten_codegen.cpp
index f77b2fb00ce9b..245e727ee0ac4 100644
--- a/clang/test/OpenMP/flatten_codegen.cpp
+++ b/clang/test/OpenMP/flatten_codegen.cpp
@@ -62,4 +62,30 @@ extern "C" void baz() {
       body(i, j);
 }
 
+// Unroll applied to the single loop generated by flatten.
+//
+// CHECK-LABEL: define {{.*}}void @unroll_on_flatten(
+// CHECK:   %.flatten.iv = alloca
+// CHECK:   call void @body(
+extern "C" void unroll_on_flatten() {
+#pragma omp unroll partial(2)
+#pragma omp flatten
+  for (int i = 0; i < 4; ++i)
+    for (int j = 0; j < 2; ++j)
+      body(i, j);
+}
+
+// Tile applied to the single loop generated by flatten.
+//
+// CHECK-LABEL: define {{.*}}void @tile_on_flatten(
+// CHECK:   %.flatten.iv = alloca
+// CHECK:   call void @body(
+extern "C" void tile_on_flatten() {
+#pragma omp tile sizes(2)
+#pragma omp flatten
+  for (int i = 0; i < 4; ++i)
+    for (int j = 0; j < 2; ++j)
+      body(i, j);
+}
+
 #endif
diff --git a/clang/test/OpenMP/flatten_depth_ast_print.cpp b/clang/test/OpenMP/flatten_depth_ast_print.cpp
index ceaaa68e1f79a..da5fce45bf11f 100644
--- a/clang/test/OpenMP/flatten_depth_ast_print.cpp
+++ b/clang/test/OpenMP/flatten_depth_ast_print.cpp
@@ -71,4 +71,27 @@ void inst() {
   foo_tmpl<int>();
 }
 
+// depth as a non-type template parameter.
+// PRINT-LABEL: template <int D> void foo_depth_d()
+// DUMP-LABEL:  FunctionTemplateDecl {{.*}} foo_depth_d
+template <int D>
+void foo_depth_d() {
+  // PRINT:     #pragma omp flatten depth(D)
+  // DUMP:      OMPFlattenDirective
+  // DUMP-NEXT: OMPDepthClause
+  #pragma omp flatten depth(D)
+  for (int i = 0; i < 4; ++i)
+    for (int j = 0; j < 4; ++j)
+      for (int k = 0; k < 4; ++k)
+        body(i, j, k);
+}
+
+// PRINT-LABEL: template<> void foo_depth_d<3>()
+// DUMP-LABEL:  FunctionDecl {{.*}} foo_depth_d 'void ()' implicit_instantiation
+// DUMP:        OMPFlattenDirective
+// DUMP-NEXT:   OMPDepthClause
+void inst_d() {
+  foo_depth_d<3>();
+}
+
 #endif
diff --git a/clang/test/OpenMP/flatten_messages.cpp b/clang/test/OpenMP/flatten_messages.cpp
index a022e0e8999d2..ffb6152da1654 100644
--- a/clang/test/OpenMP/flatten_messages.cpp
+++ b/clang/test/OpenMP/flatten_messages.cpp
@@ -86,4 +86,43 @@ void func(int n) {
   for (int i = 0; i < 7; ++i)
     for (int j = 0; j < 9; ++j)
       body(i, j);
+
+  // Without a depth clause, only the outermost two loops are flattened. Warn
+  // when a deeper perfect nest is left partially unflattened.
+  // expected-warning at +1 {{'flatten' without a 'depth' clause only combines 2 loops, but 3 or more loops are perfectly nested}}
+  #pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      for (int k = 0; k < n; ++k)
+        body(i, j, k);
+
+  // Flatten applied to reverse/unroll generated loops: those transforms wrap
+  // helper statements around the remaining inner loop, so the nest is no
+  // longer perfectly nested from flatten's point of view.
+  #pragma omp flatten
+  #pragma omp reverse
+  for (int i = 0; i < n; ++i)
+    // expected-error at +1 {{statement after '#pragma omp flatten' must be a for loop}}
+    for (int j = 0; j < n; ++j)
+      body(i, j);
+
+  // Stacked flatten: the inner flatten injects helper statements, so the outer
+  // flatten does not see a perfect nest of remaining loops.
+  #pragma omp flatten
+  // expected-warning at +1 {{'flatten' without a 'depth' clause only combines 2 loops, but 3 or more loops are perfectly nested}}
+  #pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      // expected-error at +1 {{statement after '#pragma omp flatten' must be a for loop}}
+      for (int k = 0; k < n; ++k)
+        body(i, j, k);
+
+  // Interchange needs two canonical loops; flatten produces one, so the
+  // leftover inner loop is no longer a perfect nest for interchange.
+  #pragma omp interchange
+  #pragma omp flatten
+  for (int i = 0; i < n; ++i)
+    for (int j = 0; j < n; ++j)
+      // expected-error at +1 {{statement after '#pragma omp interchange' must be a for loop}}
+      body(i, j);
 }
diff --git a/clang/test/OpenMP/fuse_depth_messages.cpp b/clang/test/OpenMP/fuse_depth_messages.cpp
new file mode 100644
index 0000000000000..2ecfde6153016
--- /dev/null
+++ b/clang/test/OpenMP/fuse_depth_messages.cpp
@@ -0,0 +1,12 @@
+// RUN: %clang_cc1 -triple x86_64-pc-linux-gnu -std=c++20 -fopenmp -fopenmp-version=61 -fsyntax-only -verify %s
+
+void func() {
+  // expected-error at +1 {{'depth' clause is not yet supported on '#pragma omp fuse' directive}}
+  #pragma omp fuse depth(2)
+  {
+    for (int i = 0; i < 7; ++i)
+      ;
+    for (int j = 0; j < 9; ++j)
+      ;
+  }
+}
diff --git a/openmp/runtime/test/transform/flatten/depth-intfor.c b/openmp/runtime/test/transform/flatten/depth-intfor.c
index 7211e02e25313..ccdc903ca4cca 100644
--- a/openmp/runtime/test/transform/flatten/depth-intfor.c
+++ b/openmp/runtime/test/transform/flatten/depth-intfor.c
@@ -1,4 +1,5 @@
-// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+// RUN: %libomp-compile -fopenmp-version=61 && %libomp-run | FileCheck %s
+// --match-full-lines
 
 // 'depth(3)' fully flattens a three-deep perfectly nested loop into one loop.
 // This verifies that the flattened loop preserves the exact row-major
diff --git a/openmp/runtime/test/transform/flatten/depth-partial.c b/openmp/runtime/test/transform/flatten/depth-partial.c
index 2fda3a04cbd84..473c306570d39 100644
--- a/openmp/runtime/test/transform/flatten/depth-partial.c
+++ b/openmp/runtime/test/transform/flatten/depth-partial.c
@@ -1,4 +1,5 @@
-// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+// RUN: %libomp-compile -fopenmp-version=61 && %libomp-run | FileCheck %s
+// --match-full-lines
 
 // A depth(2) clause on a three-deep loop nest flattens only the outermost two
 // loops; the innermost loop is left untouched and executes normally inside the
diff --git a/openmp/runtime/test/transform/flatten/empty.c b/openmp/runtime/test/transform/flatten/empty.c
index ecf1e8ef622f2..6a378887d86ce 100644
--- a/openmp/runtime/test/transform/flatten/empty.c
+++ b/openmp/runtime/test/transform/flatten/empty.c
@@ -28,6 +28,13 @@ int main() {
   printf("single-begin\n");
   flatten(1, 1);
   printf("single-end\n");
+
+  printf("neg-outer-begin\n");
+#pragma omp flatten
+  for (int i = 0; i < -1; ++i)
+    for (int j = 0; j < 3; ++j)
+      printf("i=%d j=%d\n", i, j);
+  printf("neg-outer-end\n");
   return EXIT_SUCCESS;
 }
 
@@ -40,3 +47,5 @@ int main() {
 // CHECK-NEXT: single-begin
 // CHECK-NEXT: i=0 j=0
 // CHECK-NEXT: single-end
+// CHECK-NEXT: neg-outer-begin
+// CHECK-NEXT: neg-outer-end
diff --git a/openmp/runtime/test/transform/flatten/lit.local.cfg b/openmp/runtime/test/transform/flatten/lit.local.cfg
deleted file mode 100644
index e05bd627df1e9..0000000000000
--- a/openmp/runtime/test/transform/flatten/lit.local.cfg
+++ /dev/null
@@ -1,5 +0,0 @@
-# The flatten directive's depth clause requires OpenMP 6.1.
-for i, (pattern, replacement) in enumerate(config.substitutions):
-    if pattern == "%openmp_flags":
-        config.substitutions[i] = (pattern, replacement + " -fopenmp-version=61")
-        break
diff --git a/openmp/runtime/test/transform/flatten/parallel-wsloop-intfor.c b/openmp/runtime/test/transform/flatten/parallel-wsloop-intfor.c
new file mode 100644
index 0000000000000..02cf256958a15
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/parallel-wsloop-intfor.c
@@ -0,0 +1,79 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+// Flatten changes which iterations a worksharing loop assigns to which
+// thread. With schedule(static,1) and two threads:
+//   * without flatten, the outer loop (2 iterations) is distributed
+//   * with flatten, the product space (4 iterations) is distributed
+// Results are printed per thread after the parallel region so FileCheck
+// is not racy.
+
+#ifndef HEADER
+#define HEADER
+
+#include <omp.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+enum { NThreads = 2, MaxIters = 4 };
+
+static void dump(const char *tag, int count[NThreads],
+                 int pairs[NThreads][MaxIters][2]) {
+  printf("%s\n", tag);
+  for (int t = 0; t < NThreads; ++t) {
+    printf("tid=%d count=%d\n", t, count[t]);
+    for (int k = 0; k < count[t]; ++k)
+      printf("tid=%d i=%d j=%d\n", t, pairs[t][k][0], pairs[t][k][1]);
+  }
+}
+
+int main() {
+  int count[NThreads];
+  int pairs[NThreads][MaxIters][2];
+
+  count[0] = count[1] = 0;
+#pragma omp parallel for schedule(static, 1) num_threads(2)
+#pragma omp flatten
+  for (int i = 0; i < 2; ++i)
+    for (int j = 0; j < 2; ++j) {
+      int t = omp_get_thread_num();
+      int c = count[t]++;
+      pairs[t][c][0] = i;
+      pairs[t][c][1] = j;
+    }
+  dump("with-flatten", count, pairs);
+
+  count[0] = count[1] = 0;
+#pragma omp parallel for schedule(static, 1) num_threads(2)
+  for (int i = 0; i < 2; ++i)
+    for (int j = 0; j < 2; ++j) {
+      int t = omp_get_thread_num();
+      int c = count[t]++;
+      pairs[t][c][0] = i;
+      pairs[t][c][1] = j;
+    }
+  dump("without-flatten", count, pairs);
+
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// Flattened product space has 4 iterations; static,1 with 2 threads gives
+// each thread 2 chunks: (0,0)+(1,0) and (0,1)+(1,1).
+// CHECK:      with-flatten
+// CHECK-NEXT: tid=0 count=2
+// CHECK-NEXT: tid=0 i=0 j=0
+// CHECK-NEXT: tid=0 i=1 j=0
+// CHECK-NEXT: tid=1 count=2
+// CHECK-NEXT: tid=1 i=0 j=1
+// CHECK-NEXT: tid=1 i=1 j=1
+
+// Without flatten the outer loop has 2 iterations, so each thread gets one
+// value of i and both j.
+// CHECK:      without-flatten
+// CHECK-NEXT: tid=0 count=2
+// CHECK-NEXT: tid=0 i=0 j=0
+// CHECK-NEXT: tid=0 i=0 j=1
+// CHECK-NEXT: tid=1 count=2
+// CHECK-NEXT: tid=1 i=1 j=0
+// CHECK-NEXT: tid=1 i=1 j=1
diff --git a/openmp/runtime/test/transform/flatten/reverse-on-flatten.c b/openmp/runtime/test/transform/flatten/reverse-on-flatten.c
new file mode 100644
index 0000000000000..99f85f0277d68
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/reverse-on-flatten.c
@@ -0,0 +1,29 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+int main() {
+  printf("do\n");
+#pragma omp reverse
+#pragma omp flatten
+  for (int i = 0; i < 2; ++i)
+    for (int j = 0; j < 2; ++j)
+      printf("i=%d j=%d\n", i, j);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// Reverse consumes the single loop produced by flatten, so the product
+// space is visited in reverse.
+// CHECK:      do
+// CHECK-NEXT: i=1 j=1
+// CHECK-NEXT: i=1 j=0
+// CHECK-NEXT: i=0 j=1
+// CHECK-NEXT: i=0 j=0
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/flatten/tile-on-flatten.c b/openmp/runtime/test/transform/flatten/tile-on-flatten.c
new file mode 100644
index 0000000000000..d56733e53dc4a
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/tile-on-flatten.c
@@ -0,0 +1,29 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+int main() {
+  printf("do\n");
+#pragma omp tile sizes(2)
+#pragma omp flatten
+  for (int i = 0; i < 2; ++i)
+    for (int j = 0; j < 2; ++j)
+      printf("i=%d j=%d\n", i, j);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// Tile consumes the single loop produced by flatten. For this nest the
+// tiled 1-D product still visits every (i, j) in row-major order.
+// CHECK:      do
+// CHECK-NEXT: i=0 j=0
+// CHECK-NEXT: i=0 j=1
+// CHECK-NEXT: i=1 j=0
+// CHECK-NEXT: i=1 j=1
+// CHECK-NEXT: done
diff --git a/openmp/runtime/test/transform/flatten/unroll-on-flatten.c b/openmp/runtime/test/transform/flatten/unroll-on-flatten.c
new file mode 100644
index 0000000000000..f6715a2570bfe
--- /dev/null
+++ b/openmp/runtime/test/transform/flatten/unroll-on-flatten.c
@@ -0,0 +1,29 @@
+// RUN: %libomp-compile-and-run | FileCheck %s --match-full-lines
+
+#ifndef HEADER
+#define HEADER
+
+#include <stdlib.h>
+#include <stdio.h>
+
+int main() {
+  printf("do\n");
+#pragma omp unroll partial(2)
+#pragma omp flatten
+  for (int i = 0; i < 2; ++i)
+    for (int j = 0; j < 2; ++j)
+      printf("i=%d j=%d\n", i, j);
+  printf("done\n");
+  return EXIT_SUCCESS;
+}
+
+#endif /* HEADER */
+
+// Unroll consumes the single loop produced by flatten and keeps the
+// original row-major visit order.
+// CHECK:      do
+// CHECK-NEXT: i=0 j=0
+// CHECK-NEXT: i=0 j=1
+// CHECK-NEXT: i=1 j=0
+// CHECK-NEXT: i=1 j=1
+// CHECK-NEXT: done


From openmp-commits at lists.llvm.org  Tue Aug 18 09:56:50 2026
From: openmp-commits at lists.llvm.org (Larry Meadows via Openmp-commits)
Date: Tue, 18 Aug 2026 09:56:50 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMPOpt] Ask the runtime how
 many of a block's threads can be workers (PR #217080)
Message-ID: <llvm.org/llvm/llvm-project/pull/217080@github.com>


https://github.com/lfmeadow created https://github.com/llvm/llvm-project/pull/217080

**NFC here**, and the diff is mostly regenerated test expectations.

The custom state machine gates a thread on `InitCB < BlockHwSize - WarpSize`, reconstructing the number of worker threads from the block size on the assumption that the main thread occupies a whole warp above them:

```llvm
%block.hw_size = call i32 @__kmpc_get_hardware_num_threads_in_block()
%warp.size = call i32 @__kmpc_get_warp_size()
%block.size = sub i32 %block.hw_size, %warp.size
%thread.is_main_or_worker = icmp slt i32 %0, %block.size
```

The DeviceRTL already computes that number in `mapping::getMaxTeamThreads()`, and its own generic state machine gates on it in `shouldEnterStateMachine()`. This exports it as `__kmpc_get_max_team_threads()` and calls that instead:

```llvm
%max_team_threads = call i32 @__kmpc_get_max_team_threads()
%thread.is_main_or_worker = icmp slt i32 %0, %max_team_threads
```

so the compiler's state machine and the runtime's agree by construction rather than by arithmetic that has to be kept in step with the launch geometry. In generic mode `getMaxTeamThreads()` is `BlockSize - WarpSize`, so the value is unchanged.

### Why it is worth doing if it changes nothing here

It is not NFC for a toolchain whose launch geometry differs. In ROCm, `CGOpenMPRuntimeGPU::ComputeGenericWorkgroupSize` starts a single extra thread rather than a warp — *"Only one additional thread is started, not an entire warp"* — so `thread_limit(1024)` on a 64-lane target launches 961 threads and the runtime reports 960 workers, while the state machine's arithmetic says `961 - 64 = 897`. The 63 threads in between are in neither group: the state machine returns immediately for them, and the parallel region still hands them iterations, so a kernel silently drops work. We carry a runtime regression test for that downstream, where the difference is observable; here it would pass either way.

Asking the runtime removes the assumption rather than adding a second one, and it keeps the two state machines — the compiler's and the runtime's — reading the same number.

A device runtime predating the new entry point is handled by the existing `runtimeFnsAvailable()` check, which skips the rewrite rather than emitting a call to a function that is not there. The two calls this replaces keep their other uses in the pass, including the constant folding of `__kmpc_get_hardware_num_threads_in_block`.

`check-llvm` is clean: 47336 passed, 77 expectedly failed, no unexpected failures.

cc @jdoerfert @jhuber6


>From 60b054d45ac9886ee7731e29069ee50219eef003 Mon Sep 17 00:00:00 2001
From: Larry Meadows <Lawrence.Meadows at amd.com>
Date: Tue, 18 Aug 2026 11:56:16 -0500
Subject: [PATCH] [OpenMPOpt] Ask the runtime how many of a block's threads can
 be workers

The custom state machine gates a thread on InitCB < BlockHwSize - WarpSize,
reconstructing the number of worker threads from the block size on the
assumption that the main thread occupies a whole warp above them. The DeviceRTL
already computes that number, in mapping::getMaxTeamThreads(), and its own
generic state machine gates on it in shouldEnterStateMachine(). Export it as
__kmpc_get_max_team_threads() and call that instead, so the compiler's state
machine and the runtime's agree by construction rather than by arithmetic that
has to be kept in step with the launch geometry.

This is NFC here: getMaxTeamThreads() in generic mode is BlockSize - WarpSize,
the same three instructions folded into one call. It is not NFC for a toolchain
whose launch geometry differs. In ROCm, CGOpenMPRuntimeGPU starts a single extra
thread rather than a warp -- "Only one additional thread is started, not an
entire warp" -- so thread_limit(1024) on a 64-lane target launches 961 threads
and the runtime reports 960 workers, while the state machine's own arithmetic
says 961 - 64 = 897. The threads in between are in neither group: the state
machine returns immediately for them, and the parallel region still hands them
iterations. Asking the runtime removes the assumption rather than adding a
second one.

A device runtime that predates the new entry point is handled by the existing
runtimeFnsAvailable() check, which skips the rewrite rather than emitting a call
to a function that is not there.

The two calls this replaces, __kmpc_get_hardware_num_threads_in_block and
__kmpc_get_warp_size, keep their other uses in the pass, including the constant
folding of the former.

Co-authored-by: Cursor <cursoragent at cursor.com>
---
 .../include/llvm/Frontend/OpenMP/OMPKinds.def |  2 +
 llvm/lib/Transforms/IPO/OpenMPOpt.cpp         | 52 ++++++--------
 .../OpenMP/custom_state_machines.ll           | 72 +++++++------------
 llvm/test/Transforms/OpenMP/spmdization.ll    | 64 +++++------------
 .../Transforms/OpenMP/spmdization_guarding.ll |  6 +-
 .../Transforms/OpenMP/spmdization_indirect.ll | 16 ++---
 ...zation_no_guarding_two_reaching_kernels.ll | 18 ++---
 openmp/device/include/Interface.h             |  3 +
 openmp/device/src/Mapping.cpp                 |  4 ++
 9 files changed, 83 insertions(+), 154 deletions(-)

diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
index ea1cef97900f5..730027f0fb477 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
@@ -228,6 +228,7 @@ __OMP_RTL(__kmpc_omp_reg_task_with_affinity, false, Int32, IdentPtr, Int32,
 __OMP_RTL(__kmpc_get_hardware_num_blocks, false, Int32, )
 __OMP_RTL(__kmpc_get_hardware_num_threads_in_block, false, Int32, )
 __OMP_RTL(__kmpc_get_warp_size, false, Int32, )
+__OMP_RTL(__kmpc_get_max_team_threads, false, Int32, )
 
 __OMP_RTL(omp_get_thread_num, false, Int32, )
 __OMP_RTL(omp_get_num_threads, false, Int32, )
@@ -721,6 +722,7 @@ __OMP_RTL_ATTRS(__kmpc_omp_reg_task_with_affinity, DefaultAttrs, SExt,
 __OMP_RTL_ATTRS(__kmpc_get_hardware_num_blocks, GetterAttrs, ZExt, ParamAttrs())
 __OMP_RTL_ATTRS(__kmpc_get_hardware_num_threads_in_block, GetterAttrs, ZExt, ParamAttrs())
 __OMP_RTL_ATTRS(__kmpc_get_warp_size, GetterAttrs, ZExt, ParamAttrs())
+__OMP_RTL_ATTRS(__kmpc_get_max_team_threads, GetterAttrs, ZExt, ParamAttrs())
 
 __OMP_RTL_ATTRS(omp_get_thread_num, GetterAttrs, SExt, ParamAttrs())
 __OMP_RTL_ATTRS(omp_get_num_threads, GetterAttrs, SExt, ParamAttrs())
diff --git a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
index 22a5efb402504..0f8c6b991e763 100644
--- a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
+++ b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
@@ -3904,8 +3904,7 @@ struct AAKernelInfoFunction : AAKernelInfo {
     Attributor::VirtualUseCallbackTy CustomStateMachineUseCB =
         [&](Attributor &A, const AbstractAttribute *QueryingAA) {
           // Whenever we create a custom state machine we will insert calls to
-          // __kmpc_get_hardware_num_threads_in_block,
-          // __kmpc_get_warp_size,
+          // __kmpc_get_max_team_threads,
           // __kmpc_barrier_simple_generic,
           // __kmpc_kernel_parallel, and
           // __kmpc_kernel_end_parallel.
@@ -3920,9 +3919,8 @@ struct AAKernelInfoFunction : AAKernelInfo {
 
     // Not needed if we are pre-runtime merge.
     if (!KernelInitCB->getCalledFunction()->isDeclaration()) {
-      RegisterVirtualUse(OMPRTL___kmpc_get_hardware_num_threads_in_block,
+      RegisterVirtualUse(OMPRTL___kmpc_get_max_team_threads,
                          CustomStateMachineUseCB);
-      RegisterVirtualUse(OMPRTL___kmpc_get_warp_size, CustomStateMachineUseCB);
       RegisterVirtualUse(OMPRTL___kmpc_barrier_simple_generic,
                          CustomStateMachineUseCB);
       RegisterVirtualUse(OMPRTL___kmpc_kernel_parallel,
@@ -4381,10 +4379,10 @@ struct AAKernelInfoFunction : AAKernelInfo {
       return false;
 
     auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
-    if (!OMPInfoCache.runtimeFnsAvailable(
-            {OMPRTL___kmpc_get_hardware_num_threads_in_block,
-             OMPRTL___kmpc_get_warp_size, OMPRTL___kmpc_barrier_simple_generic,
-             OMPRTL___kmpc_kernel_parallel, OMPRTL___kmpc_kernel_end_parallel}))
+    if (!OMPInfoCache.runtimeFnsAvailable({OMPRTL___kmpc_get_max_team_threads,
+                                           OMPRTL___kmpc_barrier_simple_generic,
+                                           OMPRTL___kmpc_kernel_parallel,
+                                           OMPRTL___kmpc_kernel_end_parallel}))
       return false;
 
     ConstantStruct *ExistingKernelEnvC =
@@ -4463,13 +4461,10 @@ struct AAKernelInfoFunction : AAKernelInfo {
     // Create all the blocks:
     //
     //                       InitCB = __kmpc_target_init(...)
-    //                       BlockHwSize =
-    //                         __kmpc_get_hardware_num_threads_in_block();
-    //                       WarpSize = __kmpc_get_warp_size();
-    //                       BlockSize = BlockHwSize - WarpSize;
+    //                       MaxTeamThreads = __kmpc_get_max_team_threads();
     // IsWorkerCheckBB:      bool IsWorker = InitCB != -1;
     //                       if (IsWorker) {
-    //                         if (InitCB >= BlockSize) return;
+    //                         if (InitCB >= MaxTeamThreads) return;
     // SMBeginBB:               __kmpc_barrier_simple_generic(...);
     //                         void *WorkFn;
     //                         bool Active = __kmpc_kernel_parallel(&WorkFn);
@@ -4534,26 +4529,23 @@ struct AAKernelInfoFunction : AAKernelInfo {
     IsWorker->setDebugLoc(DLoc);
     CondBrInst::Create(IsWorker, IsWorkerCheckBB, UserCodeEntryBB, InitBB);
 
+    // How many of the block's threads can be worker threads is a property of
+    // the launch geometry, which the runtime knows and the block size alone
+    // does not determine: a target may host the main thread in a whole warp
+    // above the workers, or in a single thread above them. Ask the runtime
+    // rather than subtracting a warp here, or the workers in between are left
+    // in neither group, waiting for no parallel region while it is handed
+    // iterations.
     Module &M = *Kernel->getParent();
-    FunctionCallee BlockHwSizeFn =
+    FunctionCallee MaxTeamThreadsFn =
         OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
-            M, OMPRTL___kmpc_get_hardware_num_threads_in_block);
-    FunctionCallee WarpSizeFn =
-        OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
-            M, OMPRTL___kmpc_get_warp_size);
-    CallInst *BlockHwSize =
-        CallInst::Create(BlockHwSizeFn, "block.hw_size", IsWorkerCheckBB);
-    OMPInfoCache.setCallingConvention(BlockHwSizeFn, BlockHwSize);
-    BlockHwSize->setDebugLoc(DLoc);
-    CallInst *WarpSize =
-        CallInst::Create(WarpSizeFn, "warp.size", IsWorkerCheckBB);
-    OMPInfoCache.setCallingConvention(WarpSizeFn, WarpSize);
-    WarpSize->setDebugLoc(DLoc);
-    Instruction *BlockSize = BinaryOperator::CreateSub(
-        BlockHwSize, WarpSize, "block.size", IsWorkerCheckBB);
-    BlockSize->setDebugLoc(DLoc);
+            M, OMPRTL___kmpc_get_max_team_threads);
+    CallInst *MaxTeamThreads =
+        CallInst::Create(MaxTeamThreadsFn, "max_team_threads", IsWorkerCheckBB);
+    OMPInfoCache.setCallingConvention(MaxTeamThreadsFn, MaxTeamThreads);
+    MaxTeamThreads->setDebugLoc(DLoc);
     Instruction *IsMainOrWorker = ICmpInst::Create(
-        ICmpInst::ICmp, llvm::CmpInst::ICMP_SLT, KernelInitCB, BlockSize,
+        ICmpInst::ICmp, llvm::CmpInst::ICMP_SLT, KernelInitCB, MaxTeamThreads,
         "thread.is_main_or_worker", IsWorkerCheckBB);
     IsMainOrWorker->setDebugLoc(DLoc);
     CondBrInst::Create(IsMainOrWorker, StateMachineBeginBB,
diff --git a/llvm/test/Transforms/OpenMP/custom_state_machines.ll b/llvm/test/Transforms/OpenMP/custom_state_machines.ll
index a03a4b7f3fa67..ac35d7a4a9765 100644
--- a/llvm/test/Transforms/OpenMP/custom_state_machines.ll
+++ b/llvm/test/Transforms/OpenMP/custom_state_machines.ll
@@ -1152,10 +1152,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; AMDGPU:       is_worker_check:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; AMDGPU-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; AMDGPU:       worker_state_machine.begin:
 ; AMDGPU-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -1273,10 +1271,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; AMDGPU:       is_worker_check:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; AMDGPU-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; AMDGPU:       worker_state_machine.begin:
 ; AMDGPU-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -1420,10 +1416,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; AMDGPU:       is_worker_check:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; AMDGPU-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; AMDGPU:       worker_state_machine.begin:
 ; AMDGPU-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -1542,10 +1536,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; AMDGPU:       is_worker_check:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; AMDGPU-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; AMDGPU:       worker_state_machine.begin:
 ; AMDGPU-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -1662,10 +1654,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; AMDGPU:       is_worker_check:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; AMDGPU-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; AMDGPU:       worker_state_machine.begin:
 ; AMDGPU-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -1854,10 +1844,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; AMDGPU:       is_worker_check:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; AMDGPU-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; AMDGPU:       worker_state_machine.begin:
 ; AMDGPU-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -2078,10 +2066,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; NVPTX:       is_worker_check:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; NVPTX-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; NVPTX:       worker_state_machine.begin:
 ; NVPTX-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -2198,10 +2184,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; NVPTX:       is_worker_check:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; NVPTX-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; NVPTX:       worker_state_machine.begin:
 ; NVPTX-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -2344,10 +2328,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; NVPTX:       is_worker_check:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; NVPTX-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; NVPTX:       worker_state_machine.begin:
 ; NVPTX-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -2465,10 +2447,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; NVPTX:       is_worker_check:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; NVPTX-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; NVPTX:       worker_state_machine.begin:
 ; NVPTX-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -2584,10 +2564,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; NVPTX:       is_worker_check:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; NVPTX-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; NVPTX:       worker_state_machine.begin:
 ; NVPTX-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -2775,10 +2753,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; NVPTX:       is_worker_check:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; NVPTX-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; NVPTX:       worker_state_machine.begin:
 ; NVPTX-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
diff --git a/llvm/test/Transforms/OpenMP/spmdization.ll b/llvm/test/Transforms/OpenMP/spmdization.ll
index 7d53a808ef4d3..88a4d53ae9c82 100644
--- a/llvm/test/Transforms/OpenMP/spmdization.ll
+++ b/llvm/test/Transforms/OpenMP/spmdization.ll
@@ -271,9 +271,7 @@ define internal void @__omp_offloading_fd02_2044372e_sequential_loop_l5__debug()
 ; AMDGPU-DISABLED1-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-DISABLED1-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; AMDGPU-DISABLED1:       [[IS_WORKER_CHECK]]:
-; AMDGPU-DISABLED1-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-DISABLED1-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; AMDGPU-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; AMDGPU-DISABLED1-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; AMDGPU-DISABLED1-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; AMDGPU-DISABLED1:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -343,9 +341,7 @@ define internal void @__omp_offloading_fd02_2044372e_sequential_loop_l5__debug()
 ; NVPTX-DISABLED1-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-DISABLED1-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; NVPTX-DISABLED1:       [[IS_WORKER_CHECK]]:
-; NVPTX-DISABLED1-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-DISABLED1-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; NVPTX-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; NVPTX-DISABLED1-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; NVPTX-DISABLED1-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; NVPTX-DISABLED1:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -753,9 +749,7 @@ define weak ptx_kernel void @__omp_offloading_fd02_2044372e_sequential_loop_to_s
 ; AMDGPU-DISABLED1-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-DISABLED1-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; AMDGPU-DISABLED1:       [[IS_WORKER_CHECK]]:
-; AMDGPU-DISABLED1-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-DISABLED1-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; AMDGPU-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; AMDGPU-DISABLED1-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; AMDGPU-DISABLED1-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; AMDGPU-DISABLED1:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -825,9 +819,7 @@ define weak ptx_kernel void @__omp_offloading_fd02_2044372e_sequential_loop_to_s
 ; NVPTX-DISABLED1-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-DISABLED1-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; NVPTX-DISABLED1:       [[IS_WORKER_CHECK]]:
-; NVPTX-DISABLED1-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-DISABLED1-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; NVPTX-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; NVPTX-DISABLED1-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; NVPTX-DISABLED1-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; NVPTX-DISABLED1:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -1253,9 +1245,7 @@ define weak ptx_kernel void @__omp_offloading_fd02_2044372e_sequential_loop_to_s
 ; AMDGPU-DISABLED1-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-DISABLED1-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; AMDGPU-DISABLED1:       [[IS_WORKER_CHECK]]:
-; AMDGPU-DISABLED1-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-DISABLED1-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; AMDGPU-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; AMDGPU-DISABLED1-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; AMDGPU-DISABLED1-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; AMDGPU-DISABLED1:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -1325,9 +1315,7 @@ define weak ptx_kernel void @__omp_offloading_fd02_2044372e_sequential_loop_to_s
 ; NVPTX-DISABLED1-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-DISABLED1-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; NVPTX-DISABLED1:       [[IS_WORKER_CHECK]]:
-; NVPTX-DISABLED1-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-DISABLED1-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; NVPTX-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; NVPTX-DISABLED1-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; NVPTX-DISABLED1-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; NVPTX-DISABLED1:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -1779,9 +1767,7 @@ define weak ptx_kernel void @__omp_offloading_fd02_2044372e_sequential_loop_to_s
 ; AMDGPU-DISABLED1-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-DISABLED1-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; AMDGPU-DISABLED1:       [[IS_WORKER_CHECK]]:
-; AMDGPU-DISABLED1-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-DISABLED1-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; AMDGPU-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; AMDGPU-DISABLED1-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; AMDGPU-DISABLED1-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; AMDGPU-DISABLED1:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -1851,9 +1837,7 @@ define weak ptx_kernel void @__omp_offloading_fd02_2044372e_sequential_loop_to_s
 ; NVPTX-DISABLED1-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-DISABLED1-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; NVPTX-DISABLED1:       [[IS_WORKER_CHECK]]:
-; NVPTX-DISABLED1-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-DISABLED1-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; NVPTX-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; NVPTX-DISABLED1-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; NVPTX-DISABLED1-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; NVPTX-DISABLED1:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -2300,9 +2284,7 @@ define weak ptx_kernel void @__omp_offloading_fd02_2044372e_do_not_spmdize_targe
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; AMDGPU:       [[IS_WORKER_CHECK]]:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; AMDGPU:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -2348,9 +2330,7 @@ define weak ptx_kernel void @__omp_offloading_fd02_2044372e_do_not_spmdize_targe
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; NVPTX:       [[IS_WORKER_CHECK]]:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; NVPTX:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -2395,9 +2375,7 @@ define weak ptx_kernel void @__omp_offloading_fd02_2044372e_do_not_spmdize_targe
 ; AMDGPU-DISABLED1-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-DISABLED1-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; AMDGPU-DISABLED1:       [[IS_WORKER_CHECK]]:
-; AMDGPU-DISABLED1-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-DISABLED1-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; AMDGPU-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; AMDGPU-DISABLED1-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; AMDGPU-DISABLED1-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; AMDGPU-DISABLED1:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -2461,9 +2439,7 @@ define weak ptx_kernel void @__omp_offloading_fd02_2044372e_do_not_spmdize_targe
 ; NVPTX-DISABLED1-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-DISABLED1-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; NVPTX-DISABLED1:       [[IS_WORKER_CHECK]]:
-; NVPTX-DISABLED1-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-DISABLED1-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; NVPTX-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; NVPTX-DISABLED1-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; NVPTX-DISABLED1-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; NVPTX-DISABLED1:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -2589,9 +2565,7 @@ define weak ptx_kernel void @__omp_offloading_fd02_2044372e_do_not_spmdize_task_
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; AMDGPU:       [[IS_WORKER_CHECK]]:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; AMDGPU:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -2643,9 +2617,7 @@ define weak ptx_kernel void @__omp_offloading_fd02_2044372e_do_not_spmdize_task_
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; NVPTX:       [[IS_WORKER_CHECK]]:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; NVPTX:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -2696,9 +2668,7 @@ define weak ptx_kernel void @__omp_offloading_fd02_2044372e_do_not_spmdize_task_
 ; AMDGPU-DISABLED1-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-DISABLED1-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; AMDGPU-DISABLED1:       [[IS_WORKER_CHECK]]:
-; AMDGPU-DISABLED1-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-DISABLED1-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; AMDGPU-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; AMDGPU-DISABLED1-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; AMDGPU-DISABLED1-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; AMDGPU-DISABLED1:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -2768,9 +2738,7 @@ define weak ptx_kernel void @__omp_offloading_fd02_2044372e_do_not_spmdize_task_
 ; NVPTX-DISABLED1-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-DISABLED1-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; NVPTX-DISABLED1:       [[IS_WORKER_CHECK]]:
-; NVPTX-DISABLED1-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-DISABLED1-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; NVPTX-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; NVPTX-DISABLED1-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; NVPTX-DISABLED1-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; NVPTX-DISABLED1:       [[WORKER_STATE_MACHINE_BEGIN]]:
diff --git a/llvm/test/Transforms/OpenMP/spmdization_guarding.ll b/llvm/test/Transforms/OpenMP/spmdization_guarding.ll
index 2fb1f878cce7b..4dae6889beb9d 100644
--- a/llvm/test/Transforms/OpenMP/spmdization_guarding.ll
+++ b/llvm/test/Transforms/OpenMP/spmdization_guarding.ll
@@ -201,10 +201,8 @@ define weak ptx_kernel void @__omp_offloading_2a_fbfa7a_sequential_loop_l6(ptr %
 ; CHECK-DISABLED-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; CHECK-DISABLED-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; CHECK-DISABLED:       is_worker_check:
-; CHECK-DISABLED-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; CHECK-DISABLED-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; CHECK-DISABLED-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; CHECK-DISABLED-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; CHECK-DISABLED-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; CHECK-DISABLED-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; CHECK-DISABLED-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; CHECK-DISABLED:       worker_state_machine.begin:
 ; CHECK-DISABLED-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
diff --git a/llvm/test/Transforms/OpenMP/spmdization_indirect.ll b/llvm/test/Transforms/OpenMP/spmdization_indirect.ll
index 1de719e1b85a8..71847b72ae24f 100644
--- a/llvm/test/Transforms/OpenMP/spmdization_indirect.ll
+++ b/llvm/test/Transforms/OpenMP/spmdization_indirect.ll
@@ -395,9 +395,7 @@ define weak ptx_kernel void @spmd_and_non_spmd_callee(i1 %c) #0 {
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; AMDGPU:       [[IS_WORKER_CHECK]]:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; AMDGPU:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -458,9 +456,7 @@ define weak ptx_kernel void @spmd_and_non_spmd_callee(i1 %c) #0 {
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; NVPTX:       [[IS_WORKER_CHECK]]:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; NVPTX:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -751,9 +747,7 @@ define weak ptx_kernel void @spmd_and_non_spmd_callees_metadata(ptr %fp) #0 {
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; AMDGPU:       [[IS_WORKER_CHECK]]:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; AMDGPU:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -813,9 +807,7 @@ define weak ptx_kernel void @spmd_and_non_spmd_callees_metadata(ptr %fp) #0 {
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; NVPTX:       [[IS_WORKER_CHECK]]:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; NVPTX:       [[WORKER_STATE_MACHINE_BEGIN]]:
diff --git a/llvm/test/Transforms/OpenMP/spmdization_no_guarding_two_reaching_kernels.ll b/llvm/test/Transforms/OpenMP/spmdization_no_guarding_two_reaching_kernels.ll
index 4f64839deaf53..dc08df96dfd64 100644
--- a/llvm/test/Transforms/OpenMP/spmdization_no_guarding_two_reaching_kernels.ll
+++ b/llvm/test/Transforms/OpenMP/spmdization_no_guarding_two_reaching_kernels.ll
@@ -78,10 +78,8 @@ define weak ptx_kernel void @__omp_offloading_2b_10393b5_spmd_l12(ptr %dyn) #0 {
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; CHECK-DISABLE-SPMDIZATION:       is_worker_check:
-; CHECK-DISABLE-SPMDIZATION-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; CHECK-DISABLE-SPMDIZATION-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; CHECK-DISABLE-SPMDIZATION-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; CHECK-DISABLE-SPMDIZATION-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; CHECK-DISABLE-SPMDIZATION-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; CHECK-DISABLE-SPMDIZATION-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; CHECK-DISABLE-SPMDIZATION:       worker_state_machine.begin:
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -155,10 +153,8 @@ define weak ptx_kernel void @__omp_offloading_2b_10393b5_generic_l20(ptr %dyn) #
 ; CHECK-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; CHECK-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; CHECK:       is_worker_check:
-; CHECK-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; CHECK-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; CHECK-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; CHECK-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; CHECK-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; CHECK-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; CHECK-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; CHECK:       worker_state_machine.begin:
 ; CHECK-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -197,10 +193,8 @@ define weak ptx_kernel void @__omp_offloading_2b_10393b5_generic_l20(ptr %dyn) #
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; CHECK-DISABLE-SPMDIZATION:       is_worker_check:
-; CHECK-DISABLE-SPMDIZATION-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; CHECK-DISABLE-SPMDIZATION-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; CHECK-DISABLE-SPMDIZATION-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; CHECK-DISABLE-SPMDIZATION-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; CHECK-DISABLE-SPMDIZATION-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; CHECK-DISABLE-SPMDIZATION-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; CHECK-DISABLE-SPMDIZATION:       worker_state_machine.begin:
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
diff --git a/openmp/device/include/Interface.h b/openmp/device/include/Interface.h
index 601694871597b..07ad56e8dbef0 100644
--- a/openmp/device/include/Interface.h
+++ b/openmp/device/include/Interface.h
@@ -217,6 +217,9 @@ uint32_t __kmpc_get_hardware_num_threads_in_block();
 /// External interface to get the warp size.
 uint32_t __kmpc_get_warp_size();
 
+/// External interface to get the maximum number of threads in a team.
+uint32_t __kmpc_get_max_team_threads();
+
 /// Kernel
 ///
 ///{
diff --git a/openmp/device/src/Mapping.cpp b/openmp/device/src/Mapping.cpp
index 1096e5f496590..f65ccb5203056 100644
--- a/openmp/device/src/Mapping.cpp
+++ b/openmp/device/src/Mapping.cpp
@@ -175,6 +175,10 @@ extern "C" {
 [[gnu::noinline]] uint32_t __kmpc_get_warp_size() {
   return mapping::getWarpSize();
 }
+
+[[gnu::noinline]] uint32_t __kmpc_get_max_team_threads() {
+  return mapping::getMaxTeamThreads();
+}
 }
 
 #define _TGT_KERNEL_LANGUAGE(NAME, MAPPER_NAME)                                \


From openmp-commits at lists.llvm.org  Tue Aug 18 09:57:31 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Tue, 18 Aug 2026 09:57:31 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMPOpt] Ask the runtime how
 many of a block's threads can be workers (PR #217080)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/217080@github.com>
Message-ID: <6a848efb.4987c784.36d8d6.3407@mx.google.com>


llvmorg-github-actions[bot] wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-llvm-transforms

Author: Larry Meadows (lfmeadow)

<details>
<summary>Changes</summary>

**NFC here**, and the diff is mostly regenerated test expectations.

The custom state machine gates a thread on `InitCB &lt; BlockHwSize - WarpSize`, reconstructing the number of worker threads from the block size on the assumption that the main thread occupies a whole warp above them:

```llvm
%block.hw_size = call i32 @<!-- -->__kmpc_get_hardware_num_threads_in_block()
%warp.size = call i32 @<!-- -->__kmpc_get_warp_size()
%block.size = sub i32 %block.hw_size, %warp.size
%thread.is_main_or_worker = icmp slt i32 %0, %block.size
```

The DeviceRTL already computes that number in `mapping::getMaxTeamThreads()`, and its own generic state machine gates on it in `shouldEnterStateMachine()`. This exports it as `__kmpc_get_max_team_threads()` and calls that instead:

```llvm
%max_team_threads = call i32 @<!-- -->__kmpc_get_max_team_threads()
%thread.is_main_or_worker = icmp slt i32 %0, %max_team_threads
```

so the compiler's state machine and the runtime's agree by construction rather than by arithmetic that has to be kept in step with the launch geometry. In generic mode `getMaxTeamThreads()` is `BlockSize - WarpSize`, so the value is unchanged.

### Why it is worth doing if it changes nothing here

It is not NFC for a toolchain whose launch geometry differs. In ROCm, `CGOpenMPRuntimeGPU::ComputeGenericWorkgroupSize` starts a single extra thread rather than a warp — *"Only one additional thread is started, not an entire warp"* — so `thread_limit(1024)` on a 64-lane target launches 961 threads and the runtime reports 960 workers, while the state machine's arithmetic says `961 - 64 = 897`. The 63 threads in between are in neither group: the state machine returns immediately for them, and the parallel region still hands them iterations, so a kernel silently drops work. We carry a runtime regression test for that downstream, where the difference is observable; here it would pass either way.

Asking the runtime removes the assumption rather than adding a second one, and it keeps the two state machines — the compiler's and the runtime's — reading the same number.

A device runtime predating the new entry point is handled by the existing `runtimeFnsAvailable()` check, which skips the rewrite rather than emitting a call to a function that is not there. The two calls this replaces keep their other uses in the pass, including the constant folding of `__kmpc_get_hardware_num_threads_in_block`.

`check-llvm` is clean: 47336 passed, 77 expectedly failed, no unexpected failures.

cc @<!-- -->jdoerfert @<!-- -->jhuber6


---

Patch is 46.05 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/217080.diff


9 Files Affected:

- (modified) llvm/include/llvm/Frontend/OpenMP/OMPKinds.def (+2) 
- (modified) llvm/lib/Transforms/IPO/OpenMPOpt.cpp (+22-30) 
- (modified) llvm/test/Transforms/OpenMP/custom_state_machines.ll (+24-48) 
- (modified) llvm/test/Transforms/OpenMP/spmdization.ll (+16-48) 
- (modified) llvm/test/Transforms/OpenMP/spmdization_guarding.ll (+2-4) 
- (modified) llvm/test/Transforms/OpenMP/spmdization_indirect.ll (+4-12) 
- (modified) llvm/test/Transforms/OpenMP/spmdization_no_guarding_two_reaching_kernels.ll (+6-12) 
- (modified) openmp/device/include/Interface.h (+3) 
- (modified) openmp/device/src/Mapping.cpp (+4) 


``````````diff
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
index ea1cef97900f5..730027f0fb477 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
@@ -228,6 +228,7 @@ __OMP_RTL(__kmpc_omp_reg_task_with_affinity, false, Int32, IdentPtr, Int32,
 __OMP_RTL(__kmpc_get_hardware_num_blocks, false, Int32, )
 __OMP_RTL(__kmpc_get_hardware_num_threads_in_block, false, Int32, )
 __OMP_RTL(__kmpc_get_warp_size, false, Int32, )
+__OMP_RTL(__kmpc_get_max_team_threads, false, Int32, )
 
 __OMP_RTL(omp_get_thread_num, false, Int32, )
 __OMP_RTL(omp_get_num_threads, false, Int32, )
@@ -721,6 +722,7 @@ __OMP_RTL_ATTRS(__kmpc_omp_reg_task_with_affinity, DefaultAttrs, SExt,
 __OMP_RTL_ATTRS(__kmpc_get_hardware_num_blocks, GetterAttrs, ZExt, ParamAttrs())
 __OMP_RTL_ATTRS(__kmpc_get_hardware_num_threads_in_block, GetterAttrs, ZExt, ParamAttrs())
 __OMP_RTL_ATTRS(__kmpc_get_warp_size, GetterAttrs, ZExt, ParamAttrs())
+__OMP_RTL_ATTRS(__kmpc_get_max_team_threads, GetterAttrs, ZExt, ParamAttrs())
 
 __OMP_RTL_ATTRS(omp_get_thread_num, GetterAttrs, SExt, ParamAttrs())
 __OMP_RTL_ATTRS(omp_get_num_threads, GetterAttrs, SExt, ParamAttrs())
diff --git a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
index 22a5efb402504..0f8c6b991e763 100644
--- a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
+++ b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
@@ -3904,8 +3904,7 @@ struct AAKernelInfoFunction : AAKernelInfo {
     Attributor::VirtualUseCallbackTy CustomStateMachineUseCB =
         [&](Attributor &A, const AbstractAttribute *QueryingAA) {
           // Whenever we create a custom state machine we will insert calls to
-          // __kmpc_get_hardware_num_threads_in_block,
-          // __kmpc_get_warp_size,
+          // __kmpc_get_max_team_threads,
           // __kmpc_barrier_simple_generic,
           // __kmpc_kernel_parallel, and
           // __kmpc_kernel_end_parallel.
@@ -3920,9 +3919,8 @@ struct AAKernelInfoFunction : AAKernelInfo {
 
     // Not needed if we are pre-runtime merge.
     if (!KernelInitCB->getCalledFunction()->isDeclaration()) {
-      RegisterVirtualUse(OMPRTL___kmpc_get_hardware_num_threads_in_block,
+      RegisterVirtualUse(OMPRTL___kmpc_get_max_team_threads,
                          CustomStateMachineUseCB);
-      RegisterVirtualUse(OMPRTL___kmpc_get_warp_size, CustomStateMachineUseCB);
       RegisterVirtualUse(OMPRTL___kmpc_barrier_simple_generic,
                          CustomStateMachineUseCB);
       RegisterVirtualUse(OMPRTL___kmpc_kernel_parallel,
@@ -4381,10 +4379,10 @@ struct AAKernelInfoFunction : AAKernelInfo {
       return false;
 
     auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
-    if (!OMPInfoCache.runtimeFnsAvailable(
-            {OMPRTL___kmpc_get_hardware_num_threads_in_block,
-             OMPRTL___kmpc_get_warp_size, OMPRTL___kmpc_barrier_simple_generic,
-             OMPRTL___kmpc_kernel_parallel, OMPRTL___kmpc_kernel_end_parallel}))
+    if (!OMPInfoCache.runtimeFnsAvailable({OMPRTL___kmpc_get_max_team_threads,
+                                           OMPRTL___kmpc_barrier_simple_generic,
+                                           OMPRTL___kmpc_kernel_parallel,
+                                           OMPRTL___kmpc_kernel_end_parallel}))
       return false;
 
     ConstantStruct *ExistingKernelEnvC =
@@ -4463,13 +4461,10 @@ struct AAKernelInfoFunction : AAKernelInfo {
     // Create all the blocks:
     //
     //                       InitCB = __kmpc_target_init(...)
-    //                       BlockHwSize =
-    //                         __kmpc_get_hardware_num_threads_in_block();
-    //                       WarpSize = __kmpc_get_warp_size();
-    //                       BlockSize = BlockHwSize - WarpSize;
+    //                       MaxTeamThreads = __kmpc_get_max_team_threads();
     // IsWorkerCheckBB:      bool IsWorker = InitCB != -1;
     //                       if (IsWorker) {
-    //                         if (InitCB >= BlockSize) return;
+    //                         if (InitCB >= MaxTeamThreads) return;
     // SMBeginBB:               __kmpc_barrier_simple_generic(...);
     //                         void *WorkFn;
     //                         bool Active = __kmpc_kernel_parallel(&WorkFn);
@@ -4534,26 +4529,23 @@ struct AAKernelInfoFunction : AAKernelInfo {
     IsWorker->setDebugLoc(DLoc);
     CondBrInst::Create(IsWorker, IsWorkerCheckBB, UserCodeEntryBB, InitBB);
 
+    // How many of the block's threads can be worker threads is a property of
+    // the launch geometry, which the runtime knows and the block size alone
+    // does not determine: a target may host the main thread in a whole warp
+    // above the workers, or in a single thread above them. Ask the runtime
+    // rather than subtracting a warp here, or the workers in between are left
+    // in neither group, waiting for no parallel region while it is handed
+    // iterations.
     Module &M = *Kernel->getParent();
-    FunctionCallee BlockHwSizeFn =
+    FunctionCallee MaxTeamThreadsFn =
         OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
-            M, OMPRTL___kmpc_get_hardware_num_threads_in_block);
-    FunctionCallee WarpSizeFn =
-        OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
-            M, OMPRTL___kmpc_get_warp_size);
-    CallInst *BlockHwSize =
-        CallInst::Create(BlockHwSizeFn, "block.hw_size", IsWorkerCheckBB);
-    OMPInfoCache.setCallingConvention(BlockHwSizeFn, BlockHwSize);
-    BlockHwSize->setDebugLoc(DLoc);
-    CallInst *WarpSize =
-        CallInst::Create(WarpSizeFn, "warp.size", IsWorkerCheckBB);
-    OMPInfoCache.setCallingConvention(WarpSizeFn, WarpSize);
-    WarpSize->setDebugLoc(DLoc);
-    Instruction *BlockSize = BinaryOperator::CreateSub(
-        BlockHwSize, WarpSize, "block.size", IsWorkerCheckBB);
-    BlockSize->setDebugLoc(DLoc);
+            M, OMPRTL___kmpc_get_max_team_threads);
+    CallInst *MaxTeamThreads =
+        CallInst::Create(MaxTeamThreadsFn, "max_team_threads", IsWorkerCheckBB);
+    OMPInfoCache.setCallingConvention(MaxTeamThreadsFn, MaxTeamThreads);
+    MaxTeamThreads->setDebugLoc(DLoc);
     Instruction *IsMainOrWorker = ICmpInst::Create(
-        ICmpInst::ICmp, llvm::CmpInst::ICMP_SLT, KernelInitCB, BlockSize,
+        ICmpInst::ICmp, llvm::CmpInst::ICMP_SLT, KernelInitCB, MaxTeamThreads,
         "thread.is_main_or_worker", IsWorkerCheckBB);
     IsMainOrWorker->setDebugLoc(DLoc);
     CondBrInst::Create(IsMainOrWorker, StateMachineBeginBB,
diff --git a/llvm/test/Transforms/OpenMP/custom_state_machines.ll b/llvm/test/Transforms/OpenMP/custom_state_machines.ll
index a03a4b7f3fa67..ac35d7a4a9765 100644
--- a/llvm/test/Transforms/OpenMP/custom_state_machines.ll
+++ b/llvm/test/Transforms/OpenMP/custom_state_machines.ll
@@ -1152,10 +1152,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; AMDGPU:       is_worker_check:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; AMDGPU-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; AMDGPU:       worker_state_machine.begin:
 ; AMDGPU-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -1273,10 +1271,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; AMDGPU:       is_worker_check:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; AMDGPU-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; AMDGPU:       worker_state_machine.begin:
 ; AMDGPU-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -1420,10 +1416,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; AMDGPU:       is_worker_check:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; AMDGPU-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; AMDGPU:       worker_state_machine.begin:
 ; AMDGPU-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -1542,10 +1536,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; AMDGPU:       is_worker_check:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; AMDGPU-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; AMDGPU:       worker_state_machine.begin:
 ; AMDGPU-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -1662,10 +1654,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; AMDGPU:       is_worker_check:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; AMDGPU-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; AMDGPU:       worker_state_machine.begin:
 ; AMDGPU-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -1854,10 +1844,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; AMDGPU:       is_worker_check:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; AMDGPU-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; AMDGPU:       worker_state_machine.begin:
 ; AMDGPU-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -2078,10 +2066,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; NVPTX:       is_worker_check:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; NVPTX-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; NVPTX:       worker_state_machine.begin:
 ; NVPTX-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -2198,10 +2184,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; NVPTX:       is_worker_check:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; NVPTX-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; NVPTX:       worker_state_machine.begin:
 ; NVPTX-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -2344,10 +2328,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; NVPTX:       is_worker_check:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; NVPTX-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; NVPTX:       worker_state_machine.begin:
 ; NVPTX-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -2465,10 +2447,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; NVPTX:       is_worker_check:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; NVPTX-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; NVPTX:       worker_state_machine.begin:
 ; NVPTX-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -2584,10 +2564,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; NVPTX:       is_worker_check:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; NVPTX-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; NVPTX:       worker_state_machine.begin:
 ; NVPTX-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -2775,10 +2753,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; NVPTX:       is_worker_check:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; NVPTX-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; NVPTX:       worker_state_machine.begin:
 ; NVPTX-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
diff --git a/llvm/test/Transforms/OpenMP/spmdization.ll b/llvm/test/Transforms/OpenMP/spmdization.ll
index 7d53a808ef4d3..88a4d53ae9c82 100644
--- a/llvm/test/Transforms/OpenMP/spmdization.ll
+++ b/llvm/test/Transforms/OpenMP/spmdization.ll
@@ -271,9 +271,7 @@ define internal void @__omp_offloading_fd02_2044372e_sequential_loop_l5_...
[truncated]

``````````

</details>


https://github.com/llvm/llvm-project/pull/217080

From openmp-commits at lists.llvm.org  Tue Aug 18 09:57:32 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Tue, 18 Aug 2026 09:57:32 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMPOpt] Ask the runtime how
 many of a block's threads can be workers (PR #217080)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/217080@github.com>
Message-ID: <6a848efc.e5907290.132c56.22d4@mx.google.com>


llvmorg-github-actions[bot] wrote:


<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-flang-openmp

Author: Larry Meadows (lfmeadow)

<details>
<summary>Changes</summary>

**NFC here**, and the diff is mostly regenerated test expectations.

The custom state machine gates a thread on `InitCB &lt; BlockHwSize - WarpSize`, reconstructing the number of worker threads from the block size on the assumption that the main thread occupies a whole warp above them:

```llvm
%block.hw_size = call i32 @<!-- -->__kmpc_get_hardware_num_threads_in_block()
%warp.size = call i32 @<!-- -->__kmpc_get_warp_size()
%block.size = sub i32 %block.hw_size, %warp.size
%thread.is_main_or_worker = icmp slt i32 %0, %block.size
```

The DeviceRTL already computes that number in `mapping::getMaxTeamThreads()`, and its own generic state machine gates on it in `shouldEnterStateMachine()`. This exports it as `__kmpc_get_max_team_threads()` and calls that instead:

```llvm
%max_team_threads = call i32 @<!-- -->__kmpc_get_max_team_threads()
%thread.is_main_or_worker = icmp slt i32 %0, %max_team_threads
```

so the compiler's state machine and the runtime's agree by construction rather than by arithmetic that has to be kept in step with the launch geometry. In generic mode `getMaxTeamThreads()` is `BlockSize - WarpSize`, so the value is unchanged.

### Why it is worth doing if it changes nothing here

It is not NFC for a toolchain whose launch geometry differs. In ROCm, `CGOpenMPRuntimeGPU::ComputeGenericWorkgroupSize` starts a single extra thread rather than a warp — *"Only one additional thread is started, not an entire warp"* — so `thread_limit(1024)` on a 64-lane target launches 961 threads and the runtime reports 960 workers, while the state machine's arithmetic says `961 - 64 = 897`. The 63 threads in between are in neither group: the state machine returns immediately for them, and the parallel region still hands them iterations, so a kernel silently drops work. We carry a runtime regression test for that downstream, where the difference is observable; here it would pass either way.

Asking the runtime removes the assumption rather than adding a second one, and it keeps the two state machines — the compiler's and the runtime's — reading the same number.

A device runtime predating the new entry point is handled by the existing `runtimeFnsAvailable()` check, which skips the rewrite rather than emitting a call to a function that is not there. The two calls this replaces keep their other uses in the pass, including the constant folding of `__kmpc_get_hardware_num_threads_in_block`.

`check-llvm` is clean: 47336 passed, 77 expectedly failed, no unexpected failures.

cc @<!-- -->jdoerfert @<!-- -->jhuber6


---

Patch is 46.05 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/217080.diff


9 Files Affected:

- (modified) llvm/include/llvm/Frontend/OpenMP/OMPKinds.def (+2) 
- (modified) llvm/lib/Transforms/IPO/OpenMPOpt.cpp (+22-30) 
- (modified) llvm/test/Transforms/OpenMP/custom_state_machines.ll (+24-48) 
- (modified) llvm/test/Transforms/OpenMP/spmdization.ll (+16-48) 
- (modified) llvm/test/Transforms/OpenMP/spmdization_guarding.ll (+2-4) 
- (modified) llvm/test/Transforms/OpenMP/spmdization_indirect.ll (+4-12) 
- (modified) llvm/test/Transforms/OpenMP/spmdization_no_guarding_two_reaching_kernels.ll (+6-12) 
- (modified) openmp/device/include/Interface.h (+3) 
- (modified) openmp/device/src/Mapping.cpp (+4) 


``````````diff
diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
index ea1cef97900f5..730027f0fb477 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
@@ -228,6 +228,7 @@ __OMP_RTL(__kmpc_omp_reg_task_with_affinity, false, Int32, IdentPtr, Int32,
 __OMP_RTL(__kmpc_get_hardware_num_blocks, false, Int32, )
 __OMP_RTL(__kmpc_get_hardware_num_threads_in_block, false, Int32, )
 __OMP_RTL(__kmpc_get_warp_size, false, Int32, )
+__OMP_RTL(__kmpc_get_max_team_threads, false, Int32, )
 
 __OMP_RTL(omp_get_thread_num, false, Int32, )
 __OMP_RTL(omp_get_num_threads, false, Int32, )
@@ -721,6 +722,7 @@ __OMP_RTL_ATTRS(__kmpc_omp_reg_task_with_affinity, DefaultAttrs, SExt,
 __OMP_RTL_ATTRS(__kmpc_get_hardware_num_blocks, GetterAttrs, ZExt, ParamAttrs())
 __OMP_RTL_ATTRS(__kmpc_get_hardware_num_threads_in_block, GetterAttrs, ZExt, ParamAttrs())
 __OMP_RTL_ATTRS(__kmpc_get_warp_size, GetterAttrs, ZExt, ParamAttrs())
+__OMP_RTL_ATTRS(__kmpc_get_max_team_threads, GetterAttrs, ZExt, ParamAttrs())
 
 __OMP_RTL_ATTRS(omp_get_thread_num, GetterAttrs, SExt, ParamAttrs())
 __OMP_RTL_ATTRS(omp_get_num_threads, GetterAttrs, SExt, ParamAttrs())
diff --git a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
index 22a5efb402504..0f8c6b991e763 100644
--- a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
+++ b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
@@ -3904,8 +3904,7 @@ struct AAKernelInfoFunction : AAKernelInfo {
     Attributor::VirtualUseCallbackTy CustomStateMachineUseCB =
         [&](Attributor &A, const AbstractAttribute *QueryingAA) {
           // Whenever we create a custom state machine we will insert calls to
-          // __kmpc_get_hardware_num_threads_in_block,
-          // __kmpc_get_warp_size,
+          // __kmpc_get_max_team_threads,
           // __kmpc_barrier_simple_generic,
           // __kmpc_kernel_parallel, and
           // __kmpc_kernel_end_parallel.
@@ -3920,9 +3919,8 @@ struct AAKernelInfoFunction : AAKernelInfo {
 
     // Not needed if we are pre-runtime merge.
     if (!KernelInitCB->getCalledFunction()->isDeclaration()) {
-      RegisterVirtualUse(OMPRTL___kmpc_get_hardware_num_threads_in_block,
+      RegisterVirtualUse(OMPRTL___kmpc_get_max_team_threads,
                          CustomStateMachineUseCB);
-      RegisterVirtualUse(OMPRTL___kmpc_get_warp_size, CustomStateMachineUseCB);
       RegisterVirtualUse(OMPRTL___kmpc_barrier_simple_generic,
                          CustomStateMachineUseCB);
       RegisterVirtualUse(OMPRTL___kmpc_kernel_parallel,
@@ -4381,10 +4379,10 @@ struct AAKernelInfoFunction : AAKernelInfo {
       return false;
 
     auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
-    if (!OMPInfoCache.runtimeFnsAvailable(
-            {OMPRTL___kmpc_get_hardware_num_threads_in_block,
-             OMPRTL___kmpc_get_warp_size, OMPRTL___kmpc_barrier_simple_generic,
-             OMPRTL___kmpc_kernel_parallel, OMPRTL___kmpc_kernel_end_parallel}))
+    if (!OMPInfoCache.runtimeFnsAvailable({OMPRTL___kmpc_get_max_team_threads,
+                                           OMPRTL___kmpc_barrier_simple_generic,
+                                           OMPRTL___kmpc_kernel_parallel,
+                                           OMPRTL___kmpc_kernel_end_parallel}))
       return false;
 
     ConstantStruct *ExistingKernelEnvC =
@@ -4463,13 +4461,10 @@ struct AAKernelInfoFunction : AAKernelInfo {
     // Create all the blocks:
     //
     //                       InitCB = __kmpc_target_init(...)
-    //                       BlockHwSize =
-    //                         __kmpc_get_hardware_num_threads_in_block();
-    //                       WarpSize = __kmpc_get_warp_size();
-    //                       BlockSize = BlockHwSize - WarpSize;
+    //                       MaxTeamThreads = __kmpc_get_max_team_threads();
     // IsWorkerCheckBB:      bool IsWorker = InitCB != -1;
     //                       if (IsWorker) {
-    //                         if (InitCB >= BlockSize) return;
+    //                         if (InitCB >= MaxTeamThreads) return;
     // SMBeginBB:               __kmpc_barrier_simple_generic(...);
     //                         void *WorkFn;
     //                         bool Active = __kmpc_kernel_parallel(&WorkFn);
@@ -4534,26 +4529,23 @@ struct AAKernelInfoFunction : AAKernelInfo {
     IsWorker->setDebugLoc(DLoc);
     CondBrInst::Create(IsWorker, IsWorkerCheckBB, UserCodeEntryBB, InitBB);
 
+    // How many of the block's threads can be worker threads is a property of
+    // the launch geometry, which the runtime knows and the block size alone
+    // does not determine: a target may host the main thread in a whole warp
+    // above the workers, or in a single thread above them. Ask the runtime
+    // rather than subtracting a warp here, or the workers in between are left
+    // in neither group, waiting for no parallel region while it is handed
+    // iterations.
     Module &M = *Kernel->getParent();
-    FunctionCallee BlockHwSizeFn =
+    FunctionCallee MaxTeamThreadsFn =
         OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
-            M, OMPRTL___kmpc_get_hardware_num_threads_in_block);
-    FunctionCallee WarpSizeFn =
-        OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
-            M, OMPRTL___kmpc_get_warp_size);
-    CallInst *BlockHwSize =
-        CallInst::Create(BlockHwSizeFn, "block.hw_size", IsWorkerCheckBB);
-    OMPInfoCache.setCallingConvention(BlockHwSizeFn, BlockHwSize);
-    BlockHwSize->setDebugLoc(DLoc);
-    CallInst *WarpSize =
-        CallInst::Create(WarpSizeFn, "warp.size", IsWorkerCheckBB);
-    OMPInfoCache.setCallingConvention(WarpSizeFn, WarpSize);
-    WarpSize->setDebugLoc(DLoc);
-    Instruction *BlockSize = BinaryOperator::CreateSub(
-        BlockHwSize, WarpSize, "block.size", IsWorkerCheckBB);
-    BlockSize->setDebugLoc(DLoc);
+            M, OMPRTL___kmpc_get_max_team_threads);
+    CallInst *MaxTeamThreads =
+        CallInst::Create(MaxTeamThreadsFn, "max_team_threads", IsWorkerCheckBB);
+    OMPInfoCache.setCallingConvention(MaxTeamThreadsFn, MaxTeamThreads);
+    MaxTeamThreads->setDebugLoc(DLoc);
     Instruction *IsMainOrWorker = ICmpInst::Create(
-        ICmpInst::ICmp, llvm::CmpInst::ICMP_SLT, KernelInitCB, BlockSize,
+        ICmpInst::ICmp, llvm::CmpInst::ICMP_SLT, KernelInitCB, MaxTeamThreads,
         "thread.is_main_or_worker", IsWorkerCheckBB);
     IsMainOrWorker->setDebugLoc(DLoc);
     CondBrInst::Create(IsMainOrWorker, StateMachineBeginBB,
diff --git a/llvm/test/Transforms/OpenMP/custom_state_machines.ll b/llvm/test/Transforms/OpenMP/custom_state_machines.ll
index a03a4b7f3fa67..ac35d7a4a9765 100644
--- a/llvm/test/Transforms/OpenMP/custom_state_machines.ll
+++ b/llvm/test/Transforms/OpenMP/custom_state_machines.ll
@@ -1152,10 +1152,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; AMDGPU:       is_worker_check:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; AMDGPU-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; AMDGPU:       worker_state_machine.begin:
 ; AMDGPU-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -1273,10 +1271,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; AMDGPU:       is_worker_check:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; AMDGPU-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; AMDGPU:       worker_state_machine.begin:
 ; AMDGPU-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -1420,10 +1416,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; AMDGPU:       is_worker_check:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; AMDGPU-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; AMDGPU:       worker_state_machine.begin:
 ; AMDGPU-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -1542,10 +1536,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; AMDGPU:       is_worker_check:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; AMDGPU-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; AMDGPU:       worker_state_machine.begin:
 ; AMDGPU-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -1662,10 +1654,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; AMDGPU:       is_worker_check:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; AMDGPU-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; AMDGPU:       worker_state_machine.begin:
 ; AMDGPU-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -1854,10 +1844,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; AMDGPU:       is_worker_check:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; AMDGPU-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; AMDGPU:       worker_state_machine.begin:
 ; AMDGPU-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -2078,10 +2066,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; NVPTX:       is_worker_check:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; NVPTX-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; NVPTX:       worker_state_machine.begin:
 ; NVPTX-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -2198,10 +2184,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; NVPTX:       is_worker_check:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; NVPTX-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; NVPTX:       worker_state_machine.begin:
 ; NVPTX-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -2344,10 +2328,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; NVPTX:       is_worker_check:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; NVPTX-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; NVPTX:       worker_state_machine.begin:
 ; NVPTX-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -2465,10 +2447,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; NVPTX:       is_worker_check:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; NVPTX-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; NVPTX:       worker_state_machine.begin:
 ; NVPTX-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -2584,10 +2564,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; NVPTX:       is_worker_check:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; NVPTX-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; NVPTX:       worker_state_machine.begin:
 ; NVPTX-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -2775,10 +2753,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; NVPTX:       is_worker_check:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; NVPTX-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; NVPTX:       worker_state_machine.begin:
 ; NVPTX-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
diff --git a/llvm/test/Transforms/OpenMP/spmdization.ll b/llvm/test/Transforms/OpenMP/spmdization.ll
index 7d53a808ef4d3..88a4d53ae9c82 100644
--- a/llvm/test/Transforms/OpenMP/spmdization.ll
+++ b/llvm/test/Transforms/OpenMP/spmdization.ll
@@ -271,9 +271,7 @@ define internal void @__omp_offloading_fd02_2044372e_sequential_loop_l5_...
[truncated]

``````````

</details>


https://github.com/llvm/llvm-project/pull/217080

From openmp-commits at lists.llvm.org  Tue Aug 18 10:20:11 2026
From: openmp-commits at lists.llvm.org (Larry Meadows via Openmp-commits)
Date: Tue, 18 Aug 2026 10:20:11 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMPOpt] Ask the runtime how
 many of a block's threads can be workers (PR #217080)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/217080@github.com>
Message-ID: <6a84944b.6c8668f2.38979c.01e9@mx.google.com>


https://github.com/lfmeadow converted_to_draft https://github.com/llvm/llvm-project/pull/217080

From openmp-commits at lists.llvm.org  Tue Aug 18 11:18:32 2026
From: openmp-commits at lists.llvm.org (Larry Meadows via Openmp-commits)
Date: Tue, 18 Aug 2026 11:18:32 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMPOpt] Ask the runtime how
 many of a block's threads can be workers (PR #217080)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/217080@github.com>
Message-ID: <6a84a1f8.d681cab9.3ab686.974c@mx.google.com>


https://github.com/lfmeadow edited https://github.com/llvm/llvm-project/pull/217080

From openmp-commits at lists.llvm.org  Tue Aug 18 12:47:57 2026
From: openmp-commits at lists.llvm.org (=?UTF-8?Q?Jan_Andr=C3=A9_Reuter?= via Openmp-commits)
Date: Tue, 18 Aug 2026 12:47:57 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [OpenMP][OMPT] Emit `dispatch` callback
 for serial loops (PR #207773)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/207773@github.com>
Message-ID: <6a84b6ed.510f576d.207701.f9fe@mx.google.com>


Thyre wrote:

> Thanks @Thyre. I gave this a quick glance and it appears to me that the change for `work` could be an NFC patch and then the new functionality for `dispatch` would be more targeted. If that is indeed the case, I'd appreciate if you can separate the two changes. Thanks!

You're correct. I think your suggestion is a good idea. I'll look into it :smile: 

https://github.com/llvm/llvm-project/pull/207773

From openmp-commits at lists.llvm.org  Tue Aug 18 21:59:41 2026
From: openmp-commits at lists.llvm.org (David Justo via Openmp-commits)
Date: Tue, 18 Aug 2026 21:59:41 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [NFC][CodeQL] Suppress intentional ITT
 collector library loading (PR #217116)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/217116@github.com>
Message-ID: <6a85383d.6904917c.2590d7.b2f1@mx.google.com>


davidmrdavid wrote:

@AndreyChurbanov - as you were a recent contributor to this file, would you be able to help me with a review? Thanks!

https://github.com/llvm/llvm-project/pull/217116

From openmp-commits at lists.llvm.org  Tue Aug 18 22:55:03 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Tue, 18 Aug 2026 22:55:03 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [NFC][CodeQL] Suppress intentional ITT
 collector library loading (PR #217116)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/217116@github.com>
Message-ID: <6a854537.1e1ac41d.39475c.c6f4@mx.google.com>


jprotze wrote:

The issue might be related to https://github.com/llvm/llvm-project/pull/208652/changes. I think line 1066 should use secure_getenv if available. Is there a similar API replacement for GetEnvironmentVariableA?

https://github.com/llvm/llvm-project/pull/217116

From openmp-commits at lists.llvm.org  Wed Aug 19 00:59:02 2026
From: openmp-commits at lists.llvm.org (Michael Klemm via Openmp-commits)
Date: Wed, 19 Aug 2026 00:59:02 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [libomp] Atomically claim library
	unregistration (PR #216713)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/216713@github.com>
Message-ID: <6a856246.492e23e4.2bdba3.4913@mx.google.com>


https://github.com/mjklemm approved this pull request.

LGTM, but @TerryLWilmarth or @hansangbae should have a look, too.

https://github.com/llvm/llvm-project/pull/216713

From openmp-commits at lists.llvm.org  Wed Aug 19 03:09:32 2026
From: openmp-commits at lists.llvm.org (=?UTF-8?Q?Jan_Andr=C3=A9_Reuter?= via Openmp-commits)
Date: Wed, 19 Aug 2026 03:09:32 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [NFC][OpenMP][OMPT] Move
 `for_static_init` work callbacks into macro (PR #217259)
Message-ID: <llvm.org/llvm/llvm-project/pull/217259@github.com>


https://github.com/Thyre created https://github.com/llvm/llvm-project/pull/217259

`__kmp_for_static_init` uses a very similar pattern for dispatching the work callback to an attached tool throughout the function. Hence, avoid the duplication by moving the pattern into a macro, taking the number of loop iterations as an argument. Without OMPT, this macro is a no-op.

>From c1a901cf0a4bf5036e2d26663cea8442b4093715 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Andr=C3=A9=20Reuter?= <jan at zyten.de>
Date: Wed, 19 Aug 2026 11:02:54 +0200
Subject: [PATCH] [NFC][OpenMP] Refactor `for_static_init` work callbacks into
 macro
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`__kmp_for_static_init` uses a very similar pattern for dispatching the work
callback to an attached tool throughout the function. Hence, reduce the
duplication by moving the pattern into a macro, taking the number of loop
iterations as an argument. Without OMPT, this macro is a no-op.

Signed-off-by: Jan André Reuter <jan at zyten.de>
---
 openmp/runtime/src/kmp_sched.cpp | 38 ++++++++++----------------------
 1 file changed, 12 insertions(+), 26 deletions(-)

diff --git a/openmp/runtime/src/kmp_sched.cpp b/openmp/runtime/src/kmp_sched.cpp
index 2b1bb6f595f9a..bc5d4fb57fcb5 100644
--- a/openmp/runtime/src/kmp_sched.cpp
+++ b/openmp/runtime/src/kmp_sched.cpp
@@ -128,6 +128,14 @@ static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid,
       KMP_DEBUG_ASSERT(ompt_work_type);
     }
   }
+#define OMPT_LOOP_BEGIN(count)                                                 \
+  if (ompt_enabled.ompt_callback_work) {                                       \
+    ompt_callbacks.ompt_callback(ompt_callback_work)(                          \
+        ompt_work_type, ompt_scope_begin, &(team_info->parallel_data),         \
+        &(task_info->task_data), count, codeptr);                              \
+  }
+#else
+#define OMPT_LOOP_BEGIN(count) // no-op
 #endif
 
   KMP_DEBUG_ASSERT(plastiter && plower && pupper && pstride);
@@ -181,13 +189,7 @@ static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid,
 #endif
     KE_TRACE(10, ("__kmpc_for_static_init: T#%d return\n", global_tid));
 
-#if OMPT_SUPPORT && OMPT_OPTIONAL
-    if (ompt_enabled.ompt_callback_work) {
-      ompt_callbacks.ompt_callback(ompt_callback_work)(
-          ompt_work_type, ompt_scope_begin, &(team_info->parallel_data),
-          &(task_info->task_data), 0, codeptr);
-    }
-#endif
+    OMPT_LOOP_BEGIN(0);
     KMP_STATS_LOOP_END(OMP_loop_static_iterations);
     return;
   }
@@ -235,13 +237,7 @@ static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid,
 #endif
     KE_TRACE(10, ("__kmpc_for_static_init: T#%d return\n", global_tid));
 
-#if OMPT_SUPPORT && OMPT_OPTIONAL
-    if (ompt_enabled.ompt_callback_work) {
-      ompt_callbacks.ompt_callback(ompt_callback_work)(
-          ompt_work_type, ompt_scope_begin, &(team_info->parallel_data),
-          &(task_info->task_data), *pstride, codeptr);
-    }
-#endif
+    OMPT_LOOP_BEGIN(*pstride);
     KMP_STATS_LOOP_END(OMP_loop_static_iterations);
     return;
   }
@@ -265,13 +261,7 @@ static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid,
 #endif
     KE_TRACE(10, ("__kmpc_for_static_init: T#%d return\n", global_tid));
 
-#if OMPT_SUPPORT && OMPT_OPTIONAL
-    if (ompt_enabled.ompt_callback_work) {
-      ompt_callbacks.ompt_callback(ompt_callback_work)(
-          ompt_work_type, ompt_scope_begin, &(team_info->parallel_data),
-          &(task_info->task_data), *pstride, codeptr);
-    }
-#endif
+    OMPT_LOOP_BEGIN(*pstride);
     KMP_STATS_LOOP_END(OMP_loop_static_iterations);
     return;
   }
@@ -445,12 +435,8 @@ static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid,
 #endif
   KE_TRACE(10, ("__kmpc_for_static_init: T#%d return\n", global_tid));
 
+  OMPT_LOOP_BEGIN(*pstride);
 #if OMPT_SUPPORT && OMPT_OPTIONAL
-  if (ompt_enabled.ompt_callback_work) {
-    ompt_callbacks.ompt_callback(ompt_callback_work)(
-        ompt_work_type, ompt_scope_begin, &(team_info->parallel_data),
-        &(task_info->task_data), trip_count, codeptr);
-  }
   if (ompt_enabled.ompt_callback_dispatch) {
     ompt_dispatch_t dispatch_type;
     ompt_data_t instance = ompt_data_none;


From openmp-commits at lists.llvm.org  Wed Aug 19 03:10:30 2026
From: openmp-commits at lists.llvm.org (Larry Meadows via Openmp-commits)
Date: Wed, 19 Aug 2026 03:10:30 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMPOpt] Ask the runtime how
 many of a block's threads can be workers (PR #217080)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/217080@github.com>
Message-ID: <6a858116.af6ba3ed.1ee561.7167@mx.google.com>


https://github.com/lfmeadow updated https://github.com/llvm/llvm-project/pull/217080

>From 5d6a409a60443e0bf60eb2bea6cc3de391addcc6 Mon Sep 17 00:00:00 2001
From: Larry Meadows <Lawrence.Meadows at amd.com>
Date: Tue, 18 Aug 2026 11:56:16 -0500
Subject: [PATCH 1/2] [OpenMPOpt] Ask the runtime how many of a block's threads
 can be workers

The custom state machine gates a thread on InitCB < BlockHwSize - WarpSize,
reconstructing the number of worker threads from the block size on the
assumption that the main thread occupies a whole warp above them. The DeviceRTL
already computes that number, in mapping::getMaxTeamThreads(), and its own
generic state machine gates on it in shouldEnterStateMachine(). Export it as
__kmpc_get_max_team_threads() and call that instead, so the compiler's state
machine and the runtime's agree by construction rather than by arithmetic that
has to be kept in step with the launch geometry.

This is NFC here: getMaxTeamThreads() in generic mode is BlockSize - WarpSize,
the same three instructions folded into one call. It is not NFC for a toolchain
whose launch geometry differs. In ROCm, CGOpenMPRuntimeGPU starts a single extra
thread rather than a warp -- "Only one additional thread is started, not an
entire warp" -- so thread_limit(1024) on a 64-lane target launches 961 threads
and the runtime reports 960 workers, while the state machine's own arithmetic
says 961 - 64 = 897. The threads in between are in neither group: the state
machine returns immediately for them, and the parallel region still hands them
iterations. Asking the runtime removes the assumption rather than adding a
second one.

A device runtime that predates the new entry point is handled by the existing
runtimeFnsAvailable() check, which skips the rewrite rather than emitting a call
to a function that is not there.

The two calls this replaces, __kmpc_get_hardware_num_threads_in_block and
__kmpc_get_warp_size, keep their other uses in the pass, including the constant
folding of the former.

Co-authored-by: Cursor <cursoragent at cursor.com>
---
 .../include/llvm/Frontend/OpenMP/OMPKinds.def |  2 +
 llvm/lib/Transforms/IPO/OpenMPOpt.cpp         | 52 ++++++--------
 .../OpenMP/custom_state_machines.ll           | 72 +++++++------------
 llvm/test/Transforms/OpenMP/spmdization.ll    | 64 +++++------------
 .../Transforms/OpenMP/spmdization_guarding.ll |  6 +-
 .../Transforms/OpenMP/spmdization_indirect.ll | 16 ++---
 ...zation_no_guarding_two_reaching_kernels.ll | 18 ++---
 openmp/device/include/Interface.h             |  3 +
 openmp/device/src/Mapping.cpp                 |  4 ++
 9 files changed, 83 insertions(+), 154 deletions(-)

diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
index ea1cef97900f5..730027f0fb477 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
@@ -228,6 +228,7 @@ __OMP_RTL(__kmpc_omp_reg_task_with_affinity, false, Int32, IdentPtr, Int32,
 __OMP_RTL(__kmpc_get_hardware_num_blocks, false, Int32, )
 __OMP_RTL(__kmpc_get_hardware_num_threads_in_block, false, Int32, )
 __OMP_RTL(__kmpc_get_warp_size, false, Int32, )
+__OMP_RTL(__kmpc_get_max_team_threads, false, Int32, )
 
 __OMP_RTL(omp_get_thread_num, false, Int32, )
 __OMP_RTL(omp_get_num_threads, false, Int32, )
@@ -721,6 +722,7 @@ __OMP_RTL_ATTRS(__kmpc_omp_reg_task_with_affinity, DefaultAttrs, SExt,
 __OMP_RTL_ATTRS(__kmpc_get_hardware_num_blocks, GetterAttrs, ZExt, ParamAttrs())
 __OMP_RTL_ATTRS(__kmpc_get_hardware_num_threads_in_block, GetterAttrs, ZExt, ParamAttrs())
 __OMP_RTL_ATTRS(__kmpc_get_warp_size, GetterAttrs, ZExt, ParamAttrs())
+__OMP_RTL_ATTRS(__kmpc_get_max_team_threads, GetterAttrs, ZExt, ParamAttrs())
 
 __OMP_RTL_ATTRS(omp_get_thread_num, GetterAttrs, SExt, ParamAttrs())
 __OMP_RTL_ATTRS(omp_get_num_threads, GetterAttrs, SExt, ParamAttrs())
diff --git a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
index 22a5efb402504..0f8c6b991e763 100644
--- a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
+++ b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp
@@ -3904,8 +3904,7 @@ struct AAKernelInfoFunction : AAKernelInfo {
     Attributor::VirtualUseCallbackTy CustomStateMachineUseCB =
         [&](Attributor &A, const AbstractAttribute *QueryingAA) {
           // Whenever we create a custom state machine we will insert calls to
-          // __kmpc_get_hardware_num_threads_in_block,
-          // __kmpc_get_warp_size,
+          // __kmpc_get_max_team_threads,
           // __kmpc_barrier_simple_generic,
           // __kmpc_kernel_parallel, and
           // __kmpc_kernel_end_parallel.
@@ -3920,9 +3919,8 @@ struct AAKernelInfoFunction : AAKernelInfo {
 
     // Not needed if we are pre-runtime merge.
     if (!KernelInitCB->getCalledFunction()->isDeclaration()) {
-      RegisterVirtualUse(OMPRTL___kmpc_get_hardware_num_threads_in_block,
+      RegisterVirtualUse(OMPRTL___kmpc_get_max_team_threads,
                          CustomStateMachineUseCB);
-      RegisterVirtualUse(OMPRTL___kmpc_get_warp_size, CustomStateMachineUseCB);
       RegisterVirtualUse(OMPRTL___kmpc_barrier_simple_generic,
                          CustomStateMachineUseCB);
       RegisterVirtualUse(OMPRTL___kmpc_kernel_parallel,
@@ -4381,10 +4379,10 @@ struct AAKernelInfoFunction : AAKernelInfo {
       return false;
 
     auto &OMPInfoCache = static_cast<OMPInformationCache &>(A.getInfoCache());
-    if (!OMPInfoCache.runtimeFnsAvailable(
-            {OMPRTL___kmpc_get_hardware_num_threads_in_block,
-             OMPRTL___kmpc_get_warp_size, OMPRTL___kmpc_barrier_simple_generic,
-             OMPRTL___kmpc_kernel_parallel, OMPRTL___kmpc_kernel_end_parallel}))
+    if (!OMPInfoCache.runtimeFnsAvailable({OMPRTL___kmpc_get_max_team_threads,
+                                           OMPRTL___kmpc_barrier_simple_generic,
+                                           OMPRTL___kmpc_kernel_parallel,
+                                           OMPRTL___kmpc_kernel_end_parallel}))
       return false;
 
     ConstantStruct *ExistingKernelEnvC =
@@ -4463,13 +4461,10 @@ struct AAKernelInfoFunction : AAKernelInfo {
     // Create all the blocks:
     //
     //                       InitCB = __kmpc_target_init(...)
-    //                       BlockHwSize =
-    //                         __kmpc_get_hardware_num_threads_in_block();
-    //                       WarpSize = __kmpc_get_warp_size();
-    //                       BlockSize = BlockHwSize - WarpSize;
+    //                       MaxTeamThreads = __kmpc_get_max_team_threads();
     // IsWorkerCheckBB:      bool IsWorker = InitCB != -1;
     //                       if (IsWorker) {
-    //                         if (InitCB >= BlockSize) return;
+    //                         if (InitCB >= MaxTeamThreads) return;
     // SMBeginBB:               __kmpc_barrier_simple_generic(...);
     //                         void *WorkFn;
     //                         bool Active = __kmpc_kernel_parallel(&WorkFn);
@@ -4534,26 +4529,23 @@ struct AAKernelInfoFunction : AAKernelInfo {
     IsWorker->setDebugLoc(DLoc);
     CondBrInst::Create(IsWorker, IsWorkerCheckBB, UserCodeEntryBB, InitBB);
 
+    // How many of the block's threads can be worker threads is a property of
+    // the launch geometry, which the runtime knows and the block size alone
+    // does not determine: a target may host the main thread in a whole warp
+    // above the workers, or in a single thread above them. Ask the runtime
+    // rather than subtracting a warp here, or the workers in between are left
+    // in neither group, waiting for no parallel region while it is handed
+    // iterations.
     Module &M = *Kernel->getParent();
-    FunctionCallee BlockHwSizeFn =
+    FunctionCallee MaxTeamThreadsFn =
         OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
-            M, OMPRTL___kmpc_get_hardware_num_threads_in_block);
-    FunctionCallee WarpSizeFn =
-        OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
-            M, OMPRTL___kmpc_get_warp_size);
-    CallInst *BlockHwSize =
-        CallInst::Create(BlockHwSizeFn, "block.hw_size", IsWorkerCheckBB);
-    OMPInfoCache.setCallingConvention(BlockHwSizeFn, BlockHwSize);
-    BlockHwSize->setDebugLoc(DLoc);
-    CallInst *WarpSize =
-        CallInst::Create(WarpSizeFn, "warp.size", IsWorkerCheckBB);
-    OMPInfoCache.setCallingConvention(WarpSizeFn, WarpSize);
-    WarpSize->setDebugLoc(DLoc);
-    Instruction *BlockSize = BinaryOperator::CreateSub(
-        BlockHwSize, WarpSize, "block.size", IsWorkerCheckBB);
-    BlockSize->setDebugLoc(DLoc);
+            M, OMPRTL___kmpc_get_max_team_threads);
+    CallInst *MaxTeamThreads =
+        CallInst::Create(MaxTeamThreadsFn, "max_team_threads", IsWorkerCheckBB);
+    OMPInfoCache.setCallingConvention(MaxTeamThreadsFn, MaxTeamThreads);
+    MaxTeamThreads->setDebugLoc(DLoc);
     Instruction *IsMainOrWorker = ICmpInst::Create(
-        ICmpInst::ICmp, llvm::CmpInst::ICMP_SLT, KernelInitCB, BlockSize,
+        ICmpInst::ICmp, llvm::CmpInst::ICMP_SLT, KernelInitCB, MaxTeamThreads,
         "thread.is_main_or_worker", IsWorkerCheckBB);
     IsMainOrWorker->setDebugLoc(DLoc);
     CondBrInst::Create(IsMainOrWorker, StateMachineBeginBB,
diff --git a/llvm/test/Transforms/OpenMP/custom_state_machines.ll b/llvm/test/Transforms/OpenMP/custom_state_machines.ll
index a03a4b7f3fa67..ac35d7a4a9765 100644
--- a/llvm/test/Transforms/OpenMP/custom_state_machines.ll
+++ b/llvm/test/Transforms/OpenMP/custom_state_machines.ll
@@ -1152,10 +1152,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; AMDGPU:       is_worker_check:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; AMDGPU-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; AMDGPU:       worker_state_machine.begin:
 ; AMDGPU-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -1273,10 +1271,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; AMDGPU:       is_worker_check:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; AMDGPU-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; AMDGPU:       worker_state_machine.begin:
 ; AMDGPU-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -1420,10 +1416,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; AMDGPU:       is_worker_check:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; AMDGPU-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; AMDGPU:       worker_state_machine.begin:
 ; AMDGPU-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -1542,10 +1536,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; AMDGPU:       is_worker_check:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; AMDGPU-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; AMDGPU:       worker_state_machine.begin:
 ; AMDGPU-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -1662,10 +1654,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; AMDGPU:       is_worker_check:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; AMDGPU-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; AMDGPU:       worker_state_machine.begin:
 ; AMDGPU-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -1854,10 +1844,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; AMDGPU:       is_worker_check:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; AMDGPU-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; AMDGPU:       worker_state_machine.begin:
 ; AMDGPU-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -2078,10 +2066,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; NVPTX:       is_worker_check:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; NVPTX-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; NVPTX:       worker_state_machine.begin:
 ; NVPTX-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -2198,10 +2184,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; NVPTX:       is_worker_check:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; NVPTX-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; NVPTX:       worker_state_machine.begin:
 ; NVPTX-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -2344,10 +2328,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; NVPTX:       is_worker_check:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; NVPTX-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; NVPTX:       worker_state_machine.begin:
 ; NVPTX-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -2465,10 +2447,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; NVPTX:       is_worker_check:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; NVPTX-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; NVPTX:       worker_state_machine.begin:
 ; NVPTX-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -2584,10 +2564,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; NVPTX:       is_worker_check:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; NVPTX-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; NVPTX:       worker_state_machine.begin:
 ; NVPTX-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -2775,10 +2753,8 @@ attributes #9 = { convergent nounwind willreturn memory(read) }
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; NVPTX:       is_worker_check:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; NVPTX-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; NVPTX:       worker_state_machine.begin:
 ; NVPTX-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
diff --git a/llvm/test/Transforms/OpenMP/spmdization.ll b/llvm/test/Transforms/OpenMP/spmdization.ll
index 7d53a808ef4d3..88a4d53ae9c82 100644
--- a/llvm/test/Transforms/OpenMP/spmdization.ll
+++ b/llvm/test/Transforms/OpenMP/spmdization.ll
@@ -271,9 +271,7 @@ define internal void @__omp_offloading_fd02_2044372e_sequential_loop_l5__debug()
 ; AMDGPU-DISABLED1-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-DISABLED1-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; AMDGPU-DISABLED1:       [[IS_WORKER_CHECK]]:
-; AMDGPU-DISABLED1-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-DISABLED1-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; AMDGPU-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; AMDGPU-DISABLED1-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; AMDGPU-DISABLED1-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; AMDGPU-DISABLED1:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -343,9 +341,7 @@ define internal void @__omp_offloading_fd02_2044372e_sequential_loop_l5__debug()
 ; NVPTX-DISABLED1-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-DISABLED1-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; NVPTX-DISABLED1:       [[IS_WORKER_CHECK]]:
-; NVPTX-DISABLED1-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-DISABLED1-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; NVPTX-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; NVPTX-DISABLED1-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; NVPTX-DISABLED1-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; NVPTX-DISABLED1:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -753,9 +749,7 @@ define weak ptx_kernel void @__omp_offloading_fd02_2044372e_sequential_loop_to_s
 ; AMDGPU-DISABLED1-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-DISABLED1-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; AMDGPU-DISABLED1:       [[IS_WORKER_CHECK]]:
-; AMDGPU-DISABLED1-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-DISABLED1-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; AMDGPU-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; AMDGPU-DISABLED1-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; AMDGPU-DISABLED1-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; AMDGPU-DISABLED1:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -825,9 +819,7 @@ define weak ptx_kernel void @__omp_offloading_fd02_2044372e_sequential_loop_to_s
 ; NVPTX-DISABLED1-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-DISABLED1-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; NVPTX-DISABLED1:       [[IS_WORKER_CHECK]]:
-; NVPTX-DISABLED1-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-DISABLED1-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; NVPTX-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; NVPTX-DISABLED1-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; NVPTX-DISABLED1-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; NVPTX-DISABLED1:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -1253,9 +1245,7 @@ define weak ptx_kernel void @__omp_offloading_fd02_2044372e_sequential_loop_to_s
 ; AMDGPU-DISABLED1-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-DISABLED1-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; AMDGPU-DISABLED1:       [[IS_WORKER_CHECK]]:
-; AMDGPU-DISABLED1-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-DISABLED1-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; AMDGPU-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; AMDGPU-DISABLED1-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; AMDGPU-DISABLED1-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; AMDGPU-DISABLED1:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -1325,9 +1315,7 @@ define weak ptx_kernel void @__omp_offloading_fd02_2044372e_sequential_loop_to_s
 ; NVPTX-DISABLED1-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-DISABLED1-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; NVPTX-DISABLED1:       [[IS_WORKER_CHECK]]:
-; NVPTX-DISABLED1-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-DISABLED1-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; NVPTX-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; NVPTX-DISABLED1-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; NVPTX-DISABLED1-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; NVPTX-DISABLED1:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -1779,9 +1767,7 @@ define weak ptx_kernel void @__omp_offloading_fd02_2044372e_sequential_loop_to_s
 ; AMDGPU-DISABLED1-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-DISABLED1-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; AMDGPU-DISABLED1:       [[IS_WORKER_CHECK]]:
-; AMDGPU-DISABLED1-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-DISABLED1-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; AMDGPU-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; AMDGPU-DISABLED1-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; AMDGPU-DISABLED1-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; AMDGPU-DISABLED1:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -1851,9 +1837,7 @@ define weak ptx_kernel void @__omp_offloading_fd02_2044372e_sequential_loop_to_s
 ; NVPTX-DISABLED1-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-DISABLED1-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; NVPTX-DISABLED1:       [[IS_WORKER_CHECK]]:
-; NVPTX-DISABLED1-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-DISABLED1-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; NVPTX-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; NVPTX-DISABLED1-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; NVPTX-DISABLED1-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; NVPTX-DISABLED1:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -2300,9 +2284,7 @@ define weak ptx_kernel void @__omp_offloading_fd02_2044372e_do_not_spmdize_targe
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; AMDGPU:       [[IS_WORKER_CHECK]]:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; AMDGPU:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -2348,9 +2330,7 @@ define weak ptx_kernel void @__omp_offloading_fd02_2044372e_do_not_spmdize_targe
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; NVPTX:       [[IS_WORKER_CHECK]]:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; NVPTX:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -2395,9 +2375,7 @@ define weak ptx_kernel void @__omp_offloading_fd02_2044372e_do_not_spmdize_targe
 ; AMDGPU-DISABLED1-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-DISABLED1-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; AMDGPU-DISABLED1:       [[IS_WORKER_CHECK]]:
-; AMDGPU-DISABLED1-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-DISABLED1-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; AMDGPU-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; AMDGPU-DISABLED1-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; AMDGPU-DISABLED1-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; AMDGPU-DISABLED1:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -2461,9 +2439,7 @@ define weak ptx_kernel void @__omp_offloading_fd02_2044372e_do_not_spmdize_targe
 ; NVPTX-DISABLED1-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-DISABLED1-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; NVPTX-DISABLED1:       [[IS_WORKER_CHECK]]:
-; NVPTX-DISABLED1-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-DISABLED1-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; NVPTX-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; NVPTX-DISABLED1-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; NVPTX-DISABLED1-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; NVPTX-DISABLED1:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -2589,9 +2565,7 @@ define weak ptx_kernel void @__omp_offloading_fd02_2044372e_do_not_spmdize_task_
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; AMDGPU:       [[IS_WORKER_CHECK]]:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; AMDGPU:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -2643,9 +2617,7 @@ define weak ptx_kernel void @__omp_offloading_fd02_2044372e_do_not_spmdize_task_
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; NVPTX:       [[IS_WORKER_CHECK]]:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; NVPTX:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -2696,9 +2668,7 @@ define weak ptx_kernel void @__omp_offloading_fd02_2044372e_do_not_spmdize_task_
 ; AMDGPU-DISABLED1-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-DISABLED1-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; AMDGPU-DISABLED1:       [[IS_WORKER_CHECK]]:
-; AMDGPU-DISABLED1-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-DISABLED1-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; AMDGPU-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; AMDGPU-DISABLED1-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; AMDGPU-DISABLED1-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; AMDGPU-DISABLED1:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -2768,9 +2738,7 @@ define weak ptx_kernel void @__omp_offloading_fd02_2044372e_do_not_spmdize_task_
 ; NVPTX-DISABLED1-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-DISABLED1-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; NVPTX-DISABLED1:       [[IS_WORKER_CHECK]]:
-; NVPTX-DISABLED1-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-DISABLED1-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; NVPTX-DISABLED1-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; NVPTX-DISABLED1-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; NVPTX-DISABLED1-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; NVPTX-DISABLED1:       [[WORKER_STATE_MACHINE_BEGIN]]:
diff --git a/llvm/test/Transforms/OpenMP/spmdization_guarding.ll b/llvm/test/Transforms/OpenMP/spmdization_guarding.ll
index 2fb1f878cce7b..4dae6889beb9d 100644
--- a/llvm/test/Transforms/OpenMP/spmdization_guarding.ll
+++ b/llvm/test/Transforms/OpenMP/spmdization_guarding.ll
@@ -201,10 +201,8 @@ define weak ptx_kernel void @__omp_offloading_2a_fbfa7a_sequential_loop_l6(ptr %
 ; CHECK-DISABLED-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; CHECK-DISABLED-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; CHECK-DISABLED:       is_worker_check:
-; CHECK-DISABLED-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; CHECK-DISABLED-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; CHECK-DISABLED-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; CHECK-DISABLED-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; CHECK-DISABLED-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; CHECK-DISABLED-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; CHECK-DISABLED-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; CHECK-DISABLED:       worker_state_machine.begin:
 ; CHECK-DISABLED-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
diff --git a/llvm/test/Transforms/OpenMP/spmdization_indirect.ll b/llvm/test/Transforms/OpenMP/spmdization_indirect.ll
index 1de719e1b85a8..71847b72ae24f 100644
--- a/llvm/test/Transforms/OpenMP/spmdization_indirect.ll
+++ b/llvm/test/Transforms/OpenMP/spmdization_indirect.ll
@@ -395,9 +395,7 @@ define weak ptx_kernel void @spmd_and_non_spmd_callee(i1 %c) #0 {
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; AMDGPU:       [[IS_WORKER_CHECK]]:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; AMDGPU:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -458,9 +456,7 @@ define weak ptx_kernel void @spmd_and_non_spmd_callee(i1 %c) #0 {
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; NVPTX:       [[IS_WORKER_CHECK]]:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; NVPTX:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -751,9 +747,7 @@ define weak ptx_kernel void @spmd_and_non_spmd_callees_metadata(ptr %fp) #0 {
 ; AMDGPU-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; AMDGPU:       [[IS_WORKER_CHECK]]:
-; AMDGPU-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; AMDGPU-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; AMDGPU-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; AMDGPU-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; AMDGPU-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; AMDGPU:       [[WORKER_STATE_MACHINE_BEGIN]]:
@@ -813,9 +807,7 @@ define weak ptx_kernel void @spmd_and_non_spmd_callees_metadata(ptr %fp) #0 {
 ; NVPTX-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_WORKER]], label %[[IS_WORKER_CHECK:.*]], label %[[THREAD_USER_CODE_CHECK:.*]]
 ; NVPTX:       [[IS_WORKER_CHECK]]:
-; NVPTX-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; NVPTX-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
+; NVPTX-NEXT:    [[BLOCK_SIZE:%.*]] = call i32 @__kmpc_get_max_team_threads()
 ; NVPTX-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
 ; NVPTX-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label %[[WORKER_STATE_MACHINE_BEGIN:.*]], label %[[WORKER_STATE_MACHINE_FINISHED:.*]]
 ; NVPTX:       [[WORKER_STATE_MACHINE_BEGIN]]:
diff --git a/llvm/test/Transforms/OpenMP/spmdization_no_guarding_two_reaching_kernels.ll b/llvm/test/Transforms/OpenMP/spmdization_no_guarding_two_reaching_kernels.ll
index 4f64839deaf53..dc08df96dfd64 100644
--- a/llvm/test/Transforms/OpenMP/spmdization_no_guarding_two_reaching_kernels.ll
+++ b/llvm/test/Transforms/OpenMP/spmdization_no_guarding_two_reaching_kernels.ll
@@ -78,10 +78,8 @@ define weak ptx_kernel void @__omp_offloading_2b_10393b5_spmd_l12(ptr %dyn) #0 {
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; CHECK-DISABLE-SPMDIZATION:       is_worker_check:
-; CHECK-DISABLE-SPMDIZATION-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; CHECK-DISABLE-SPMDIZATION-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; CHECK-DISABLE-SPMDIZATION-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; CHECK-DISABLE-SPMDIZATION-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; CHECK-DISABLE-SPMDIZATION-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; CHECK-DISABLE-SPMDIZATION-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; CHECK-DISABLE-SPMDIZATION:       worker_state_machine.begin:
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -155,10 +153,8 @@ define weak ptx_kernel void @__omp_offloading_2b_10393b5_generic_l20(ptr %dyn) #
 ; CHECK-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; CHECK-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; CHECK:       is_worker_check:
-; CHECK-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; CHECK-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; CHECK-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; CHECK-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; CHECK-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; CHECK-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; CHECK-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; CHECK:       worker_state_machine.begin:
 ; CHECK-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
@@ -197,10 +193,8 @@ define weak ptx_kernel void @__omp_offloading_2b_10393b5_generic_l20(ptr %dyn) #
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    [[THREAD_IS_WORKER:%.*]] = icmp ne i32 [[TMP0]], -1
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    br i1 [[THREAD_IS_WORKER]], label [[IS_WORKER_CHECK:%.*]], label [[THREAD_USER_CODE_CHECK:%.*]]
 ; CHECK-DISABLE-SPMDIZATION:       is_worker_check:
-; CHECK-DISABLE-SPMDIZATION-NEXT:    [[BLOCK_HW_SIZE:%.*]] = call i32 @__kmpc_get_hardware_num_threads_in_block()
-; CHECK-DISABLE-SPMDIZATION-NEXT:    [[WARP_SIZE:%.*]] = call i32 @__kmpc_get_warp_size()
-; CHECK-DISABLE-SPMDIZATION-NEXT:    [[BLOCK_SIZE:%.*]] = sub i32 [[BLOCK_HW_SIZE]], [[WARP_SIZE]]
-; CHECK-DISABLE-SPMDIZATION-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[BLOCK_SIZE]]
+; CHECK-DISABLE-SPMDIZATION-NEXT:    [[MAX_TEAM_THREADS:%.*]] = call i32 @__kmpc_get_max_team_threads()
+; CHECK-DISABLE-SPMDIZATION-NEXT:    [[THREAD_IS_MAIN_OR_WORKER:%.*]] = icmp slt i32 [[TMP0]], [[MAX_TEAM_THREADS]]
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    br i1 [[THREAD_IS_MAIN_OR_WORKER]], label [[WORKER_STATE_MACHINE_BEGIN:%.*]], label [[WORKER_STATE_MACHINE_FINISHED:%.*]]
 ; CHECK-DISABLE-SPMDIZATION:       worker_state_machine.begin:
 ; CHECK-DISABLE-SPMDIZATION-NEXT:    call void @__kmpc_barrier_simple_generic(ptr @[[GLOB1]], i32 [[TMP0]])
diff --git a/openmp/device/include/Interface.h b/openmp/device/include/Interface.h
index 601694871597b..07ad56e8dbef0 100644
--- a/openmp/device/include/Interface.h
+++ b/openmp/device/include/Interface.h
@@ -217,6 +217,9 @@ uint32_t __kmpc_get_hardware_num_threads_in_block();
 /// External interface to get the warp size.
 uint32_t __kmpc_get_warp_size();
 
+/// External interface to get the maximum number of threads in a team.
+uint32_t __kmpc_get_max_team_threads();
+
 /// Kernel
 ///
 ///{
diff --git a/openmp/device/src/Mapping.cpp b/openmp/device/src/Mapping.cpp
index 1096e5f496590..f65ccb5203056 100644
--- a/openmp/device/src/Mapping.cpp
+++ b/openmp/device/src/Mapping.cpp
@@ -175,6 +175,10 @@ extern "C" {
 [[gnu::noinline]] uint32_t __kmpc_get_warp_size() {
   return mapping::getWarpSize();
 }
+
+[[gnu::noinline]] uint32_t __kmpc_get_max_team_threads() {
+  return mapping::getMaxTeamThreads();
+}
 }
 
 #define _TGT_KERNEL_LANGUAGE(NAME, MAPPER_NAME)                                \

>From 76cf6380e0e69821eb4346abc6f3766839b4b16b Mon Sep 17 00:00:00 2001
From: Larry Meadows <Lawrence.Meadows at amd.com>
Date: Wed, 19 Aug 2026 05:10:16 -0500
Subject: [PATCH 2/2] Say what __kmpc_get_max_team_threads returns

"The maximum number of threads in a team" reads as the block size. It is not:
in generic mode the threads hosting the main thread are not available to the
team, and how many those are is the runtime's to decide, which is the reason
the compiler asks instead of subtracting a warp itself.

Co-authored-by: Cursor <cursoragent at cursor.com>
---
 openmp/device/include/Interface.h | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/openmp/device/include/Interface.h b/openmp/device/include/Interface.h
index 07ad56e8dbef0..dabec09f94524 100644
--- a/openmp/device/include/Interface.h
+++ b/openmp/device/include/Interface.h
@@ -217,7 +217,11 @@ uint32_t __kmpc_get_hardware_num_threads_in_block();
 /// External interface to get the warp size.
 uint32_t __kmpc_get_warp_size();
 
-/// External interface to get the maximum number of threads in a team.
+/// External interface to get the number of a block's threads that can execute a
+/// team's parallel region. This is not the block size: in generic mode the
+/// threads hosting the main thread are not available to the team. How many
+/// those are is the runtime's to decide, so callers must ask rather than
+/// subtract for themselves.
 uint32_t __kmpc_get_max_team_threads();
 
 /// Kernel


From openmp-commits at lists.llvm.org  Wed Aug 19 03:10:54 2026
From: openmp-commits at lists.llvm.org (Larry Meadows via Openmp-commits)
Date: Wed, 19 Aug 2026 03:10:54 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMPOpt] Ask the runtime how
 many of a block's threads can be workers (PR #217080)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/217080@github.com>
Message-ID: <6a85812e.af4834b4.25a1a0.7e80@mx.google.com>


================
@@ -4534,26 +4529,23 @@ struct AAKernelInfoFunction : AAKernelInfo {
     IsWorker->setDebugLoc(DLoc);
     CondBrInst::Create(IsWorker, IsWorkerCheckBB, UserCodeEntryBB, InitBB);
 
+    // How many of the block's threads can be worker threads is a property of
+    // the launch geometry, which the runtime knows and the block size alone
+    // does not determine: a target may host the main thread in a whole warp
+    // above the workers, or in a single thread above them. Ask the runtime
+    // rather than subtracting a warp here, or the workers in between are left
+    // in neither group, waiting for no parallel region while it is handed
+    // iterations.
     Module &M = *Kernel->getParent();
-    FunctionCallee BlockHwSizeFn =
+    FunctionCallee MaxTeamThreadsFn =
         OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
-            M, OMPRTL___kmpc_get_hardware_num_threads_in_block);
-    FunctionCallee WarpSizeFn =
-        OMPInfoCache.OMPBuilder.getOrCreateRuntimeFunction(
-            M, OMPRTL___kmpc_get_warp_size);
-    CallInst *BlockHwSize =
-        CallInst::Create(BlockHwSizeFn, "block.hw_size", IsWorkerCheckBB);
-    OMPInfoCache.setCallingConvention(BlockHwSizeFn, BlockHwSize);
-    BlockHwSize->setDebugLoc(DLoc);
-    CallInst *WarpSize =
-        CallInst::Create(WarpSizeFn, "warp.size", IsWorkerCheckBB);
-    OMPInfoCache.setCallingConvention(WarpSizeFn, WarpSize);
-    WarpSize->setDebugLoc(DLoc);
-    Instruction *BlockSize = BinaryOperator::CreateSub(
-        BlockHwSize, WarpSize, "block.size", IsWorkerCheckBB);
-    BlockSize->setDebugLoc(DLoc);
+            M, OMPRTL___kmpc_get_max_team_threads);
+    CallInst *MaxTeamThreads =
+        CallInst::Create(MaxTeamThreadsFn, "max_team_threads", IsWorkerCheckBB);
+    OMPInfoCache.setCallingConvention(MaxTeamThreadsFn, MaxTeamThreads);
+    MaxTeamThreads->setDebugLoc(DLoc);
----------------
lfmeadow wrote:

The premise is right — `AAFoldRuntimeCall` folds `__kmpc_get_hardware_num_threads_in_block` to `omp_target_thread_limit`, and `__kmpc_get_max_team_threads` is not registered — but there is no folding opportunity being lost, because the bound was never a constant to begin with.

`__kmpc_get_warp_size` is not one of the folded runtime calls, so in the old code the subtraction always had an opaque operand. The in-tree expectations on `main` show it:

```llvm
%block.hw_size = call i32 @__kmpc_get_hardware_num_threads_in_block()
%warp.size = call i32 @__kmpc_get_warp_size()
%block.size = sub i32 %block.hw_size, %warp.size
%thread_is_main_or_worker = icmp slt i32 %0, %block.size
```

Even where `omp_target_thread_limit` is present and `%block.hw_size` folds, `%block.size` stays dynamic and the comparison never becomes constant. One opaque call in place of two is if anything less work. Folding elsewhere is unaffected, since `AAFoldRuntimeCall` works per call site and user-code calls to the old getter still fold.

Teaching the folder to treat the two equivalently would be actively wrong: `omp_target_thread_limit` is the block size, and the whole point of this patch is that the worker bound is *not* the block size — the threads hosting the main thread are excluded, by a convention the runtime owns. Folding the new getter to the thread limit would reintroduce the bug the patch fixes, which downstream shows up as workers stranded between the two bounds.


https://github.com/llvm/llvm-project/pull/217080

From openmp-commits at lists.llvm.org  Wed Aug 19 03:10:54 2026
From: openmp-commits at lists.llvm.org (Larry Meadows via Openmp-commits)
Date: Wed, 19 Aug 2026 03:10:54 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMPOpt] Ask the runtime how
 many of a block's threads can be workers (PR #217080)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/217080@github.com>
Message-ID: <6a85812e.e7e80b80.16e6cf.c78d@mx.google.com>


================
@@ -217,6 +217,9 @@ uint32_t __kmpc_get_hardware_num_threads_in_block();
 /// External interface to get the warp size.
 uint32_t __kmpc_get_warp_size();
 
+/// External interface to get the maximum number of threads in a team.
+uint32_t __kmpc_get_max_team_threads();
+
----------------
lfmeadow wrote:

Agreed, "the maximum number of threads in a team" reads as the block size and it is not that. Reworded to say the threads hosting the main thread are unavailable to the team in generic mode, and that how many those are is the runtime's to decide — which is the reason the compiler asks instead of subtracting a warp for itself.


https://github.com/llvm/llvm-project/pull/217080

From openmp-commits at lists.llvm.org  Wed Aug 19 03:16:57 2026
From: openmp-commits at lists.llvm.org (Larry Meadows via Openmp-commits)
Date: Wed, 19 Aug 2026 03:16:57 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMPOpt] Ask the runtime how
 many of a block's threads can be workers (PR #217080)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/217080@github.com>
Message-ID: <6a858299.160324b0.9e5b0.b4c2@mx.google.com>


https://github.com/lfmeadow edited https://github.com/llvm/llvm-project/pull/217080

From openmp-commits at lists.llvm.org  Wed Aug 19 04:07:03 2026
From: openmp-commits at lists.llvm.org (=?UTF-8?Q?Jan_Andr=C3=A9_Reuter?= via Openmp-commits)
Date: Wed, 19 Aug 2026 04:07:03 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [NFC][OpenMP][OMPT] Move
 `for_static_init` work callbacks into macro (PR #217259)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/217259@github.com>
Message-ID: <6a858e57.6dfd78af.23c9be.8811@mx.google.com>


https://github.com/Thyre updated https://github.com/llvm/llvm-project/pull/217259

>From 6015cc86dd02fef7b2b1559519bd496d3a6ad847 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jan=20Andr=C3=A9=20Reuter?= <jan at zyten.de>
Date: Wed, 19 Aug 2026 11:02:54 +0200
Subject: [PATCH] [NFC][OpenMP] Refactor `for_static_init` work callbacks into
 macro
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`__kmp_for_static_init` uses a very similar pattern for dispatching the work
callback to an attached tool throughout the function. Hence, reduce the
duplication by moving the pattern into a macro, taking the number of loop
iterations as an argument. Without OMPT, this macro is a no-op.

Signed-off-by: Jan André Reuter <jan at zyten.de>
---
 openmp/runtime/src/kmp_sched.cpp | 38 ++++++++++----------------------
 1 file changed, 12 insertions(+), 26 deletions(-)

diff --git a/openmp/runtime/src/kmp_sched.cpp b/openmp/runtime/src/kmp_sched.cpp
index 2b1bb6f595f9a..7b6b54d274a6f 100644
--- a/openmp/runtime/src/kmp_sched.cpp
+++ b/openmp/runtime/src/kmp_sched.cpp
@@ -128,6 +128,14 @@ static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid,
       KMP_DEBUG_ASSERT(ompt_work_type);
     }
   }
+#define OMPT_LOOP_BEGIN(count)                                                 \
+  if (ompt_enabled.ompt_callback_work) {                                       \
+    ompt_callbacks.ompt_callback(ompt_callback_work)(                          \
+        ompt_work_type, ompt_scope_begin, &(team_info->parallel_data),         \
+        &(task_info->task_data), count, codeptr);                              \
+  }
+#else
+#define OMPT_LOOP_BEGIN(count) // no-op
 #endif
 
   KMP_DEBUG_ASSERT(plastiter && plower && pupper && pstride);
@@ -181,13 +189,7 @@ static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid,
 #endif
     KE_TRACE(10, ("__kmpc_for_static_init: T#%d return\n", global_tid));
 
-#if OMPT_SUPPORT && OMPT_OPTIONAL
-    if (ompt_enabled.ompt_callback_work) {
-      ompt_callbacks.ompt_callback(ompt_callback_work)(
-          ompt_work_type, ompt_scope_begin, &(team_info->parallel_data),
-          &(task_info->task_data), 0, codeptr);
-    }
-#endif
+    OMPT_LOOP_BEGIN(0);
     KMP_STATS_LOOP_END(OMP_loop_static_iterations);
     return;
   }
@@ -235,13 +237,7 @@ static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid,
 #endif
     KE_TRACE(10, ("__kmpc_for_static_init: T#%d return\n", global_tid));
 
-#if OMPT_SUPPORT && OMPT_OPTIONAL
-    if (ompt_enabled.ompt_callback_work) {
-      ompt_callbacks.ompt_callback(ompt_callback_work)(
-          ompt_work_type, ompt_scope_begin, &(team_info->parallel_data),
-          &(task_info->task_data), *pstride, codeptr);
-    }
-#endif
+    OMPT_LOOP_BEGIN(*pstride);
     KMP_STATS_LOOP_END(OMP_loop_static_iterations);
     return;
   }
@@ -265,13 +261,7 @@ static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid,
 #endif
     KE_TRACE(10, ("__kmpc_for_static_init: T#%d return\n", global_tid));
 
-#if OMPT_SUPPORT && OMPT_OPTIONAL
-    if (ompt_enabled.ompt_callback_work) {
-      ompt_callbacks.ompt_callback(ompt_callback_work)(
-          ompt_work_type, ompt_scope_begin, &(team_info->parallel_data),
-          &(task_info->task_data), *pstride, codeptr);
-    }
-#endif
+    OMPT_LOOP_BEGIN(*pstride);
     KMP_STATS_LOOP_END(OMP_loop_static_iterations);
     return;
   }
@@ -445,12 +435,8 @@ static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid,
 #endif
   KE_TRACE(10, ("__kmpc_for_static_init: T#%d return\n", global_tid));
 
+  OMPT_LOOP_BEGIN(trip_count);
 #if OMPT_SUPPORT && OMPT_OPTIONAL
-  if (ompt_enabled.ompt_callback_work) {
-    ompt_callbacks.ompt_callback(ompt_callback_work)(
-        ompt_work_type, ompt_scope_begin, &(team_info->parallel_data),
-        &(task_info->task_data), trip_count, codeptr);
-  }
   if (ompt_enabled.ompt_callback_dispatch) {
     ompt_dispatch_t dispatch_type;
     ompt_data_t instance = ompt_data_none;


From openmp-commits at lists.llvm.org  Wed Aug 19 05:19:55 2026
From: openmp-commits at lists.llvm.org (Larry Meadows via Openmp-commits)
Date: Wed, 19 Aug 2026 05:19:55 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMPOpt] Ask the runtime how
 many of a block's threads can be workers (PR #217080)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/217080@github.com>
Message-ID: <6a859f6b.d76d81af.1755f0.a78e@mx.google.com>


lfmeadow wrote:

I'm sorry, this was premature.

https://github.com/llvm/llvm-project/pull/217080

From openmp-commits at lists.llvm.org  Wed Aug 19 05:19:56 2026
From: openmp-commits at lists.llvm.org (Larry Meadows via Openmp-commits)
Date: Wed, 19 Aug 2026 05:19:56 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [OpenMPOpt] Ask the runtime how
 many of a block's threads can be workers (PR #217080)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/217080@github.com>
Message-ID: <6a859f6c.541ea9c2.11b9a7.3e5c@mx.google.com>


https://github.com/lfmeadow closed https://github.com/llvm/llvm-project/pull/217080

From openmp-commits at lists.llvm.org  Wed Aug 19 10:37:09 2026
From: openmp-commits at lists.llvm.org (Jan Patrick Lehr via Openmp-commits)
Date: Wed, 19 Aug 2026 10:37:09 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [NFC][OpenMP][OMPT] Move
 `for_static_init` work callbacks into macro (PR #217259)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/217259@github.com>
Message-ID: <6a85e9c5.510f576d.37de7c.e5de@mx.google.com>


https://github.com/jplehr approved this pull request.

Thanks for splitting this into an extra nfci 

https://github.com/llvm/llvm-project/pull/217259

From openmp-commits at lists.llvm.org  Wed Aug 19 14:00:23 2026
From: openmp-commits at lists.llvm.org (David Justo via Openmp-commits)
Date: Wed, 19 Aug 2026 14:00:23 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [NFC][CodeQL] Suppress intentional ITT
 collector library loading (PR #217116)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/217116@github.com>
Message-ID: <6a861967.af4834b4.25a1a0.0bfe@mx.google.com>


davidmrdavid wrote:

@jprotze - I just learned about `secure_getenv` so take this with a graint of salt - I am not aware of a 'more secure' alternative to `GetEnvironmentVariableA`. I feel that if one existed, it'd be listed in the mslearn page here: https://learn.microsoft.com/en-us/windows/win32/api/processenv/nf-processenv-getenvironmentvariablea

I think it should be 'ok' to suppress this warning, at least until another static analyzer complains, at which point maybe there's more reason to look into an alternative.

https://github.com/llvm/llvm-project/pull/217116

From openmp-commits at lists.llvm.org  Thu Aug 20 08:55:01 2026
From: openmp-commits at lists.llvm.org (Michael Klemm via Openmp-commits)
Date: Thu, 20 Aug 2026 08:55:01 -0700 (PDT)
Subject: [Openmp-commits] [flang] [llvm] [openmp] [OpenMP][Flang] Add
 integer-kind wrappers to omp_lib (PR #213505)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213505@github.com>
Message-ID: <6a872355.0318b02e.39cad5.3d15@mx.google.com>


https://github.com/mjklemm approved this pull request.

LGTM.

https://github.com/llvm/llvm-project/pull/213505

From openmp-commits at lists.llvm.org  Thu Aug 20 08:56:55 2026
From: openmp-commits at lists.llvm.org (Michael Klemm via Openmp-commits)
Date: Thu, 20 Aug 2026 08:56:55 -0700 (PDT)
Subject: [Openmp-commits] [flang] [llvm] [openmp] [OpenMP][Flang] Add
 integer-kind wrappers to omp_lib (PR #213505)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213505@github.com>
Message-ID: <6a8723c7.4a342cbc.1dab76.434f@mx.google.com>


mjklemm wrote:

This PR is a breaking change, as it updates the module hash.  This is fine.  We should plan for a subsequent NFC PR that moves the `private` interfaces to the C stubs into an implementation module to not break the `omp_lib` module hash everytime the internals are updated:

```Fortran
submodule (omp_lib) omp_lib_impl
  use omp_lib_kinds
  use omp_lib_c_iface
contains
  ! bodies ....
end submodule omp_lib_impl
```

https://github.com/llvm/llvm-project/pull/213505

From openmp-commits at lists.llvm.org  Thu Aug 20 10:50:22 2026
From: openmp-commits at lists.llvm.org (Alexey Bataev via Openmp-commits)
Date: Thu, 20 Aug 2026 10:50:22 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [Clang][OpenMP] Add
 `#pragma omp flatten` loop directive + `depth` clause (PR #206977)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206977@github.com>
Message-ID: <6a873e5e.510f576d.37de7c.e8af@mx.google.com>


alexey-bataev wrote:

Will it handle this test:
```
#pragma omp flatten
for (int i = 0; i < -1; ++i)   // also: runtime n,m both negative
  for (int j = 0; j < -1; ++j)
    body(i, j);                // must execute 0 times; flatten executes it once
```
?

https://github.com/llvm/llvm-project/pull/206977

From openmp-commits at lists.llvm.org  Thu Aug 20 10:51:54 2026
From: openmp-commits at lists.llvm.org (Alexey Bataev via Openmp-commits)
Date: Thu, 20 Aug 2026 10:51:54 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [Clang][OpenMP] Add
 `#pragma omp flatten` loop directive + `depth` clause (PR #206977)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206977@github.com>
Message-ID: <6a873eba.633cb3bb.39123d.2f47@mx.google.com>


================
@@ -16528,6 +16534,277 @@ StmtResult SemaOpenMP::ActOnOpenMPInterchangeDirective(
                                          buildPreInits(Context, PreInits));
 }
 
+StmtResult
+SemaOpenMP::ActOnOpenMPFlattenDirective(ArrayRef<OMPClause *> Clauses,
+                                        Stmt *AStmt, SourceLocation StartLoc,
+                                        SourceLocation EndLoc) {
+  ASTContext &Context = getASTContext();
+  DeclContext *CurContext = SemaRef.CurContext;
+  Scope *CurScope = SemaRef.getCurScope();
+
+  // Empty statement should only be possible if there already was an error.
+  if (!AStmt)
+    return StmtError();
+
+  // flatten without 'depth' clause combines two loops; 'depth(k)' selects k.
+  unsigned NumLoops = 2;
+  bool DepthIsDependent = false;
+  const auto *DepthClause =
+      OMPExecutableDirective::getSingleClause<OMPDepthClause>(Clauses);
+  if (DepthClause) {
+    Expr *DepthExpr = DepthClause->getDepth();
+    if (DepthExpr && DepthExpr->isInstantiationDependent()) {
+      DepthIsDependent = true;
+    } else if (DepthExpr) {
+      Expr::EvalResult EvalResult;
+      if (DepthExpr->EvaluateAsInt(EvalResult, Context))
+        NumLoops = EvalResult.Val.getInt().getLimitedValue();
----------------
alexey-bataev wrote:

```suggestion
        NumLoops = EvalResult.Val.getInt().getLimitedValue(std::numeric_limits<unsigned>::max());
```

https://github.com/llvm/llvm-project/pull/206977

From openmp-commits at lists.llvm.org  Thu Aug 20 10:51:54 2026
From: openmp-commits at lists.llvm.org (Alexey Bataev via Openmp-commits)
Date: Thu, 20 Aug 2026 10:51:54 -0700 (PDT)
Subject: [Openmp-commits] [clang] [llvm] [openmp] [Clang][OpenMP] Add
 `#pragma omp flatten` loop directive + `depth` clause (PR #206977)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/206977@github.com>
Message-ID: <6a873eba.3865fcc3.3a8777.194c@mx.google.com>


================
@@ -0,0 +1,12 @@
+// RUN: c-index-test -test-load-source local %s -fopenmp=libomp -fopenmp-version=60 | FileCheck %s
+
+void test(void) {
----------------
alexey-bataev wrote:

What about this:
```
#pragma omp flatten
for (int i = 0; i < 3; ++i)
  for (int j = 0; j < 0; ++j)
    body(i, j);
```
?

https://github.com/llvm/llvm-project/pull/206977

From openmp-commits at lists.llvm.org  Thu Aug 20 11:18:36 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Thu, 20 Aug 2026 11:18:36 -0700 (PDT)
Subject: [Openmp-commits] [openmp] 0b2e935 - [NFCI][OpenMP][OMPT] Move
 `for_static_init` work callbacks into macro (#217259)
Message-ID: <6a8744fc.d07e16d6.329d01.fa8b@mx.google.com>


Author: Jan André Reuter
Date: 2026-08-20T20:18:30+02:00
New Revision: 0b2e935b198a58439a42c2fda15819bcfea93b16

URL: https://github.com/llvm/llvm-project/commit/0b2e935b198a58439a42c2fda15819bcfea93b16
DIFF: https://github.com/llvm/llvm-project/commit/0b2e935b198a58439a42c2fda15819bcfea93b16.diff

LOG: [NFCI][OpenMP][OMPT] Move `for_static_init` work callbacks into macro (#217259)

`__kmp_for_static_init` uses a very similar pattern for dispatching the
work callback to an attached tool throughout the function. Hence, avoid
the duplication by moving the pattern into a macro, taking the number of
loop iterations as an argument. Without OMPT, this macro is a no-op.

Signed-off-by: Jan André Reuter <jan at zyten.de>

Added: 
    

Modified: 
    openmp/runtime/src/kmp_sched.cpp

Removed: 
    


################################################################################
diff  --git a/openmp/runtime/src/kmp_sched.cpp b/openmp/runtime/src/kmp_sched.cpp
index 2b1bb6f595f9a..7b6b54d274a6f 100644
--- a/openmp/runtime/src/kmp_sched.cpp
+++ b/openmp/runtime/src/kmp_sched.cpp
@@ -128,6 +128,14 @@ static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid,
       KMP_DEBUG_ASSERT(ompt_work_type);
     }
   }
+#define OMPT_LOOP_BEGIN(count)                                                 \
+  if (ompt_enabled.ompt_callback_work) {                                       \
+    ompt_callbacks.ompt_callback(ompt_callback_work)(                          \
+        ompt_work_type, ompt_scope_begin, &(team_info->parallel_data),         \
+        &(task_info->task_data), count, codeptr);                              \
+  }
+#else
+#define OMPT_LOOP_BEGIN(count) // no-op
 #endif
 
   KMP_DEBUG_ASSERT(plastiter && plower && pupper && pstride);
@@ -181,13 +189,7 @@ static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid,
 #endif
     KE_TRACE(10, ("__kmpc_for_static_init: T#%d return\n", global_tid));
 
-#if OMPT_SUPPORT && OMPT_OPTIONAL
-    if (ompt_enabled.ompt_callback_work) {
-      ompt_callbacks.ompt_callback(ompt_callback_work)(
-          ompt_work_type, ompt_scope_begin, &(team_info->parallel_data),
-          &(task_info->task_data), 0, codeptr);
-    }
-#endif
+    OMPT_LOOP_BEGIN(0);
     KMP_STATS_LOOP_END(OMP_loop_static_iterations);
     return;
   }
@@ -235,13 +237,7 @@ static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid,
 #endif
     KE_TRACE(10, ("__kmpc_for_static_init: T#%d return\n", global_tid));
 
-#if OMPT_SUPPORT && OMPT_OPTIONAL
-    if (ompt_enabled.ompt_callback_work) {
-      ompt_callbacks.ompt_callback(ompt_callback_work)(
-          ompt_work_type, ompt_scope_begin, &(team_info->parallel_data),
-          &(task_info->task_data), *pstride, codeptr);
-    }
-#endif
+    OMPT_LOOP_BEGIN(*pstride);
     KMP_STATS_LOOP_END(OMP_loop_static_iterations);
     return;
   }
@@ -265,13 +261,7 @@ static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid,
 #endif
     KE_TRACE(10, ("__kmpc_for_static_init: T#%d return\n", global_tid));
 
-#if OMPT_SUPPORT && OMPT_OPTIONAL
-    if (ompt_enabled.ompt_callback_work) {
-      ompt_callbacks.ompt_callback(ompt_callback_work)(
-          ompt_work_type, ompt_scope_begin, &(team_info->parallel_data),
-          &(task_info->task_data), *pstride, codeptr);
-    }
-#endif
+    OMPT_LOOP_BEGIN(*pstride);
     KMP_STATS_LOOP_END(OMP_loop_static_iterations);
     return;
   }
@@ -445,12 +435,8 @@ static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid,
 #endif
   KE_TRACE(10, ("__kmpc_for_static_init: T#%d return\n", global_tid));
 
+  OMPT_LOOP_BEGIN(trip_count);
 #if OMPT_SUPPORT && OMPT_OPTIONAL
-  if (ompt_enabled.ompt_callback_work) {
-    ompt_callbacks.ompt_callback(ompt_callback_work)(
-        ompt_work_type, ompt_scope_begin, &(team_info->parallel_data),
-        &(task_info->task_data), trip_count, codeptr);
-  }
   if (ompt_enabled.ompt_callback_dispatch) {
     ompt_dispatch_t dispatch_type;
     ompt_data_t instance = ompt_data_none;


        

From openmp-commits at lists.llvm.org  Thu Aug 20 11:18:38 2026
From: openmp-commits at lists.llvm.org (=?UTF-8?Q?Jan_Andr=C3=A9_Reuter?= via Openmp-commits)
Date: Thu, 20 Aug 2026 11:18:38 -0700 (PDT)
Subject: [Openmp-commits] [openmp] [NFCI][OpenMP][OMPT] Move
 `for_static_init` work callbacks into macro (PR #217259)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/217259@github.com>
Message-ID: <6a8744fe.8e75e0cb.2b427.d446@mx.google.com>


https://github.com/Thyre closed https://github.com/llvm/llvm-project/pull/217259

From openmp-commits at lists.llvm.org  Fri Aug 21 21:29:24 2026
From: openmp-commits at lists.llvm.org (via Openmp-commits)
Date: Fri, 21 Aug 2026 21:29:24 -0700 (PDT)
Subject: [Openmp-commits] [flang] [llvm] [openmp] [OpenMP][Flang] Add
 integer-kind wrappers to omp_lib (PR #213505)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/213505@github.com>
Message-ID: <6a8925a4.ed8a271b.2d6122.dfa4@mx.google.com>


shivaramaarao wrote:

@Saieiei hope this is fine to submit. let me know otherwise

https://github.com/llvm/llvm-project/pull/213505

From openmp-commits at lists.llvm.org  Fri Aug 21 23:37:16 2026
From: openmp-commits at lists.llvm.org (Robert Imschweiler via Openmp-commits)
Date: Fri, 21 Aug 2026 23:37:16 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [offload] Pool host and shared
	allocations (PR #214754)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/214754@github.com>
Message-ID: <6a89439c.e22746c9.27e136.c36b@mx.google.com>


================
@@ -4529,6 +4535,12 @@ Expected<void *> AMDGPUDeviceTy::allocate(size_t Size, void *,
     // Enable all valid kernel agents to access the buffer.
     if (auto Err = MemoryPool->enableAccess(Alloc, Size, Agents))
       return std::move(Err);
+
+    // Register host / shared memory as pinned memory, so that transfers reading
+    // from it can take a device-accessible path.
+    if (Kind == TARGET_ALLOC_HOST || Kind == TARGET_ALLOC_SHARED)
+      if (auto Err = PinnedAllocs.registerHostBuffer(Alloc, Alloc, Size))
+        return std::move(Err);
----------------
ro-i wrote:

Afaics, the other plugins have not yet been using `PinnedAllocs` yet, only the amdgpu plugin:

```
$ grep -rn PinnedAllocs
plugins-nextgen/amdgpu/src/rtl.cpp:2845:            PinnedAllocs.getDeviceAccessiblePtrFromPinnedBuffer(HstPtr)) {
plugins-nextgen/amdgpu/src/rtl.cpp:2904:            PinnedAllocs.getDeviceAccessiblePtrFromPinnedBuffer(HstPtr)) {
plugins-nextgen/common/include/PluginInterface.h:1057:    return PinnedAllocs.registerMemory(HstPtr, Size, LockMemory);
plugins-nextgen/common/include/PluginInterface.h:1062:    return PinnedAllocs.unregisterMemory(HstPtr, UnlockMemory);
plugins-nextgen/common/include/PluginInterface.h:1078:    auto Err = PinnedAllocs.registerMemory(HstPtr, Size, LockMappedBuffers);
plugins-nextgen/common/include/PluginInterface.h:1088:    auto Err = PinnedAllocs.unregisterMemory(HstPtr, LockMappedBuffers);
plugins-nextgen/common/include/PluginInterface.h:1484:  PinnedAllocationMapTy PinnedAllocs;
plugins-nextgen/common/src/PluginInterface.cpp:462:      PinnedAllocs(*this), RPCServer(nullptr) {
plugins-nextgen/common/src/PluginInterface.cpp:1008:    if (auto Err = PinnedAllocs.registerHostBuffer(Alloc, Alloc, Size))
plugins-nextgen/common/src/PluginInterface.cpp:1084:    if (auto Err = PinnedAllocs.unregisterHostBuffer(TgtPtr))
```

https://github.com/llvm/llvm-project/pull/214754

From openmp-commits at lists.llvm.org  Fri Aug 21 23:37:36 2026
From: openmp-commits at lists.llvm.org (Robert Imschweiler via Openmp-commits)
Date: Fri, 21 Aug 2026 23:37:36 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [offload] Pool host and shared
	allocations (PR #214754)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/214754@github.com>
Message-ID: <6a8943b0.c4d3d80a.183bf.4fea@mx.google.com>


================
@@ -4529,6 +4535,12 @@ Expected<void *> AMDGPUDeviceTy::allocate(size_t Size, void *,
     // Enable all valid kernel agents to access the buffer.
     if (auto Err = MemoryPool->enableAccess(Alloc, Size, Agents))
       return std::move(Err);
+
+    // Register host / shared memory as pinned memory, so that transfers reading
+    // from it can take a device-accessible path.
+    if (Kind == TARGET_ALLOC_HOST || Kind == TARGET_ALLOC_SHARED)
+      if (auto Err = PinnedAllocs.registerHostBuffer(Alloc, Alloc, Size))
+        return std::move(Err);
----------------
ro-i wrote:

So, the others had the (un)registration, but didn't do anything with it

https://github.com/llvm/llvm-project/pull/214754

From openmp-commits at lists.llvm.org  Fri Aug 21 23:39:16 2026
From: openmp-commits at lists.llvm.org (Robert Imschweiler via Openmp-commits)
Date: Fri, 21 Aug 2026 23:39:16 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [offload] Pool host and shared
	allocations (PR #214754)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/214754@github.com>
Message-ID: <6a894414.ef5297aa.3477d5.b0a7@mx.google.com>


ro-i wrote:

> We could keep track of users of an unpinned side and co-locate ourselves anyway. Just a thought.

You mean: don't allocate a pinned page per memory chunk and instead try to fit multiple on one pinned page? That sounds like a good optimization, but I feel like it would be out-of-scope for my current stack.

https://github.com/llvm/llvm-project/pull/214754

From openmp-commits at lists.llvm.org  Fri Aug 21 23:46:33 2026
From: openmp-commits at lists.llvm.org (Robert Imschweiler via Openmp-commits)
Date: Fri, 21 Aug 2026 23:46:33 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [offload] Pool host and shared
	allocations (PR #214754)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/214754@github.com>
Message-ID: <6a8945c9.507fc713.35c652.b426@mx.google.com>


https://github.com/ro-i edited https://github.com/llvm/llvm-project/pull/214754

From openmp-commits at lists.llvm.org  Sun Aug 23 11:52:14 2026
From: openmp-commits at lists.llvm.org (Kevin Sala Penades via Openmp-commits)
Date: Sun, 23 Aug 2026 11:52:14 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [offload] Pool host and shared
	allocations (PR #214754)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/214754@github.com>
Message-ID: <6a8b415e.510f576d.1e8f6d.9a51@mx.google.com>


================
@@ -4529,6 +4535,12 @@ Expected<void *> AMDGPUDeviceTy::allocate(size_t Size, void *,
     // Enable all valid kernel agents to access the buffer.
     if (auto Err = MemoryPool->enableAccess(Alloc, Size, Agents))
       return std::move(Err);
+
+    // Register host / shared memory as pinned memory, so that transfers reading
+    // from it can take a device-accessible path.
+    if (Kind == TARGET_ALLOC_HOST || Kind == TARGET_ALLOC_SHARED)
+      if (auto Err = PinnedAllocs.registerHostBuffer(Alloc, Alloc, Size))
+        return std::move(Err);
----------------
kevinsala wrote:

Thanks for the clarification!

https://github.com/llvm/llvm-project/pull/214754

From openmp-commits at lists.llvm.org  Sun Aug 23 11:52:53 2026
From: openmp-commits at lists.llvm.org (Kevin Sala Penades via Openmp-commits)
Date: Sun, 23 Aug 2026 11:52:53 -0700 (PDT)
Subject: [Openmp-commits] [llvm] [openmp] [offload] Pool host and shared
	allocations (PR #214754)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/214754@github.com>
Message-ID: <6a8b4185.1e1ac41d.2d530b.a0ab@mx.google.com>


https://github.com/kevinsala approved this pull request.

LGTM

https://github.com/llvm/llvm-project/pull/214754

From openmp-commits at lists.llvm.org  Sun Aug 23 16:44:57 2026
From: openmp-commits at lists.llvm.org (Petr Hosek via Openmp-commits)
Date: Sun, 23 Aug 2026 16:44:57 -0700 (PDT)
Subject: [Openmp-commits] [compiler-rt] [libclc] [libcxx] [libcxxabi]
 [libunwind] [llvm] [openmp] [CMake] Remove the use of AddLLVM from runtimes
 (PR #217502)
In-Reply-To: <llvm.org/llvm/llvm-project/pull/217502@github.com>
Message-ID: <6a8b85f9.d2cb4adc.3b4790.50b8@mx.google.com>


https://github.com/petrhosek updated https://github.com/llvm/llvm-project/pull/217502

>From a6090bf4ee29d8c06803e894065b88f0d54c0467 Mon Sep 17 00:00:00 2001
From: Petr Hosek <phosek at google.com>
Date: Thu, 20 Aug 2026 01:36:18 +0000
Subject: [PATCH 1/3] [CMake] Remove the use of AddLLVM from runtimes

AddLLVM primarily contains utility functions needed for LLVM based
tools, but runtimes are not LLVM tools. The only part of AddLLVM that's
relevant to runtimes is the support for lit. This change extracts the
lit support into a new top-level AddLit module and replaces all uses of
AddLLVM in runtimes with this new module.
---
 cmake/Modules/AddLit.cmake                    | 427 +++++++++++++++++
 cmake/Modules/GetSubprojectTitle.cmake        |  14 +
 compiler-rt/CMakeLists.txt                    |   2 +-
 .../CompilerRTMockLLVMCMakeConfig.cmake       |   2 +-
 compiler-rt/cmake/config-ix.cmake             |   1 -
 compiler-rt/test/CMakeLists.txt               |   2 +-
 compiler-rt/unittests/CMakeLists.txt          |   2 +-
 libclc/CMakeLists.txt                         |  33 +-
 libcxx/test/CMakeLists.txt                    |   2 +-
 libcxxabi/test/CMakeLists.txt                 |   2 +-
 libunwind/test/CMakeLists.txt                 |   2 +-
 llvm/cmake/modules/AddLLVM.cmake              | 442 +-----------------
 llvm/cmake/modules/AddOCaml.cmake             |   1 +
 llvm/cmake/modules/AddSphinxTarget.cmake      |   1 +
 llvm/cmake/modules/CMakeLists.txt             |   8 +
 llvm/cmake/modules/CrossCompile.cmake         |   1 +
 .../modules/LLVMDistributionSupport.cmake     |   1 +
 llvm/cmake/modules/TableGen.cmake             |   1 +
 llvm/runtimes/CMakeLists.txt                  |   7 +
 runtimes/CMakeLists.txt                       |   2 +-
 20 files changed, 482 insertions(+), 471 deletions(-)
 create mode 100644 cmake/Modules/AddLit.cmake
 create mode 100644 cmake/Modules/GetSubprojectTitle.cmake

diff --git a/cmake/Modules/AddLit.cmake b/cmake/Modules/AddLit.cmake
new file mode 100644
index 0000000000000..2595ee504cb8f
--- /dev/null
+++ b/cmake/Modules/AddLit.cmake
@@ -0,0 +1,427 @@
+include(GetSubprojectTitle)
+
+
+# This function canonicalize the CMake variables passed by names
+# from CMake boolean to 0/1 suitable for passing into Python or C++,
+# in place.
+function(llvm_canonicalize_cmake_booleans)
+  foreach(var ${ARGN})
+    if(${var})
+      set(${var} 1 PARENT_SCOPE)
+    else()
+      set(${var} 0 PARENT_SCOPE)
+    endif()
+  endforeach()
+endfunction(llvm_canonicalize_cmake_booleans)
+
+macro(set_llvm_build_mode)
+  # Configuration-time: See Unit/lit.site.cfg.in
+  if (CMAKE_CFG_INTDIR STREQUAL ".")
+    set(LLVM_BUILD_MODE ".")
+  else ()
+    set(LLVM_BUILD_MODE "%(build_mode)s")
+  endif ()
+endmacro()
+
+# Takes a list of path names in pathlist and a base directory, and returns
+# a list of paths relative to the base directory in out_pathlist.
+# Paths that are on a different drive than the basedir (on Windows) or that
+# contain symlinks are returned absolute.
+# Use with LLVM_LIT_PATH_FUNCTION below.
+function(make_paths_relative out_pathlist basedir pathlist)
+  # Passing ARG_PATH_VALUES as-is to execute_process() makes cmake strip
+  # empty list entries. So escape the ;s in the list and do the splitting
+  # ourselves. cmake has no relpath function, so use Python for that.
+  string(REPLACE ";" "\\;" pathlist_escaped "${pathlist}")
+  execute_process(COMMAND "${Python3_EXECUTABLE}" "-c" "\n
+import os, sys\n
+base = sys.argv[1]
+def haslink(p):\n
+    if not p or p == os.path.dirname(p): return False\n
+    return os.path.islink(p) or haslink(os.path.dirname(p))\n
+def relpath(p):\n
+    if not p: return ''\n
+    if os.path.splitdrive(p)[0] != os.path.splitdrive(base)[0]: return p\n
+    if haslink(p) or haslink(base): return p\n
+    return os.path.relpath(p, base)\n
+if len(sys.argv) < 3: sys.exit(0)\n
+sys.stdout.write(';'.join(relpath(p) for p in sys.argv[2].split(';')))"
+    ${basedir}
+    ${pathlist_escaped}
+    OUTPUT_VARIABLE pathlist_relative
+    ERROR_VARIABLE error
+    RESULT_VARIABLE result)
+  if (NOT result EQUAL 0)
+    message(FATAL_ERROR "make_paths_relative() failed due to error '${result}', with stderr\n${error}")
+  endif()
+  set(${out_pathlist} "${pathlist_relative}" PARENT_SCOPE)
+endfunction()
+
+# Converts a file that's relative to the current python file to an absolute
+# path. Since this uses __file__, it has to be emitted into python files that
+# use it and can't be in a lit module. Use with make_paths_relative().
+string(CONCAT LLVM_LIT_PATH_FUNCTION
+  "# Allow generated file to be relocatable.\n"
+  "import os\n"
+  "import platform\n"
+  "def path(p):\n"
+  "    if not p: return ''\n"
+  "    # Follows lit.util.abs_path_preserve_drive, which cannot be imported here.\n"
+  "    if platform.system() == 'Windows':\n"
+  "        return os.path.abspath(os.path.join(os.path.dirname(__file__), p))\n"
+  "    else:\n"
+  "        return os.path.realpath(os.path.join(os.path.dirname(__file__), p))\n"
+  )
+
+# This function provides an automatic way to 'configure'-like generate a file
+# based on a set of common and custom variables, specifically targeting the
+# variables needed for the 'lit.site.cfg' files. This function bundles the
+# common variables that any Lit instance is likely to need, and custom
+# variables can be passed in.
+# The keyword PATHS is followed by a list of cmake variable names that are
+# mentioned as `path("@varname@")` in the lit.cfg.py.in file. Variables in that
+# list are treated as paths that are relative to the directory the generated
+# lit.cfg.py file is in, and the `path()` function converts the relative
+# path back to absolute form. This makes it possible to move a build directory
+# containing lit.cfg.py files from one machine to another.
+function(configure_lit_site_cfg site_in site_out)
+  cmake_parse_arguments(ARG "" "" "MAIN_CONFIG;PATHS" ${ARGN})
+
+  if ("${ARG_MAIN_CONFIG}" STREQUAL "")
+    get_filename_component(INPUT_DIR ${site_in} DIRECTORY)
+    set(ARG_MAIN_CONFIG "${INPUT_DIR}/lit.cfg")
+  endif()
+
+  foreach(c ${LLVM_TARGETS_TO_BUILD})
+    set(TARGETS_BUILT "${TARGETS_BUILT} ${c}")
+  endforeach(c)
+  set(TARGETS_TO_BUILD ${TARGETS_BUILT})
+
+  set(SHLIBEXT "${LTDL_SHLIB_EXT}")
+
+  set_llvm_build_mode()
+
+  # For standalone builds of subprojects, these might not be the build tree but
+  # a provided binary tree.
+  set(LLVM_SOURCE_DIR ${LLVM_MAIN_SRC_DIR})
+  set(LLVM_BINARY_DIR ${LLVM_BINARY_DIR})
+  string(REPLACE "${CMAKE_CFG_INTDIR}" "${LLVM_BUILD_MODE}" LLVM_TOOLS_DIR "${LLVM_TOOLS_BINARY_DIR}")
+  string(REPLACE "${CMAKE_CFG_INTDIR}" "${LLVM_BUILD_MODE}" LLVM_LIBS_DIR  "${LLVM_LIBRARY_DIR}")
+  # Like LLVM_{TOOLS,LIBS}_DIR, but pointing at the build tree.
+  string(REPLACE "${CMAKE_CFG_INTDIR}" "${LLVM_BUILD_MODE}" CURRENT_TOOLS_DIR "${LLVM_RUNTIME_OUTPUT_INTDIR}")
+  string(REPLACE "${CMAKE_CFG_INTDIR}" "${LLVM_BUILD_MODE}" CURRENT_LIBS_DIR  "${LLVM_LIBRARY_OUTPUT_INTDIR}")
+  string(REPLACE "${CMAKE_CFG_INTDIR}" "${LLVM_BUILD_MODE}" SHLIBDIR "${LLVM_SHLIB_OUTPUT_INTDIR}")
+
+  # FIXME: "ENABLE_SHARED" doesn't make sense, since it is used just for
+  # plugins. We may rename it.
+  if(LLVM_ENABLE_PLUGINS)
+    set(ENABLE_SHARED "1")
+  else()
+    set(ENABLE_SHARED "0")
+  endif()
+
+  if(LLVM_ENABLE_ASSERTIONS)
+    set(ENABLE_ASSERTIONS "1")
+  else()
+    set(ENABLE_ASSERTIONS "0")
+  endif()
+
+  set(HOST_OS ${CMAKE_SYSTEM_NAME})
+  set(HOST_ARCH ${CMAKE_SYSTEM_PROCESSOR})
+
+  set(HOST_CC "${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_ARG1}")
+  set(HOST_CXX "${CMAKE_CXX_COMPILER} ${CMAKE_CXX_COMPILER_ARG1}")
+  set(HOST_LDFLAGS "${CMAKE_EXE_LINKER_FLAGS}")
+
+  string(CONCAT LIT_SITE_CFG_IN_HEADER
+    "# Autogenerated from ${site_in}\n# Do not edit!\n\n"
+    "${LLVM_LIT_PATH_FUNCTION}"
+    )
+
+  # Override config_target_triple (and the env)
+  if(LLVM_TARGET_TRIPLE_ENV)
+    # This is expanded into the heading.
+    string(CONCAT LIT_SITE_CFG_IN_HEADER "${LIT_SITE_CFG_IN_HEADER}"
+      "import os\n"
+      "target_env = \"${LLVM_TARGET_TRIPLE_ENV}\"\n"
+      "config.target_triple = config.environment[target_env] = os.environ.get(target_env, \"${LLVM_TARGET_TRIPLE}\")\n"
+      )
+
+    # This is expanded to; config.target_triple = ""+config.target_triple+""
+    set(LLVM_TARGET_TRIPLE "\"+config.target_triple+\"")
+  endif()
+
+  if (ARG_PATHS)
+    # Walk ARG_PATHS and collect the current value of the variables in there.
+    # list(APPEND) ignores empty elements exactly if the list is empty,
+    # so start the list with a dummy element and drop it, to make sure that
+    # even empty values make it into the values list.
+    set(ARG_PATH_VALUES "dummy")
+    foreach(path ${ARG_PATHS})
+      list(APPEND ARG_PATH_VALUES "${${path}}")
+    endforeach()
+    list(REMOVE_AT ARG_PATH_VALUES 0)
+
+    get_filename_component(OUTPUT_DIR ${site_out} DIRECTORY)
+    make_paths_relative(
+        ARG_PATH_VALUES_RELATIVE "${OUTPUT_DIR}" "${ARG_PATH_VALUES}")
+
+    list(LENGTH ARG_PATHS len_paths)
+    list(LENGTH ARG_PATH_VALUES len_path_values)
+    list(LENGTH ARG_PATH_VALUES_RELATIVE len_path_value_rels)
+    if ((NOT ${len_paths} EQUAL ${len_path_values}) OR
+        (NOT ${len_paths} EQUAL ${len_path_value_rels}))
+      message(SEND_ERROR "PATHS lengths got confused")
+    endif()
+
+    # Transform variables mentioned in ARG_PATHS to relative paths for
+    # the configure_file() call. Variables are copied to subscopeds by cmake,
+    # so this only modifies the local copy of the variables.
+    math(EXPR arg_path_limit "${len_paths} - 1")
+    foreach(i RANGE ${arg_path_limit})
+      list(GET ARG_PATHS ${i} val1)
+      list(GET ARG_PATH_VALUES_RELATIVE ${i} val2)
+      set(${val1} ${val2})
+    endforeach()
+  endif()
+
+  configure_file(${site_in} ${site_out} @ONLY)
+
+  if (EXISTS "${ARG_MAIN_CONFIG}")
+    # Remember main config / generated site config for llvm-lit.in.
+    get_property(LLVM_LIT_CONFIG_FILES GLOBAL PROPERTY LLVM_LIT_CONFIG_FILES)
+    list(APPEND LLVM_LIT_CONFIG_FILES "${ARG_MAIN_CONFIG}" "${site_out}")
+    set_property(GLOBAL PROPERTY LLVM_LIT_CONFIG_FILES ${LLVM_LIT_CONFIG_FILES})
+  endif()
+endfunction()
+
+function(dump_all_cmake_variables)
+  get_cmake_property(_variableNames VARIABLES)
+  foreach (_variableName ${_variableNames})
+    message(STATUS "${_variableName}=${${_variableName}}")
+  endforeach()
+endfunction()
+
+function(get_llvm_lit_path base_dir file_name)
+  cmake_parse_arguments(ARG "ALLOW_EXTERNAL" "" "" ${ARGN})
+
+  if (ARG_ALLOW_EXTERNAL)
+    set (LLVM_EXTERNAL_LIT "" CACHE STRING "Command used to spawn lit")
+    if ("${LLVM_EXTERNAL_LIT}" STREQUAL "")
+      set(LLVM_EXTERNAL_LIT "${LLVM_DEFAULT_EXTERNAL_LIT}")
+    endif()
+
+    if (NOT "${LLVM_EXTERNAL_LIT}" STREQUAL "")
+      if (EXISTS ${LLVM_EXTERNAL_LIT})
+        get_filename_component(LIT_FILE_NAME ${LLVM_EXTERNAL_LIT} NAME)
+        get_filename_component(LIT_BASE_DIR ${LLVM_EXTERNAL_LIT} DIRECTORY)
+        set(${file_name} ${LIT_FILE_NAME} PARENT_SCOPE)
+        set(${base_dir} ${LIT_BASE_DIR} PARENT_SCOPE)
+        return()
+      elseif (NOT DEFINED CACHE{LLVM_EXTERNAL_LIT_MISSING_WARNED_ONCE})
+        message(WARNING "LLVM_EXTERNAL_LIT set to ${LLVM_EXTERNAL_LIT}, but the path does not exist.")
+        set(LLVM_EXTERNAL_LIT_MISSING_WARNED_ONCE YES CACHE INTERNAL "")
+      endif()
+    endif()
+  endif()
+
+  set(lit_file_name "llvm-lit")
+  if (CMAKE_HOST_WIN32 AND NOT CYGWIN)
+    # llvm-lit needs suffix.py for multiprocess to find a main module.
+    set(lit_file_name "${lit_file_name}.py")
+  endif ()
+  set(${file_name} ${lit_file_name} PARENT_SCOPE)
+
+  get_property(LLVM_LIT_BASE_DIR GLOBAL PROPERTY LLVM_LIT_BASE_DIR)
+  if (NOT "${LLVM_LIT_BASE_DIR}" STREQUAL "")
+    set(${base_dir} ${LLVM_LIT_BASE_DIR} PARENT_SCOPE)
+  endif()
+
+  # Allow individual projects to provide an override
+  if (NOT "${LLVM_LIT_OUTPUT_DIR}" STREQUAL "")
+    set(LLVM_LIT_BASE_DIR ${LLVM_LIT_OUTPUT_DIR})
+  elseif(NOT "${LLVM_RUNTIME_OUTPUT_INTDIR}" STREQUAL "")
+    set(LLVM_LIT_BASE_DIR ${LLVM_RUNTIME_OUTPUT_INTDIR})
+  else()
+    set(LLVM_LIT_BASE_DIR "")
+  endif()
+
+  # Cache this so we don't have to do it again and have subsequent calls
+  # potentially disagree on the value.
+  set_property(GLOBAL PROPERTY LLVM_LIT_BASE_DIR ${LLVM_LIT_BASE_DIR})
+  set(${base_dir} ${LLVM_LIT_BASE_DIR} PARENT_SCOPE)
+endfunction()
+
+# A raw function to create a lit target. This is used to implement the testuite
+# management functions.
+function(add_lit_target target comment)
+  cmake_parse_arguments(ARG "" "" "PARAMS;DEPENDS;ARGS" ${ARGN})
+  set(LIT_ARGS "${ARG_ARGS} ${LLVM_LIT_ARGS}")
+  separate_arguments(LIT_ARGS)
+  if (NOT CMAKE_CFG_INTDIR STREQUAL ".")
+    list(APPEND LIT_ARGS --param build_mode=${CMAKE_CFG_INTDIR})
+  endif ()
+
+  # Get the path to the lit to *run* tests with.  This can be overriden by
+  # the user by specifying -DLLVM_EXTERNAL_LIT=<path-to-lit.py>
+  get_llvm_lit_path(
+    lit_base_dir
+    lit_file_name
+    ALLOW_EXTERNAL
+    )
+
+  set(LIT_COMMAND "${Python3_EXECUTABLE};${lit_base_dir}/${lit_file_name}")
+  list(APPEND LIT_COMMAND ${LIT_ARGS})
+  foreach(param ${ARG_PARAMS})
+    list(APPEND LIT_COMMAND --param ${param})
+  endforeach()
+  if (ARG_UNPARSED_ARGUMENTS)
+    add_custom_target(${target}
+      COMMAND ${LIT_COMMAND} ${ARG_UNPARSED_ARGUMENTS}
+      COMMENT "${comment}"
+      USES_TERMINAL
+      )
+  else()
+    add_custom_target(${target}
+      COMMAND ${CMAKE_COMMAND} -E echo "${target} does nothing, no tools built.")
+    message(STATUS "${target} does nothing.")
+  endif()
+  get_subproject_title(subproject_title)
+  set_target_properties(${target} PROPERTIES FOLDER "${subproject_title}/Tests")
+
+  if (ARG_DEPENDS)
+    add_dependencies(${target} ${ARG_DEPENDS})
+  endif()
+
+  # Tests should be excluded from "Build Solution".
+  set_target_properties(${target} PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD ON)
+  set_target_properties(${target} PROPERTIES XCODE_GENERATE_SCHEME ON)
+endfunction()
+
+# Convert a target name like check-clang to a variable name like CLANG.
+function(umbrella_lit_testsuite_var target outvar)
+  if (NOT target MATCHES "^check-")
+    message(FATAL_ERROR "umbrella lit suites must be check-*, not '${target}'")
+  endif()
+  string(SUBSTRING "${target}" 6 -1 var)
+  string(REPLACE "-" "_" var ${var})
+  string(TOUPPER "${var}" var)
+  set(${outvar} "${var}" PARENT_SCOPE)
+endfunction()
+
+# Start recording all lit test suites for a combined 'check-foo' target.
+# The recording continues until umbrella_lit_testsuite_end() creates the target.
+function(umbrella_lit_testsuite_begin target)
+  umbrella_lit_testsuite_var(${target} name)
+  set_property(GLOBAL APPEND PROPERTY LLVM_LIT_UMBRELLAS ${name})
+endfunction()
+
+# Create a combined 'check-foo' target for a set of related test suites.
+# It runs all suites added since the matching umbrella_lit_testsuite_end() call.
+# Tests marked EXCLUDE_FROM_CHECK_ALL are not gathered.
+function(umbrella_lit_testsuite_end target)
+  umbrella_lit_testsuite_var(${target} name)
+
+  get_property(testsuites GLOBAL PROPERTY LLVM_${name}_LIT_TESTSUITES)
+  get_property(params GLOBAL PROPERTY LLVM_${name}_LIT_PARAMS)
+  get_property(depends GLOBAL PROPERTY LLVM_${name}_LIT_DEPENDS)
+  get_property(extra_args GLOBAL PROPERTY LLVM_${name}_LIT_EXTRA_ARGS)
+  # Additional test targets are not gathered, but may be set externally.
+  get_property(additional_test_targets
+               GLOBAL PROPERTY LLVM_${name}_ADDITIONAL_TEST_TARGETS)
+
+  string(TOLOWER ${name} name)
+  add_lit_target(${target}
+    "Running ${name} regression tests"
+    ${testsuites}
+    PARAMS ${params}
+    DEPENDS ${depends} ${additional_test_targets}
+    ARGS ${extra_args}
+    )
+endfunction()
+
+# A function to add a set of lit test suites to be driven through 'check-*' targets.
+function(add_lit_testsuite target comment)
+  cmake_parse_arguments(ARG "EXCLUDE_FROM_CHECK_ALL" "" "PARAMS;DEPENDS;ARGS" ${ARGN})
+
+  # EXCLUDE_FROM_ALL excludes the test ${target} out of check-all.
+  if(NOT ARG_EXCLUDE_FROM_CHECK_ALL)
+    get_property(gather_names GLOBAL PROPERTY LLVM_LIT_UMBRELLAS)
+    foreach(name ${gather_names})
+    # Register the testsuites, params and depends for the umbrella check rule.
+      set_property(GLOBAL APPEND PROPERTY LLVM_${name}_LIT_TESTSUITES ${ARG_UNPARSED_ARGUMENTS})
+      set_property(GLOBAL APPEND PROPERTY LLVM_${name}_LIT_PARAMS ${ARG_PARAMS})
+      set_property(GLOBAL APPEND PROPERTY LLVM_${name}_LIT_DEPENDS ${ARG_DEPENDS})
+      set_property(GLOBAL APPEND PROPERTY LLVM_${name}_LIT_EXTRA_ARGS ${ARG_ARGS})
+    endforeach()
+  endif()
+
+  # Produce a specific suffixed check rule.
+  add_lit_target(${target} ${comment}
+    ${ARG_UNPARSED_ARGUMENTS}
+    PARAMS ${ARG_PARAMS}
+    DEPENDS ${ARG_DEPENDS}
+    ARGS ${ARG_ARGS}
+    )
+endfunction()
+
+function(add_lit_testsuites project directory)
+  if (NOT LLVM_ENABLE_IDE)
+    cmake_parse_arguments(ARG
+      "EXCLUDE_FROM_CHECK_ALL"
+      "FOLDER;BINARY_DIR"
+      "PARAMS;DEPENDS;ARGS;SKIP"
+      ${ARGN}
+      )
+
+    if (NOT ARG_FOLDER)
+      get_subproject_title(subproject_title)
+      set(ARG_FOLDER "${subproject_title}/Tests/LIT Testsuites")
+    endif()
+
+    # Search recursively for test directories by assuming anything not
+    # in a directory called Inputs contains tests.
+    file(GLOB_RECURSE to_process LIST_DIRECTORIES true ${directory}/*)
+    foreach(lit_suite ${to_process})
+      if(NOT IS_DIRECTORY ${lit_suite})
+        continue()
+      endif()
+      string(FIND ${lit_suite} Inputs is_inputs)
+      string(FIND ${lit_suite} Output is_output)
+      if (NOT (is_inputs EQUAL -1 AND is_output EQUAL -1))
+        continue()
+      endif()
+
+      # Create a check- target for the directory.
+      string(REPLACE "${directory}/" "" name_slash ${lit_suite})
+      set(_should_skip FALSE)
+      foreach(skip IN LISTS ARG_SKIP)
+        if(name_slash MATCHES "${skip}")
+          set(_should_skip TRUE)
+          break()
+        endif()
+      endforeach()
+      if (_should_skip)
+        continue()
+      endif()
+      if (name_slash)
+        set(filter ${name_slash})
+        string(REPLACE "/" "-" name_slash ${name_slash})
+        string(REPLACE "\\" "-" name_dashes ${name_slash})
+        string(TOLOWER "${project}-${name_dashes}" name_var)
+        set(lit_args ${lit_suite})
+        if (ARG_BINARY_DIR)
+          set(lit_args ${ARG_BINARY_DIR} --filter=${filter})
+        endif()
+        add_lit_target("check-${name_var}" "Running lit suite ${lit_suite}"
+          ${lit_args}
+          ${EXCLUDE_FROM_CHECK_ALL}
+          PARAMS ${ARG_PARAMS}
+          DEPENDS ${ARG_DEPENDS}
+          ARGS ${ARG_ARGS}
+        )
+        set_target_properties(check-${name_var} PROPERTIES FOLDER ${ARG_FOLDER})
+      endif()
+    endforeach()
+  endif()
+endfunction()
diff --git a/cmake/Modules/GetSubprojectTitle.cmake b/cmake/Modules/GetSubprojectTitle.cmake
new file mode 100644
index 0000000000000..00890f87d2d94
--- /dev/null
+++ b/cmake/Modules/GetSubprojectTitle.cmake
@@ -0,0 +1,14 @@
+# get_subproject_title(titlevar)
+#   Set ${outvar} to the title of the current LLVM subproject (Clang, MLIR ...)
+#
+# The title is set in the subproject's top-level using the variable
+# LLVM_SUBPROJECT_TITLE. If it does not exist, it is assumed it is LLVM itself.
+# The title is not semantically significant, but use to create folders in
+# CMake-generated IDE projects (Visual Studio/XCode).
+function(get_subproject_title outvar)
+  if (LLVM_SUBPROJECT_TITLE)
+    set(${outvar} "${LLVM_SUBPROJECT_TITLE}" PARENT_SCOPE)
+  else ()
+    set(${outvar} "LLVM" PARENT_SCOPE)
+  endif ()
+endfunction(get_subproject_title)
diff --git a/compiler-rt/CMakeLists.txt b/compiler-rt/CMakeLists.txt
index 9e45a43b34287..95989bf20ed38 100644
--- a/compiler-rt/CMakeLists.txt
+++ b/compiler-rt/CMakeLists.txt
@@ -919,7 +919,7 @@ if(COMPILER_RT_INCLUDE_TESTS)
     # by specifying -DLLVM_EXTERNAL_LIT, but we generate it regardless.
     if (EXISTS ${LLVM_MAIN_SRC_DIR}/utils/llvm-lit)
       # Needed for lit support in standalone builds.
-      include(AddLLVM)
+      include(AddLit)
       add_subdirectory(${LLVM_MAIN_SRC_DIR}/utils/llvm-lit ${CMAKE_CURRENT_BINARY_DIR}/llvm-lit)
       # Ensure that the testsuite uses the local lit rather than
       # LLVM_INSTALL_DIR/bin/llvm-lit (which probably does not exist).
diff --git a/compiler-rt/cmake/Modules/CompilerRTMockLLVMCMakeConfig.cmake b/compiler-rt/cmake/Modules/CompilerRTMockLLVMCMakeConfig.cmake
index be14992b2944f..afce40f1e9077 100644
--- a/compiler-rt/cmake/Modules/CompilerRTMockLLVMCMakeConfig.cmake
+++ b/compiler-rt/cmake/Modules/CompilerRTMockLLVMCMakeConfig.cmake
@@ -76,5 +76,5 @@ endfunction()
 
 macro(compiler_rt_mock_llvm_cmake_config_include_cmake_files)
   # Some compiler-rt CMake code needs to call code in this file.
-  include("${LLVM_CMAKE_DIR}/AddLLVM.cmake")
+  include(AddLit)
 endmacro()
diff --git a/compiler-rt/cmake/config-ix.cmake b/compiler-rt/cmake/config-ix.cmake
index 3b86ba26f8a84..02ca4be011579 100644
--- a/compiler-rt/cmake/config-ix.cmake
+++ b/compiler-rt/cmake/config-ix.cmake
@@ -1,5 +1,4 @@
 include(CMakePushCheckState)
-include(AddLLVM)
 include(CheckCCompilerFlag)
 include(CheckCXXCompilerFlag)
 include(CheckIncludeFiles)
diff --git a/compiler-rt/test/CMakeLists.txt b/compiler-rt/test/CMakeLists.txt
index fa2847c1b7dd0..1e4bee6135e07 100644
--- a/compiler-rt/test/CMakeLists.txt
+++ b/compiler-rt/test/CMakeLists.txt
@@ -1,5 +1,5 @@
 # Needed for lit support in standalone builds.
-include(AddLLVM)
+include(AddLit)
 
 pythonize_bool(COMPILER_RT_TEST_STANDALONE_BUILD_LIBS)
 
diff --git a/compiler-rt/unittests/CMakeLists.txt b/compiler-rt/unittests/CMakeLists.txt
index 88f9865a6e7d7..b01354ca7655c 100644
--- a/compiler-rt/unittests/CMakeLists.txt
+++ b/compiler-rt/unittests/CMakeLists.txt
@@ -1,5 +1,5 @@
 # Needed for lit support in standalone builds.
-include(AddLLVM)
+include(AddLit)
 
 configure_compiler_rt_lit_site_cfg(
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.common.unit.configured.in
diff --git a/libclc/CMakeLists.txt b/libclc/CMakeLists.txt
index 5829bd9b39eac..66d5d89debb11 100644
--- a/libclc/CMakeLists.txt
+++ b/libclc/CMakeLists.txt
@@ -56,13 +56,7 @@ if( LIBCLC_STANDALONE_BUILD OR CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DI
   set( LIBCLC_STANDALONE_BUILD TRUE )
 
   find_package(LLVM REQUIRED HINTS "${LLVM_CMAKE_DIR}")
-  include(AddLLVM)
-
-  foreach( tool IN ITEMS llvm-link opt )
-    find_program( LLVM_TOOL_${tool} ${tool} PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH )
-    set( ${tool}_exe ${LLVM_TOOL_${tool}} )
-    set( ${tool}_target )
-  endforeach()
+  include(AddLit)
 
   # Setup the paths where libclc runtimes should be stored.
   set( LIBCLC_OUTPUT_LIBRARY_DIR ${CMAKE_CURRENT_BINARY_DIR} )
@@ -82,9 +76,6 @@ else()
     set( PACKAGE_VERSION "${LLVM_PACKAGE_VERSION}" )
   endif()
 
-  get_host_tool_path( llvm-link LLVM_LINK llvm-link_exe llvm-link_target )
-  get_host_tool_path( opt OPT opt_exe opt_target )
-
   # Setup the paths where libclc runtimes should be stored. By default, in an
   # in-tree build we place the libraries in clang's resource driectory.
   include(GetClangResourceDir)
@@ -95,16 +86,20 @@ else()
   )
 endif()
 
+foreach( tool IN ITEMS llvm-link opt )
+  find_program( LLVM_TOOL_${tool} ${tool} PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH )
+  if( NOT LLVM_TOOL_${tool} )
+    message( FATAL_ERROR "libclc toolchain incomplete - missing tool ${tool}!" )
+  endif()
+  set( ${tool}_exe ${LLVM_TOOL_${tool}} )
+endforeach()
+
 if( NOT LIBCLC_USE_SPIRV_BACKEND )
   # llvm-spirv is an optional dependency, used to build spirv-* targets when
   # the SPIR-V backend hasn't been requested. It may be provided in-tree or
   # externally.
-  if( TARGET llvm-spirv )
-    get_host_tool_path( llvm-spirv LLVM_SPIRV llvm-spirv_exe llvm-spirv_target )
-  else()
-    find_program( LLVM_SPIRV llvm-spirv HINTS ${LLVM_TOOLS_BINARY_DIR} )
-    set( llvm-spirv_exe "${LLVM_SPIRV}" )
-  endif()
+  find_program( LLVM_SPIRV llvm-spirv HINTS ${LLVM_TOOLS_BINARY_DIR} )
+  set( llvm-spirv_exe "${LLVM_SPIRV}" )
 endif()
 
 message(STATUS "libclc target '${LIBCLC_TARGET}' is enabled")
@@ -124,12 +119,6 @@ if(LIBCLC_TARGET_ARCH IN_LIST LIBCLC_ARCHS_SPIRV)
   endif()
 endif()
 
-foreach( tool IN ITEMS opt llvm-link )
-  if( NOT EXISTS "${${tool}_exe}" AND "${${tool}_target}" STREQUAL "" )
-    message( FATAL_ERROR "libclc toolchain incomplete - missing tool ${tool}!" )
-  endif()
-endforeach()
-
 # Determine the clang target triple.
 set(clang_triple ${LIBCLC_TARGET})
 
diff --git a/libcxx/test/CMakeLists.txt b/libcxx/test/CMakeLists.txt
index 8fd34c086a992..de50d6863c9e3 100644
--- a/libcxx/test/CMakeLists.txt
+++ b/libcxx/test/CMakeLists.txt
@@ -53,7 +53,7 @@ endif()
 
 serialize_lit_params_list(SERIALIZED_LIT_PARAMS LIBCXX_TEST_PARAMS)
 
-include(AddLLVM) # for configure_lit_site_cfg and add_lit_testsuite
+include(AddLit) # for configure_lit_site_cfg and add_lit_testsuite
 
 # The test suite is driven by a generated lit.site.cfg that loads the various other configuration
 # files needed by it. In order:
diff --git a/libcxxabi/test/CMakeLists.txt b/libcxxabi/test/CMakeLists.txt
index bc0902054a3be..7a6af2f3f3e69 100644
--- a/libcxxabi/test/CMakeLists.txt
+++ b/libcxxabi/test/CMakeLists.txt
@@ -1,4 +1,4 @@
-include(AddLLVM) # for configure_lit_site_cfg and add_lit_testsuite
+include(AddLit) # for configure_lit_site_cfg and add_lit_testsuite
 include(HandleLitArguments)
 macro(pythonize_bool var)
   if (${var})
diff --git a/libunwind/test/CMakeLists.txt b/libunwind/test/CMakeLists.txt
index 780be2f13b85c..fb1c8e456a1c1 100644
--- a/libunwind/test/CMakeLists.txt
+++ b/libunwind/test/CMakeLists.txt
@@ -1,4 +1,4 @@
-include(AddLLVM) # for add_lit_testsuite
+include(AddLit) # for add_lit_testsuite
 include(HandleLitArguments)
 macro(pythonize_bool var)
   if (${var})
diff --git a/llvm/cmake/modules/AddLLVM.cmake b/llvm/cmake/modules/AddLLVM.cmake
index a267166eb6c2d..20545b55220d1 100644
--- a/llvm/cmake/modules/AddLLVM.cmake
+++ b/llvm/cmake/modules/AddLLVM.cmake
@@ -3,21 +3,8 @@ include(LLVMDistributionSupport)
 include(LLVMProcessSources)
 include(LLVM-Config)
 include(DetermineGCCCompatible)
-
-# get_subproject_title(titlevar)
-#   Set ${outvar} to the title of the current LLVM subproject (Clang, MLIR ...)
-#
-# The title is set in the subproject's top-level using the variable
-# LLVM_SUBPROJECT_TITLE. If it does not exist, it is assumed it is LLVM itself.
-# The title is not semantically significant, but use to create folders in
-# CMake-generated IDE projects (Visual Studio/XCode).
-function(get_subproject_title outvar)
-  if (LLVM_SUBPROJECT_TITLE)
-    set(${outvar} "${LLVM_SUBPROJECT_TITLE}" PARENT_SCOPE)
-  else ()
-    set(${outvar} "LLVM" PARENT_SCOPE)
-  endif ()
-endfunction(get_subproject_title)
+include(GetSubprojectTitle)
+include(AddLit)
 
 # Determine required flags to enable/disable RTTI.
 set(LLVM_CXXFLAGS_RTTI_DISABLE "")
@@ -1967,431 +1954,6 @@ function(add_benchmark benchmark_name)
   target_link_libraries(${benchmark_name} PRIVATE benchmark)
 endfunction()
 
-# This function canonicalize the CMake variables passed by names
-# from CMake boolean to 0/1 suitable for passing into Python or C++,
-# in place.
-function(llvm_canonicalize_cmake_booleans)
-  foreach(var ${ARGN})
-    if(${var})
-      set(${var} 1 PARENT_SCOPE)
-    else()
-      set(${var} 0 PARENT_SCOPE)
-    endif()
-  endforeach()
-endfunction(llvm_canonicalize_cmake_booleans)
-
-macro(set_llvm_build_mode)
-  # Configuration-time: See Unit/lit.site.cfg.in
-  if (CMAKE_CFG_INTDIR STREQUAL ".")
-    set(LLVM_BUILD_MODE ".")
-  else ()
-    set(LLVM_BUILD_MODE "%(build_mode)s")
-  endif ()
-endmacro()
-
-# Takes a list of path names in pathlist and a base directory, and returns
-# a list of paths relative to the base directory in out_pathlist.
-# Paths that are on a different drive than the basedir (on Windows) or that
-# contain symlinks are returned absolute.
-# Use with LLVM_LIT_PATH_FUNCTION below.
-function(make_paths_relative out_pathlist basedir pathlist)
-  # Passing ARG_PATH_VALUES as-is to execute_process() makes cmake strip
-  # empty list entries. So escape the ;s in the list and do the splitting
-  # ourselves. cmake has no relpath function, so use Python for that.
-  string(REPLACE ";" "\\;" pathlist_escaped "${pathlist}")
-  execute_process(COMMAND "${Python3_EXECUTABLE}" "-c" "\n
-import os, sys\n
-base = sys.argv[1]
-def haslink(p):\n
-    if not p or p == os.path.dirname(p): return False\n
-    return os.path.islink(p) or haslink(os.path.dirname(p))\n
-def relpath(p):\n
-    if not p: return ''\n
-    if os.path.splitdrive(p)[0] != os.path.splitdrive(base)[0]: return p\n
-    if haslink(p) or haslink(base): return p\n
-    return os.path.relpath(p, base)\n
-if len(sys.argv) < 3: sys.exit(0)\n
-sys.stdout.write(';'.join(relpath(p) for p in sys.argv[2].split(';')))"
-    ${basedir}
-    ${pathlist_escaped}
-    OUTPUT_VARIABLE pathlist_relative
-    ERROR_VARIABLE error
-    RESULT_VARIABLE result)
-  if (NOT result EQUAL 0)
-    message(FATAL_ERROR "make_paths_relative() failed due to error '${result}', with stderr\n${error}")
-  endif()
-  set(${out_pathlist} "${pathlist_relative}" PARENT_SCOPE)
-endfunction()
-
-# Converts a file that's relative to the current python file to an absolute
-# path. Since this uses __file__, it has to be emitted into python files that
-# use it and can't be in a lit module. Use with make_paths_relative().
-string(CONCAT LLVM_LIT_PATH_FUNCTION
-  "# Allow generated file to be relocatable.\n"
-  "import os\n"
-  "import platform\n"
-  "def path(p):\n"
-  "    if not p: return ''\n"
-  "    # Follows lit.util.abs_path_preserve_drive, which cannot be imported here.\n"
-  "    if platform.system() == 'Windows':\n"
-  "        return os.path.abspath(os.path.join(os.path.dirname(__file__), p))\n"
-  "    else:\n"
-  "        return os.path.realpath(os.path.join(os.path.dirname(__file__), p))\n"
-  )
-
-# This function provides an automatic way to 'configure'-like generate a file
-# based on a set of common and custom variables, specifically targeting the
-# variables needed for the 'lit.site.cfg' files. This function bundles the
-# common variables that any Lit instance is likely to need, and custom
-# variables can be passed in.
-# The keyword PATHS is followed by a list of cmake variable names that are
-# mentioned as `path("@varname@")` in the lit.cfg.py.in file. Variables in that
-# list are treated as paths that are relative to the directory the generated
-# lit.cfg.py file is in, and the `path()` function converts the relative
-# path back to absolute form. This makes it possible to move a build directory
-# containing lit.cfg.py files from one machine to another.
-function(configure_lit_site_cfg site_in site_out)
-  cmake_parse_arguments(ARG "" "" "MAIN_CONFIG;PATHS" ${ARGN})
-
-  if ("${ARG_MAIN_CONFIG}" STREQUAL "")
-    get_filename_component(INPUT_DIR ${site_in} DIRECTORY)
-    set(ARG_MAIN_CONFIG "${INPUT_DIR}/lit.cfg")
-  endif()
-
-  foreach(c ${LLVM_TARGETS_TO_BUILD})
-    set(TARGETS_BUILT "${TARGETS_BUILT} ${c}")
-  endforeach(c)
-  set(TARGETS_TO_BUILD ${TARGETS_BUILT})
-
-  set(SHLIBEXT "${LTDL_SHLIB_EXT}")
-
-  set_llvm_build_mode()
-
-  # For standalone builds of subprojects, these might not be the build tree but
-  # a provided binary tree.
-  set(LLVM_SOURCE_DIR ${LLVM_MAIN_SRC_DIR})
-  set(LLVM_BINARY_DIR ${LLVM_BINARY_DIR})
-  string(REPLACE "${CMAKE_CFG_INTDIR}" "${LLVM_BUILD_MODE}" LLVM_TOOLS_DIR "${LLVM_TOOLS_BINARY_DIR}")
-  string(REPLACE "${CMAKE_CFG_INTDIR}" "${LLVM_BUILD_MODE}" LLVM_LIBS_DIR  "${LLVM_LIBRARY_DIR}")
-  # Like LLVM_{TOOLS,LIBS}_DIR, but pointing at the build tree.
-  string(REPLACE "${CMAKE_CFG_INTDIR}" "${LLVM_BUILD_MODE}" CURRENT_TOOLS_DIR "${LLVM_RUNTIME_OUTPUT_INTDIR}")
-  string(REPLACE "${CMAKE_CFG_INTDIR}" "${LLVM_BUILD_MODE}" CURRENT_LIBS_DIR  "${LLVM_LIBRARY_OUTPUT_INTDIR}")
-  string(REPLACE "${CMAKE_CFG_INTDIR}" "${LLVM_BUILD_MODE}" SHLIBDIR "${LLVM_SHLIB_OUTPUT_INTDIR}")
-
-  # FIXME: "ENABLE_SHARED" doesn't make sense, since it is used just for
-  # plugins. We may rename it.
-  if(LLVM_ENABLE_PLUGINS)
-    set(ENABLE_SHARED "1")
-  else()
-    set(ENABLE_SHARED "0")
-  endif()
-
-  if(LLVM_ENABLE_ASSERTIONS)
-    set(ENABLE_ASSERTIONS "1")
-  else()
-    set(ENABLE_ASSERTIONS "0")
-  endif()
-
-  set(HOST_OS ${CMAKE_SYSTEM_NAME})
-  set(HOST_ARCH ${CMAKE_SYSTEM_PROCESSOR})
-
-  set(HOST_CC "${CMAKE_C_COMPILER} ${CMAKE_C_COMPILER_ARG1}")
-  set(HOST_CXX "${CMAKE_CXX_COMPILER} ${CMAKE_CXX_COMPILER_ARG1}")
-  set(HOST_LDFLAGS "${CMAKE_EXE_LINKER_FLAGS}")
-
-  string(CONCAT LIT_SITE_CFG_IN_HEADER
-    "# Autogenerated from ${site_in}\n# Do not edit!\n\n"
-    "${LLVM_LIT_PATH_FUNCTION}"
-    )
-
-  # Override config_target_triple (and the env)
-  if(LLVM_TARGET_TRIPLE_ENV)
-    # This is expanded into the heading.
-    string(CONCAT LIT_SITE_CFG_IN_HEADER "${LIT_SITE_CFG_IN_HEADER}"
-      "import os\n"
-      "target_env = \"${LLVM_TARGET_TRIPLE_ENV}\"\n"
-      "config.target_triple = config.environment[target_env] = os.environ.get(target_env, \"${LLVM_TARGET_TRIPLE}\")\n"
-      )
-
-    # This is expanded to; config.target_triple = ""+config.target_triple+""
-    set(LLVM_TARGET_TRIPLE "\"+config.target_triple+\"")
-  endif()
-
-  if (ARG_PATHS)
-    # Walk ARG_PATHS and collect the current value of the variables in there.
-    # list(APPEND) ignores empty elements exactly if the list is empty,
-    # so start the list with a dummy element and drop it, to make sure that
-    # even empty values make it into the values list.
-    set(ARG_PATH_VALUES "dummy")
-    foreach(path ${ARG_PATHS})
-      list(APPEND ARG_PATH_VALUES "${${path}}")
-    endforeach()
-    list(REMOVE_AT ARG_PATH_VALUES 0)
-
-    get_filename_component(OUTPUT_DIR ${site_out} DIRECTORY)
-    make_paths_relative(
-        ARG_PATH_VALUES_RELATIVE "${OUTPUT_DIR}" "${ARG_PATH_VALUES}")
-
-    list(LENGTH ARG_PATHS len_paths)
-    list(LENGTH ARG_PATH_VALUES len_path_values)
-    list(LENGTH ARG_PATH_VALUES_RELATIVE len_path_value_rels)
-    if ((NOT ${len_paths} EQUAL ${len_path_values}) OR
-        (NOT ${len_paths} EQUAL ${len_path_value_rels}))
-      message(SEND_ERROR "PATHS lengths got confused")
-    endif()
-
-    # Transform variables mentioned in ARG_PATHS to relative paths for
-    # the configure_file() call. Variables are copied to subscopeds by cmake,
-    # so this only modifies the local copy of the variables.
-    math(EXPR arg_path_limit "${len_paths} - 1")
-    foreach(i RANGE ${arg_path_limit})
-      list(GET ARG_PATHS ${i} val1)
-      list(GET ARG_PATH_VALUES_RELATIVE ${i} val2)
-      set(${val1} ${val2})
-    endforeach()
-  endif()
-
-  configure_file(${site_in} ${site_out} @ONLY)
-
-  if (EXISTS "${ARG_MAIN_CONFIG}")
-    # Remember main config / generated site config for llvm-lit.in.
-    get_property(LLVM_LIT_CONFIG_FILES GLOBAL PROPERTY LLVM_LIT_CONFIG_FILES)
-    list(APPEND LLVM_LIT_CONFIG_FILES "${ARG_MAIN_CONFIG}" "${site_out}")
-    set_property(GLOBAL PROPERTY LLVM_LIT_CONFIG_FILES ${LLVM_LIT_CONFIG_FILES})
-  endif()
-endfunction()
-
-function(dump_all_cmake_variables)
-  get_cmake_property(_variableNames VARIABLES)
-  foreach (_variableName ${_variableNames})
-    message(STATUS "${_variableName}=${${_variableName}}")
-  endforeach()
-endfunction()
-
-function(get_llvm_lit_path base_dir file_name)
-  cmake_parse_arguments(ARG "ALLOW_EXTERNAL" "" "" ${ARGN})
-
-  if (ARG_ALLOW_EXTERNAL)
-    set (LLVM_EXTERNAL_LIT "" CACHE STRING "Command used to spawn lit")
-    if ("${LLVM_EXTERNAL_LIT}" STREQUAL "")
-      set(LLVM_EXTERNAL_LIT "${LLVM_DEFAULT_EXTERNAL_LIT}")
-    endif()
-
-    if (NOT "${LLVM_EXTERNAL_LIT}" STREQUAL "")
-      if (EXISTS ${LLVM_EXTERNAL_LIT})
-        get_filename_component(LIT_FILE_NAME ${LLVM_EXTERNAL_LIT} NAME)
-        get_filename_component(LIT_BASE_DIR ${LLVM_EXTERNAL_LIT} DIRECTORY)
-        set(${file_name} ${LIT_FILE_NAME} PARENT_SCOPE)
-        set(${base_dir} ${LIT_BASE_DIR} PARENT_SCOPE)
-        return()
-      elseif (NOT DEFINED CACHE{LLVM_EXTERNAL_LIT_MISSING_WARNED_ONCE})
-        message(WARNING "LLVM_EXTERNAL_LIT set to ${LLVM_EXTERNAL_LIT}, but the path does not exist.")
-        set(LLVM_EXTERNAL_LIT_MISSING_WARNED_ONCE YES CACHE INTERNAL "")
-      endif()
-    endif()
-  endif()
-
-  set(lit_file_name "llvm-lit")
-  if (CMAKE_HOST_WIN32 AND NOT CYGWIN)
-    # llvm-lit needs suffix.py for multiprocess to find a main module.
-    set(lit_file_name "${lit_file_name}.py")
-  endif ()
-  set(${file_name} ${lit_file_name} PARENT_SCOPE)
-
-  get_property(LLVM_LIT_BASE_DIR GLOBAL PROPERTY LLVM_LIT_BASE_DIR)
-  if (NOT "${LLVM_LIT_BASE_DIR}" STREQUAL "")
-    set(${base_dir} ${LLVM_LIT_BASE_DIR} PARENT_SCOPE)
-  endif()
-
-  # Allow individual projects to provide an override
-  if (NOT "${LLVM_LIT_OUTPUT_DIR}" STREQUAL "")
-    set(LLVM_LIT_BASE_DIR ${LLVM_LIT_OUTPUT_DIR})
-  elseif(NOT "${LLVM_RUNTIME_OUTPUT_INTDIR}" STREQUAL "")
-    set(LLVM_LIT_BASE_DIR ${LLVM_RUNTIME_OUTPUT_INTDIR})
-  else()
-    set(LLVM_LIT_BASE_DIR "")
-  endif()
-
-  # Cache this so we don't have to do it again and have subsequent calls
-  # potentially disagree on the value.
-  set_property(GLOBAL PROPERTY LLVM_LIT_BASE_DIR ${LLVM_LIT_BASE_DIR})
-  set(${base_dir} ${LLVM_LIT_BASE_DIR} PARENT_SCOPE)
-endfunction()
-
-# A raw function to create a lit target. This is used to implement the testuite
-# management functions.
-function(add_lit_target target comment)
-  cmake_parse_arguments(ARG "" "" "PARAMS;DEPENDS;ARGS" ${ARGN})
-  set(LIT_ARGS "${ARG_ARGS} ${LLVM_LIT_ARGS}")
-  separate_arguments(LIT_ARGS)
-  if (NOT CMAKE_CFG_INTDIR STREQUAL ".")
-    list(APPEND LIT_ARGS --param build_mode=${CMAKE_CFG_INTDIR})
-  endif ()
-
-  # Get the path to the lit to *run* tests with.  This can be overriden by
-  # the user by specifying -DLLVM_EXTERNAL_LIT=<path-to-lit.py>
-  get_llvm_lit_path(
-    lit_base_dir
-    lit_file_name
-    ALLOW_EXTERNAL
-    )
-
-  set(LIT_COMMAND "${Python3_EXECUTABLE};${lit_base_dir}/${lit_file_name}")
-  list(APPEND LIT_COMMAND ${LIT_ARGS})
-  foreach(param ${ARG_PARAMS})
-    list(APPEND LIT_COMMAND --param ${param})
-  endforeach()
-  if (ARG_UNPARSED_ARGUMENTS)
-    add_custom_target(${target}
-      COMMAND ${LIT_COMMAND} ${ARG_UNPARSED_ARGUMENTS}
-      COMMENT "${comment}"
-      USES_TERMINAL
-      )
-  else()
-    add_custom_target(${target}
-      COMMAND ${CMAKE_COMMAND} -E echo "${target} does nothing, no tools built.")
-    message(STATUS "${target} does nothing.")
-  endif()
-  get_subproject_title(subproject_title)
-  set_target_properties(${target} PROPERTIES FOLDER "${subproject_title}/Tests")
-
-  if (ARG_DEPENDS)
-    add_dependencies(${target} ${ARG_DEPENDS})
-  endif()
-
-  # Tests should be excluded from "Build Solution".
-  set_target_properties(${target} PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD ON)
-  set_target_properties(${target} PROPERTIES XCODE_GENERATE_SCHEME ON)
-endfunction()
-
-# Convert a target name like check-clang to a variable name like CLANG.
-function(umbrella_lit_testsuite_var target outvar)
-  if (NOT target MATCHES "^check-")
-    message(FATAL_ERROR "umbrella lit suites must be check-*, not '${target}'")
-  endif()
-  string(SUBSTRING "${target}" 6 -1 var)
-  string(REPLACE "-" "_" var ${var})
-  string(TOUPPER "${var}" var)
-  set(${outvar} "${var}" PARENT_SCOPE)
-endfunction()
-
-# Start recording all lit test suites for a combined 'check-foo' target.
-# The recording continues until umbrella_lit_testsuite_end() creates the target.
-function(umbrella_lit_testsuite_begin target)
-  umbrella_lit_testsuite_var(${target} name)
-  set_property(GLOBAL APPEND PROPERTY LLVM_LIT_UMBRELLAS ${name})
-endfunction()
-
-# Create a combined 'check-foo' target for a set of related test suites.
-# It runs all suites added since the matching umbrella_lit_testsuite_end() call.
-# Tests marked EXCLUDE_FROM_CHECK_ALL are not gathered.
-function(umbrella_lit_testsuite_end target)
-  umbrella_lit_testsuite_var(${target} name)
-
-  get_property(testsuites GLOBAL PROPERTY LLVM_${name}_LIT_TESTSUITES)
-  get_property(params GLOBAL PROPERTY LLVM_${name}_LIT_PARAMS)
-  get_property(depends GLOBAL PROPERTY LLVM_${name}_LIT_DEPENDS)
-  get_property(extra_args GLOBAL PROPERTY LLVM_${name}_LIT_EXTRA_ARGS)
-  # Additional test targets are not gathered, but may be set externally.
-  get_property(additional_test_targets
-               GLOBAL PROPERTY LLVM_${name}_ADDITIONAL_TEST_TARGETS)
-
-  string(TOLOWER ${name} name)
-  add_lit_target(${target}
-    "Running ${name} regression tests"
-    ${testsuites}
-    PARAMS ${params}
-    DEPENDS ${depends} ${additional_test_targets}
-    ARGS ${extra_args}
-    )
-endfunction()
-
-# A function to add a set of lit test suites to be driven through 'check-*' targets.
-function(add_lit_testsuite target comment)
-  cmake_parse_arguments(ARG "EXCLUDE_FROM_CHECK_ALL" "" "PARAMS;DEPENDS;ARGS" ${ARGN})
-
-  # EXCLUDE_FROM_ALL excludes the test ${target} out of check-all.
-  if(NOT ARG_EXCLUDE_FROM_CHECK_ALL)
-    get_property(gather_names GLOBAL PROPERTY LLVM_LIT_UMBRELLAS)
-    foreach(name ${gather_names})
-    # Register the testsuites, params and depends for the umbrella check rule.
-      set_property(GLOBAL APPEND PROPERTY LLVM_${name}_LIT_TESTSUITES ${ARG_UNPARSED_ARGUMENTS})
-      set_property(GLOBAL APPEND PROPERTY LLVM_${name}_LIT_PARAMS ${ARG_PARAMS})
-      set_property(GLOBAL APPEND PROPERTY LLVM_${name}_LIT_DEPENDS ${ARG_DEPENDS})
-      set_property(GLOBAL APPEND PROPERTY LLVM_${name}_LIT_EXTRA_ARGS ${ARG_ARGS})
-    endforeach()
-  endif()
-
-  # Produce a specific suffixed check rule.
-  add_lit_target(${target} ${comment}
-    ${ARG_UNPARSED_ARGUMENTS}
-    PARAMS ${ARG_PARAMS}
-    DEPENDS ${ARG_DEPENDS}
-    ARGS ${ARG_ARGS}
-    )
-endfunction()
-
-function(add_lit_testsuites project directory)
-  if (NOT LLVM_ENABLE_IDE)
-    cmake_parse_arguments(ARG
-      "EXCLUDE_FROM_CHECK_ALL"
-      "FOLDER;BINARY_DIR"
-      "PARAMS;DEPENDS;ARGS;SKIP"
-      ${ARGN}
-      )
-
-    if (NOT ARG_FOLDER)
-      get_subproject_title(subproject_title)
-      set(ARG_FOLDER "${subproject_title}/Tests/LIT Testsuites")
-    endif()
-
-    # Search recursively for test directories by assuming anything not
-    # in a directory called Inputs contains tests.
-    file(GLOB_RECURSE to_process LIST_DIRECTORIES true ${directory}/*)
-    foreach(lit_suite ${to_process})
-      if(NOT IS_DIRECTORY ${lit_suite})
-        continue()
-      endif()
-      string(FIND ${lit_suite} Inputs is_inputs)
-      string(FIND ${lit_suite} Output is_output)
-      if (NOT (is_inputs EQUAL -1 AND is_output EQUAL -1))
-        continue()
-      endif()
-
-      # Create a check- target for the directory.
-      string(REPLACE "${directory}/" "" name_slash ${lit_suite})
-      set(_should_skip FALSE)
-      foreach(skip IN LISTS ARG_SKIP)
-        if(name_slash MATCHES "${skip}")
-          set(_should_skip TRUE)
-          break()
-        endif()
-      endforeach()
-      if (_should_skip)
-        continue()
-      endif()
-      if (name_slash)
-        set(filter ${name_slash})
-        string(REPLACE "/" "-" name_slash ${name_slash})
-        string(REPLACE "\\" "-" name_dashes ${name_slash})
-        string(TOLOWER "${project}-${name_dashes}" name_var)
-        set(lit_args ${lit_suite})
-        if (ARG_BINARY_DIR)
-          set(lit_args ${ARG_BINARY_DIR} --filter=${filter})
-        endif()
-        add_lit_target("check-${name_var}" "Running lit suite ${lit_suite}"
-          ${lit_args}
-          ${EXCLUDE_FROM_CHECK_ALL}
-          PARAMS ${ARG_PARAMS}
-          DEPENDS ${ARG_DEPENDS}
-          ARGS ${ARG_ARGS}
-        )
-        set_target_properties(check-${name_var} PROPERTIES FOLDER ${ARG_FOLDER})
-      endif()
-    endforeach()
-  endif()
-endfunction()
-
 function(llvm_install_library_symlink name dest type)
   cmake_parse_arguments(ARG "FULL_DEST" "COMPONENT" "" ${ARGN})
   foreach(path ${CMAKE_MODULE_PATH})
diff --git a/llvm/cmake/modules/AddOCaml.cmake b/llvm/cmake/modules/AddOCaml.cmake
index 9a393ae3b7474..ef4171cdbd8d0 100644
--- a/llvm/cmake/modules/AddOCaml.cmake
+++ b/llvm/cmake/modules/AddOCaml.cmake
@@ -1,4 +1,5 @@
 # CMake build rules for the OCaml language.
+include(GetSubprojectTitle)
 # Assumes FindOCaml is used.
 # http://ocaml.org/
 #
diff --git a/llvm/cmake/modules/AddSphinxTarget.cmake b/llvm/cmake/modules/AddSphinxTarget.cmake
index 02a869b7941a9..55e8d556347e0 100644
--- a/llvm/cmake/modules/AddSphinxTarget.cmake
+++ b/llvm/cmake/modules/AddSphinxTarget.cmake
@@ -1,4 +1,5 @@
 include(GNUInstallDirs)
+include(GetSubprojectTitle)
 
 # Create sphinx target
 if (LLVM_ENABLE_SPHINX)
diff --git a/llvm/cmake/modules/CMakeLists.txt b/llvm/cmake/modules/CMakeLists.txt
index ef4cfa3acdb59..d9c8f0c69188c 100644
--- a/llvm/cmake/modules/CMakeLists.txt
+++ b/llvm/cmake/modules/CMakeLists.txt
@@ -113,6 +113,12 @@ file(COPY .
   PATTERN CMakeFiles EXCLUDE
   PATTERN llvm-driver-template.cpp.in
   )
+file(COPY
+  ${LLVM_COMMON_CMAKE_UTILS}/Modules/AddLit.cmake
+  ${LLVM_COMMON_CMAKE_UTILS}/Modules/GetSubprojectTitle.cmake
+  DESTINATION ${llvm_cmake_builddir}
+  NO_SOURCE_PERMISSIONS
+  )
 
 #
 # Generate LLVMConfig.cmake for the install tree.
@@ -170,6 +176,8 @@ if (NOT LLVM_INSTALL_TOOLCHAIN_ONLY)
     ${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/LLVMConfig.cmake
     ${llvm_cmake_builddir}/LLVMConfigVersion.cmake
     LLVM-Config.cmake
+    ${LLVM_COMMON_CMAKE_UTILS}/Modules/AddLit.cmake
+    ${LLVM_COMMON_CMAKE_UTILS}/Modules/GetSubprojectTitle.cmake
     DESTINATION ${LLVM_INSTALL_PACKAGE_DIR}
     COMPONENT cmake-exports)
 
diff --git a/llvm/cmake/modules/CrossCompile.cmake b/llvm/cmake/modules/CrossCompile.cmake
index 0a0be5336c74c..ae6ae68bbbea4 100644
--- a/llvm/cmake/modules/CrossCompile.cmake
+++ b/llvm/cmake/modules/CrossCompile.cmake
@@ -1,4 +1,5 @@
 include(AddLLVM)
+include(GetSubprojectTitle)
 include(LLVMExternalProjectUtils)
 
 
diff --git a/llvm/cmake/modules/LLVMDistributionSupport.cmake b/llvm/cmake/modules/LLVMDistributionSupport.cmake
index 03c6773570231..d0dad77932658 100644
--- a/llvm/cmake/modules/LLVMDistributionSupport.cmake
+++ b/llvm/cmake/modules/LLVMDistributionSupport.cmake
@@ -1,5 +1,6 @@
 # Utility functions for packaging an LLVM distribution. See the
 # BuildingADistribution documentation for more details.
+include(GetSubprojectTitle)
 
 # These functions assume a number of conventions that are common across all LLVM
 # subprojects:
diff --git a/llvm/cmake/modules/TableGen.cmake b/llvm/cmake/modules/TableGen.cmake
index 397089a2d3efe..2b85ccf083f73 100644
--- a/llvm/cmake/modules/TableGen.cmake
+++ b/llvm/cmake/modules/TableGen.cmake
@@ -3,6 +3,7 @@
 # Extra parameters for `tblgen' may come after `ofn' parameter.
 # Adds the name of the generated file to TABLEGEN_OUTPUT.
 include(LLVMDistributionSupport)
+include(GetSubprojectTitle)
 
 function(tablegen project ofn)
   cmake_parse_arguments(ARG "" "" "DEPENDS;EXTRA_INCLUDES" ${ARGN})
diff --git a/llvm/runtimes/CMakeLists.txt b/llvm/runtimes/CMakeLists.txt
index c2607393d1bb9..ca1fcb7aa8f50 100644
--- a/llvm/runtimes/CMakeLists.txt
+++ b/llvm/runtimes/CMakeLists.txt
@@ -636,6 +636,13 @@ if(build_runtimes)
       endif()
     endif()
   endif()
+  if("libclc" IN_LIST LLVM_ENABLE_RUNTIMES)
+    foreach(dep opt llvm-link clang)
+      if(TARGET ${dep})
+        list(APPEND extra_deps ${dep})
+      endif()
+    endforeach()
+  endif()
   if("libsycl" IN_LIST LLVM_ENABLE_RUNTIMES)
     foreach(dep clang-linker-wrapper clang-sycl-linker clang llvm-offload-binary)
       if(TARGET ${dep})
diff --git a/runtimes/CMakeLists.txt b/runtimes/CMakeLists.txt
index 61cf25af22870..3c5fcfbbbdb64 100644
--- a/runtimes/CMakeLists.txt
+++ b/runtimes/CMakeLists.txt
@@ -223,7 +223,7 @@ endif()
 set(LLVM_COMPILER_CHECKED ON)
 
 # Handle common options used by all runtimes.
-include(AddLLVM)
+include(AddLit)
 include(HandleLLVMOptions)
 
 # Loot at the PATH first to avoid a version mismatch between the command-line

>From 7c1b3e2908bd5fdae59aa9e0d8a9da2f0297eb5e Mon Sep 17 00:00:00 2001
From: Petr Hosek <phosek at google.com>
Date: Sun, 23 Aug 2026 22:30:34 +0000
Subject: [PATCH 2/3] Address review feedback and builder failures

---
 libclc/CMakeLists.txt                |   2 +-
 libclc/cmake/modules/AddLibclc.cmake |   4 +-
 third-party/unittest/CMakeLists.txt  | 135 +++++++++++++--------------
 3 files changed, 70 insertions(+), 71 deletions(-)

diff --git a/libclc/CMakeLists.txt b/libclc/CMakeLists.txt
index 66d5d89debb11..4aa677abcc93a 100644
--- a/libclc/CMakeLists.txt
+++ b/libclc/CMakeLists.txt
@@ -87,7 +87,7 @@ else()
 endif()
 
 foreach( tool IN ITEMS llvm-link opt )
-  find_program( LLVM_TOOL_${tool} ${tool} PATHS ${LLVM_TOOLS_BINARY_DIR} NO_DEFAULT_PATH )
+  find_program( LLVM_TOOL_${tool} ${tool} PATHS ${LLVM_NATIVE_TOOL_DIR} NO_DEFAULT_PATH )
   if( NOT LLVM_TOOL_${tool} )
     message( FATAL_ERROR "libclc toolchain incomplete - missing tool ${tool}!" )
   endif()
diff --git a/libclc/cmake/modules/AddLibclc.cmake b/libclc/cmake/modules/AddLibclc.cmake
index d84101f71d2f9..0eb3648aac5da 100644
--- a/libclc/cmake/modules/AddLibclc.cmake
+++ b/libclc/cmake/modules/AddLibclc.cmake
@@ -128,7 +128,7 @@ function(libclc_add_library target_name)
   # Link the object files, using a temporary library for internalization.
   set(linked_bc ${CMAKE_CURRENT_BINARY_DIR}/${target_name}.linked.bc)
   set(link_cmd ${llvm-link_exe})
-  set(link_deps ${llvm-link_target})
+  set(link_deps "")
 
   add_library(${target_name}-src STATIC)
   target_link_libraries(${target_name}-src PRIVATE ${ARG_SOURCE_TARGET})
@@ -183,7 +183,7 @@ function(libclc_add_library target_name)
     set(builtins_lib ${library_dir}/${ARG_OUTPUT_FILENAME}.bc)
     add_custom_command(OUTPUT ${builtins_lib}
       COMMAND ${opt_exe} ${ARG_OPT_FLAGS} -o ${builtins_lib} ${linked_bc}
-      DEPENDS ${opt_target} ${linked_bc}
+      DEPENDS ${linked_bc}
     )
   endif()
 
diff --git a/third-party/unittest/CMakeLists.txt b/third-party/unittest/CMakeLists.txt
index 79535a1de4616..51fa8bdcfa105 100644
--- a/third-party/unittest/CMakeLists.txt
+++ b/third-party/unittest/CMakeLists.txt
@@ -13,32 +13,6 @@
 
 set(LLVM_SUBPROJECT_TITLE "Third-Party/Google Test")
 
-if(LLVM_RUNTIMES_BUILD)
-  # This instance of GTest is use for unittests for the runtime libraries. It
-  # must not link to LLVMSupport (used for llvm::raw_ostream and llvm::cl
-  # support) of the host build: It may be a different architecture and/or
-  # using a different C++ ABI such as libcxx/libstdc++.
-  set(GTEST_LLVM_COMPONENTS "")
-
-  # We cannot use llvm_gtest for the target; it would clash with
-  # find_package(LLVM) with LLVM_EXPORT_GTEST=ON. Instead, we define an alias
-  # default_gtest that points to llvm_gtest in the LLVM build and
-  # runtimes_gtest in an runtimes build.
-  set(gtest_name "runtimes_gtest")
-
-  # Override locally; never install the runtimes-GTest.
-  set(LLVM_INSTALL_GTEST OFF)
-
-  # Build the library containing main() so unittests need less boilerplate.
-  # UnitTestMain/TestMain.cpp always needs LLVMSupport, use GTest's original
-  # main when unavailable.
-  set(gtest_main_src googletest/src/gtest_main.cc)
-else()
-  set(GTEST_LLVM_COMPONENTS "Support")
-  set(gtest_name "llvm_gtest")
-  set(gtest_main_src UnitTestMain/TestMain.cpp)
-endif()
-
 if(WIN32)
   add_definitions(-DGTEST_OS_WINDOWS=1)
 endif()
@@ -72,44 +46,51 @@ if (LLVM_INSTALL_GTEST)
   set(BUILDTREE_ONLY "")
 endif ()
 
-add_llvm_library("${gtest_name}"
-  googletest/src/gtest-all.cc
-  googlemock/src/gmock-all.cc
+if (LLVM_RUNTIMES_BUILD)
+  # We cannot use llvm_gtest for the target; it would clash with
+  # find_package(LLVM) with LLVM_EXPORT_GTEST=ON. Instead, we define an alias
+  # default_gtest that points to llvm_gtest in the LLVM build and
+  # runtimes_gtest in an runtimes build.
+  set(gtest_name "runtimes_gtest")
 
-  LINK_LIBS
-  ${LIBS}
+  # This instance of GTest is use for unittests for the runtime libraries. It
+  # must not link to LLVMSupport (used for llvm::raw_ostream and llvm::cl
+  # support) of the host build: It may be a different architecture and/or
+  # using a different C++ ABI such as libcxx/libstdc++.
+  add_library("${gtest_name}" STATIC
+    googletest/src/gtest-all.cc
+    googlemock/src/gmock-all.cc
+  )
+  target_compile_definitions("${gtest_name}" PUBLIC GTEST_NO_LLVM_SUPPORT=1)
+  target_link_libraries("${gtest_name}"
+    PUBLIC # STATIC -OR- PRIVATE for SHARED
+      ${LIBS}
+  )
 
-  LINK_COMPONENTS
-  ${GTEST_LLVM_COMPONENTS}
+  add_library("${gtest_name}_main" STATIC
+    googletest/src/gtest_main.cc
+  )
+  target_link_libraries("${gtest_name}_main"
+    PUBLIC
+      ${gtest_name}
+  )
+else()
+  set(gtest_name "llvm_gtest")
 
-  # This is a library meant only for the build tree.
-  ${BUILDTREE_ONLY}
-)
+  add_llvm_library("${gtest_name}"
+    googletest/src/gtest-all.cc
+    googlemock/src/gmock-all.cc
 
-# The googletest and googlemock sources don't presently use the 'override'
-# keyword, which leads to lots of warnings from -Wsuggest-override. Disable
-# that warning here for any targets that link to gtest.
-if(CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
-  add_definitions("-Wno-suggest-override")
-  set_target_properties("${gtest_name}" PROPERTIES INTERFACE_COMPILE_OPTIONS "-Wno-suggest-override")
-endif()
+    LINK_LIBS
+    ${LIBS}
 
-if (NOT LLVM_ENABLE_THREADS)
-  target_compile_definitions("${gtest_name}" PUBLIC GTEST_HAS_PTHREAD=0)
-endif ()
+    LINK_COMPONENTS
+    Support
 
-target_include_directories("${gtest_name}"
-  PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/googletest/include>
-         $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/googlemock/include>
-         $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}/>
-         $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}/llvm-gtest/>
-         $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}/llvm-gmock/>
-  PRIVATE googletest googlemock
+    # This is a library meant only for the build tree.
+    ${BUILDTREE_ONLY}
   )
 
-if(LLVM_RUNTIMES_BUILD)
-  target_compile_definitions("${gtest_name}" PUBLIC GTEST_NO_LLVM_SUPPORT=1)
-else()
   # When used from the buildtree, also force use of buildtree LLVM headers,
   # (instead locally installed version)
   # FIXME: Shouldn't this be done for all LLVM libraries? Currently, LLVM uses a
@@ -118,27 +99,45 @@ else()
   target_include_directories("${gtest_name}" BEFORE
     PUBLIC $<BUILD_INTERFACE:${LLVM_SOURCE_DIR}/include>
            $<BUILD_INTERFACE:${LLVM_BINARY_DIR}/include>
-    )
-endif()
-
+  )
 
-add_llvm_library("${gtest_name}_main"
-  ${gtest_main_src}
+  add_llvm_library("${gtest_name}_main"
+    UnitTestMain/TestMain.cpp
 
-  LINK_LIBS
-  "${gtest_name}"
+    LINK_LIBS
+    "${gtest_name}"
 
-  LINK_COMPONENTS
-  ${GTEST_LLVM_COMPONENTS}
+    LINK_COMPONENTS
+    Support
 
-  ${BUILDTREE_ONLY}
-)
+    ${BUILDTREE_ONLY}
+  )
 
-if (LLVM_INSTALL_GTEST)
   install(DIRECTORY googletest/include/gtest/ DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/llvm-gtest/gtest/" COMPONENT "${gtest_name}")
   install(DIRECTORY googlemock/include/gmock/ DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/llvm-gmock/gmock/" COMPONENT "${gtest_name}")
 endif()
 
+# The googletest and googlemock sources don't presently use the 'override'
+# keyword, which leads to lots of warnings from -Wsuggest-override. Disable
+# that warning here for any targets that link to gtest.
+if(CXX_SUPPORTS_SUGGEST_OVERRIDE_FLAG)
+  add_definitions("-Wno-suggest-override")
+  set_target_properties("${gtest_name}" PROPERTIES INTERFACE_COMPILE_OPTIONS "-Wno-suggest-override")
+endif()
+
+if (NOT LLVM_ENABLE_THREADS)
+  target_compile_definitions("${gtest_name}" PUBLIC GTEST_HAS_PTHREAD=0)
+endif ()
+
+target_include_directories("${gtest_name}"
+  PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/googletest/include>
+         $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/googlemock/include>
+         $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}/>
+         $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}/llvm-gtest/>
+         $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}/llvm-gmock/>
+  PRIVATE googletest googlemock
+  )
+
 # When LLVM_LINK_LLVM_DYLIB is enabled, libLLVM.so is added to the interface
 # link libraries for gtest and gtest_main.  This means that any target, like
 # unittests for example, that links against gtest will be forced to link

>From 9af9c8443186762afab94e1e711aba258eeb51c8 Mon Sep 17 00:00:00 2001
From: Petr Hosek <phosek at google.com>
Date: Sun, 23 Aug 2026 23:44:19 +0000
Subject: [PATCH 3/3] Drop LLVM_EXPORTS from openmp runtime

---
 openmp/runtime/src/CMakeLists.txt | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/openmp/runtime/src/CMakeLists.txt b/openmp/runtime/src/CMakeLists.txt
index 69def2b03b1b4..9f801af7e6e21 100644
--- a/openmp/runtime/src/CMakeLists.txt
+++ b/openmp/runtime/src/CMakeLists.txt
@@ -405,11 +405,6 @@ if(NOT WIN32 AND NOT APPLE)
 endif()
 add_dependencies(libomp-micro-tests libomp-test-deps)
 
-# `omp` needs to be exported if in-tree build.
-set(export_to_llvmexports)
-get_target_export_arg(omp LLVM export_to_llvmexports)
-set_property(GLOBAL APPEND PROPERTY LLVM_EXPORTS omp)
-
 # Install rules
 # We want to install libomp in ${DESTDIR}/${CMAKE_INSTALL_FULL_LIBDIR}
 # We want to install headers in ${DESTDIR}/${CMAKE_INSTALL_FULL_INCLUDEDIR}


